From 919c9a80fb39c8b0727eb5b14baf25075c0d0cf7 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 12 Nov 2026 20:17:04 -0700 Subject: [PATCH 01/16] test(oracle-gaps): law 15 owes a stream log that the first line names The maintainer's decision of 2026-11-12: an agent that runs a check in the background learns where to read its stream from the run itself. A run that starts a check names `/stream.log` in its first line on both streams (`oracle-gaps: note log `), and that file holds the stderr's lines, in order, cold, through the daemon, and after a stop or an abort. The driver records each run's first lines and whether its log equals its stderr; two calibrations give a log that misses a line and a naming line after another line. Co-authored-by: Isaac --- .../gap-report-oracle.property.test.mjs | 84 ++++++++++++++++++- packages/oracles/scripts/e2e-shards.mjs | 3 + 2 files changed, 83 insertions(+), 4 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index f3f3a63..1328130 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -330,7 +330,12 @@ * The run's last line is its done line: `completed, gaps[ * (provisional)], report ` on stdout, `refused[, report * ]` after a refusal's block, or `stopped ()` - * after a stop. A check through the daemon streams the same lines. + * after a stop. A check through the daemon streams the same lines. The + * run's first line on both streams names its stream log, `oracle-gaps: + * note log /stream.log`, and that file holds the stderr's lines, in + * order, cold and through the daemon (the maintainer's decision of + * 2026-11-12: an agent that runs a check in the background learns from + * the run itself where to read it). * * Authority: the maintainer's decisions in the gap-check design runs v11, v14, * and v19 (2026-11-04 and 2026-11-05; the design notes are kept by the project @@ -699,7 +704,7 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { writeFixture(dir, files) const before = new Set(filesUnder(dir)) const pids = mkdtempSync(join(realpathSync(tmpdir()), `gap-stop-`)) - const { args } = cliArgs(dir, files, { + const { args, out } = cliArgs(dir, files, { ...how, outName: `out-stop`, contexts: 1, @@ -809,6 +814,16 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { [dir, ``], ]), ), + outDir: namedScratch(out, [ + [how.cli ?? here, ``], + [dir, ``], + ]), + log: streamLogOf(out, stderr, stdout, (text) => + namedScratch(text, [ + [how.cli ?? here, ``], + [dir, ``], + ]), + ), // The session directories in the package once it has ended: a stopped // run leaves its own, and the next run removes it. sessions: readdirSync(dir).filter((name) => @@ -950,6 +965,31 @@ const streamedGap = * or a heartbeat that names a mutant came before the first verdict; * - `last`: the last line, which ends a run that did not complete. */ +/** + * What the run's stream log says (the maintainer's decision of 2026-11-12: + * the CLI writes its stream to `/stream.log`, and its first line on + * both streams names that log), as relations: the first line of each + * stream, with scratch paths named, whether the log exists, and whether its + * lines are the stderr's lines, in order. + */ +function streamLogOf(out, stderr, stdout, name) { + const lines = (text) => `${text ?? ``}`.split(`\n`).filter(Boolean) + let log + try { + log = readFileSync(join(out, `stream.log`), `utf8`) + } catch { + // No log: the run wrote none. + } + return { + stderrFirst: name(lines(stderr)[0] ?? ``), + stdoutFirst: name(lines(stdout)[0] ?? ``), + exists: log !== undefined, + equal: + log !== undefined && + JSON.stringify(lines(log)) === JSON.stringify(lines(stderr)), + } +} + function streamOf(lines, name) { const kinds = new RegExp(`^oracle-gaps: (?:${streamKinds.join(`|`)}) `) const strays = new Set() @@ -1076,6 +1116,10 @@ function readRun(dir, out, status, stderr, cli, stdout) { // The output directory the run was given, as the trace names it: the // final line names its report there. outDir: namedScratch(out, scratch), + // The run's stream log, and the first line that names it. + log: streamLogOf(out, stderr, stdout, (text) => + namedScratch(text, scratch), + ), // Whether the run captured its tests' own console output there. captured: (() => { try { @@ -1297,6 +1341,9 @@ async function incremental(dir, plan, cli, copy, observe, states) { // The stream through the daemon (law 15): both streams' lines. stream: run.stream, stdoutStream: run.stdoutStream, + // Its stream log, and the output directory it names. + log: run.log, + outDir: run.outDir, // The first line of a failure, for the message; readRun has named // the scratch package and CLI copy in it. stderr: run.stderr?.split(`\n`)[0], @@ -2490,6 +2537,19 @@ function endsBreach(run) { [`stderr`, run.stream], [`stdout`, run.stdoutStream], ] + // The run's first line on each stream names its stream log, and the log + // holds the stderr's lines, in order (the maintainer's decision of + // 2026-11-12), for every run that starts a check: each of these does. + const note = `oracle-gaps: note log ${run.outDir}/stream.log` + if (run.log.stderrFirst !== note || run.log.stdoutFirst !== note) + return { + difference: `the run's first lines are ${JSON.stringify([run.log.stderrFirst, run.log.stdoutFirst])}, not ${JSON.stringify(note)} on each stream`, + } + if (!run.log.exists) return { difference: `the run wrote no stream.log` } + if (!run.log.equal) + return { + difference: `the run's stream.log is not its stderr, line for line`, + } for (const [where, stream] of streams) if (stream.strays.length > 0) return { @@ -2954,6 +3014,22 @@ calibrations.push( }, ), }, + { + name: `the stream log misses a line`, + law: `greppable stream`, + corrupt: corruptCli((cli) => ({ + ...cli, + log: { ...cli.log, equal: false }, + })), + }, + { + name: `the line that names the stream log comes after another line`, + law: `greppable stream`, + corrupt: corruptCli((cli) => ({ + ...cli, + log: { ...cli.log, stderrFirst: `oracle-gaps: config vite.config.ts` }, + })), + }, { name: `a stopped run ends without its done line`, law: `greppable stream`, @@ -3246,8 +3322,8 @@ const withDeclaration = (constructed) => oracleProperty({ id: `oracle-gaps.report`, law: { - statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds the runs that confirm a mutant's cut-off copies by one budget per mutant and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, - authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10); the maintainer's request for a greppable stream (2026-11-12, relayed by the coordinator with its decisions of that day: the same prefixes on stdout and stderr, a done line for a stopped run, and no --max-gaps).`, + statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds the runs that confirm a mutant's cut-off copies by one budget per mutant and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, a first line on both streams that names the stream log in the output directory, which holds the stderr's lines in order, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, + authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10); the maintainer's request for a greppable stream (2026-11-12, relayed by the coordinator with its decisions of that day: the same prefixes on stdout and stderr, a done line for a stopped run, and no --max-gaps); the maintainer's decision of 2026-11-12 that the CLI writes its stream to the output directory's stream.log and names it in its first line.`, limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come only between checks, so a file saved during a check is not generated (the runner invalidates a known input that changed while a run ran and does not store that run, which only inspection fixes; a witness needs a step that edits a file between a unit's start and its end), and a module that a run first loads and that changes during that same run is stored under its later hash; law 9's edits leave out files read through fs, other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export.`, }, histories: { diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index 986c404..9361d23 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -167,6 +167,9 @@ export const shards = { // 79 s and 70 s; the daemon-notes one, bound to its history, 2 s (it // took 433 s searching for one). `calibration: a streamed line has no prefix`, + // The stream log's calibrations, not yet measured. + `calibration: the stream log misses a line`, + `calibration: the line that names the stream log comes after another line`, `calibration: a refusal through the daemon drops the cold run's notes`, // Bound to its history (re-review N2), not yet measured. `calibration: a run names a test-output log it never captured`, From 17ea63427021a076d78fcb48d6f0423a750e1c70 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 07:24:39 -0700 Subject: [PATCH 02/16] test(oracle-gaps): a refusal after a check started names its stream log first The maintainer's decision of 2026-11-12 names the stream log first in every run that starts a check, refusals in it included. The refusal law (#46) owed stdout = the done line alone; it now owes [note log, done line] when the check started, and [done line] for a refusal of the run's arguments, which comes before any check (outsideSource, and the static oracle's argument refusals). Law 15's log checks apply to runs that start a check. Co-authored-by: Isaac --- .../oracle-gaps/gap-report-model-oracle.mjs | 27 ++++++++---- .../gap-report-oracle.property.test.mjs | 41 ++++++++++++------- 2 files changed, 46 insertions(+), 22 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 1958c9f..063cd3b 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -3815,7 +3815,11 @@ function refusal(plan) { if (plan.fault === `badEntry`) return { cause: `${SOURCE}#nope` } if (plan.fault === `missingSource`) return { cause: `src/missing.ts` } if (plan.fault === `nonLiteral`) return { cause: `b.test.ts` } - if (plan.fault === `outsideSource`) return { cause: `outside the package` } + // A `--source` outside the package is an argument the CLI refuses before + // it starts a check (the argument rules that the static oracle judges), + // so its stdout has no line that names a stream log. + if (plan.fault === `outsideSource`) + return { cause: `outside the package`, arguments: true } // The analysis reads the test files that the package's Vitest config // includes (index.mjs, step 2), so a config that does not load is named // before anything the analysis would find. The cause names the config @@ -5174,8 +5178,13 @@ function endingOf(exit, stderr, out, name = (text) => text, stdout = ``) { * The first way a run's ending breaks the refusal law, or undefined. The * cause must name `cause`; with `outKnown`, the run knew its output * directory, so the block names it and the directory holds the refusal. + * `logNote` is the line that names the run's stream log, owed first on + * stdout when the run refused after it started a check, and undefined for + * a refusal of its arguments, before any check (the maintainer's decision + * of 2026-11-12 that every run that starts a check names its stream log + * first). */ -function refusalBreach(ending, { cause, outKnown }) { +function refusalBreach(ending, { cause, outKnown, logNote }) { if (ending.exit !== 2) return `the refusal exited ${ending.exit}, not 2` if (ending.stackFrames) return `the refusal printed a stack trace` if (ending.cause === undefined) @@ -5194,12 +5203,14 @@ function refusalBreach(ending, { cause, outKnown }) { !/^refused(?:, report .+)?$/.test(ending.done) ) return `the refusal's last line is not \`oracle-gaps: done refused\`` - // The done line ends stdout too, as its only line. - if ( - ending.stdout.length !== 1 || - ending.stdout[0] !== `oracle-gaps: done ${ending.done}` - ) - return `the refusal's stdout is ${JSON.stringify(ending.stdout)}, not its done line` + // The done line ends stdout too, as its only line after the line that + // names the stream log, if the run started a check. + const owedStdout = [ + ...(logNote === undefined ? [] : [logNote]), + `oracle-gaps: done ${ending.done}`, + ] + if (JSON.stringify(ending.stdout) !== JSON.stringify(owedStdout)) + return `the refusal's stdout is ${JSON.stringify(ending.stdout)}, not ${JSON.stringify(owedStdout)}` if (!outKnown) return undefined if (!ending.namesOutput) return `the refusal does not name its report in its output directory: done ${ending.done}` diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 1328130..85ed4d3 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -331,11 +331,14 @@ * (provisional)], report ` on stdout, `refused[, report * ]` after a refusal's block, or `stopped ()` * after a stop. A check through the daemon streams the same lines. The - * run's first line on both streams names its stream log, `oracle-gaps: - * note log /stream.log`, and that file holds the stderr's lines, in - * order, cold and through the daemon (the maintainer's decision of - * 2026-11-12: an agent that runs a check in the background learns from - * the run itself where to read it). + * first line, on both streams, of a run that starts a check names its + * stream log, `oracle-gaps: note log /stream.log`, and that file + * holds the stderr's lines, in order, cold and through the daemon (the + * maintainer's decision of 2026-11-12: an agent that runs a check in the + * background learns from the run itself where to read it). A refusal + * after the check started has that line and then its done line on + * stdout; a refusal of the run's arguments comes before any check, and + * its stdout is its done line alone. * * Authority: the maintainer's decisions in the gap-check design runs v11, v14, * and v19 (2026-11-04 and 2026-11-05; the design notes are kept by the project @@ -2061,7 +2064,10 @@ function incrementalCheck(observations) { // A refusal through the daemon ends as a cold refusal does (law 12): // the CLI that asked the daemon writes it to its own output directory. if (cold.refusal !== undefined) { - const breach = refusalBreach(daemon.ending, { outKnown: true }) + const breach = refusalBreach(daemon.ending, { + outKnown: true, + logNote: logNoteOf(daemon), + }) if (breach) return violation( `incremental`, @@ -2198,6 +2204,7 @@ function check(plan, observations, expected) { const breach = refusalBreach(cli.ending, { cause: expected.refused.cause, outKnown: true, + logNote: expected.refused.arguments ? undefined : logNoteOf(cli), }) if (breach) return violation(`cli`, breach, `report-shaped ending`) return incrementalCheck(observations) ?? daemonCheck(observations) ?? pass @@ -2483,7 +2490,7 @@ function check(plan, observations, expected) { function streamBreach(checkpoint, run, expected, plan) { const breach = (difference, extra) => violation(checkpoint, difference, `greppable stream`, extra) - const ends = endsBreach(run) + const ends = endsBreach(run, { checked: !expected.refused?.arguments }) if (ends) return breach(ends.difference, ends.extra) if (run.stream.mutantBeforeVerdict) return breach( @@ -2526,27 +2533,33 @@ function streamBreach(checkpoint, run, expected, plan) { return undefined } +/** The line that names a run's stream log. */ +const logNoteOf = (run) => `oracle-gaps: note log ${run.outDir}/stream.log` + /** * What law 15 judges of every run, on both streams, whatever its outcome: * no stray line, and exactly one done line, which ends each stream, the * same on both, of the kind its exit status says (0 completed, 2 refused, - * 3 error, and a signal's code stopped). Undefined when it holds. + * 3 error, and a signal's code stopped). With `checked`, the run started a + * check, so its first line names its stream log. Undefined when it holds. */ -function endsBreach(run) { +function endsBreach(run, { checked = true } = {}) { const streams = [ [`stderr`, run.stream], [`stdout`, run.stdoutStream], ] // The run's first line on each stream names its stream log, and the log // holds the stderr's lines, in order (the maintainer's decision of - // 2026-11-12), for every run that starts a check: each of these does. - const note = `oracle-gaps: note log ${run.outDir}/stream.log` - if (run.log.stderrFirst !== note || run.log.stdoutFirst !== note) + // 2026-11-12), for every run that starts a check. A refusal of the run's + // arguments comes before any check, and names no log. + const note = logNoteOf(run) + if (checked && (run.log.stderrFirst !== note || run.log.stdoutFirst !== note)) return { difference: `the run's first lines are ${JSON.stringify([run.log.stderrFirst, run.log.stdoutFirst])}, not ${JSON.stringify(note)} on each stream`, } - if (!run.log.exists) return { difference: `the run wrote no stream.log` } - if (!run.log.equal) + if (checked && !run.log.exists) + return { difference: `the run wrote no stream.log` } + if (checked && !run.log.equal) return { difference: `the run's stream.log is not its stderr, line for line`, } From ceeeaf6cbf468ef3840684400aee432d0bd776f4 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 07:24:39 -0700 Subject: [PATCH 03/16] feat(oracle-gaps): write the stream to /stream.log and name it first A run that starts a check creates `/stream.log`, writes `oracle-gaps: note log ` as its first line on both streams, and writes every stderr line to the log too: the cold check's lines, a daemon reply's stderr, an internal error's or refusal's ending, the completed done line, and the supervisor's own done line for a stopped or crashed run. Co-authored-by: Isaac --- .changeset/gap-cli-stream-log.md | 5 ++ README.md | 4 ++ packages/oracles/oracle-gaps/index.mjs | 64 ++++++++++++++++++-------- 3 files changed, 55 insertions(+), 18 deletions(-) create mode 100644 .changeset/gap-cli-stream-log.md diff --git a/.changeset/gap-cli-stream-log.md b/.changeset/gap-cli-stream-log.md new file mode 100644 index 0000000..cb23854 --- /dev/null +++ b/.changeset/gap-cli-stream-log.md @@ -0,0 +1,5 @@ +--- +'@tanstack/oracles': patch +--- + +The `oracle-gaps` CLI now writes the lines it streams on stderr to `stream.log` in the output directory too, in order, cold and through `--daemon`, and the first line of a check on both streams names that file: `oracle-gaps: note log /stream.log`. An agent that starts a check in the background can read the run's progress and its done line there. diff --git a/README.md b/README.md index 0a5001b..02664e7 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,10 @@ every commit with [pkg.pr.new](https://pkg.pr.new). to the process's stdout and stderr goes to `test-output.log` in the output directory, which a note names; `summary.md` holds the report for reading. A run through the daemon prints the same lines, at the end of its check. + The first line of a check, on both streams, is `oracle-gaps: note log +/stream.log`: the run writes its stderr lines to that file too, in + order, so a check started in the background can be read there while it + runs and after it ends. `--help` is not a check, so its usage text has no prefix. For contributors: two environment variables in the `ORACLE_GAPS_TEST_` diff --git a/packages/oracles/oracle-gaps/index.mjs b/packages/oracles/oracle-gaps/index.mjs index d2639fe..1e873aa 100755 --- a/packages/oracles/oracle-gaps/index.mjs +++ b/packages/oracles/oracle-gaps/index.mjs @@ -122,8 +122,9 @@ // ended 2 s later is killed, and so is any process left in the group when // the run ends. The run kills its group itself if the CLI dies first. import { spawn } from 'node:child_process' -import { realpathSync } from 'node:fs' +import { appendFileSync, mkdirSync, realpathSync, writeFileSync } from 'node:fs' import { constants } from 'node:os' +import { join } from 'node:path' import { pathToFileURL } from 'node:url' import { parseArgs, usage } from './args.mjs' import { codeIdentity, openSession } from './check.mjs' @@ -134,6 +135,19 @@ import { memoryStore } from './evidence.mjs' // How long the group has to end after a stop signal before it is killed. const STOP_GRACE_MS = 2000 +// The stream log (the maintainer's decision of 2026-11-12): a run that +// starts a check writes its stderr to `/stream.log` too, line for +// line, and names that file in its first line on both streams, so an agent +// that runs a check in the background learns from the run where to read +// it. Undefined until the run starts a check. +let streamLog +const logPathOf = (out) => join(out, `stream.log`) +/** Writes `text` to stderr and to the stream log. */ +const writeStream = (text) => { + process.stderr.write(text) + if (streamLog) appendFileSync(streamLog, text) +} + /** Signals every process of group `group`; false when none is left. */ function signalGroup(group, signal) { try { @@ -149,7 +163,9 @@ function signalGroup(group, signal) { * with it: with its exit code, or, once a stop signal has ended its group, * with 128 plus the signal's number. */ -function superviseRun() { +function superviseRun(out) { + // The supervisor's own done line goes to the run's stream log too. + streamLog = logPathOf(out) let child let stopped // How the run ended: its exit code, or the signal that ended it. @@ -192,11 +208,16 @@ function superviseRun() { const line = stopped ? `oracle-gaps: done stopped (${stopped})\n` : `oracle-gaps: done error, the run's process exited with ${how}${outOfMemory ? `: it ran out of memory; give Node more (NODE_OPTIONS=--max-old-space-size=) or run fewer --contexts` : ``}\n` - process.stdout.write(line, () => + process.stdout.write(line, () => { + try { + appendFileSync(streamLog, line) + } catch { + // A run stopped before it wrote its log has none to end. + } process.stderr.write(line, () => process.exit(stopped ? 128 + constants.signals[stopped] : 3), - ), - ) + ) + }) } if (child === undefined || closed) return decide() child.once(`close`, decide) @@ -271,9 +292,9 @@ function watchSupervisor(supervisor) { function asksForCheck(argv) { try { const options = parseArgs(argv) - return !options.help && !options.stop + return options.help || options.stop ? undefined : options } catch { - return false + return undefined } } @@ -281,7 +302,9 @@ function asksForCheck(argv) { async function checkCold(options) { const session = await openSession(options.pkg) try { - return await session.check(options, memoryStore()) + return await session.check(options, memoryStore(), (line) => + writeStream(`${line}\n`), + ) } finally { await session.close() } @@ -294,7 +317,7 @@ async function checkThroughDaemon(options) { process.argv.slice(2), codeIdentity(), ) - if (reply.stderr) process.stderr.write(reply.stderr) + if (reply.stderr) writeStream(reply.stderr) if (reply.failed) throw received(reply.failed) return reply.stdout } @@ -306,13 +329,11 @@ const main = // Windows has no process groups: there the run is the CLI's own process. const supervised = process.platform !== `win32` const supervisor = process.env.ORACLE_GAPS_GROUP -if ( - main && - supervised && - supervisor === undefined && - asksForCheck(process.argv.slice(2)) -) - superviseRun() +const checking = + main && supervised && supervisor === undefined + ? asksForCheck(process.argv.slice(2)) + : undefined +if (checking) superviseRun(checking.out) else if (main) { if (supervisor !== undefined) watchSupervisor(Number(supervisor)) // A test that runs this CLI starts a run of its own, in a group of its own. @@ -356,7 +377,7 @@ else if (main) { if (ended) return const ending = endRun(error, options?.out) holdStops() - process.stderr.write(ending.stderr) + writeStream(ending.stderr) process.stdout.write(`${ending.done}\n`) end(ending.exitCode) } @@ -369,6 +390,13 @@ else if (main) { if (options.help) console.log(usage) else if (options.stop) await stopDaemon(options.pkg) else { + // The first line on both streams names the stream log. + mkdirSync(options.out, { recursive: true }) + streamLog = logPathOf(options.out) + writeFileSync(streamLog, ``) + const note = `oracle-gaps: note log ${streamLog}\n` + process.stdout.write(note) + writeStream(note) const summary = await (options.daemon ? checkThroughDaemon : checkCold)( options, ) @@ -376,7 +404,7 @@ else if (main) { // The summary ends with the done line (report.mjs), which ends stderr // too. console.log(summary) - process.stderr.write(`${summary.split(`\n`).at(-1)}\n`) + writeStream(`${summary.split(`\n`).at(-1)}\n`) } end(0) } catch (error) { From 5a92a3e56effa82e98f0d57415d0a8c2cbefe960 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 10:23:58 -0700 Subject: [PATCH 04/16] fix(oracle-gaps): own the stream log in the supervisor; refuse a second run on a live --out The supervisor opens /stream.log and writes the note before it spawns the child; it resolves --out once and passes it to the child and to the daemon request; it tees the child's stderr, V8 lines included, to the log. A run whose log cannot be opened is refused before any check (exit 2, one done line), and a second run on a live output directory is refused without writing into it (maintainer decision, 2026-11-13). Oracle: out-mode (explicit or default) and log-blocked and overlap histories, with the law that a completed run's results.json sits beside its stream log and that a refused run writes nothing into a live directory. Stop histories use the stop-only witness. Co-authored-by: Isaac --- .changeset/gap-cli-stream-log.md | 2 +- README.md | 10 +- packages/oracles/oracle-gaps/ending.mjs | 4 +- .../oracle-gaps/gap-report-model-oracle.mjs | 84 +++- .../gap-report-oracle.property.test.mjs | 370 ++++++++++++++++-- ...gap-report-static-oracle.property.test.mjs | 37 +- packages/oracles/oracle-gaps/index.mjs | 129 +++++- packages/oracles/scripts/e2e-shards.mjs | 14 + 8 files changed, 568 insertions(+), 82 deletions(-) diff --git a/.changeset/gap-cli-stream-log.md b/.changeset/gap-cli-stream-log.md index cb23854..1c8c225 100644 --- a/.changeset/gap-cli-stream-log.md +++ b/.changeset/gap-cli-stream-log.md @@ -2,4 +2,4 @@ '@tanstack/oracles': patch --- -The `oracle-gaps` CLI now writes the lines it streams on stderr to `stream.log` in the output directory too, in order, cold and through `--daemon`, and the first line of a check on both streams names that file: `oracle-gaps: note log /stream.log`. An agent that starts a check in the background can read the run's progress and its done line there. +The `oracle-gaps` CLI now writes the lines it streams on stderr to `stream.log` in the output directory too, in order, cold and through `--daemon`, and the first line of a check on both streams names that file: `oracle-gaps: note log /stream.log`. An agent that starts a cold check in the background can read the run's progress and its done line there while it runs. A check through `--daemon` prints its lines at the end of its check, so its progress is readable there only after it ends. diff --git a/README.md b/README.md index d89f87e..655a7d9 100644 --- a/README.md +++ b/README.md @@ -119,10 +119,12 @@ every commit with [pkg.pr.new](https://pkg.pr.new). to the process's stdout and stderr goes to `test-output.log` in the output directory, which a note names; `summary.md` holds the report for reading. A run through the daemon prints the same lines, at the end of its check. - The first line of a check, on both streams, is `oracle-gaps: note log -/stream.log`: the run writes its stderr lines to that file too, in - order, so a check started in the background can be read there while it - runs and after it ends. + The first line of a check, on both streams, is + `oracle-gaps: note log /stream.log`: the run writes its stderr lines + to that file too, in order, so a check started in the background can be + read there while it runs and after it ends. A cold run's lines reach the + file as they are printed; a run through `--daemon` prints its lines at the + end of its check, so the file fills then. `--help` is not a check, so its usage text has no prefix. For contributors: two environment variables in the `ORACLE_GAPS_TEST_` diff --git a/packages/oracles/oracle-gaps/ending.mjs b/packages/oracles/oracle-gaps/ending.mjs index f26a94c..e915354 100644 --- a/packages/oracles/oracle-gaps/ending.mjs +++ b/packages/oracles/oracle-gaps/ending.mjs @@ -37,8 +37,10 @@ export function endRun(error, out = error?.out) { // The last line says the run refused, and where its report is once the // run knows its output directory (the stream's done line, stream.mjs); // it ends stdout too (index.mjs). + // A reportless refusal (another run uses the directory) writes nothing + // there: the directory's files are the other run's. const done = write( - out, + error.reportless ? undefined : out, results, `Refused: ${message}\n\nFix: ${error.fix}\n`, ) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 08f0b89..bcbe550 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -541,9 +541,12 @@ const drawn = fc.record({ // The budget witness alone (`budget`) costs two 10 s confirmations per // mutant in every CLI run of a history, so generated histories rarely // draw it; its constructed history (the daemon recheck) always does. + // `stop` is the stop witness alone (`w_stop`), which a run that the driver + // stops needs: the capture witness's other outcome classes are not run. witness: fc.oneof( { weight: 30, arbitrary: fc.constantFrom(false, true, `capture`) }, { weight: 1, arbitrary: fc.constant(`budget`) }, + { weight: 1, arbitrary: fc.constant(`stop`) }, ), authored: fc.option( fc.record({ @@ -690,6 +693,39 @@ const drawn = fc.record({ // A second run with `--only`, of one owned function's mutants: its // baseline runs only their owners (law 14). It costs a run of its own. onlyRun: fc.constantFrom(false, false, false, false, false, true), + // Where the run writes: the output directory `--out` names (`explicit`), or + // the one the CLI picks when `--out` is left out (`default`, args.mjs), whose + // stream log, results, and done line must still agree (law 15). + outMode: fc.constantFrom(`explicit`, `explicit`, `explicit`, `default`), + // The run's `/stream.log` is a directory, so the log cannot be written + // (a problem the author can fix: the run refuses before it checks). + logBlocked: fc.constantFrom( + false, + false, + false, + false, + false, + false, + false, + false, + true, + ), + // A second run starts on the first run's output directory while the first + // is live (the coordinator's decision of 2026-11-13 on F4): the second + // refuses, and the first's stream log stays its own. + overlap: fc.constantFrom( + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + ), // Oracle file a names its own Vitest environment in a pragma // (`// @vitest-environment`), another than the config's, and fails to load // outside it: every copy of it must run in that environment too. @@ -2477,6 +2513,7 @@ function resolveHistory(history) { // so only histories of the `budget` mode carry it. (history.witness === true && w.id !== `w_slowpair`) || (history.witness === `capture` && captureWitness.has(w.id)) || + (history.witness === `stop` && w.id === `w_stop`) || (history.witness === `budget` && w.id === `w_slowpair`), ) .map((w) => ({ @@ -2837,6 +2874,12 @@ function resolveHistory(history) { depSubpath: history.depSubpath === true, unbuiltDep: history.unbuiltDep === true, budget: history.budget ?? `two`, + outMode: history.outMode ?? `explicit`, + // Both name the run's `--out` directory before it starts, so they need + // an explicit one (a default directory is chosen by the CLI). + logBlocked: + history.logBlocked === true && outModeOf(history) === `explicit`, + overlap: history.overlap === true && outModeOf(history) === `explicit`, stopRuns: history.stopRuns ?? [], onlyRun: history.onlyRun === true, pragma: history.pragma === true, @@ -4083,7 +4126,16 @@ function refusal(plan) { if (plan.fault === `brokenVitest`) return { cause: `vitest`, runtime: true } if (plan.fault === `projects`) return { cause: `test.projects`, runtime: true } + // A stream log that cannot be written is refused before the check starts + // (the CLI opens the log before its first line, index.mjs), so its stdout + // owes its done line alone. + if (plan.logBlocked === true) + return { cause: `the stream log`, arguments: true } if (plan.fault === `badEntry`) return { cause: `${SOURCE}#nope` } + // An owns entry that names a missing file (a test file's declaration) is + // found by the analysis, after the check has started, so this refusal + // owes the stream log's note. The argument-stage case, a `--source` that + // is not a file of the package, is the static oracle's `absentSource`. if (plan.fault === `missingSource`) return { cause: `src/missing.ts` } if (plan.fault === `nonLiteral`) return { cause: `b.test.ts` } // A `--source` outside the package is an argument the CLI refuses before @@ -4187,6 +4239,7 @@ const declarationFaults = [ `nonLiteral`, `outsideSource`, ] +const outModeOf = (history) => history.outMode ?? `explicit` const reachesRun = (history) => !declarationFaults.includes(history.fault ?? `none`) @@ -4239,21 +4292,22 @@ function model(plan) { // fault, a Vitest that does not load, a config that does not load, and a // config that defines test.projects, which a loaded context shows. Every // other run names it once, refused or not. - const configLines = [ - `outsideSource`, - `brokenVitest`, - `configThrows`, - `configNoDefault`, - `configNull`, - `configSyntax`, - `projects`, - ].includes(plan.fault) - ? [] - : [ - plan.configPath === undefined - ? `oracle-gaps: config none, Vitest's default discovery` - : `oracle-gaps: config ${plan.configPath}`, - ] + const configLines = + [ + `outsideSource`, + `brokenVitest`, + `configThrows`, + `configNoDefault`, + `configNull`, + `configSyntax`, + `projects`, + ].includes(plan.fault) || plan.logBlocked === true + ? [] + : [ + plan.configPath === undefined + ? `oracle-gaps: config none, Vitest's default discovery` + : `oracle-gaps: config ${plan.configPath}`, + ] if (refused) return { refused, configLines } const sites = sitesOf(plan) const owners = ownersOf(plan) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 380b12e..cddd89c 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -588,14 +588,17 @@ function cliArgs( sources = `${SOURCE},${LIB}`, budget = `two`, outName = daemon ? `out-daemon` : `out`, + // `default` leaves `--out` out, so the CLI picks its own output + // directory (args.mjs): the run's first stdout line names it. + outMode = `explicit`, config, contexts = 4, only, pool, } = {}, ) { - const out = join(dir, outName) - rmSync(out, { recursive: true, force: true }) + const out = outMode === `default` ? undefined : join(dir, outName) + if (out !== undefined) rmSync(out, { recursive: true, force: true }) // The CLI runs from the package directory. A daemon check names its paths // relative to there, as an author types them, and the daemon must read // them as the CLI's caller meant them. @@ -628,9 +631,8 @@ function cliArgs( // one copy: one slow owner's confirmation fits it, two do not (law 4). `--mutant-limit`, String(MUTANT_LIMIT_MS), - `--out`, - path(out), ] + if (out !== undefined) args.push(`--out`, path(out)) if (only !== undefined) args.push(`--only`, only.join(`,`)) if (pool !== undefined) args.push(`--pool`, pool) if (ordinary) args.push(`--ordinary`) @@ -655,6 +657,7 @@ function runCli(dir, files, how = {}) { writeFixture(dir, files) const before = new Set(filesUnder(dir)) const { args, out } = cliArgs(dir, files, how) + if (how.logBlocked) mkdirSync(join(out, `stream.log`), { recursive: true }) const child = spawnSync(process.execPath, args, { cwd: repo, encoding: `utf8`, @@ -727,7 +730,7 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { writeFixture(dir, files) const before = new Set(filesUnder(dir)) const pids = mkdtempSync(join(realpathSync(tmpdir()), `gap-stop-`)) - const { args, out } = cliArgs(dir, files, { + const { args, out: given } = cliArgs(dir, files, { ...how, outName: `out-stop`, contexts: 1, @@ -813,7 +816,18 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { } if (!ended) child.kill(`SIGKILL`) rmSync(pids, { recursive: true, force: true }) - return { + const out = + given ?? + `${stdout}` + .split(`\n`)[0] + .match(/^oracle-gaps: note log (.*)\/stream\.log$/)?.[1] ?? + join(dir, `no-output`) + const scratchNames = [ + [how.cli ?? here, ``], + [dir, ``], + ...(given === undefined ? [[out, ``]] : []), + ] + const result = { signal, pool, spun, @@ -826,26 +840,14 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { exit: exitCode, // Both streams' lines (law 15), with the scratch paths named. stream: streamOf(stderr.split(`\n`), (text) => - namedScratch(text, [ - [how.cli ?? here, ``], - [dir, ``], - ]), + namedScratch(text, scratchNames), ), stdoutStream: streamOf(stdout.split(`\n`), (text) => - namedScratch(text, [ - [how.cli ?? here, ``], - [dir, ``], - ]), + namedScratch(text, scratchNames), ), - outDir: namedScratch(out, [ - [how.cli ?? here, ``], - [dir, ``], - ]), + outDir: namedScratch(out, scratchNames), log: streamLogOf(out, stderr, stdout, (text) => - namedScratch(text, [ - [how.cli ?? here, ``], - [dir, ``], - ]), + namedScratch(text, scratchNames), ), // The session directories in the package once it has ended: a stopped // run leaves its own, and the next run removes it. @@ -857,6 +859,9 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { (file) => !/^\.oracle-gaps-\d+\//.test(file), ), } + // A default output directory is the run's own scratch. + if (given === undefined) rmSync(out, { recursive: true, force: true }) + return result } /** @@ -905,6 +910,83 @@ function filesUnder(root, within = ``) { ) } +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)) + +/** + * Runs the CLI twice on one output directory, the second while the first is + * live (the first has printed its note, so it has its stream log). The first + * run is the history's `cli`, and the second's ending is the `overlap` + * observation (the coordinator's decision of 2026-11-13 on F4: a second run + * refuses, and the first's stream log stays its own). + */ +async function overlappingRuns(dir, files, how) { + writeFixture(dir, files) + const before = new Set(filesUnder(dir)) + const { args, out } = cliArgs(dir, files, how) + const start = () => { + const child = spawn(process.execPath, args, { + cwd: repo, + env: cleanEnv(), + stdio: [`ignore`, `pipe`, `pipe`], + }) + const got = { stdout: ``, stderr: `` } + child.stdout.on(`data`, (chunk) => { + got.stdout += chunk + }) + child.stderr.on(`data`, (chunk) => { + got.stderr += chunk + }) + const status = new Promise((resolvePromise) => { + const timer = setTimeout(() => child.kill(), 300_000) + child.on(`close`, (code) => { + clearTimeout(timer) + resolvePromise(code) + }) + }) + return { got, status } + } + const first = start() + // The first run's note is its first stdout line: its stream log exists. + for ( + let waited = 0; + !first.got.stdout.includes(`oracle-gaps: note log`) && waited < 120_000; + waited += 20 + ) + await sleep(20) + const second = start() + const secondStatus = await second.status + const firstStatus = await first.status + const ended = readRun( + dir, + out, + firstStatus, + first.got.stderr, + how.cli ?? here, + first.got.stdout, + ) + const refused = readRun( + dir, + out, + secondStatus, + second.got.stderr, + how.cli ?? here, + second.got.stdout, + ) + const replay = first.got.stderr + .split(`\n`) + .find((line) => line.startsWith(` replay: `)) + ?.slice(` replay: `.length) + return { + cli: { ...ended, replay, strayWrites: strays(dir, before) }, + overlap: { + exit: refused.exit, + ending: refused.ending, + stream: refused.stream, + stdoutStream: refused.stdoutStream, + }, + } +} + // Where a run may write in the package: only the output directories the // driver names with `--out` (`out`, `out-daemon`, `out-stop`, and // `out-only`). The daemon's state, the workspace dependencies' one-file @@ -996,7 +1078,9 @@ const streamedGap = * lines are the stderr's lines, in order. */ function streamLogOf(out, stderr, stdout, name) { - const lines = (text) => `${text ?? ``}`.split(`\n`).filter(Boolean) + // The lines of a text, its final newline ending the last line. A blank line + // is a line: the log must equal the stderr's lines exactly (F10). + const lines = (text) => `${text ?? ``}`.replace(/\n$/, ``).split(`\n`) let log try { log = readFileSync(join(out, `stream.log`), `utf8`) @@ -1060,11 +1144,40 @@ function streamOf(lines, name) { } function readRun(dir, out, status, stderr, cli, stdout) { + // A run without `--out` writes to the directory its stream log is in, which + // its first stdout line names (law 15); that directory is removed after the + // run is read, as the run's own scratch. + const chosen = + out ?? + `${stdout ?? ``}` + .split(`\n`)[0] + .match(/^oracle-gaps: note log (.*)\/stream\.log$/)?.[1] + try { + return readRunIn( + dir, + chosen ?? join(dir, `no-output`), + status, + stderr, + cli, + stdout, + out === undefined ? chosen : undefined, + ) + } finally { + if (out === undefined && chosen !== undefined) + rmSync(chosen, { recursive: true, force: true }) + } +} + +// `derived` is the output directory the CLI chose (no `--out`), which is named +// `` so that two runs of one history, each with its own directory, record +// the same trace (law 9's determinism). +function readRunIn(dir, out, status, stderr, cli, stdout, derived) { // The output names the CLI directory that ran and the package, as scratch // directories do, before it is cut to its last characters. const scratch = [ [cli, ``], [dir, ``], + ...(derived === undefined ? [] : [[derived, ``]]), ] const named = status === 0 ? undefined : namedScratch(`${stderr}`, scratch) let results @@ -1332,6 +1445,7 @@ async function incremental(dir, plan, cli, copy, observe, states) { const key = fingerprint(files, state, step === `otherInstall`) const how = { seed: state.seed, + outMode: plan.outMode, cli: step === `otherInstall` ? otherCli() : copy, ordinary: plan.ordinary, sources: sourceArgument(plan, `${dir}.link`), @@ -1357,6 +1471,8 @@ async function incremental(dir, plan, cli, copy, observe, states) { : await daemonChecks(step, files, how) if (!cold.has(key)) cold.set(key, runCli(dir, files, how)) const coldRun = cold.get(key) + if (plan.outMode === `default`) + states.add(`a daemon check with the default output directory`) for (const [i, run] of [daemon, ...others].entries()) observe(`incremental`, { step: i === 0 ? step : `${step} (a second check at once)`, @@ -1377,6 +1493,7 @@ async function incremental(dir, plan, cli, copy, observe, states) { // Its stream log, and the output directory it names. log: run.log, outDir: run.outDir, + reported: run.reported, // The first line of a failure, for the message; readRun has named // the scratch package and CLI copy in it. stderr: run.stderr?.split(`\n`)[0], @@ -1647,10 +1764,18 @@ async function driver(plan, observe) { ordinary: plan.ordinary, sources: sourceArgument(plan, `${dir}.link`), budget: plan.budget, + outMode: plan.outMode, + logBlocked: plan.logBlocked, cli: plan.cliPath === `spaced` ? copy : here, ...configArgs(plan), } - const { replay, ...cli } = runCli(dir, files, how) + // Two runs on one output directory: the second's refusal is its own + // observation (overlappingRuns). + const overlapping = plan.overlap + ? await overlappingRuns(dir, files, how) + : undefined + const { replay, ...cli } = overlapping?.cli ?? runCli(dir, files, how) + if (overlapping) observe(`overlap`, overlapping.overlap) // How many units were computed or reused says how the result was // produced, not what it is, so it is not observed. observe(`cli`, { ...cli, evidence: undefined }) @@ -1659,6 +1784,12 @@ async function driver(plan, observe) { observe(`replay`, { command: replay, ran: replayed(dir, replay) }) // Reach comes from what the CLI reported, mapped onto the plan. const states = new Set() + if (plan.outMode === `default`) + states.add(`a run with the default output directory`) + if (plan.logBlocked && cli.exit === 2) + states.add(`a stream log that cannot be written`) + if (overlapping && overlapping.overlap.exit === 2) + states.add(`two runs on one output directory`) if (cli.exit === 0 && !plan.consoleOutput) states.add(`a completed run whose tests wrote nothing`) // Law 14: a partial run of one owned function's mutants, whose baseline @@ -1712,6 +1843,8 @@ async function driver(plan, observe) { }, } observe(`stopped`, stopped) + if (plan.outMode === `default`) + states.add(`a stop with the default output directory`) if (previousSessions.length > 0) states.add(`a run after a stopped one that left its session`) previousSessions = run.sessions @@ -2221,6 +2354,24 @@ function check(plan, observations, expected) { `config`, { actual: cli.configLines, expected: expected.configLines }, ) + // A second run on a live output directory refuses before it writes + // anything (F4, the coordinator's decision of 2026-11-13): the refusal + // names the directory, is one block with its fix, and its stdout is its + // done line alone. The first run's own laws are judged on `cli`. + const overlap = observations.find((o) => o.checkpoint === `overlap`)?.value + if (overlap !== undefined) { + if (overlap.exit !== 2) + return violation( + `overlap`, + `the second run on a live output directory exited ${overlap.exit}, not refusing it`, + `ownership`, + ) + const breach = refusalBreach(overlap.ending, { + cause: `another run is using`, + outKnown: false, + }) + if (breach) return violation(`overlap`, breach, `report-shaped ending`) + } const stream = streamBreach(`cli`, cli, expected, plan) if (stream) return stream // Through the daemon, each check streams the same kinds of line, and a @@ -2290,6 +2441,14 @@ function check(plan, observations, expected) { logNote: expected.refused.arguments ? undefined : logNoteOf(cli), }) if (breach) return violation(`cli`, breach, `report-shaped ending`) + // A refusal of the arguments, or before the check starts, writes no + // stream log (F8, the coordinator's review of #57). + if (expected.refused.arguments && cli.log.exists) + return violation( + `cli`, + `the refusal before the check started wrote a stream log`, + `greppable stream`, + ) return incrementalCheck(observations) ?? daemonCheck(observations) ?? pass } if (cli.exit !== 0 || !cli.reported) @@ -2644,6 +2803,13 @@ function endsBreach(run, { checked = true } = {}) { return { difference: `the run's stream.log is not its stderr, line for line`, } + // A completed run's results.json sits beside its stream log, in the + // directory the log names (law 15's stream log, the maintainer's decision + // of 2026-11-12). + if (checked && run.exit === 0 && run.reported === false) + return { + difference: `the run completed, but its results.json is not beside its stream log`, + } for (const [where, stream] of streams) if (stream.strays.length > 0) return { @@ -3109,20 +3275,31 @@ calibrations.push( ), }, { + // A completed run that starts a check, so its log is judged (law 15). name: `the stream log misses a line`, law: `greppable stream`, - corrupt: corruptCli((cli) => ({ - ...cli, - log: { ...cli.log, equal: false }, - })), + history: { constructed: `two authored mutants` }, + corrupt: corruptCli((cli) => + cli.exit !== 0 + ? undefined + : { ...cli, log: { ...cli.log, equal: false } }, + ), }, { name: `the line that names the stream log comes after another line`, law: `greppable stream`, - corrupt: corruptCli((cli) => ({ - ...cli, - log: { ...cli.log, stderrFirst: `oracle-gaps: config vite.config.ts` }, - })), + history: { constructed: `two authored mutants` }, + corrupt: corruptCli((cli) => + cli.exit !== 0 + ? undefined + : { + ...cli, + log: { + ...cli.log, + stderrFirst: `oracle-gaps: config vite.config.ts`, + }, + }, + ), }, { name: `a stopped run ends without its done line`, @@ -3188,6 +3365,11 @@ const staticStates = [ `owned through bind`, ] const required = [ + `a stream log that cannot be written`, + `two runs on one output directory`, + `a run with the default output directory`, + `a stop with the default output directory`, + `a daemon check with the default output directory`, `killed by a second owner`, `killed`, `random-only kill`, @@ -3488,6 +3670,130 @@ oracleProperty({ consoleOutput: true, }), }, + { + // Without `--out`, the CLI picks the run's output directory + // (args.mjs). A run stopped while its config warms must still leave + // its stream log beside its results, with the done line last on both + // streams (law 15). The stop witness runs alone (witness `stop`), and a history + // with the witness module takes no steps. + name: `a stop with the default output directory`, + enters: [ + `a run with the default output directory`, + `a stop with the default output directory`, + ], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `push`, quote: `'` }]], + containers: [], + overloads: [], + owners: [`a`], + library: [], + libraryOwners: [], + runes: [], + wholeFile: `none`, + witness: `stop`, + tests: [{ ...judging(`a`, `fixed seed`, 1), fn: 0 }], + authored: null, + fault: `none`, + stability: `none`, + steps: null, + ordinary: false, + outMode: `default`, + budget: `two`, + stopRuns: [{ signal: `SIGINT`, pool: `threads`, when: `warm` }], + }), + }, + { + // Through the daemon, a check that the CLI parses without `--out` + // would write its results to a directory of its own, apart from the + // stream log the CLI names (law 15). + name: `a daemon check with the default output directory`, + enters: [ + `a run with the default output directory`, + `a daemon check with the default output directory`, + ], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `push`, quote: `'` }]], + containers: [], + overloads: [], + owners: [`a`], + library: [], + libraryOwners: [], + runes: [], + wholeFile: `none`, + witness: false, + tests: [{ ...judging(`a`, `fixed seed`, 1), fn: 0 }], + authored: null, + fault: `none`, + stability: `none`, + steps: [`test`], + ordinary: false, + outMode: `default`, + budget: `two`, + }), + }, + { + // A stream log that is a directory cannot be written: the CLI refuses + // before its first line (index.mjs opens the log first), with its + // block, its fix, and its report in the output directory, and its + // stdout is its done line alone (law 12, law 15, and F3 of the + // coordinator's review of #57). No stack trace, and exit 2 or 3. + name: `a stream log that cannot be written`, + enters: [`a stream log that cannot be written`], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `push`, quote: `'` }]], + containers: [], + overloads: [], + owners: [`a`], + library: [], + libraryOwners: [], + runes: [], + wholeFile: `none`, + witness: false, + tests: [{ ...judging(`a`, `fixed seed`, 1), fn: 0 }], + authored: null, + fault: `none`, + stability: `none`, + steps: null, + ordinary: false, + budget: `two`, + outMode: `explicit`, + logBlocked: true, + overlap: false, + }), + }, + { + // A second run on the first's output directory while the first is + // live refuses (F4, the coordinator's decision of 2026-11-13): the + // first run's stream log still equals its own stderr, line for line, + // and the refusal reports no directory it does not own. + name: `two runs on one output directory`, + enters: [`two runs on one output directory`], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `push`, quote: `'` }]], + containers: [], + overloads: [], + owners: [`a`], + library: [], + libraryOwners: [], + runes: [], + wholeFile: `none`, + witness: false, + tests: [{ ...judging(`a`, `fixed seed`, 1), fn: 0 }], + authored: null, + fault: `none`, + stability: `none`, + steps: null, + ordinary: false, + budget: `two`, + outMode: `explicit`, + logBlocked: false, + overlap: true, + }), + }, { // The budget witness checked through the daemon twice with nothing // changed between: the second check reuses the confirmations the diff --git a/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs index d2ef2d0..07849a8 100644 --- a/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs @@ -89,6 +89,7 @@ * target. An authored mutant's `source` field is used as written. */ import { + existsSync, mkdirSync, mkdtempSync, readFileSync, @@ -371,6 +372,14 @@ const matcherOwns = fc.oneof( ) // Each history of the shared grammar, with its argument fault and its // matcher's `owns`. +// The end-to-end dimensions the static oracle does not draw: `logBlocked` +// (a stream log the run cannot write) and `overlap` (a second run on a live +// output directory). The static run creates no stream log and starts no +// second CLI process, so these are off for it by deliberate scope; the +// end-to-end oracle (gap-report-oracle) owns both (F3 and F4 of the review of +// #57). +const staticResolve = (history) => + resolveHistory({ ...history, logBlocked: false, overlap: false }) const argumentHistories = fc.tuple(histories, badArguments, matcherOwns).map( ([history, bad, matcher]) => ({ ...history, @@ -489,13 +498,19 @@ function argumentsRunOf(dir, plan) { encoding: `utf8`, timeout: 120_000, }) - return endingOf( - run.status, - run.stderr, - out, - (text) => text.replaceAll(dir, ``), - run.stdout, - ) + return { + ...endingOf( + run.status, + run.stderr, + out, + (text) => text.replaceAll(dir, ``), + run.stdout, + ), + // A refusal of the arguments, or before a check starts, writes no stream + // log (F8, the coordinator's review of #57). Judged where the run names + // its output directory; a default one lives in the OS temp directory. + streamLog: out !== undefined && existsSync(join(out, `stream.log`)), + } } // ## The derived class, run @@ -1082,6 +1097,12 @@ function check(plan, observations, expected) { `with ${bad.kind} arguments${bad.daemon ? ` and --daemon` : ``}, ${breach}`, `report-shaped ending`, ) + if (argumentsRun.streamLog) + return violation( + `arguments`, + `with ${bad.kind} arguments${bad.daemon ? ` and --daemon` : ``}, the refusal wrote a stream log`, + `greppable stream`, + ) // An owner that fails, and a config the runner cannot load or honor, are // found only when the runner starts, so the analysis must not refuse for // them: it refuses as it would for the same plan without the fault. @@ -1785,7 +1806,7 @@ oracleProperty({ }, ]), }, - resolve: resolveHistory, + resolve: staticResolve, model, driver, check, diff --git a/packages/oracles/oracle-gaps/index.mjs b/packages/oracles/oracle-gaps/index.mjs index 1e873aa..349b96e 100755 --- a/packages/oracles/oracle-gaps/index.mjs +++ b/packages/oracles/oracle-gaps/index.mjs @@ -122,14 +122,21 @@ // ended 2 s later is killed, and so is any process left in the group when // the run ends. The run kills its group itself if the CLI dies first. import { spawn } from 'node:child_process' -import { appendFileSync, mkdirSync, realpathSync, writeFileSync } from 'node:fs' +import { + appendFileSync, + mkdirSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync, +} from 'node:fs' import { constants } from 'node:os' import { join } from 'node:path' import { pathToFileURL } from 'node:url' import { parseArgs, usage } from './args.mjs' import { codeIdentity, openSession } from './check.mjs' import { received, requestDaemon, stopDaemon } from './daemon.mjs' -import { endRun } from './ending.mjs' +import { endRun, Refusal } from './ending.mjs' import { memoryStore } from './evidence.mjs' // How long the group has to end after a stop signal before it is killed. @@ -139,14 +146,86 @@ const STOP_GRACE_MS = 2000 // starts a check writes its stderr to `/stream.log` too, line for // line, and names that file in its first line on both streams, so an agent // that runs a check in the background learns from the run where to read -// it. Undefined until the run starts a check. +// it. The supervisor opens it before the child starts (startLog), and then +// is the only writer of the child's stderr to it (superviseRun). Undefined +// until the log is open. let streamLog const logPathOf = (out) => join(out, `stream.log`) -/** Writes `text` to stderr and to the stream log. */ +/** Writes `text` to stderr and, once the log is open, to the stream log. */ const writeStream = (text) => { process.stderr.write(text) if (streamLog) appendFileSync(streamLog, text) } +const cannotWrite = (path, error) => + new Refusal( + `the stream log ${path} cannot be written (${error.code ?? error.message})`, + `Remove ${path}, or pass --out a directory whose stream.log can be written.`, + ) +/** Whether process `pid` runs. */ +const alive = (pid) => { + try { + process.kill(pid, 0) + return true + } catch (error) { + return error.code === `EPERM` + } +} +/** + * Claims output directory `out` for this process: a run keeps it while it + * lives, and a second run on it refuses, reportless so that the live run's + * files stay its own (the maintainer's decision of 2026-11-13). A claim by a + * run that has ended is taken over. + */ +function claimOut(out) { + const claim = join(out, `run.pid`) + for (;;) { + try { + writeFileSync(claim, `${process.pid}\n`, { flag: `wx` }) + return + } catch (error) { + if (error.code !== `EEXIST`) throw cannotWrite(claim, error) + } + let holder + try { + holder = Number(readFileSync(claim, `utf8`)) + } catch (error) { + if (error.code !== `ENOENT`) throw cannotWrite(claim, error) + continue + } + if (holder !== process.pid && holder > 0 && alive(holder)) { + const refusal = new Refusal( + `another run is using ${out}`, + `Pass --out a directory no run is using, or wait for the run that uses it to end.`, + ) + refusal.reportless = true + throw refusal + } + rmSync(claim, { force: true }) + } +} +/** + * Opens the run's stream log in `out` and writes its first line, on both + * streams, which names the log. Throws a Refusal when the output directory + * is in use or the log cannot be written; nothing is written then. + */ +function startLog(out) { + const path = logPathOf(out) + try { + mkdirSync(out, { recursive: true }) + } catch (error) { + throw cannotWrite(path, error) + } + claimOut(out) + try { + writeFileSync(path, ``) + } catch (error) { + throw cannotWrite(path, error) + } + streamLog = path + const note = `oracle-gaps: note log ${path}\n` + process.stdout.write(note) + writeStream(note) +} /** Signals every process of group `group`; false when none is left. */ function signalGroup(group, signal) { @@ -164,8 +243,6 @@ function signalGroup(group, signal) { * with 128 plus the signal's number. */ function superviseRun(out) { - // The supervisor's own done line goes to the run's stream log too. - streamLog = logPathOf(out) let child let stopped // How the run ended: its exit code, or the signal that ended it. @@ -209,11 +286,8 @@ function superviseRun(out) { ? `oracle-gaps: done stopped (${stopped})\n` : `oracle-gaps: done error, the run's process exited with ${how}${outOfMemory ? `: it ran out of memory; give Node more (NODE_OPTIONS=--max-old-space-size=) or run fewer --contexts` : ``}\n` process.stdout.write(line, () => { - try { - appendFileSync(streamLog, line) - } catch { - // A run stopped before it wrote its log has none to end. - } + // The log was opened before the run started (startLog). + appendFileSync(streamLog, line) process.stderr.write(line, () => process.exit(stopped ? 128 + constants.signals[stopped] : 3), ) @@ -249,6 +323,10 @@ function superviseRun(out) { [ ...process.execArgv.filter((flag) => !/^--inspect/.test(flag)), ...process.argv.slice(1), + // The out directory, resolved once by this process (asksForCheck): the + // run and a daemon's check write there, not to a directory of their own. + `--out`, + out, ], { detached: true, @@ -259,10 +337,12 @@ function superviseRun(out) { child.on(`message`, (message) => { if (message === `ending`) ending = true }) - // The run's stderr, passed on as it comes, and read for V8's - // out-of-memory message. + // The run's stderr, passed on as it comes, written to the stream log as it + // is (V8's own lines included, which the run does not write itself), and + // read for V8's out-of-memory message. child.stderr.on(`data`, (chunk) => { process.stderr.write(chunk) + appendFileSync(streamLog, chunk) stderrTail = (stderrTail + chunk).slice(-200) if (/heap out of memory/i.test(stderrTail)) outOfMemory = true }) @@ -333,7 +413,18 @@ const checking = main && supervised && supervisor === undefined ? asksForCheck(process.argv.slice(2)) : undefined -if (checking) superviseRun(checking.out) +// A check the supervisor cannot open its log for is run in this process, which +// refuses it the same way (below). +const supervising = (() => { + if (checking === undefined) return false + try { + startLog(checking.out) + return true + } catch { + return false + } +})() +if (supervising) superviseRun(checking.out) else if (main) { if (supervisor !== undefined) watchSupervisor(Number(supervisor)) // A test that runs this CLI starts a run of its own, in a group of its own. @@ -390,13 +481,9 @@ else if (main) { if (options.help) console.log(usage) else if (options.stop) await stopDaemon(options.pkg) else { - // The first line on both streams names the stream log. - mkdirSync(options.out, { recursive: true }) - streamLog = logPathOf(options.out) - writeFileSync(streamLog, ``) - const note = `oracle-gaps: note log ${streamLog}\n` - process.stdout.write(note) - writeStream(note) + // The supervisor opened the log (or, on Windows, there is none and this + // process runs the check); its child writes stderr alone. + if (supervisor === undefined) startLog(options.out) const summary = await (options.daemon ? checkThroughDaemon : checkCold)( options, ) diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index 9361d23..c25528f 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -41,6 +41,10 @@ const suite = `oracle-gaps.report` export const shards = { // 522 s in the measured run, about 221 s idle. 'e2e-1': [ + `fixed seed 20261105: a stop with the default output directory`, + `fixed seed 20261105: a daemon check with the default output directory`, + `fixed seed 20261105: a stream log that cannot be written`, + `fixed seed 20261105: two runs on one output directory`, `fixed seed 20261105: a .cjs config that calls require`, `fixed seed 20261105: a declaration fault: missingSource`, `fixed seed 20261105: a declaration fault: noTypescript`, @@ -106,6 +110,10 @@ export const shards = { ], // 522 s in the measured run, about 221 s idle. 'e2e-3': [ + `random seed: a stop with the default output directory`, + `random seed: a daemon check with the default output directory`, + `random seed: a stream log that cannot be written`, + `random seed: two runs on one output directory`, `fixed seed 20261105: a config fault in a run that owns nothing`, `fixed seed 20261105: a declaration fault: brokenOwner`, `fixed seed 20261105: a declaration fault: configThrows`, @@ -181,6 +189,10 @@ export const shards = { ], // 522 s in the measured run, about 221 s idle. 'e2e-5': [ + `determinism: constructed history: a stop with the default output directory`, + `determinism: constructed history: a daemon check with the default output directory`, + `determinism: constructed history: a stream log that cannot be written`, + `determinism: constructed history: two runs on one output directory`, `fixed seed 20261105: an edit sequence through the daemon`, `fixed seed 20261105: ordinary tests left out by default`, `fixed seed 20261105: two authored mutants`, @@ -267,6 +279,8 @@ function listed() { const run = spawnSync( `pnpm`, [ + // --silent keeps pnpm's workspace banner off stdout, which is JSON. + `--silent`, `exec`, `vitest`, `list`, From fd5655d1c7555b77f0c1e4bd1aa5bc6bd63989ff Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 10:59:20 -0700 Subject: [PATCH 05/16] wip(oracle-gaps): checkpoint at the pre-restructure CLI with the oracle fixes Oracle changes: out-mode, stop-only and stop-at-spawn dimensions appended to the history grammar (earlier fields unchanged on the sampled histories); the five new states as own-history states; overlap generated only for an explicit out, a live first run, and a fault-free history; the overlap wait ends on the first run's exit; constructed fault histories reset the end-to-end dimensions their generated base may draw. CLI: pre-restructure index.mjs from 5a92a3e. Co-authored-by: Isaac --- .../oracle-gaps/gap-report-model-oracle.mjs | 116 ++++++++++++------ .../gap-report-oracle.property.test.mjs | 97 +++++++++++++-- packages/oracles/scripts/e2e-shards.mjs | 3 + 3 files changed, 167 insertions(+), 49 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index bcbe550..6c44396 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -541,12 +541,9 @@ const drawn = fc.record({ // The budget witness alone (`budget`) costs two 10 s confirmations per // mutant in every CLI run of a history, so generated histories rarely // draw it; its constructed history (the daemon recheck) always does. - // `stop` is the stop witness alone (`w_stop`), which a run that the driver - // stops needs: the capture witness's other outcome classes are not run. witness: fc.oneof( { weight: 30, arbitrary: fc.constantFrom(false, true, `capture`) }, { weight: 1, arbitrary: fc.constant(`budget`) }, - { weight: 1, arbitrary: fc.constant(`stop`) }, ), authored: fc.option( fc.record({ @@ -693,39 +690,6 @@ const drawn = fc.record({ // A second run with `--only`, of one owned function's mutants: its // baseline runs only their owners (law 14). It costs a run of its own. onlyRun: fc.constantFrom(false, false, false, false, false, true), - // Where the run writes: the output directory `--out` names (`explicit`), or - // the one the CLI picks when `--out` is left out (`default`, args.mjs), whose - // stream log, results, and done line must still agree (law 15). - outMode: fc.constantFrom(`explicit`, `explicit`, `explicit`, `default`), - // The run's `/stream.log` is a directory, so the log cannot be written - // (a problem the author can fix: the run refuses before it checks). - logBlocked: fc.constantFrom( - false, - false, - false, - false, - false, - false, - false, - false, - true, - ), - // A second run starts on the first run's output directory while the first - // is live (the coordinator's decision of 2026-11-13 on F4): the second - // refuses, and the first's stream log stays its own. - overlap: fc.constantFrom( - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - ), // Oracle file a names its own Vitest environment in a pragma // (`// @vitest-environment`), another than the config's, and fails to load // outside it: every copy of it must run in that environment too. @@ -805,6 +769,66 @@ const drawn = fc.record({ }), // Who names the importer's function in an owns entry, if any. importer: fc.constantFrom(`none`, `a`, `b`), + // Where the run writes: the output directory `--out` names (`explicit`), or + // the one the CLI picks when `--out` is left out (`default`, args.mjs), whose + // stream log, results, and done line must still agree (law 15). + outMode: fc.constantFrom(`explicit`, `explicit`, `explicit`, `default`), + // The run's `/stream.log` is a directory, so the log cannot be written + // (a problem the author can fix: the run refuses before it checks). + logBlocked: fc.constantFrom( + false, + false, + false, + false, + false, + false, + false, + false, + true, + ), + // A second run starts on the first run's output directory while the first + // is live (the coordinator's decision of 2026-11-13 on F4): the second + // refuses, and the first's stream log stays its own. + overlap: fc.constantFrom( + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + ), + // The stop witness alone (`w_stop`), which a run the driver stops needs. A + // field of its own, so the witness options above keep their weights. + stopWitness: fc.constantFrom( + false, + false, + false, + false, + false, + false, + false, + false, + true, + ), + // A stop sent as soon as the CLI spawns, before its own run has started (P1: + // the note is written before the child exists, so the stopped line follows + // it). The driver adds this stop to the run's stops. + spawnStop: fc.constantFrom( + false, + false, + false, + false, + false, + false, + false, + false, + true, + ), }) // The edit sequence over the witness module, the one list of steps that a @@ -816,6 +840,10 @@ const drawn = fc.record({ // nothing changed, which reuses the confirmations that the first check ran. const witnessSteps = [`test`, `helper`, `revert`] const budgetSteps = [`recheck`] +// The stop witness alone takes no steps, so it counts only where the history +// has none; a history with steps and the field is a legal history without it. +const stopsOnly = (history) => + history.stopWitness === true && history.steps === null const keepsSteps = (history) => { if (history.witness === false || history.steps === null) return true const kept = history.witness === `budget` ? budgetSteps : witnessSteps @@ -2505,7 +2533,7 @@ function resolveHistory(history) { ...exported, ...importer, ...dotGen, - ...(history.witness + ...(history.witness || stopsOnly(history) ? witness .filter( (w) => @@ -2513,7 +2541,7 @@ function resolveHistory(history) { // so only histories of the `budget` mode carry it. (history.witness === true && w.id !== `w_slowpair`) || (history.witness === `capture` && captureWitness.has(w.id)) || - (history.witness === `stop` && w.id === `w_stop`) || + (stopsOnly(history) && w.id === `w_stop`) || (history.witness === `budget` && w.id === `w_slowpair`), ) .map((w) => ({ @@ -2879,8 +2907,18 @@ function resolveHistory(history) { // an explicit one (a default directory is chosen by the CLI). logBlocked: history.logBlocked === true && outModeOf(history) === `explicit`, - overlap: history.overlap === true && outModeOf(history) === `explicit`, + // A second run needs a live first run, which has its note and stream log. + // A blocked log refuses the first run before it is live, and a refusing + // fault (an argument, declaration, or config fault) ends it before its note + // too, so the overlap is generated only for a run that reaches its check + // (decision of 2026-11-13, review of #57: O1 and O2). + overlap: + history.overlap === true && + outModeOf(history) === `explicit` && + history.logBlocked !== true && + (history.fault ?? `none`) === `none`, stopRuns: history.stopRuns ?? [], + spawnStop: history.spawnStop === true, onlyRun: history.onlyRun === true, pragma: history.pragma === true, cliPath: history.cliPath ?? `plain`, diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index cddd89c..1cb305b 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -719,6 +719,9 @@ const strays = (dir, before, running = false) => * one done line and no refusal. */ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { + // `spawn`: the signal comes as the CLI starts, once its own code runs (a + // signal during Node's own start has no handler yet; Limits of #57). + const atSpawn = when === `spawn` const after = when === `after` const warm = when === `warm` // With `abort`, no signal comes: a test of file a aborts the run's @@ -786,11 +789,13 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { if (abort) await Promise.race([exit, sleep(120_000)]) const spun = abort ? exited - : after - ? stdout.includes(`oracle-gaps: done completed`) && !exited - : warm - ? existsSync(join(pids, `warming`)) && !exited - : readdirSync(pids).length >= 2 + : atSpawn + ? !exited + : after + ? stdout.includes(`oracle-gaps: done completed`) && !exited + : warm + ? existsSync(join(pids, `warming`)) && !exited + : readdirSync(pids).length >= 2 const strayWhileRunning = strays(dir, before, true) if (!abort) child.kill(signal) const ended = await Promise.race([exit, sleep(30_000).then(() => false)]) @@ -946,13 +951,25 @@ async function overlappingRuns(dir, files, how) { return { got, status } } const first = start() - // The first run's note is its first stdout line: its stream log exists. + let firstEnded = false + first.status.then(() => { + firstEnded = true + }) + // The first run's note is its first stdout line: its stream log exists. A + // first run that ends first never had a live output directory, so the + // overlap cannot be observed: the driver says so, rather than waiting. for ( let waited = 0; - !first.got.stdout.includes(`oracle-gaps: note log`) && waited < 120_000; + !first.got.stdout.includes(`oracle-gaps: note log`) && + !firstEnded && + waited < 120_000; waited += 20 ) await sleep(20) + if (!first.got.stdout.includes(`oracle-gaps: note log`)) + throw new Error( + `the first run ended before its stream log existed; an overlap needs a live first run`, + ) const second = start() const secondStatus = await second.status const firstStatus = await first.status @@ -1832,7 +1849,12 @@ async function driver(plan, observe) { // session directories the package has once the run ended, and how many // of them the previous stopped run had left. if (cli.exit === 0 && stopIds.length > 0) - for (const stop of plan.stopRuns) { + for (const stop of [ + ...plan.stopRuns, + ...(plan.spawnStop + ? [{ signal: `SIGINT`, pool: `threads`, when: `spawn` }] + : []), + ]) { const run = await stoppedRun(dir, files, how, stopIds, stop) const stopped = { ...run, @@ -1853,6 +1875,10 @@ async function driver(plan, observe) { states.add(`a stop signal after the completed line`) continue } + if (stop.when === `spawn`) { + states.add(`a stop as the CLI starts`) + continue + } if (stop.when === `warm`) { states.add(`a stop signal while the config warms`) continue @@ -2367,7 +2393,7 @@ function check(plan, observations, expected) { `ownership`, ) const breach = refusalBreach(overlap.ending, { - cause: `another run is using`, + cause: `another run (pid `, outKnown: false, }) if (breach) return violation(`overlap`, breach, `report-shaped ending`) @@ -3365,6 +3391,7 @@ const staticStates = [ `owned through bind`, ] const required = [ + `a stop as the CLI starts`, `a stream log that cannot be written`, `two runs on one output directory`, `a run with the default output directory`, @@ -3536,7 +3563,19 @@ const faults = [ `setupThrows`, ...analyzedFaults, ] +// The states that each named constructed history enters, and only that +// history: they are not witnessed by "every outcome class" (a named history +// of its own must enter them, or registration fails). +const ownHistoryStates = [ + `a stop as the CLI starts`, + `a stream log that cannot be written`, + `two runs on one output directory`, + `a run with the default output directory`, + `a stop with the default output directory`, + `a daemon check with the default output directory`, +] const separate = [ + ...ownHistoryStates, `ordinary tests left out by default`, `two authored mutants judged apart`, `a package with a prebuilt addon deep in its tree ran in child processes`, @@ -3595,6 +3634,12 @@ const withDeclaration = (constructed) => build: (generated) => item.build({ ...generated, + // The end-to-end dimensions a generated base may draw: a construction + // that does not name them runs without them (review of #57, apiLess). + logBlocked: false, + overlap: false, + spawnStop: false, + stopWitness: false, declaration: index % 2 === 0 ? `wrapper` : `oracleProperty`, ownerless: false, owners: [`a`, ...generated.owners.slice(1)], @@ -3691,7 +3736,8 @@ oracleProperty({ libraryOwners: [], runes: [], wholeFile: `none`, - witness: `stop`, + witness: false, + stopWitness: true, tests: [{ ...judging(`a`, `fixed seed`, 1), fn: 0 }], authored: null, fault: `none`, @@ -3733,6 +3779,37 @@ oracleProperty({ budget: `two`, }), }, + { + // A stop sent as the CLI spawns, before its note's run has started: + // the note is written first, and the stopped line follows it on both + // streams, with one done line (P1 of the review of #57). + name: `a stop as the CLI starts`, + enters: [`a stop as the CLI starts`], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `push`, quote: `'` }]], + containers: [], + overloads: [], + owners: [`a`], + library: [], + libraryOwners: [], + runes: [], + wholeFile: `none`, + witness: false, + stopWitness: true, + tests: [{ ...judging(`a`, `fixed seed`, 1), fn: 0 }], + authored: null, + fault: `none`, + stability: `none`, + steps: null, + ordinary: false, + outMode: `explicit`, + logBlocked: false, + overlap: false, + spawnStop: true, + budget: `two`, + }), + }, { // A stream log that is a directory cannot be written: the CLI refuses // before its first line (index.mjs opens the log first), with its diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index c25528f..cbe86c5 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -78,6 +78,7 @@ export const shards = { ], // 522 s in the measured run, about 221 s idle. 'e2e-2': [ + `fixed seed 20261105: a stop at spawn`, `fixed seed 20261105: a declaration fault: setupThrows`, `fixed seed 20261105: a package with no config`, `fixed seed 20261105: an ambiguous authored mutant`, @@ -110,6 +111,7 @@ export const shards = { ], // 522 s in the measured run, about 221 s idle. 'e2e-3': [ + `determinism: constructed history: a stop at spawn`, `random seed: a stop with the default output directory`, `random seed: a daemon check with the default output directory`, `random seed: a stream log that cannot be written`, @@ -144,6 +146,7 @@ export const shards = { ], // 522 s in the measured run, about 221 s idle. 'e2e-4': [ + `random seed: a stop at spawn`, `fixed seed 20261105: a .ts config in a package with no module type that calls require`, `fixed seed 20261105: a config that --config names`, `fixed seed 20261105: a declaration fault: apiLessTypescript`, From 4d6cc15cdde866e2b6c4ae55d0f92d53eaaf2258 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 11:13:05 -0700 Subject: [PATCH 06/16] fix(oracle-gaps): claim the output directory atomically and release run.pid on exit A run claims its output directory with run.pid: a temp file that already holds the pid, linked to run.pid (fails if it exists), so two starts cannot both claim. The claim is released on every exit but SIGKILL. A claim whose pid is gone is taken over; a live one refuses the second run and names the pid. Oracle: the spawn-delay law (outcome a or b, judged by the started marker, added with the next commit), the overlap hold guard (a test-only hook), the stop history that waits for the CLI to start, and the appended history fields. Co-authored-by: Isaac --- README.md | 2 +- .../oracle-gaps/gap-report-model-oracle.mjs | 6 ++ .../gap-report-oracle.property.test.mjs | 68 ++++++++++++++--- packages/oracles/oracle-gaps/index.mjs | 74 ++++++++++++------- packages/oracles/scripts/e2e-shards.mjs | 6 +- 5 files changed, 114 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 655a7d9..0d19451 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ every commit with [pkg.pr.new](https://pkg.pr.new). For contributors: two environment variables in the `ORACLE_GAPS_TEST_` namespace are test-only hooks for the CLI's own oracle, not settings for - users. `ORACLE_GAPS_TEST_HOLD_EXIT_MS` holds the run's exit after its done + users. `ORACLE_GAPS_TEST_STARTED` names a file that the CLI writes as its first project statement, before any project module loads, so a test can tell a stop that came before the CLI started from one that came during its loads (test-only, like the hooks below). `ORACLE_GAPS_TEST_HOLD_EXIT_MS` holds the run's exit after its done line, and `ORACLE_GAPS_TEST_WARM_PROBE` makes the first config warm-up wait where a stop signal can reach it. diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 6c44396..6aa882e 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -829,6 +829,10 @@ const drawn = fc.record({ false, true, ), + // The signal of a spawn stop and when it lands, 0 to 150 ms after the CLI + // spawns (the spawn-delay law, review of #57). + spawnSignal: fc.constantFrom(`SIGINT`, `SIGINT`, `SIGTERM`), + spawnDelay: fc.integer({ min: 0, max: 150 }), }) // The edit sequence over the witness module, the one list of steps that a @@ -2919,6 +2923,8 @@ function resolveHistory(history) { (history.fault ?? `none`) === `none`, stopRuns: history.stopRuns ?? [], spawnStop: history.spawnStop === true, + spawnSignal: history.spawnSignal ?? `SIGINT`, + spawnDelay: history.spawnDelay ?? 0, onlyRun: history.onlyRun === true, pragma: history.pragma === true, cliPath: history.cliPath ?? `plain`, diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 1cb305b..ad17e9a 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -718,7 +718,13 @@ const strays = (dir, before, running = false) => * Vitest's does; re-review N1). The run then ends stopped, with exactly * one done line and no refusal. */ -async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { +async function stoppedRun( + dir, + files, + how, + ids, + { signal, pool, when, delay = 0 }, +) { // `spawn`: the signal comes as the CLI starts, once its own code runs (a // signal during Node's own start has no handler yet; Limits of #57). const atSpawn = when === `spawn` @@ -740,15 +746,23 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { only: ids, pool: abort ? `threads` : pool, }) + // The started marker: the CLI writes it as its first statement (index.mjs). + const startedPath = join(pids, `started`) const child = spawn(process.execPath, args, { cwd: repo, - env: after - ? { ...cleanEnv(), ORACLE_GAPS_TEST_HOLD_EXIT_MS: `500` } - : warm - ? { ...cleanEnv(), ORACLE_GAPS_TEST_WARM_PROBE: join(pids, `warming`) } - : abort - ? { ...cleanEnv(), GAP_ABORT: `1` } - : { ...cleanEnv(), GAP_STOP_DIR: pids }, + env: { + ...(atSpawn ? { ORACLE_GAPS_TEST_STARTED: startedPath } : {}), + ...(after + ? { ...cleanEnv(), ORACLE_GAPS_TEST_HOLD_EXIT_MS: `500` } + : warm + ? { + ...cleanEnv(), + ORACLE_GAPS_TEST_WARM_PROBE: join(pids, `warming`), + } + : abort + ? { ...cleanEnv(), GAP_ABORT: `1` } + : { ...cleanEnv(), GAP_STOP_DIR: pids }), + }, stdio: [`ignore`, `pipe`, `pipe`], }) let stdout = `` @@ -773,6 +787,7 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { let waited = 0; !exited && !abort && + !atSpawn && (after ? !stdout.includes(`oracle-gaps: done completed`) : warm @@ -797,6 +812,7 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { ? existsSync(join(pids, `warming`)) && !exited : readdirSync(pids).length >= 2 const strayWhileRunning = strays(dir, before, true) + if (atSpawn) await sleep(delay) if (!abort) child.kill(signal) const ended = await Promise.race([exit, sleep(30_000).then(() => false)]) await sleep(1000) @@ -834,6 +850,9 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { ] const result = { signal, + // Whether the CLI wrote its started marker before the signal (outcomes a and b). + started: atSpawn ? existsSync(startedPath) : undefined, + stdoutLength: stdout.length, pool, spun, ended, @@ -928,10 +947,18 @@ async function overlappingRuns(dir, files, how) { writeFixture(dir, files) const before = new Set(filesUnder(dir)) const { args, out } = cliArgs(dir, files, how) - const start = () => { + const start = (holdMs) => { const child = spawn(process.execPath, args, { cwd: repo, - env: cleanEnv(), + // A test-only hold (ORACLE_GAPS_TEST_HOLD_EXIT_MS, index.mjs): the first + // run keeps its output claim until the second has been refused, so the + // overlap does not depend on how fast the first run ends. + env: { + ...cleanEnv(), + ...(holdMs === undefined + ? {} + : { ORACLE_GAPS_TEST_HOLD_EXIT_MS: String(holdMs) }), + }, stdio: [`ignore`, `pipe`, `pipe`], }) const got = { stdout: ``, stderr: `` } @@ -950,7 +977,7 @@ async function overlappingRuns(dir, files, how) { }) return { got, status } } - const first = start() + const first = start(5000) let firstEnded = false first.status.then(() => { firstEnded = true @@ -1852,7 +1879,14 @@ async function driver(plan, observe) { for (const stop of [ ...plan.stopRuns, ...(plan.spawnStop - ? [{ signal: `SIGINT`, pool: `threads`, when: `spawn` }] + ? [ + { + signal: plan.spawnSignal, + pool: `threads`, + when: `spawn`, + delay: plan.spawnDelay, + }, + ] : []), ]) { const run = await stoppedRun(dir, files, how, stopIds, stop) @@ -2756,6 +2790,13 @@ function check(plan, observations, expected) { function streamBreach(checkpoint, run, expected, plan) { const breach = (difference, extra) => violation(checkpoint, difference, `greppable stream`, extra) + // The spawn-delay law (review of #57): a stop before the CLI started (no + // started marker) is outcome (a): no output, no log, the exit by signal. + // Once the marker exists, the outcome is (b) in full. + if (run.started === false) + return run.stdoutLength === 0 && !run.log.exists && run.exit !== 0 + ? undefined + : breach(`a stop before the CLI started left output or a log`) const ends = endsBreach(run, { checked: !expected.refused?.arguments }) if (ends) return breach(ends.difference, ends.extra) if (run.stream.mutantBeforeVerdict) @@ -3807,6 +3848,9 @@ oracleProperty({ logBlocked: false, overlap: false, spawnStop: true, + // Pinned inside the CLI's module-load window (about 20–50 ms after + // spawn, measured): the handlers must already be installed there. + spawnDelay: 30, budget: `two`, }), }, diff --git a/packages/oracles/oracle-gaps/index.mjs b/packages/oracles/oracle-gaps/index.mjs index 349b96e..078c294 100755 --- a/packages/oracles/oracle-gaps/index.mjs +++ b/packages/oracles/oracle-gaps/index.mjs @@ -124,6 +124,7 @@ import { spawn } from 'node:child_process' import { appendFileSync, + linkSync, mkdirSync, readFileSync, realpathSync, @@ -170,37 +171,58 @@ const alive = (pid) => { return error.code === `EPERM` } } +// The pid this process claimed each output directory with, released when the +// process exits (on every exit but SIGKILL). +const claims = new Set() +process.on(`exit`, () => { + for (const claim of claims) { + try { + if (Number(readFileSync(claim, `utf8`)) === process.pid) + rmSync(claim, { force: true }) + } catch { + // Already gone. + } + } +}) /** - * Claims output directory `out` for this process: a run keeps it while it - * lives, and a second run on it refuses, reportless so that the live run's - * files stay its own (the maintainer's decision of 2026-11-13). A claim by a - * run that has ended is taken over. + * Claims output directory `out` for this process (review of #57, P2 and P3): + * a run keeps it while it lives, and a second run on it refuses without + * writing into it (the maintainer's decision of 2026-11-13). The claim is a + * temp file that already holds the pid, linked to `run.pid`, which fails when + * `run.pid` exists, so the claim is atomic. A claim whose pid no longer runs + * is taken over. */ function claimOut(out) { const claim = join(out, `run.pid`) - for (;;) { - try { - writeFileSync(claim, `${process.pid}\n`, { flag: `wx` }) - return - } catch (error) { - if (error.code !== `EEXIST`) throw cannotWrite(claim, error) - } - let holder - try { - holder = Number(readFileSync(claim, `utf8`)) - } catch (error) { - if (error.code !== `ENOENT`) throw cannotWrite(claim, error) - continue - } - if (holder !== process.pid && holder > 0 && alive(holder)) { - const refusal = new Refusal( - `another run is using ${out}`, - `Pass --out a directory no run is using, or wait for the run that uses it to end.`, - ) - refusal.reportless = true - throw refusal + const temp = `${claim}.${process.pid}.tmp` + try { + mkdirSync(out, { recursive: true }) + writeFileSync(temp, `${process.pid}\n`) + } catch (error) { + throw cannotWrite(claim, error) + } + try { + for (;;) { + try { + linkSync(temp, claim) + claims.add(claim) + return + } catch (error) { + if (error.code !== `EEXIST`) throw cannotWrite(claim, error) + } + const holder = Number(readFileSync(claim, `utf8`)) + if (holder !== process.pid && holder > 0 && alive(holder)) { + const refusal = new Refusal( + `another run (pid ${holder}) is using ${out}`, + `If no run is using it, delete ${claim}; otherwise pass --out a directory no run is using.`, + ) + refusal.reportless = true + throw refusal + } + rmSync(claim, { force: true }) } - rmSync(claim, { force: true }) + } finally { + rmSync(temp, { force: true }) } } /** diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index cbe86c5..27191c5 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -78,7 +78,7 @@ export const shards = { ], // 522 s in the measured run, about 221 s idle. 'e2e-2': [ - `fixed seed 20261105: a stop at spawn`, + `fixed seed 20261105: a stop as the CLI starts`, `fixed seed 20261105: a declaration fault: setupThrows`, `fixed seed 20261105: a package with no config`, `fixed seed 20261105: an ambiguous authored mutant`, @@ -111,7 +111,7 @@ export const shards = { ], // 522 s in the measured run, about 221 s idle. 'e2e-3': [ - `determinism: constructed history: a stop at spawn`, + `determinism: constructed history: a stop as the CLI starts`, `random seed: a stop with the default output directory`, `random seed: a daemon check with the default output directory`, `random seed: a stream log that cannot be written`, @@ -146,7 +146,7 @@ export const shards = { ], // 522 s in the measured run, about 221 s idle. 'e2e-4': [ - `random seed: a stop at spawn`, + `random seed: a stop as the CLI starts`, `fixed seed 20261105: a .ts config in a package with no module type that calls require`, `fixed seed 20261105: a config that --config names`, `fixed seed 20261105: a declaration fault: apiLessTypescript`, From a2d51b903949fe14ea90f81a34db3bc10bc48419 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 11:13:07 -0700 Subject: [PATCH 07/16] fix(oracle-gaps): install the stop handlers before any project module loads The CLI's first project statement is now the handler install, so a stop that arrives during the loads is kept and acted on once the note is written. The single run path (P4) is in the same file; the static-import variant with the same handlers passed the default-out units, so the dynamic loads are not the cause of any batch failure (review of #57). A test-only marker, ORACLE_GAPS_TEST_STARTED, is written as the first project statement so the oracle can tell a stop before the CLI started (no output is allowed) from one during its loads (the full output is required). Co-authored-by: Isaac --- packages/oracles/oracle-gaps/index.mjs | 247 +++++++++++++------------ 1 file changed, 127 insertions(+), 120 deletions(-) diff --git a/packages/oracles/oracle-gaps/index.mjs b/packages/oracles/oracle-gaps/index.mjs index 078c294..827c27d 100755 --- a/packages/oracles/oracle-gaps/index.mjs +++ b/packages/oracles/oracle-gaps/index.mjs @@ -121,6 +121,10 @@ // SIGINT or SIGTERM is sent on to the whole group; a process that has not // ended 2 s later is killed, and so is any process left in the group when // the run ends. The run kills its group itself if the CLI dies first. +// The stop handlers and the claim on the output directory come before the +// CLI's modules load (review of #57, P1): a signal during the loads is kept +// and acted on once the stream log's note is written, so a stop as the CLI starts still +// leaves the note and one done line. Only Node's own start is before them. import { spawn } from 'node:child_process' import { appendFileSync, @@ -134,23 +138,32 @@ import { import { constants } from 'node:os' import { join } from 'node:path' import { pathToFileURL } from 'node:url' -import { parseArgs, usage } from './args.mjs' -import { codeIdentity, openSession } from './check.mjs' -import { received, requestDaemon, stopDaemon } from './daemon.mjs' -import { endRun, Refusal } from './ending.mjs' -import { memoryStore } from './evidence.mjs' + +// A test-only marker (ORACLE_GAPS_TEST_STARTED): the first statement that runs +// project code writes this file, before any project module loads, so a test can +// tell a stop that came before the CLI started from one that came during its loads. +if (process.env.ORACLE_GAPS_TEST_STARTED) + writeFileSync(process.env.ORACLE_GAPS_TEST_STARTED, ``) + +let early +const keepEarly = (signal) => { + early ??= signal +} +process.on(`SIGINT`, keepEarly) +process.on(`SIGTERM`, keepEarly) +const { parseArgs, usage } = await import(`./args.mjs`) +const { endRun, Refusal } = await import(`./ending.mjs`) // How long the group has to end after a stop signal before it is killed. const STOP_GRACE_MS = 2000 // The stream log (the maintainer's decision of 2026-11-12): a run that -// starts a check writes its stderr to `/stream.log` too, line for -// line, and names that file in its first line on both streams, so an agent -// that runs a check in the background learns from the run where to read -// it. The supervisor opens it before the child starts (startLog), and then -// is the only writer of the child's stderr to it (superviseRun). Undefined -// until the log is open. +// starts a check writes its stderr to `/stream.log` too, line for line, +// and names that file in its first line on both streams. The supervisor opens +// it before the child starts (startLog), and is then the only writer of the +// child's stderr to it (superviseRun). Undefined until the log is open. let streamLog +let writingEnding = false const logPathOf = (out) => join(out, `stream.log`) /** Writes `text` to stderr and, once the log is open, to the stream log. */ const writeStream = (text) => { @@ -226,17 +239,12 @@ function claimOut(out) { } } /** - * Opens the run's stream log in `out` and writes its first line, on both - * streams, which names the log. Throws a Refusal when the output directory - * is in use or the log cannot be written; nothing is written then. + * Claims the output directory and writes the run's first line, on both + * streams, which names the log. Throws a Refusal when the directory is in use + * or the log cannot be written; nothing is written then. */ function startLog(out) { const path = logPathOf(out) - try { - mkdirSync(out, { recursive: true }) - } catch (error) { - throw cannotWrite(path, error) - } claimOut(out) try { writeFileSync(path, ``) @@ -249,20 +257,19 @@ function startLog(out) { writeStream(note) } -/** Signals every process of group `group`; false when none is left. */ -function signalGroup(group, signal) { - try { - process.kill(-group, signal) - return true - } catch { - return false - } +/** Flushes both streams, then exits with `code` (after a test's hold). */ +function exitWhenFlushed(code) { + const hold = Number(process.env.ORACLE_GAPS_TEST_HOLD_EXIT_MS ?? 0) + process.stdout.write(``, () => + process.stderr.write(``, () => setTimeout(() => process.exit(code), hold)), + ) } /** - * Runs this CLI again as a child in a process group of its own, and ends - * with it: with its exit code, or, once a stop signal has ended its group, - * with 128 plus the signal's number. + * Runs this CLI again as a child in a process group of its own, and ends with + * it: with its exit code, or, once a stop signal has ended its group, with 128 + * plus the signal's number. The child gets `--out` appended, the directory this + * process resolved (review of #57, F2). */ function superviseRun(out) { let child @@ -270,26 +277,22 @@ function superviseRun(out) { // How the run ended: its exit code, or the signal that ended it. let ended // Whether the run said that it writes its ending (a message it sends - // before it writes): then its done line is the last on each stream, and - // it ends with its own code, whatever stop reaches it. + // before it writes): then its done line is the last on each stream, and it + // ends with its own code, whatever stop reaches it. let ending = false - // Whether the run's stderr, which the CLI passes on, said that V8 ran - // out of memory. + // Whether the run's stderr, which the CLI passes on, said that V8 ran out + // of memory. let outOfMemory = false let stderrTail = `` // Whether the run's streams and channel have closed. let closed = false - // Ends the CLI once the run has ended (or the grace has passed), with - // what is left of the group killed, and once the run's stderr has been - // passed on (at most 2 s more). A run that writes its ending wrote its - // done line, so the CLI ends with its code and writes none. Every other - // end gets one done line from the CLI, on both streams: a run that a stop - // ended stopped (review of #46, F3; re-review N1); a run whose process - // ended on its own, by a signal or a code, with no ending (a V8 - // out-of-memory abort is a SIGABRT) is an error, exit 3, with the remedy - // when it ran out of memory (the coordinator's decision of 2026-11-12 on - // P10). A write to a pipe is asynchronous on macOS, so the process exits - // once it is written. + // Ends the CLI once the run has ended (or the grace has passed), with what + // is left of the group killed. A run that writes its ending wrote its done + // line, so the CLI ends with its code and writes none. Every other end gets + // one done line from the CLI, on both streams: a stop ended the run + // (stopped); a run whose process ended on its own (a signal, a code, a V8 + // out-of-memory abort) is an error, exit 3, with the remedy when it ran out + // of memory. let finishing = false const finish = () => { if (finishing) return @@ -335,18 +338,18 @@ function superviseRun(out) { finish() }, 50) } - // The stop handlers come first, so a signal that comes while the run - // starts is a stop too (re-review N3). + // The stop handlers come first, so a signal that comes while the run starts + // is a stop too. A stop that came during the CLI's loads ends the run before + // it starts: its stopped line follows the note the caller already wrote. process.on(`SIGINT`, stop) process.on(`SIGTERM`, stop) + if (early !== undefined) return stop(early) // An inspector flag would bind the CLI's own port a second time. child = spawn( process.execPath, [ ...process.execArgv.filter((flag) => !/^--inspect/.test(flag)), ...process.argv.slice(1), - // The out directory, resolved once by this process (asksForCheck): the - // run and a daemon's check write there, not to a directory of their own. `--out`, out, ], @@ -359,9 +362,9 @@ function superviseRun(out) { child.on(`message`, (message) => { if (message === `ending`) ending = true }) - // The run's stderr, passed on as it comes, written to the stream log as it - // is (V8's own lines included, which the run does not write itself), and - // read for V8's out-of-memory message. + // The run's stderr, passed on as it comes, written to the stream log as it is + // (V8's own lines included, which the run does not write itself), and read + // for V8's out-of-memory message. child.stderr.on(`data`, (chunk) => { process.stderr.write(chunk) appendFileSync(streamLog, chunk) @@ -377,9 +380,19 @@ function superviseRun(out) { }) } +/** Signals every process of group `group`; false when none is left. */ +function signalGroup(group, signal) { + try { + process.kill(-group, signal) + return true + } catch { + return false + } +} + /** - * The run's side of the group: once the CLI that started it is gone, the - * run kills its group, itself included. + * The run's side of the group: once the CLI that started it is gone, the run + * kills its group, itself included. */ function watchSupervisor(supervisor) { setInterval(() => { @@ -388,8 +401,9 @@ function watchSupervisor(supervisor) { } /** - * Whether arguments `argv` ask for a check, the only run that starts - * processes; a refusal of them is written by the run itself. + * The options of arguments `argv` when they ask for a check, the only run that + * starts processes; undefined for help, a stop, or arguments that refuse (a + * refusal is written by the run itself). */ function asksForCheck(argv) { try { @@ -402,6 +416,8 @@ function asksForCheck(argv) { /** The summary of a check run in this process. */ async function checkCold(options) { + const { openSession } = await import(`./check.mjs`) + const { memoryStore } = await import(`./evidence.mjs`) const session = await openSession(options.pkg) try { return await session.check(options, memoryStore(), (line) => @@ -414,6 +430,8 @@ async function checkCold(options) { /** The summary of a check run by the package's daemon. */ async function checkThroughDaemon(options) { + const { codeIdentity } = await import(`./check.mjs`) + const { received, requestDaemon } = await import(`./daemon.mjs`) const reply = await requestDaemon( options.pkg, process.argv.slice(2), @@ -424,6 +442,23 @@ async function checkThroughDaemon(options) { return reply.stdout } +// The run's ending, once: its refusal or error is written to its output +// directory and stdout and stderr, and the process exits with its code. +let ended = false +const holdStops = () => { + writingEnding = true + process.send?.(`ending`) +} +function endWith(error, out) { + if (ended) return + ended = true + const ending = endRun(error, out) + holdStops() + writeStream(ending.stderr) + process.stdout.write(`${ending.done}\n`) + exitWhenFlushed(ending.exitCode) +} + // A file URL escapes a space, `#`, and the like, so the script's own URL is // compared with the URL of the path it was started as. const main = @@ -431,80 +466,50 @@ const main = // Windows has no process groups: there the run is the CLI's own process. const supervised = process.platform !== `win32` const supervisor = process.env.ORACLE_GAPS_GROUP -const checking = - main && supervised && supervisor === undefined - ? asksForCheck(process.argv.slice(2)) - : undefined -// A check the supervisor cannot open its log for is run in this process, which -// refuses it the same way (below). -const supervising = (() => { - if (checking === undefined) return false - try { - startLog(checking.out) - return true - } catch { - return false + +async function run() { + // The supervisor: it claims the output directory, writes the note, and runs + // the child. A refusal of the claim or the log ends it through endWith (the + // single path for every refusal of a check). + if (supervised && supervisor === undefined) { + const checking = asksForCheck(process.argv.slice(2)) + if (checking !== undefined) { + try { + startLog(checking.out) + } catch (error) { + return endWith(error, checking.out) + } + return superviseRun(checking.out) + } } -})() -if (supervising) superviseRun(checking.out) -else if (main) { if (supervisor !== undefined) watchSupervisor(Number(supervisor)) // A test that runs this CLI starts a run of its own, in a group of its own. delete process.env.ORACLE_GAPS_GROUP - // The run's options, once its arguments parse; a refusal or an error is - // written to their output directory. - let options - let ended = false - // Ends the run once. A write to a pipe is asynchronous on macOS, and - // exiting at once cuts it off at the pipe's buffer, so the process exits - // once both streams have written everything before it. - // A test-only hook for the CLI's own oracle, not for users: - // ORACLE_GAPS_TEST_HOLD_EXIT_MS holds the exit after the done line, so the - // CLI's oracle can send a signal in that window. - const end = (exitCode) => { - if (ended) return - ended = true - const hold = Number(process.env.ORACLE_GAPS_TEST_HOLD_EXIT_MS ?? 0) - process.stdout.write(``, () => - process.stderr.write(``, () => - setTimeout(() => process.exit(exitCode), hold), - ), - ) - } - // A stop ends the run at once, with no line: an error that the stop - // causes (a config load that it cuts off) is no refusal, and the - // supervisor writes the stopped line (re-review N1). Once the run writes - // its ending, it says so to the supervisor, and a stop no longer cuts it - // off: its done line is then the last line of each stream. - let writingEnding = false + // A stop ends the run at once, with no line: an error that the stop causes + // is no refusal, and the supervisor writes the stopped line. Once the run + // writes its ending, a stop no longer cuts it off. const onStop = (signal) => { if (!writingEnding) process.exit(128 + constants.signals[signal]) } process.on(`SIGINT`, onStop) process.on(`SIGTERM`, onStop) - const holdStops = () => { - writingEnding = true - process.send?.(`ending`) - } - const fail = (error) => { - if (ended) return - const ending = endRun(error, options?.out) - holdStops() - writeStream(ending.stderr) - process.stdout.write(`${ending.done}\n`) - end(ending.exitCode) - } - // An error thrown outside the awaited check, by a timer or a promise no - // one awaits, ends the run the same way. - process.on(`uncaughtException`, fail) - process.on(`unhandledRejection`, fail) + // A stop that came before this process could handle it ends it now. + if (early !== undefined) process.exit(128 + constants.signals[early]) + // The run's options, once its arguments parse. + let options + // An error thrown outside the awaited check, by a timer or a promise no one + // awaits, ends the run the same way. + process.on(`uncaughtException`, (error) => endWith(error, options?.out)) + process.on(`unhandledRejection`, (error) => endWith(error, options?.out)) try { options = parseArgs(process.argv.slice(2)) if (options.help) console.log(usage) - else if (options.stop) await stopDaemon(options.pkg) - else { - // The supervisor opened the log (or, on Windows, there is none and this - // process runs the check); its child writes stderr alone. + else if (options.stop) { + const { stopDaemon } = await import(`./daemon.mjs`) + await stopDaemon(options.pkg) + } else { + // Windows has no supervisor: this process opens the log and runs the + // check. A child of a supervisor writes its stderr alone. if (supervisor === undefined) startLog(options.out) const summary = await (options.daemon ? checkThroughDaemon : checkCold)( options, @@ -515,8 +520,10 @@ else if (main) { console.log(summary) writeStream(`${summary.split(`\n`).at(-1)}\n`) } - end(0) + exitWhenFlushed(0) } catch (error) { - fail(error) + endWith(error, options?.out) } } + +if (main) await run() From 2a0fdda73a86a73f49e8d5a7b8fd8e7c99a68c55 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 19:41:52 -0700 Subject: [PATCH 08/16] test(oracle-gaps): the spawn stop signals once the CLI has started, drawn 0-150 ms otherwise The constructed stop history waits for the started marker (ORACLE_GAPS_TEST_STARTED) and then signals at once, so the signal lands in the CLI's loads on every machine. Generated histories keep the drawn 0-150 ms delay and signal (SIGINT or SIGTERM). Verified: the real CLI passes; the late-handler mutant (handlers after the project loads) is rejected with started-but-no-output. Co-authored-by: Isaac --- .../oracle-gaps/gap-report-model-oracle.mjs | 4 ++++ .../gap-report-oracle.property.test.mjs | 20 ++++++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 6aa882e..6b29ac7 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -833,6 +833,9 @@ const drawn = fc.record({ // spawns (the spawn-delay law, review of #57). spawnSignal: fc.constantFrom(`SIGINT`, `SIGINT`, `SIGTERM`), spawnDelay: fc.integer({ min: 0, max: 150 }), + // The spawn stop waits for the CLI's started marker, then signals at once (the + // deterministic witness, event-driven rather than timed). + spawnOnStart: fc.constantFrom(false, false, false, true), }) // The edit sequence over the witness module, the one list of steps that a @@ -2925,6 +2928,7 @@ function resolveHistory(history) { spawnStop: history.spawnStop === true, spawnSignal: history.spawnSignal ?? `SIGINT`, spawnDelay: history.spawnDelay ?? 0, + spawnOnStart: history.spawnOnStart === true, onlyRun: history.onlyRun === true, pragma: history.pragma === true, cliPath: history.cliPath ?? `plain`, diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index ad17e9a..681948a 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -723,7 +723,7 @@ async function stoppedRun( files, how, ids, - { signal, pool, when, delay = 0 }, + { signal, pool, when, delay = 0, onStart = false }, ) { // `spawn`: the signal comes as the CLI starts, once its own code runs (a // signal during Node's own start has no handler yet; Limits of #57). @@ -812,7 +812,16 @@ async function stoppedRun( ? existsSync(join(pids, `warming`)) && !exited : readdirSync(pids).length >= 2 const strayWhileRunning = strays(dir, before, true) - if (atSpawn) await sleep(delay) + // A spawn stop signals after its drawn delay, or, when it waits for the + // started marker, at once once the marker exists (event-driven, 30 s cap). + if (atSpawn && onStart) + for ( + let waited = 0; + !existsSync(startedPath) && waited < 30_000; + waited += 1 + ) + await sleep(1) + else if (atSpawn) await sleep(delay) if (!abort) child.kill(signal) const ended = await Promise.race([exit, sleep(30_000).then(() => false)]) await sleep(1000) @@ -1885,6 +1894,7 @@ async function driver(plan, observe) { pool: `threads`, when: `spawn`, delay: plan.spawnDelay, + onStart: plan.spawnOnStart, }, ] : []), @@ -3848,9 +3858,9 @@ oracleProperty({ logBlocked: false, overlap: false, spawnStop: true, - // Pinned inside the CLI's module-load window (about 20–50 ms after - // spawn, measured): the handlers must already be installed there. - spawnDelay: 30, + // The signal waits for the CLI's started marker, so it lands during + // the loads of a CLI that loads before it installs its handlers. + spawnOnStart: true, budget: `two`, }), }, From 76522ffe36efe51470d75fa473283e0e86e4de37 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 20:04:20 -0700 Subject: [PATCH 09/16] test(oracle-gaps): one decision for whether a refused run owes its note A refusal of the arguments comes before any check, so its output is its done line alone, through the daemon as in a cold run. The cold stream check, the daemon stream check, and the daemon refusal's note now share owesNote(expected), instead of the daemon path applying the note requirement to every refusal. Verified: the outsideSource unit passes on the CLI; the mutant that requires a note for argument refusals fails it. Co-authored-by: Isaac --- .../gap-report-oracle.property.test.mjs | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 681948a..0aa4df7 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -2331,7 +2331,7 @@ async function driver(plan, observe) { * Law 9: each check through the daemon reports what the cold run of the same * state reports. A refused state is compared by the cause it names. */ -function incrementalCheck(observations) { +function incrementalCheck(observations, expected) { for (const { value } of observations.filter( (o) => o.checkpoint === `incremental`, )) { @@ -2352,7 +2352,7 @@ function incrementalCheck(observations) { if (cold.refusal !== undefined) { const breach = refusalBreach(daemon.ending, { outKnown: true, - logNote: logNoteOf(daemon), + logNote: owesNote(expected) ? logNoteOf(daemon) : undefined, }) if (breach) return violation( @@ -2451,7 +2451,7 @@ function check(plan, observations, expected) { (o) => o.checkpoint === `incremental`, )) { const { daemon, cold } = value - const ends = endsBreach(daemon) + const ends = endsBreach(daemon, { checked: owesNote(expected) }) if (ends) return violation( `incremental`, @@ -2519,7 +2519,11 @@ function check(plan, observations, expected) { `the refusal before the check started wrote a stream log`, `greppable stream`, ) - return incrementalCheck(observations) ?? daemonCheck(observations) ?? pass + return ( + incrementalCheck(observations, expected) ?? + daemonCheck(observations) ?? + pass + ) } if (cli.exit !== 0 || !cli.reported) return violation( @@ -2781,7 +2785,11 @@ function check(plan, observations, expected) { { actual: after, expected: before }, ) } - return incrementalCheck(observations) ?? daemonCheck(observations) ?? pass + return ( + incrementalCheck(observations, expected) ?? + daemonCheck(observations) ?? + pass + ) } /** @@ -2797,6 +2805,11 @@ function check(plan, observations, expected) { * (stopped). `run` is a run as readRun records it, observed at * `checkpoint`. */ +// The stage at which a run refuses decides whether it owes its note (law 15, +// review of #57): a refusal of the arguments comes before any check, so its +// output is its done line alone. Both the cold run and a run through the daemon +// are judged by this one decision. +const owesNote = (expected) => !expected.refused?.arguments function streamBreach(checkpoint, run, expected, plan) { const breach = (difference, extra) => violation(checkpoint, difference, `greppable stream`, extra) @@ -2807,7 +2820,7 @@ function streamBreach(checkpoint, run, expected, plan) { return run.stdoutLength === 0 && !run.log.exists && run.exit !== 0 ? undefined : breach(`a stop before the CLI started left output or a log`) - const ends = endsBreach(run, { checked: !expected.refused?.arguments }) + const ends = endsBreach(run, { checked: owesNote(expected) }) if (ends) return breach(ends.difference, ends.extra) if (run.stream.mutantBeforeVerdict) return breach( From a784642c6af2da65cb96ab0fc6c4e18161fff0bc Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 21:27:14 -0700 Subject: [PATCH 10/16] fix(oracle-gaps): install the stop handlers before the started marker The started marker (ORACLE_GAPS_TEST_STARTED) is written after the stop handlers are installed, so a marker means the CLI can handle a stop. A test-only pause after the marker (ORACLE_GAPS_TEST_PAUSE_AFTER_STARTED, ms) widens the window for the spawn-stop witness: with the marker before the handlers, a stop during the pause ended the CLI with a marker and no output (0 of 20 passes); with the fixed order, 40 of 40 pass. Co-authored-by: Isaac --- .../gap-report-oracle.property.test.mjs | 5 +++++ packages/oracles/oracle-gaps/index.mjs | 18 ++++++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 0aa4df7..70e0b87 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -752,6 +752,11 @@ async function stoppedRun( cwd: repo, env: { ...(atSpawn ? { ORACLE_GAPS_TEST_STARTED: startedPath } : {}), + // The constructed witness pauses after the marker, so the signal it sends + // on seeing the marker lands inside the pause (review F2). + ...(atSpawn && onStart + ? { ORACLE_GAPS_TEST_PAUSE_AFTER_STARTED: `200` } + : {}), ...(after ? { ...cleanEnv(), ORACLE_GAPS_TEST_HOLD_EXIT_MS: `500` } : warm diff --git a/packages/oracles/oracle-gaps/index.mjs b/packages/oracles/oracle-gaps/index.mjs index 827c27d..962fb9e 100755 --- a/packages/oracles/oracle-gaps/index.mjs +++ b/packages/oracles/oracle-gaps/index.mjs @@ -139,18 +139,24 @@ import { constants } from 'node:os' import { join } from 'node:path' import { pathToFileURL } from 'node:url' -// A test-only marker (ORACLE_GAPS_TEST_STARTED): the first statement that runs -// project code writes this file, before any project module loads, so a test can -// tell a stop that came before the CLI started from one that came during its loads. -if (process.env.ORACLE_GAPS_TEST_STARTED) - writeFileSync(process.env.ORACLE_GAPS_TEST_STARTED, ``) - let early const keepEarly = (signal) => { early ??= signal } process.on(`SIGINT`, keepEarly) process.on(`SIGTERM`, keepEarly) +// A test-only marker (ORACLE_GAPS_TEST_STARTED): the first statement that runs +// project code writes this file, before any project module loads, so a test can +// tell a stop that came before the CLI started from one that came during its loads. +if (process.env.ORACLE_GAPS_TEST_STARTED) + writeFileSync(process.env.ORACLE_GAPS_TEST_STARTED, ``) +// A test-only pause after the marker (ORACLE_GAPS_TEST_PAUSE_AFTER_STARTED, ms): +// it blocks this thread synchronously, so a stop sent during it lands before the +// handlers below are installed (the widened witness of review F2). Inert unless set. +const pauseMs = Number(process.env.ORACLE_GAPS_TEST_PAUSE_AFTER_STARTED ?? 0) +if (pauseMs > 0) + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, pauseMs) + const { parseArgs, usage } = await import(`./args.mjs`) const { endRun, Refusal } = await import(`./ending.mjs`) From d338cb16c2b59ebc5089a9e9cbc2dc86bc7b7fe6 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 22:07:28 -0700 Subject: [PATCH 11/16] fix(oracle-gaps): take over a stale claim under a mutex, and name results.json's state A stale run.pid is taken over only by the racer that creates run.pid.takeover (a directory only one racer can make), and only after it re-reads the stale pid under the mutex. Two concurrent starts against a stale claim now yield exactly one claim (witness: 5 of 5 trials, one claimant each; the mutant without the mutex and the re-read yields two in 5 of 5). A test-only pause (ORACLE_GAPS_TEST_PAUSE_AT, inert unless set) widens the window. The results-beside-log message names the directory and whether results.json is missing, unparseable or read (batch 3, issue #76). Co-authored-by: Isaac --- .../gap-report-oracle.property.test.mjs | 12 +++- packages/oracles/oracle-gaps/index.mjs | 67 ++++++++++++++++++- 2 files changed, 75 insertions(+), 4 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 70e0b87..1f45028 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -1241,10 +1241,16 @@ function readRunIn(dir, out, status, stderr, cli, stdout, derived) { let results let report let summary = `` + // The state of `results.json` in the output directory: missing, unparseable, + // or read (review of #57, batch 3: the message names which). + let reportState = `missing` try { - report = JSON.parse(readFileSync(join(out, `results.json`), `utf8`)) + const text = readFileSync(join(out, `results.json`), `utf8`) + reportState = `unparseable` + report = JSON.parse(text) results = report.results summary = readFileSync(join(out, `summary.md`), `utf8`) + reportState = `read` } catch { // A run that wrote no report is recorded by its exit status and output. } @@ -1348,6 +1354,7 @@ function readRunIn(dir, out, status, stderr, cli, stdout, derived) { ? undefined : `${status === 3 && existsSync(join(out, `error.log`)) ? `error.log: ${namedScratch(readFileSync(join(out, `error.log`), `utf8`), scratch).slice(0, 2000)}\n` : ``}${named.slice(-1500)}`, reported: results !== undefined, + reportState, streamed, // Law 9 compares everything the run reports except how it was // produced: durations, the options it was given (the output directory @@ -1552,6 +1559,7 @@ async function incremental(dir, plan, cli, copy, observe, states) { log: run.log, outDir: run.outDir, reported: run.reported, + reportState: run.reportState, // The first line of a failure, for the message; readRun has named // the scratch package and CLI copy in it. stderr: run.stderr?.split(`\n`)[0], @@ -2903,7 +2911,7 @@ function endsBreach(run, { checked = true } = {}) { // of 2026-11-12). if (checked && run.exit === 0 && run.reported === false) return { - difference: `the run completed, but its results.json is not beside its stream log`, + difference: `the run completed, but ${run.outDir}/results.json is ${run.reportState ?? `missing or unparseable`}, not beside its stream log`, } for (const [where, stream] of streams) if (stream.strays.length > 0) diff --git a/packages/oracles/oracle-gaps/index.mjs b/packages/oracles/oracle-gaps/index.mjs index 962fb9e..c944303 100755 --- a/packages/oracles/oracle-gaps/index.mjs +++ b/packages/oracles/oracle-gaps/index.mjs @@ -213,6 +213,7 @@ process.on(`exit`, () => { */ function claimOut(out) { const claim = join(out, `run.pid`) + const takeover = `${claim}.takeover` const temp = `${claim}.${process.pid}.tmp` try { mkdirSync(out, { recursive: true }) @@ -229,7 +230,8 @@ function claimOut(out) { } catch (error) { if (error.code !== `EEXIST`) throw cannotWrite(claim, error) } - const holder = Number(readFileSync(claim, `utf8`)) + const holder = readHolder(claim) + if (holder === undefined) continue if (holder !== process.pid && holder > 0 && alive(holder)) { const refusal = new Refusal( `another run (pid ${holder}) is using ${out}`, @@ -238,12 +240,73 @@ function claimOut(out) { refusal.reportless = true throw refusal } - rmSync(claim, { force: true }) + // A stale claim: only the racer that creates the takeover mutex removes it, + // and only if it still names the stale pid it read (review F4). + testPause(`takeover`) + if (!takeMutex(takeover)) { + sleepMs(10) + continue + } + try { + if (readHolder(claim) === holder) rmSync(claim, { force: true }) + } finally { + rmSync(takeover, { recursive: true, force: true }) + } } } finally { rmSync(temp, { force: true }) } } +/** The pid a claim names, or undefined when the claim is gone. */ +function readHolder(claim) { + try { + return Number(readFileSync(claim, `utf8`)) + } catch (error) { + if (error.code === `ENOENT`) return undefined + throw cannotWrite(claim, error) + } +} +/** + * Creates the takeover mutex: a directory only one racer can make. A mutex + * whose pid is gone (a crashed racer) is removed and made again; that rare case + * is in Limits. + */ +function takeMutex(mutex) { + try { + mkdirSync(mutex) + } catch (error) { + if (error.code !== `EEXIST`) throw error + const owner = Number( + (() => { + try { + return readFileSync(join(mutex, `pid`), `utf8`) + } catch { + return `` + } + })(), + ) + if (!(owner > 0 && alive(owner))) { + rmSync(mutex, { recursive: true, force: true }) + return takeMutex(mutex) + } + return false + } + writeFileSync(join(mutex, `pid`), `${process.pid}\n`) + return true +} +/** A test-only pause (ORACLE_GAPS_TEST_PAUSE_AT, inert unless set), see README. */ +function testPause(point) { + if (process.env.ORACLE_GAPS_TEST_PAUSE_AT === point) + Atomics.wait( + new Int32Array(new SharedArrayBuffer(4)), + 0, + 0, + Number(process.env.ORACLE_GAPS_TEST_PAUSE_MS ?? 0), + ) +} +function sleepMs(ms) { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms) +} /** * Claims the output directory and writes the run's first line, on both * streams, which names the log. Throws a Refusal when the directory is in use From 6222e257373fcf99abd1d0b2a294c5288f1fd97a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 02:35:42 -0700 Subject: [PATCH 12/16] ci(oracles): place the new Windows fixed-seed units in shards The fixed-seed units added with the spawn stop, the stream log, and the overlap (five units) were not in any Windows fixed-seed shard, so the pr-shards partition check failed in the Test job. Placed in shards 2 and 3. Co-authored-by: Isaac --- packages/oracles/scripts/pr-shards.mjs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index 9e16288..bb47680 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -524,6 +524,8 @@ export const partitions = { "test.projects in a parent's config", ]), 2: anyOf([ + 'a stop with the default output directory', + 'a daemon check with the default output directory', 'a .ts config in a package with no module type that calls require', 'a declaration fault: brokenVitest', 'a declaration fault: noTypescript', @@ -533,6 +535,9 @@ export const partitions = { 'two authored mutants', ]), 3: anyOf([ + 'a stop as the CLI starts', + 'a stream log that cannot be written', + 'two runs on one output directory', 'a config fault in a run that owns nothing', 'a declaration fault: configNoDefault', 'a declaration fault: nonLiteral', From 5fa74e07e75260c2467c97b8ff1cdbc0c6cdce04 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 07:57:19 -0700 Subject: [PATCH 13/16] chore(oracle-gaps): drop the unsupervised POSIX seam The unsupervised (win32) path is being replaced by a win32 supervisor (#74), so the seam that forced it on POSIX is removed; its stop witness was not kept. The unsupervised path's stop law (no done line on a stop) is a Limit of this PR. Co-authored-by: Isaac --- packages/oracles/oracle-gaps/index.mjs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/oracles/oracle-gaps/index.mjs b/packages/oracles/oracle-gaps/index.mjs index b6d1a7d..75828f6 100755 --- a/packages/oracles/oracle-gaps/index.mjs +++ b/packages/oracles/oracle-gaps/index.mjs @@ -542,11 +542,7 @@ function endWith(error, out) { const main = import.meta.url === pathToFileURL(realpathSync(process.argv[1])).href // Windows has no process groups: there the run is the CLI's own process. -// The test seam ORACLE_GAPS_TEST_UNSUPERVISED forces the unsupervised path on -// POSIX (in-process, as win32 runs it), so its stop law is tested without Windows. -const supervised = - process.platform !== `win32` && - process.env.ORACLE_GAPS_TEST_UNSUPERVISED === undefined +const supervised = process.platform !== `win32` const supervisor = process.env.ORACLE_GAPS_GROUP // Set once the note is written (the unsupervised run has started). From 9560e86eabe95d177c0862ccfe2cb7117335b2cd Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 09:45:15 -0700 Subject: [PATCH 14/16] fix(oracle-gaps): create the takeover mutex with its pid, and never take over an unreadable claim The takeover mutex was made as a directory and its pid written after, so a racer could read it empty, take it for dead, and remove a live mutex. The pid is now written to a temp file and linked into place, so no reader sees an empty mutex. An unreadable or empty mutex or claim counts as live and is never taken over. Witness (two concurrent starts against a stale claim, a test pause between the create and the pid write): the current code gives one claimed and one crashing run in every trial; the fix gives one claimed and one refused in every trial. Co-authored-by: Isaac --- packages/oracles/oracle-gaps/index.mjs | 38 ++++++++++++++------------ 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/packages/oracles/oracle-gaps/index.mjs b/packages/oracles/oracle-gaps/index.mjs index 75828f6..99c3412 100755 --- a/packages/oracles/oracle-gaps/index.mjs +++ b/packages/oracles/oracle-gaps/index.mjs @@ -233,6 +233,11 @@ function claimOut(out) { } const holder = readHolder(claim) if (holder === undefined) continue + // An unreadable claim counts as live: wait for it, never take it over. + if (!(holder > 0)) { + sleepMs(10) + continue + } if (holder !== process.pid && holder > 0 && alive(holder)) { const refusal = new Refusal( `another run (pid ${holder}) is using ${out}`, @@ -268,32 +273,31 @@ function readHolder(claim) { } } /** - * Creates the takeover mutex: a directory only one racer can make. A mutex - * whose pid is gone (a crashed racer) is removed and made again; that rare case - * is in Limits. + * Takes the takeover mutex. The pid is written to a temp file first and linked + * into place, so no reader ever sees an empty mutex (review F4; register: #57 + * claim rules). A mutex whose pid can't be read counts as live and is never + * taken over; only a mutex whose recorded pid is dead is removed, and then made + * again. */ function takeMutex(mutex) { + const temp = `${mutex}.${process.pid}.tmp` + writeFileSync(temp, `${process.pid}\n`) try { - mkdirSync(mutex) + testPause(`mutex-create`) + linkSync(temp, mutex) + return true } catch (error) { if (error.code !== `EEXIST`) throw error - const owner = Number( - (() => { - try { - return readFileSync(join(mutex, `pid`), `utf8`) - } catch { - return `` - } - })(), - ) - if (!(owner > 0 && alive(owner))) { - rmSync(mutex, { recursive: true, force: true }) + const owner = readHolder(mutex) + if (owner === undefined) return false + if (owner > 0 && !alive(owner)) { + rmSync(mutex, { force: true }) return takeMutex(mutex) } return false + } finally { + rmSync(temp, { force: true }) } - writeFileSync(join(mutex, `pid`), `${process.pid}\n`) - return true } /** A test-only pause (ORACLE_GAPS_TEST_PAUSE_AT, inert unless set), see README. */ function testPause(point) { From c8352b8ae30f7c85efaf818c9bd2b8b926fb3b02 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 09:45:18 -0700 Subject: [PATCH 15/16] docs(oracle-gaps): document every test-only ORACLE_GAPS_TEST hook Co-authored-by: Isaac --- README.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 46ac75a..2881da5 100644 --- a/README.md +++ b/README.md @@ -131,11 +131,16 @@ agent. end of its check, so the file fills then. `--help` is not a check, so its usage text has no prefix. - For contributors: two environment variables in the `ORACLE_GAPS_TEST_` - namespace are test-only hooks for the CLI's own oracle, not settings for - users. `ORACLE_GAPS_TEST_STARTED` names a file that the CLI writes as its first project statement, before any project module loads, so a test can tell a stop that came before the CLI started from one that came during its loads (test-only, like the hooks below). `ORACLE_GAPS_TEST_HOLD_EXIT_MS` holds the run's exit after its done - line, and `ORACLE_GAPS_TEST_WARM_PROBE` makes the first config warm-up - wait where a stop signal can reach it. + For contributors: the `ORACLE_GAPS_TEST_` environment variables are test-only + hooks for the CLI's own oracle, not settings for users. Each is inert unless + set; a hook may pause at one named point and never changes an outcome. + `ORACLE_GAPS_TEST_STARTED` names a file that the CLI writes as its first project statement, before any project module loads, so a test can tell a stop that came before the CLI started from one that came during its loads. + `ORACLE_GAPS_TEST_PAUSE_AFTER_STARTED` (milliseconds) blocks the CLI for that long right after the started file is written, so a stop sent on seeing the file lands inside the pause. + `ORACLE_GAPS_TEST_PAUSE_AT` names one point at which `ORACLE_GAPS_TEST_PAUSE_MS` (milliseconds) blocks the CLI: `takeover` (inside a stale-claim takeover, after the stale pid is read) and `mutex-create` (inside the takeover mutex's creation). + `ORACLE_GAPS_TEST_STOP` names a file whose appearance makes the CLI take a stop as a SIGINT would. + `ORACLE_GAPS_TEST_HOLD_EXIT_MS` holds the run's exit after its done line, + and `ORACLE_GAPS_TEST_WARM_PROBE` makes the first config warm-up wait where a + stop signal can reach it. A few endings print no done line, or one more line: a SIGKILL of the CLI, or a signal that comes while Node is still starting the CLI, before its From 7db86c57b22ca7f4eb1d49aee8dd7b83c5532bab Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 14 Nov 2026 09:55:18 -0700 Subject: [PATCH 16/16] test(oracle-gaps): hold the exit at a named point, and delete the wall-clock hold The overlap and the spawn stop 'after' case kept the first run alive with a wall-clock hold (HOLD_EXIT_MS): under load the first run can end before the second starts, so the overlap outcome depended on timing. The hold is now a named point: after the done line the process keeps its exit (and its output claim) until ORACLE_GAPS_TEST_HOLD_UNTIL names an existing file. The driver releases it once the second run has been refused, or once the 'after' stop has been sent. HOLD_EXIT_MS is deleted. The README documents the new hook. Co-authored-by: Isaac --- README.md | 5 ++-- .../gap-report-oracle.property.test.mjs | 27 ++++++++++++------- packages/oracles/oracle-gaps/index.mjs | 16 +++++++++-- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 2881da5..de7f4ed 100644 --- a/README.md +++ b/README.md @@ -138,8 +138,9 @@ agent. `ORACLE_GAPS_TEST_PAUSE_AFTER_STARTED` (milliseconds) blocks the CLI for that long right after the started file is written, so a stop sent on seeing the file lands inside the pause. `ORACLE_GAPS_TEST_PAUSE_AT` names one point at which `ORACLE_GAPS_TEST_PAUSE_MS` (milliseconds) blocks the CLI: `takeover` (inside a stale-claim takeover, after the stale pid is read) and `mutex-create` (inside the takeover mutex's creation). `ORACLE_GAPS_TEST_STOP` names a file whose appearance makes the CLI take a stop as a SIGINT would. - `ORACLE_GAPS_TEST_HOLD_EXIT_MS` holds the run's exit after its done line, - and `ORACLE_GAPS_TEST_WARM_PROBE` makes the first config warm-up wait where a + `ORACLE_GAPS_TEST_HOLD_UNTIL` names a file: after its done line the run keeps + its exit, and its output claim, until the file exists, so a test can send a + stop or start a second run before the run ends. `ORACLE_GAPS_TEST_WARM_PROBE` makes the first config warm-up wait where a stop signal can reach it. A few endings print no done line, or one more line: a SIGKILL of the CLI, diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 7c98382..8dc3edb 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -708,7 +708,7 @@ const strays = (dir, before, running = false) => * * With `when` `after`, the run is not stopped while it runs: the witness * does not spin, the run's process holds its exit for 500 ms once it has - * printed its done line (`ORACLE_GAPS_TEST_HOLD_EXIT_MS`, a hook for this + * printed its done line (`ORACLE_GAPS_TEST_HOLD_UNTIL`, a hook for this * driver), and the driver sends the signal once that line is on stdout * (review of #46, F3). The run then ends as it completed, with exactly one * done line. With `warm`, the driver sends the signal while the run warms @@ -758,7 +758,10 @@ async function stoppedRun( ? { ORACLE_GAPS_TEST_PAUSE_AFTER_STARTED: `200` } : {}), ...(after - ? { ...cleanEnv(), ORACLE_GAPS_TEST_HOLD_EXIT_MS: `500` } + ? { + ...cleanEnv(), + ORACLE_GAPS_TEST_HOLD_UNTIL: join(pids, `after-release`), + } : warm ? { ...cleanEnv(), @@ -828,6 +831,8 @@ async function stoppedRun( await sleep(1) else if (atSpawn) await sleep(delay) if (!abort) child.kill(signal) + // The `after` stop is sent while the exit is held; the hold is released now. + if (after) writeFileSync(join(pids, `after-release`), ``) const ended = await Promise.race([exit, sleep(30_000).then(() => false)]) await sleep(1000) const alive = readdirSync(pids) @@ -961,17 +966,17 @@ async function overlappingRuns(dir, files, how) { writeFixture(dir, files) const before = new Set(filesUnder(dir)) const { args, out } = cliArgs(dir, files, how) - const start = (holdMs) => { + const start = (holdUntilFile) => { const child = spawn(process.execPath, args, { cwd: repo, - // A test-only hold (ORACLE_GAPS_TEST_HOLD_EXIT_MS, index.mjs): the first - // run keeps its output claim until the second has been refused, so the - // overlap does not depend on how fast the first run ends. + // The first run holds its exit (and its output claim) until the driver + // releases it after the second run has been refused: the overlap is + // deterministic (ORACLE_GAPS_TEST_HOLD_UNTIL, index.mjs). env: { ...cleanEnv(), - ...(holdMs === undefined + ...(holdUntilFile === undefined ? {} - : { ORACLE_GAPS_TEST_HOLD_EXIT_MS: String(holdMs) }), + : { ORACLE_GAPS_TEST_HOLD_UNTIL: holdUntilFile }), }, stdio: [`ignore`, `pipe`, `pipe`], }) @@ -991,7 +996,10 @@ async function overlappingRuns(dir, files, how) { }) return { got, status } } - const first = start(5000) + // The release file lives outside the package: a stray write there is judged. + const releaseDir = mkdtempSync(join(realpathSync(tmpdir()), `gap-release-`)) + const releaseFile = join(releaseDir, `overlap-release`) + const first = start(releaseFile) let firstEnded = false first.status.then(() => { firstEnded = true @@ -1013,6 +1021,7 @@ async function overlappingRuns(dir, files, how) { ) const second = start() const secondStatus = await second.status + writeFileSync(releaseFile, ``) const firstStatus = await first.status const ended = readRun( dir, diff --git a/packages/oracles/oracle-gaps/index.mjs b/packages/oracles/oracle-gaps/index.mjs index 99c3412..f1ee301 100755 --- a/packages/oracles/oracle-gaps/index.mjs +++ b/packages/oracles/oracle-gaps/index.mjs @@ -332,10 +332,22 @@ function startLog(out) { } /** Flushes both streams, then exits with `code` (after a test's hold). */ +/** + * A test-only hold (ORACLE_GAPS_TEST_HOLD_UNTIL, a path): after its done line the + * process keeps its exit, and its output claim, until the file exists. A test + * releases it once the overlapping run or the stop it sent has been observed, so + * the outcome does not depend on wall-clock timing. Inert unless set. + */ +function holdUntil() { + const file = process.env.ORACLE_GAPS_TEST_HOLD_UNTIL + if (file !== undefined) while (!existsSync(file)) sleepMs(20) +} function exitWhenFlushed(code) { - const hold = Number(process.env.ORACLE_GAPS_TEST_HOLD_EXIT_MS ?? 0) process.stdout.write(``, () => - process.stderr.write(``, () => setTimeout(() => process.exit(code), hold)), + process.stderr.write(``, () => { + holdUntil() + process.exit(code) + }), ) }