From f6255caf1802b223bdd6fc6c64c7e42105b19adb Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 29 Oct 2026 12:09:22 +0100 Subject: [PATCH 01/13] feat(core): run walltime benchmarks in forks Benchmarks in one process share allocator and JIT state, and rounds within one process are correlated, so a single process gives a biased view of a benchmark. Add the building blocks for integrations to run each walltime benchmark in several fresh processes (forks) and pool their rounds: - `walltimeIsolation(forks?)` picks the mode from the env: parent, fork (`CODSPEED_ISOLATED_BENCHMARK`), or one shared process when `CODSPEED_PROCESS_ISOLATION=false` - the fork count is `CODSPEED_WALLTIME_FORKS`, else the integration's `forks` option, else 3 - `runForks`/`runForksSync` re-run the benchmark command (keeping the V8 flags) once per fork and collect the samples each fork writes with `writeForkSamples`; a failing fork's exit code is passed on - `samplesToResult` builds tinybench-style statistics over the pooled samples Refs COD-3960 --- packages/core/src/index.ts | 1 + packages/core/src/isolation.ts | 213 ++++++++++++++++++ packages/core/src/walltime/utils.ts | 46 ++++ packages/core/tests/fixtures/isolation.cjs | 36 +++ packages/core/tests/isolation.integ.test.ts | 187 +++++++++++++++ .../core/tests/walltimeUtils.integ.test.ts | 31 +++ 6 files changed, 514 insertions(+) create mode 100644 packages/core/src/isolation.ts create mode 100644 packages/core/tests/fixtures/isolation.cjs create mode 100644 packages/core/tests/isolation.integ.test.ts create mode 100644 packages/core/tests/walltimeUtils.integ.test.ts diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 8dc74641..8ceb582f 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -53,6 +53,7 @@ export type { SetupInstrumentsResponse, } from "./generated/openapi"; export { getV8Flags, tryIntrospect } from "./introspection"; +export * from "./isolation"; export { getUnsupportedNodeVersionWarning, SUPPORTED_NODE_MAJORS, diff --git a/packages/core/src/isolation.ts b/packages/core/src/isolation.ts new file mode 100644 index 00000000..6f0f1d36 --- /dev/null +++ b/packages/core/src/isolation.ts @@ -0,0 +1,213 @@ +/** + * Process isolation for walltime runs: every benchmark runs in fresh copies + * of the benchmark process (forks), so the allocator state, JIT state, and + * handles that other benchmarks or setup code leave behind can't change its + * measurement. Rounds within one process are correlated (JIT tier-up, heap + * sizing), so the rounds of several forks are pooled into one result. + */ +import { spawn, spawnSync } from "child_process"; +import { randomUUID } from "crypto"; +import { readFileSync, rmSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { deserialize, serialize } from "v8"; +import { constants, gunzipSync, gzipSync } from "zlib"; + +/** Accepts `0`, `false`, `1`, or `true`; defaults to enabled. */ +export const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +/** Forks per walltime benchmark. Overrides the integration's `forks` option. */ +export const WALLTIME_FORKS_ENV = "CODSPEED_WALLTIME_FORKS"; +/** Forks per walltime benchmark when neither the env nor an option sets it. */ +export const DEFAULT_WALLTIME_FORKS = 3; +/** Set on each fork: the URI of the only benchmark it runs. */ +export const ISOLATED_BENCHMARK_ENV = "CODSPEED_ISOLATED_BENCHMARK"; +/** Set on each fork: the file where it writes its round samples. */ +export const FORK_SAMPLES_FILE_ENV = "CODSPEED_FORK_SAMPLES_FILE"; + +/** Walltime settings that every integration exposes next to its own options. */ +export interface WalltimeOptions { + /** + * Fresh processes that each walltime benchmark runs in; their rounds are + * reported together. {@link WALLTIME_FORKS_ENV} overrides it; defaults to + * {@link DEFAULT_WALLTIME_FORKS}. `CODSPEED_PROCESS_ISOLATION=false` runs + * everything in the benchmark process instead. + */ + forks?: number; +} + +export type IsolationMode = + /** All benchmarks are measured in this process. */ + | { kind: "SharedProcess" } + /** Every benchmark is measured in `forks` fresh processes, one after the other. */ + | { kind: "Forks"; forks: number } + /** A fork: measures only `uri` and writes its samples to `samplesFile`. */ + | { kind: "Fork"; uri: string; samplesFile: string }; + +/** + * The isolation of a walltime run, from the environment. `forks` is the + * integration's option, which {@link WALLTIME_FORKS_ENV} overrides. + */ +export function walltimeIsolation(forks?: number): IsolationMode { + const uri = process.env[ISOLATED_BENCHMARK_ENV]; + if (uri !== undefined) { + const samplesFile = process.env[FORK_SAMPLES_FILE_ENV]; + if (samplesFile === undefined) { + throw new Error( + `[CodSpeed] ${ISOLATED_BENCHMARK_ENV} is set without ${FORK_SAMPLES_FILE_ENV}`, + ); + } + return { kind: "Fork", uri, samplesFile }; + } + const isolation = process.env[PROCESS_ISOLATION_ENV]; + if (isolation === "0" || isolation === "false") { + return { kind: "SharedProcess" }; + } + if (isolation !== undefined && isolation !== "1" && isolation !== "true") { + throw new Error( + `[CodSpeed] ${PROCESS_ISOLATION_ENV} must be 0, false, 1, or true, got \`${isolation}\``, + ); + } + return { kind: "Forks", forks: resolveWalltimeForks(forks) }; +} + +/** + * The number of forks per walltime benchmark: {@link WALLTIME_FORKS_ENV}, + * else `option`, else {@link DEFAULT_WALLTIME_FORKS}. + */ +export function resolveWalltimeForks(option?: number): number { + const env = process.env[WALLTIME_FORKS_ENV]; + if (env !== undefined && env !== "") { + return validForks(env, WALLTIME_FORKS_ENV); + } + if (option !== undefined) { + return validForks(option, "forks"); + } + return DEFAULT_WALLTIME_FORKS; +} + +function validForks(raw: string | number, source: string): number { + const forks = Number(raw); + if (!Number.isInteger(forks) || forks < 1) { + throw new Error( + `[CodSpeed] ${source} must be a positive integer, got \`${raw}\``, + ); + } + return forks; +} + +/** The command of this process, which each fork re-runs. */ +function currentCommand(): string[] { + // execArgv keeps the V8 flags (e.g. --perf-prof) that the runner set. + return [process.execPath, ...process.execArgv, ...process.argv.slice(1)]; +} + +/** + * Runs the benchmark `uri` in `forks` fresh processes, one after the other, + * and returns the round samples of all of them, in milliseconds. Throws when + * a fork fails, without running the next one. + * + * Each fork re-runs this process's command, not fork(): Node can't fork, and a + * fork would only copy the calling thread. Call it from the process whose + * command runs the benchmarks: Vitest's main process, not a worker. + */ +export async function runForks(uri: string, forks: number): Promise { + let samples: number[] = []; + for (let fork = 0; fork < forks; fork++) { + // Not push(...): spreading a time-based run's samples overflows the stack. + samples = samples.concat(await runFork(uri)); + } + return samples; +} + +/** {@link runForks} for synchronous runs, which block anyway. */ +export function runForksSync(uri: string, forks: number): number[] { + let samples: number[] = []; + for (let fork = 0; fork < forks; fork++) { + samples = samples.concat(runForkSync(uri)); + } + return samples; +} + +/** A fresh file per fork: overlapping runForks() calls must not share one. */ +function newSamplesFile(): string { + return join( + tmpdir(), + `codspeed-samples-${process.pid}-${randomUUID()}.v8.gz`, + ); +} + +/** Runs one fork of `uri` and returns the samples it wrote. */ +async function runFork(uri: string): Promise { + const samplesFile = newSamplesFile(); + const [executable, ...args] = currentCommand(); + const child = spawn(executable, args, { + stdio: "inherit", + env: forkEnv(uri, samplesFile), + }); + const { promise, resolve, reject } = + Promise.withResolvers<[number | null, NodeJS.Signals | null]>(); + child.once("error", (error) => reject(startError(uri, error))); + child.once("close", (status, signal) => resolve([status, signal])); + const [status, signal] = await promise; + return forkSamples(uri, samplesFile, status, signal); +} + +function runForkSync(uri: string): number[] { + const samplesFile = newSamplesFile(); + const [executable, ...args] = currentCommand(); + const result = spawnSync(executable, args, { + stdio: "inherit", + env: forkEnv(uri, samplesFile), + }); + if (result.error !== undefined) { + throw startError(uri, result.error); + } + return forkSamples(uri, samplesFile, result.status, result.signal); +} + +function forkEnv(uri: string, samplesFile: string): NodeJS.ProcessEnv { + return { + ...process.env, + [ISOLATED_BENCHMARK_ENV]: uri, + [FORK_SAMPLES_FILE_ENV]: samplesFile, + }; +} + +function startError(uri: string, error: Error): Error { + return new Error( + `[CodSpeed] failed to start the process for ${uri}: ${error.message}`, + ); +} + +/** The samples a finished fork wrote, deleting the file; throws if it failed. */ +function forkSamples( + uri: string, + samplesFile: string, + status: number | null, + signal: NodeJS.Signals | null, +): number[] { + try { + if (status !== 0) { + throw new Error( + `[CodSpeed] the process for ${uri} failed with ${signal ?? `exit code ${status}`}`, + ); + } + try { + return deserialize(gunzipSync(readFileSync(samplesFile))) as number[]; + } catch (error) { + throw new Error( + `[CodSpeed] the process for ${uri} did not report its samples: ${(error as Error).message}`, + ); + } + } finally { + rmSync(samplesFile, { force: true }); + } +} + +/** Hands round samples (milliseconds) to the parent as gzip-compressed V8 data. */ +export function writeForkSamples(samplesFile: string, samples: number[]) { + writeFileSync( + samplesFile, + gzipSync(serialize(samples), { level: constants.Z_BEST_SPEED }), + ); +} diff --git a/packages/core/src/walltime/utils.ts b/packages/core/src/walltime/utils.ts index fafa9e9e..baf97181 100644 --- a/packages/core/src/walltime/utils.ts +++ b/packages/core/src/walltime/utils.ts @@ -18,3 +18,49 @@ export const msToNs = (ms: number) => ms * 1e6; * @returns the seconds */ export const msToS = (ms: number) => ms / 1e3; + +/** tinybench-style statistics of round samples, in milliseconds. */ +export interface RoundsResult { + totalTime: number; + latency: { + min: number; + max: number; + mean: number; + sd: number; + samples: number[]; + }; +} + +/** + * The result of nonempty round samples in milliseconds, e.g. the samples of + * several forks of one benchmark. Throws if no samples were recorded. + */ +export function samplesToResult(samples: number[]): RoundsResult { + const n = samples.length; + if (n === 0) { + throw new Error("[CodSpeed] Cannot compute statistics from empty samples"); + } + let totalTime = 0; + let min = Infinity; + let max = -Infinity; + for (const ms of samples) { + totalTime += ms; + if (ms < min) min = ms; + if (ms > max) max = ms; + } + const mean = totalTime / n; + const variance = + n > 1 + ? samples.reduce((sum, ms) => sum + (ms - mean) ** 2, 0) / (n - 1) + : 0; + return { + totalTime, + latency: { + min, + max, + mean, + sd: Math.sqrt(variance), + samples, + }, + }; +} diff --git a/packages/core/tests/fixtures/isolation.cjs b/packages/core/tests/fixtures/isolation.cjs new file mode 100644 index 00000000..bd2d6d2e --- /dev/null +++ b/packages/core/tests/fixtures/isolation.cjs @@ -0,0 +1,36 @@ +// Run by `isolation.integ.test.ts`. Measures benchmark "a" in forks. +const { + runForks, + runForksSync, + walltimeIsolation, + writeForkSamples, +} = require("../../dist/index.cjs.js"); + +const mode = walltimeIsolation(); + +function measure() { + console.log(`ran ${process.pid} ${JSON.stringify(process.execArgv)}`); + switch (process.env.ISOLATION_FIXTURE_ACTION) { + case "exit": + process.exit(3); + case "no-samples": + process.exit(0); + } + writeForkSamples(mode.samplesFile, [Number(process.pid), 1]); +} + +async function main() { + if (mode.kind === "Fork") { + measure(); + return; + } + const forks = Number(process.env.ISOLATION_FIXTURE_FORKS); + const samples = + process.env.ISOLATION_FIXTURE_SYNC !== undefined + ? runForksSync("a", forks) + : await runForks("a", forks); + console.log(`samples ${samples.length}`); +} + +// A rejection ends the process with exit code 1. +main(); diff --git a/packages/core/tests/isolation.integ.test.ts b/packages/core/tests/isolation.integ.test.ts new file mode 100644 index 00000000..c61d3f45 --- /dev/null +++ b/packages/core/tests/isolation.integ.test.ts @@ -0,0 +1,187 @@ +import { spawnSync } from "child_process"; +import { join } from "path"; +import { + FORK_SAMPLES_FILE_ENV, + ISOLATED_BENCHMARK_ENV, + PROCESS_ISOLATION_ENV, + resolveWalltimeForks, + WALLTIME_FORKS_ENV, + walltimeIsolation, +} from "../src/isolation"; + +const FIXTURE = join(__dirname, "fixtures", "isolation.cjs"); +/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ +const RUN_TIMEOUT_MS = 60_000; +const CHILD_EXIT_CODE = 3; +const RAN_LINE = /^ran (\d+) (.+)$/; +const SAMPLES_LINE = /^samples (\d+)$/; +/** Passed to Node to check that forks keep the flags of their parent. */ +const NODE_FLAG = "--no-warnings"; +/** Each fork of the fixture reports two samples. */ +const SAMPLES_PER_FORK = 2; + +const ENV_KEYS = [ + PROCESS_ISOLATION_ENV, + ISOLATED_BENCHMARK_ENV, + FORK_SAMPLES_FILE_ENV, + WALLTIME_FORKS_ENV, +]; +const savedEnv = Object.fromEntries( + ENV_KEYS.map((key) => [key, process.env[key]]), +); + +function setEnv(env: Record): void { + for (const key of ENV_KEYS) { + delete process.env[key]; + } + for (const [key, value] of Object.entries(env)) { + if (value !== undefined) { + process.env[key] = value; + } + } +} + +afterEach(() => setEnv(savedEnv)); + +describe("resolveWalltimeForks", () => { + it("defaults to 3 forks", () => { + setEnv({}); + expect(resolveWalltimeForks()).toBe(3); + }); + + it("takes the option over the default", () => { + setEnv({}); + expect(resolveWalltimeForks(5)).toBe(5); + }); + + it("takes the env over the option", () => { + setEnv({ [WALLTIME_FORKS_ENV]: "2" }); + expect(resolveWalltimeForks(5)).toBe(2); + }); + + it("ignores an empty env", () => { + setEnv({ [WALLTIME_FORKS_ENV]: "" }); + expect(resolveWalltimeForks(5)).toBe(5); + }); + + it.each(["0", "-1", "1.5", "abc"])("rejects the env %p", (value) => { + setEnv({ [WALLTIME_FORKS_ENV]: value }); + expect(() => resolveWalltimeForks()).toThrow(WALLTIME_FORKS_ENV); + }); + + it.each([0, -1, 1.5, NaN])("rejects the option %p", (value) => { + setEnv({}); + expect(() => resolveWalltimeForks(value)).toThrow("forks"); + }); +}); + +describe("walltimeIsolation", () => { + it.each([ + [undefined, "Forks"], + ["1", "Forks"], + ["true", "Forks"], + ["0", "SharedProcess"], + ["false", "SharedProcess"], + ])("selects %s as %s", (value, kind) => { + setEnv({ [PROCESS_ISOLATION_ENV]: value }); + expect(walltimeIsolation().kind).toBe(kind); + }); + + it("runs everything in this process when isolation is off, whatever the fork count", () => { + setEnv({ [PROCESS_ISOLATION_ENV]: "false", [WALLTIME_FORKS_ENV]: "4" }); + expect(walltimeIsolation(2).kind).toBe("SharedProcess"); + }); + + it("restricts a fork to its benchmark, whatever the isolation setting", () => { + setEnv({ + [PROCESS_ISOLATION_ENV]: "false", + [ISOLATED_BENCHMARK_ENV]: "file::bench", + [FORK_SAMPLES_FILE_ENV]: "samples.json", + }); + expect(walltimeIsolation()).toEqual({ + kind: "Fork", + uri: "file::bench", + samplesFile: "samples.json", + }); + }); +}); + +interface FixtureOptions { + forks: number; + action?: "exit" | "no-samples"; + sync?: boolean; +} + +function runFixture({ forks, action, sync }: FixtureOptions) { + const env: NodeJS.ProcessEnv = { + ...process.env, + ISOLATION_FIXTURE_ACTION: action, + ISOLATION_FIXTURE_FORKS: String(forks), + }; + for (const key of ENV_KEYS) { + delete env[key]; + } + if (sync) { + env.ISOLATION_FIXTURE_SYNC = "1"; + } + const result = spawnSync(process.execPath, [NODE_FLAG, FIXTURE], { + env, + encoding: "utf8", + timeout: RUN_TIMEOUT_MS, + }); + const lines = result.stdout.split("\n"); + const ran = lines + .map((line) => RAN_LINE.exec(line)) + .filter((match): match is RegExpExecArray => match !== null); + const samples = lines + .map((line) => SAMPLES_LINE.exec(line)) + .find((match) => match !== null); + return { + status: result.status, + stderr: result.stderr, + ran, + samples: samples ? Number(samples[1]) : undefined, + }; +} + +describe.each([ + { name: "async", sync: false }, + { name: "sync", sync: true }, +])("runForks ($name run)", ({ sync }) => { + it("runs every fork in a fresh process with the flags of its parent and pools their samples", () => { + const forks = 3; + const { status, ran, samples } = runFixture({ forks, sync }); + + expect(status).toBe(0); + expect(ran).toHaveLength(forks); + expect(new Set(ran.map((match) => match[1])).size).toBe(forks); + for (const match of ran) { + expect(JSON.parse(match[2])).toEqual([NODE_FLAG]); + } + expect(samples).toBe(forks * SAMPLES_PER_FORK); + }); + + it("fails on a failing fork and runs no further fork", () => { + const { status, stderr, ran, samples } = runFixture({ + forks: 3, + sync, + action: "exit", + }); + + expect(status).not.toBe(0); + expect(stderr).toContain(`failed with exit code ${CHILD_EXIT_CODE}`); + expect(ran).toHaveLength(1); + expect(samples).toBeUndefined(); + }); + + it("fails when a fork exits without reporting its samples", () => { + const { status, stderr } = runFixture({ + forks: 2, + sync, + action: "no-samples", + }); + + expect(status).not.toBe(0); + expect(stderr).toContain("did not report its samples"); + }); +}); diff --git a/packages/core/tests/walltimeUtils.integ.test.ts b/packages/core/tests/walltimeUtils.integ.test.ts new file mode 100644 index 00000000..132b7559 --- /dev/null +++ b/packages/core/tests/walltimeUtils.integ.test.ts @@ -0,0 +1,31 @@ +import { samplesToResult } from "../src/walltime/utils"; + +describe("samplesToResult", () => { + it("computes pooled statistics", () => { + const samples = [2, 4, 4, 4, 5, 5, 7, 9]; + const { totalTime, latency } = samplesToResult(samples); + expect(totalTime).toBe(40); + expect(latency.min).toBe(2); + expect(latency.max).toBe(9); + expect(latency.mean).toBe(5); + // sample standard deviation: sqrt(32 / 7) + expect(latency.sd).toBeCloseTo(Math.sqrt(32 / 7), 10); + }); + + it("has zero deviation for a single sample", () => { + const { totalTime, latency } = samplesToResult([3]); + expect(totalTime).toBe(3); + expect(latency).toMatchObject({ min: 3, max: 3, mean: 3, sd: 0 }); + }); + + it("handles very large sample counts without overflowing the stack", () => { + const samples = new Array(500_000).fill(1); + expect(samplesToResult(samples).totalTime).toBe(500_000); + }); + + it("rejects empty samples", () => { + expect(() => samplesToResult([])).toThrow( + "[CodSpeed] Cannot compute statistics from empty samples", + ); + }); +}); From 68883f3758cd752071aeabbe8cd8b170b0a24a57 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 29 Oct 2026 12:09:50 +0100 Subject: [PATCH 02/13] feat(tinybench): run walltime benchmarks in forks `withCodSpeed(bench, { forks })` sets the fork count of walltime runs, and `CODSPEED_WALLTIME_FORKS` overrides it. The parent runs each task in its forks and reports their pooled rounds; a fork runs only its task, hands its samples over and prints no banner or summary. Simulation runs are unchanged. Refs COD-3960 --- packages/tinybench-plugin/src/analysis.ts | 4 +- packages/tinybench-plugin/src/index.ts | 8 +- packages/tinybench-plugin/src/shared.ts | 54 ++++- packages/tinybench-plugin/src/walltime.ts | 69 ++++++- .../tests/fixtures/isolation.mjs | 32 +++ .../tests/isolation.integ.test.ts | 194 ++++++++++++++++++ 6 files changed, 341 insertions(+), 20 deletions(-) create mode 100644 packages/tinybench-plugin/tests/fixtures/isolation.mjs create mode 100644 packages/tinybench-plugin/tests/isolation.integ.test.ts diff --git a/packages/tinybench-plugin/src/analysis.ts b/packages/tinybench-plugin/src/analysis.ts index 65bd6183..54c8d975 100644 --- a/packages/tinybench-plugin/src/analysis.ts +++ b/packages/tinybench-plugin/src/analysis.ts @@ -14,7 +14,9 @@ export function setupCodspeedAnalysisBench( bench: Bench, rootCallingFile: string, ): void { - const runner = new AnalysisBenchRunner(bench, rootCallingFile); + const runner = new AnalysisBenchRunner(bench, rootCallingFile, { + kind: "SharedProcess", + }); runner.setupBenchMethods(); } diff --git a/packages/tinybench-plugin/src/index.ts b/packages/tinybench-plugin/src/index.ts index 298f97b9..bb903ae0 100644 --- a/packages/tinybench-plugin/src/index.ts +++ b/packages/tinybench-plugin/src/index.ts @@ -8,6 +8,7 @@ import { SetupInstrumentsResponse, tryIntrospect, wrapWithRootFrameSync, + type WalltimeOptions, } from "@codspeed/core"; import { Bench } from "tinybench"; import { setupCodspeedAnalysisBench } from "./analysis"; @@ -17,7 +18,10 @@ import { setupCodspeedWalltimeBench } from "./walltime"; tryIntrospect(); -export function withCodSpeed(bench: Bench): Bench { +export function withCodSpeed( + bench: Bench, + options: WalltimeOptions = {}, +): Bench { const codspeedRunnerMode = getCodspeedRunnerMode(); if (codspeedRunnerMode === "disabled") { return bench; @@ -52,7 +56,7 @@ export function withCodSpeed(bench: Bench): Bench { if (instrumentMode === "analysis") { setupCodspeedAnalysisBench(bench, rootCallingFile); } else if (instrumentMode === "walltime") { - setupCodspeedWalltimeBench(bench, rootCallingFile); + setupCodspeedWalltimeBench(bench, rootCallingFile, options); } return bench; diff --git a/packages/tinybench-plugin/src/shared.ts b/packages/tinybench-plugin/src/shared.ts index 0fb811c5..1e10b4bf 100644 --- a/packages/tinybench-plugin/src/shared.ts +++ b/packages/tinybench-plugin/src/shared.ts @@ -1,4 +1,4 @@ -import { setupCore, teardownCore } from "@codspeed/core"; +import { setupCore, teardownCore, type IsolationMode } from "@codspeed/core"; import { Bench, Task } from "tinybench"; import { CapturedTaskData, getTaskData } from "./taskData"; import { getTaskUri } from "./uri"; @@ -8,10 +8,21 @@ declare const __VERSION__: string; export abstract class BaseBenchRunner { protected bench: Bench; protected rootCallingFile: string; - - constructor(bench: Bench, rootCallingFile: string) { + /** Walltime runners pass the isolation of this process; analysis shares it. */ + protected readonly isolationMode: IsolationMode; + + constructor( + bench: Bench, + rootCallingFile: string, + isolationMode: IsolationMode, + ) { this.bench = bench; this.rootCallingFile = rootCallingFile; + this.isolationMode = isolationMode; + } + + protected get isChild(): boolean { + return this.isolationMode.kind === "Fork"; } private setupBenchRun(): void { @@ -20,6 +31,10 @@ export abstract class BaseBenchRunner { } private logStart(): void { + // The parent already printed the banner, children stay quiet. + if (this.isChild) { + return; + } console.log( `[CodSpeed] running with @codspeed/tinybench v${__VERSION__} (${this.getModeName()})`, ); @@ -45,7 +60,12 @@ export abstract class BaseBenchRunner { protected finalizeBenchRun(): Task[] { teardownCore(); - console.log(`[CodSpeed] Done running ${this.bench.tasks.length} benches.`); + // The parent already printed the summary line, children stay quiet. + if (!this.isChild) { + console.log( + `[CodSpeed] Done running ${this.bench.tasks.length} benches.`, + ); + } return this.bench.tasks; } @@ -57,10 +77,15 @@ export abstract class BaseBenchRunner { public setupBenchMethods(): void { this.bench.run = async () => { + const tasks = this.selectTasks(); + // A child that owns none of this bench's tasks must not touch the + // instruments, the bench that owns its task does. + if (this.isChild && tasks.length === 0) { + return this.bench.tasks; + } this.setupBenchRun(); - for (const task of this.bench.tasks) { - const uri = this.getTaskUri(task); + for (const { task, uri } of tasks) { await this.runTaskAsync(task, uri); } @@ -68,14 +93,27 @@ export abstract class BaseBenchRunner { }; this.bench.runSync = () => { + const tasks = this.selectTasks(); + if (this.isChild && tasks.length === 0) { + return this.bench.tasks; + } this.setupBenchRun(); - for (const task of this.bench.tasks) { - const uri = this.getTaskUri(task); + for (const { task, uri } of tasks) { this.runTaskSync(task, uri); } return this.finalizeSyncRun(); }; } + + /** The tasks this process measures: all of them, or the isolated one. */ + private selectTasks(): { task: Task; uri: string }[] { + return this.bench.tasks + .map((task) => ({ task, uri: this.getTaskUri(task) })) + .filter( + ({ uri }) => + this.isolationMode.kind !== "Fork" || this.isolationMode.uri === uri, + ); + } } diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index 90e6b700..7168cb0c 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -6,19 +6,28 @@ import { mongoMeasurement, msToNs, msToS, + runForks, + runForksSync, + samplesToResult, + walltimeIsolation, + writeForkSamples, writeWalltimeResults, type BenchmarkStats, type Benchmark as CodspeedBenchmark, + type RoundsResult, + type WalltimeOptions, } from "@codspeed/core"; -import { Bench, Task, TaskResult } from "tinybench"; +import { Bench, Task } from "tinybench"; import { getBenchOptions } from "./benchOptions"; import { BaseBenchRunner } from "./shared"; export function setupCodspeedWalltimeBench( bench: Bench, rootCallingFile: string, + options: WalltimeOptions, ): void { - const runner = new WalltimeBenchRunner(bench, rootCallingFile); + const isolationMode = walltimeIsolation(options.forks); + const runner = new WalltimeBenchRunner(bench, rootCallingFile, isolationMode); runner.installInstrumentHooks(); runner.setupBenchMethods(); } @@ -91,6 +100,13 @@ class WalltimeBenchRunner extends BaseBenchRunner { } protected async runTaskAsync(task: Task, uri: string): Promise { + const { isolationMode } = this; + if (isolationMode.kind === "Forks") { + const samples = await runForks(uri, isolationMode.forks); + setTaskResult(task, samples); + return this.registerCodspeedBenchmark(task, samplesToResult(samples)); + } + // run the warmup of the task right before its actual run if (getBenchOptions(this.bench).warmup) { await task.warmup(); @@ -103,8 +119,14 @@ class WalltimeBenchRunner extends BaseBenchRunner { this.registerCodspeedBenchmarkFromTask(task); } - // eslint-disable-next-line @typescript-eslint/no-unused-vars - protected runTaskSync(task: Task, _uri: string): void { + protected runTaskSync(task: Task, uri: string): void { + const { isolationMode } = this; + if (isolationMode.kind === "Forks") { + const samples = runForksSync(uri, isolationMode.forks); + setTaskResult(task, samples); + return this.registerCodspeedBenchmark(task, samplesToResult(samples)); + } + if (getBenchOptions(this.bench).warmup) { task.warmupSync(); } @@ -123,19 +145,30 @@ class WalltimeBenchRunner extends BaseBenchRunner { } private registerCodspeedBenchmarkFromTask(task: Task): void { - const uri = this.getTaskUri(task); - if (!task.result) { - console.warn(` ⚠ No result data available for ${uri}`); + console.warn( + ` ⚠ No result data available for ${this.getTaskUri(task)}`, + ); + return; + } + // A fork hands its samples to the parent, which reports the benchmark. + if (this.isolationMode.kind === "Fork") { + writeForkSamples(this.isolationMode.samplesFile, [ + ...task.result.latency.samples, + ]); return; } + this.registerCodspeedBenchmark(task, task.result); + } + private registerCodspeedBenchmark(task: Task, result: RoundsResult): void { + const uri = this.getTaskUri(task); const opts = getBenchOptions(this.bench); const warmupIterations = opts.warmup ? (opts.warmupIterations ?? TINYBENCH_WARMUP_DEFAULT) : 0; const stats = convertTinybenchResultToBenchmarkStats( - task.result, + result, warmupIterations, ); @@ -156,6 +189,11 @@ class WalltimeBenchRunner extends BaseBenchRunner { } private finalizeWalltimeRun(isAsync: boolean): Task[] { + // A fork only hands its samples to the parent, which reports the results. + if (this.isChild) { + return this.bench.tasks; + } + // Write results to JSON file using core function if (this.codspeedBenchmarks.length > 0) { writeWalltimeResults(this.codspeedBenchmarks, isAsync); @@ -171,7 +209,7 @@ class WalltimeBenchRunner extends BaseBenchRunner { const TINYBENCH_WARMUP_DEFAULT = 16; function convertTinybenchResultToBenchmarkStats( - result: TaskResult, + result: RoundsResult, warmupIterations: number, ): BenchmarkStats { const { min, max, mean, sd, samples } = result.latency; @@ -200,3 +238,16 @@ function convertTinybenchResultToBenchmarkStats( warmup_iters: warmupIterations, }; } + +/** + * Fills `task.result` from the pooled fork samples so `bench.table()` and + * `bench.results` work. Only tinybench 4/5 expose a way in; v6 keeps the + * result `#private`. + */ +function setTaskResult(task: Task, samples: number[]): void { + const { processRunResult } = task as unknown as { + processRunResult?: (run: { latencySamples: number[] }) => void; + }; + // tinybench sorts the samples in place. + processRunResult?.call(task, { latencySamples: [...samples] }); +} diff --git a/packages/tinybench-plugin/tests/fixtures/isolation.mjs b/packages/tinybench-plugin/tests/fixtures/isolation.mjs new file mode 100644 index 00000000..e347110d --- /dev/null +++ b/packages/tinybench-plugin/tests/fixtures/isolation.mjs @@ -0,0 +1,32 @@ +// Run by `isolation.integ.test.ts`. Prints the PID of every process that runs a +// benchmark, and the sample count of every `task.result` the parent ends with. +import { Bench } from "tinybench"; +import { withCodSpeed } from "../../dist/index.es5.js"; + +const options = process.env.ISOLATION_FIXTURE_FORKS_OPTION; +const bench = withCodSpeed( + new Bench({ iterations: 4, time: 1, warmupTime: 0 }), + options === undefined ? {} : { forks: Number(options) }, +); + +for (const name of ["first", "second"]) { + // tinybench calls the function once when it is added, in every process. + let calls = 0; + bench.add(name, () => { + if (++calls === 2) { + console.log(`ran ${process.pid} ${name}`); + } + }); +} + +if (process.env.ISOLATION_FIXTURE_SYNC !== undefined) { + bench.runSync(); +} else { + await bench.run(); +} + +if (process.env.CODSPEED_ISOLATED_BENCHMARK === undefined) { + for (const task of bench.tasks) { + console.log(`result ${task.name} ${task.result?.samples.length ?? 0}`); + } +} diff --git a/packages/tinybench-plugin/tests/isolation.integ.test.ts b/packages/tinybench-plugin/tests/isolation.integ.test.ts new file mode 100644 index 00000000..b78a5db4 --- /dev/null +++ b/packages/tinybench-plugin/tests/isolation.integ.test.ts @@ -0,0 +1,194 @@ +import { getV8Flags } from "@codspeed/core"; +import { spawn } from "child_process"; +import { once } from "events"; +import { mkdtempSync, readdirSync, readFileSync, rmSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { fileURLToPath } from "url"; +import { describe, expect, it } from "vitest"; + +const FIXTURE = fileURLToPath( + new URL("./fixtures/isolation.mjs", import.meta.url), +); +const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +const WALLTIME_FORKS_ENV = "CODSPEED_WALLTIME_FORKS"; +const BENCHMARKS = ["first", "second"]; +const DEFAULT_FORKS = 3; +/** The fixture's `iterations`: every fork measures at least this many rounds. */ +const MIN_ROUNDS_PER_FORK = 4; +/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ +const RUN_TIMEOUT_MS = 60_000; +const RAN_LINE = /^ran (\d+) (.+)$/; +const RESULT_LINE = /^result (.+) (\d+)$/; + +interface FixtureRun { + status: number | null; + /** PIDs that ran each benchmark. */ + pidsByName: Map>; + /** PID of the process the test started. */ + parentPid: number; + /** Rounds of each benchmark in the results file. */ + roundsByName: Map; + /** Samples in the parent's `task.result` of each benchmark. */ + parentSamplesByName: Map; +} + +interface FixtureOptions { + /** `CODSPEED_PROCESS_ISOLATION=false`: every benchmark shares our process. */ + shared?: boolean; + /** `forks` option of `withCodSpeed`. */ + forksOption?: number; + /** `CODSPEED_WALLTIME_FORKS`. */ + forksEnv?: number; + sync?: boolean; +} + +async function runFixture({ + shared, + forksOption, + forksEnv, + sync, +}: FixtureOptions = {}): Promise { + const profileFolder = mkdtempSync(join(tmpdir(), "codspeed-tinybench-")); + try { + const env: NodeJS.ProcessEnv = { + ...process.env, + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "walltime", + CODSPEED_PROFILE_FOLDER: profileFolder, + }; + delete env[PROCESS_ISOLATION_ENV]; + delete env[WALLTIME_FORKS_ENV]; + if (shared) { + env[PROCESS_ISOLATION_ENV] = "false"; + } + if (forksOption !== undefined) { + env.ISOLATION_FIXTURE_FORKS_OPTION = String(forksOption); + } + if (forksEnv !== undefined) { + env[WALLTIME_FORKS_ENV] = String(forksEnv); + } + if (sync) { + env.ISOLATION_FIXTURE_SYNC = "1"; + } + + // The runner starts Node with these flags, and forks must keep them. Not + // spawnSync: blocking the test worker times out its calls to Vitest. + const child = spawn(process.execPath, [...getV8Flags(), FIXTURE], { + env, + timeout: RUN_TIMEOUT_MS, + }); + let stdout = ""; + child.stdout.setEncoding("utf8").on("data", (chunk) => (stdout += chunk)); + const [status] = (await once(child, "close")) as [number | null]; + + const pidsByName = new Map>(); + const parentSamplesByName = new Map(); + for (const line of stdout.split("\n")) { + const ran = RAN_LINE.exec(line); + if (ran) { + const pids = pidsByName.get(ran[2]) ?? new Set(); + pids.add(Number(ran[1])); + pidsByName.set(ran[2], pids); + continue; + } + const result = RESULT_LINE.exec(line); + if (result) { + parentSamplesByName.set(result[1], Number(result[2])); + } + } + + const roundsByName = new Map(); + const resultsDir = join(profileFolder, "results"); + for (const file of readdirSync(resultsDir)) { + const data = JSON.parse(readFileSync(join(resultsDir, file), "utf8")); + for (const benchmark of data.benchmarks) { + roundsByName.set(benchmark.name, benchmark.stats.rounds); + } + } + return { + status, + pidsByName, + parentPid: child.pid!, + roundsByName, + parentSamplesByName, + }; + } finally { + rmSync(profileFolder, { recursive: true, force: true }); + } +} + +/** PIDs that ran a benchmark besides the one we started. */ +function forkPids(run: FixtureRun, name: string): Set { + const pids = new Set(run.pidsByName.get(name)); + pids.delete(run.parentPid); + return pids; +} + +describe("walltime forks", () => { + it.each([ + { name: "async", sync: false }, + { name: "sync", sync: true }, + ])( + "runs every benchmark in 3 forks by default and pools their rounds ($name run)", + async ({ sync }) => { + const run = await runFixture({ sync }); + + expect(run.status).toBe(0); + for (const name of BENCHMARKS) { + expect(forkPids(run, name).size).toBe(DEFAULT_FORKS); + const rounds = run.roundsByName.get(name); + expect(rounds).toBeGreaterThanOrEqual( + DEFAULT_FORKS * MIN_ROUNDS_PER_FORK, + ); + // The parent reports the pooled rounds as its own `task.result`. + expect(run.parentSamplesByName.get(name)).toBe(rounds); + } + // Forks of different benchmarks are different processes too. + const all = new Set( + BENCHMARKS.flatMap((name) => [...forkPids(run, name)]), + ); + expect(all.size).toBe(DEFAULT_FORKS * BENCHMARKS.length); + }, + RUN_TIMEOUT_MS, + ); + + it( + "takes the fork count from the withCodSpeed option", + async () => { + const run = await runFixture({ forksOption: 2 }); + + expect(run.status).toBe(0); + for (const name of BENCHMARKS) { + expect(forkPids(run, name).size).toBe(2); + } + }, + RUN_TIMEOUT_MS, + ); + + it( + "takes the fork count from the environment over the option", + async () => { + const run = await runFixture({ forksOption: 2, forksEnv: 1 }); + + expect(run.status).toBe(0); + for (const name of BENCHMARKS) { + expect(forkPids(run, name).size).toBe(1); + } + }, + RUN_TIMEOUT_MS, + ); + + it( + "runs every benchmark in the parent process when isolation is disabled", + async () => { + const run = await runFixture({ shared: true, forksEnv: 4 }); + + expect(run.status).toBe(0); + for (const name of BENCHMARKS) { + expect(run.pidsByName.get(name)).toEqual(new Set([run.parentPid])); + } + }, + RUN_TIMEOUT_MS, + ); +}); From fe13d0c892ec4646a99f94d66c890b75506e4d10 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 29 Oct 2026 12:09:50 +0100 Subject: [PATCH 03/13] feat(vitest): run walltime benchmarks in forks With Vitest 5, each walltime benchmark runs in fresh `vitest bench` processes restricted to its file. The fork count resolves as `CODSPEED_WALLTIME_FORKS`, the benchmark's or run's `forks` option, `codspeedPlugin({ forks })`, then 3. The test timeout is scaled by the env or plugin fork count, since every fork of a test runs within it. Vitest's own report shows zeroed statistics for forked benchmarks; the CodSpeed results hold the pooled rounds. Vitest 3/4 don't fork. Closes COD-3960 --- .../benches/exclusive-resource.bench.ts | 28 +++ .../vitest-plugin/src/__tests__/index.test.ts | 13 ++ .../src/__tests__/isolation.test.ts | 159 ++++++++++++++++++ packages/vitest-plugin/src/index.ts | 28 ++- packages/vitest-plugin/src/instrument.ts | 27 +++ packages/vitest-plugin/src/v5/forkContext.ts | 49 ++++++ packages/vitest-plugin/src/v5/forkReporter.ts | 57 +++++++ packages/vitest-plugin/src/v5/provider.ts | 74 +++++++- packages/vitest-plugin/src/vitestBackend.ts | 78 ++++++++- .../tests/fixtures/isolation/globalSetup.ts | 34 ++++ .../tests/fixtures/isolation/pids.bench.ts | 47 ++++++ .../tests/fixtures/isolation/vitest.config.ts | 15 ++ packages/vitest-plugin/vitest.config.ts | 3 + 13 files changed, 598 insertions(+), 14 deletions(-) create mode 100644 packages/vitest-plugin/benches/exclusive-resource.bench.ts create mode 100644 packages/vitest-plugin/src/__tests__/isolation.test.ts create mode 100644 packages/vitest-plugin/src/v5/forkContext.ts create mode 100644 packages/vitest-plugin/src/v5/forkReporter.ts create mode 100644 packages/vitest-plugin/tests/fixtures/isolation/globalSetup.ts create mode 100644 packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts create mode 100644 packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts diff --git a/packages/vitest-plugin/benches/exclusive-resource.bench.ts b/packages/vitest-plugin/benches/exclusive-resource.bench.ts new file mode 100644 index 00000000..a6d45b55 --- /dev/null +++ b/packages/vitest-plugin/benches/exclusive-resource.bench.ts @@ -0,0 +1,28 @@ +import { connect, createServer, type Server } from "node:net"; +import { afterAll, beforeAll, test } from "vitest"; + +// Fixed on purpose: a second process holding it must fail with EADDRINUSE. +const PORT = 48_123; +let server: Server; + +beforeAll(() => { + const { promise, resolve, reject } = Promise.withResolvers(); + server = createServer((socket) => socket.end()) + .once("error", reject) + .listen(PORT, resolve); + return promise; +}); + +afterAll(() => { + const { promise, resolve } = Promise.withResolvers(); + server.close(() => resolve()); + return promise; +}); + +test("exclusive port", async ({ bench }) => { + await bench("connect", async () => { + const { promise, resolve } = Promise.withResolvers(); + connect(PORT).once("close", resolve); + return promise; + }).run(); +}); diff --git a/packages/vitest-plugin/src/__tests__/index.test.ts b/packages/vitest-plugin/src/__tests__/index.test.ts index d7459fea..f697d49a 100644 --- a/packages/vitest-plugin/src/__tests__/index.test.ts +++ b/packages/vitest-plugin/src/__tests__/index.test.ts @@ -2,6 +2,7 @@ import { getV8Flags } from "@codspeed/core"; import { fromPartial } from "@total-typescript/shoehorn"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { type ViteUserConfig } from "vitest/config"; +import { type Vitest } from "vitest/node"; import codspeedPlugin from "../index"; const coreMocks = vi.hoisted(() => { @@ -162,5 +163,17 @@ describe("codSpeedPlugin", () => { expect(result.test?.benchmark?.suppressExportGetterWarnings).toBe(false); }); + + it("should register the fork reporter once across projects", () => { + vi.stubEnv("CODSPEED_RUNNER_MODE", "walltime"); + if (typeof plugin.configureVitest !== "function") + throw new Error("configureVitest is not a function"); + const vitest = fromPartial({ config: { reporters: [] } }); + for (const name of ["a (bench)", "b (bench)"]) { + plugin.configureVitest(fromPartial({ vitest, project: { name } })); + } + + expect(vitest.config.reporters).toHaveLength(1); + }); }); }); diff --git a/packages/vitest-plugin/src/__tests__/isolation.test.ts b/packages/vitest-plugin/src/__tests__/isolation.test.ts new file mode 100644 index 00000000..0169373e --- /dev/null +++ b/packages/vitest-plugin/src/__tests__/isolation.test.ts @@ -0,0 +1,159 @@ +import { spawn } from "child_process"; +import { once } from "events"; +import { + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "fs"; +import { createRequire } from "module"; +import { tmpdir } from "os"; +import { dirname, join } from "path"; +import { fileURLToPath } from "url"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +const FIXTURE_DIR = fileURLToPath( + new URL("../../tests/fixtures/isolation", import.meta.url), +); +const VITEST_BIN = join( + dirname(createRequire(import.meta.url).resolve("vitest/package.json")), + "vitest.mjs", +); +const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +const WALLTIME_FORKS_ENV = "CODSPEED_WALLTIME_FORKS"; +const DEFAULT_FORKS = 3; +/** The fixture's `iterations`: every fork measures at least this many rounds. */ +const MIN_ROUNDS_PER_FORK = 4; +const RUN_TIMEOUT_MS = 180_000; +const RAN_LINE = /^ran (\d+) (.+)$/; + +let tmpDir: string; + +beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), "codspeed-isolation-")); +}); + +afterEach(() => { + rmSync(tmpDir, { recursive: true, force: true }); +}); + +interface FixtureOptions { + isolation?: string; + forksEnv?: number; + pluginForks?: number; + firstForks?: number; + failing?: boolean; +} + +async function runFixture(options: FixtureOptions = {}) { + const logFile = join(tmpDir, "ran.log"); + writeFileSync(logFile, ""); + const profileFolder = join(tmpDir, "profile"); + const env: NodeJS.ProcessEnv = { + ...process.env, + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "walltime", + CODSPEED_PROFILE_FOLDER: profileFolder, + ISOLATION_LOG_FILE: logFile, + [PROCESS_ISOLATION_ENV]: options.isolation, + [WALLTIME_FORKS_ENV]: options.forksEnv?.toString(), + ISOLATION_FIXTURE_PLUGIN_FORKS: options.pluginForks?.toString(), + ISOLATION_FIXTURE_FIRST_FORKS: options.firstForks?.toString(), + ISOLATION_FIXTURE_FAIL: options.failing ? "1" : undefined, + }; + + // Not spawnSync: blocking this worker times out its calls to Vitest. + const child = spawn(process.execPath, [VITEST_BIN, "bench", "--run"], { + cwd: FIXTURE_DIR, + env, + timeout: RUN_TIMEOUT_MS, + stdio: "ignore", + }); + const [status] = (await once(child, "close")) as [number | null]; + + const pidsByName = new Map>(); + for (const line of readFileSync(logFile, "utf8").split("\n")) { + const match = RAN_LINE.exec(line); + if (!match) continue; + const pids = pidsByName.get(match[2]) ?? new Set(); + pids.add(Number(match[1])); + pidsByName.set(match[2], pids); + } + + const roundsByName = new Map(); + const resultsDir = join(profileFolder, "results"); + for (const file of status === 0 ? readdirSync(resultsDir) : []) { + const data = JSON.parse(readFileSync(join(resultsDir, file), "utf8")); + for (const benchmark of data.benchmarks) { + roundsByName.set(benchmark.name, benchmark.stats.rounds); + } + } + return { status, pidsByName, roundsByName }; +} + +describe("walltime forks", () => { + it( + "runs every benchmark in 3 forks by default and pools their rounds", + async () => { + const run = await runFixture(); + + expect(run.status).toBe(0); + for (const name of ["first", "second"]) { + expect(run.pidsByName.get(name)?.size).toBe(DEFAULT_FORKS); + expect(run.roundsByName.get(name)).toBeGreaterThanOrEqual( + DEFAULT_FORKS * MIN_ROUNDS_PER_FORK, + ); + } + }, + RUN_TIMEOUT_MS, + ); + + it( + "takes the benchmark's option over the plugin's", + async () => { + const run = await runFixture({ firstForks: 2, pluginForks: 1 }); + + expect(run.status).toBe(0); + expect(run.pidsByName.get("first")?.size).toBe(2); + expect(run.pidsByName.get("second")?.size).toBe(1); + }, + RUN_TIMEOUT_MS, + ); + + it( + "takes the environment over the benchmark's option", + async () => { + const run = await runFixture({ firstForks: 2, forksEnv: 1 }); + + expect(run.status).toBe(0); + expect(run.pidsByName.get("first")?.size).toBe(1); + expect(run.pidsByName.get("second")?.size).toBe(1); + }, + RUN_TIMEOUT_MS, + ); + + it( + "runs every benchmark in one process when isolation is disabled", + async () => { + const run = await runFixture({ isolation: "false" }); + + expect(run.status).toBe(0); + const pids = new Set( + [...run.pidsByName.values()].flatMap((set) => [...set]), + ); + expect(pids.size).toBe(1); + }, + RUN_TIMEOUT_MS, + ); + + it( + "fails the run when a benchmark throws in its fork", + async () => { + const run = await runFixture({ failing: true, forksEnv: 1 }); + + expect(run.status).not.toBe(0); + }, + RUN_TIMEOUT_MS, + ); +}); diff --git a/packages/vitest-plugin/src/index.ts b/packages/vitest-plugin/src/index.ts index cb3af1ae..0b7d7854 100644 --- a/packages/vitest-plugin/src/index.ts +++ b/packages/vitest-plugin/src/index.ts @@ -5,12 +5,18 @@ import { mongoMeasurement, SetupInstrumentsRequestBody, SetupInstrumentsResponse, + type WalltimeOptions, } from "@codspeed/core"; import { join } from "path"; import { Plugin } from "vite"; import { type ViteUserConfig } from "vitest/config"; +import { type Vitest } from "vitest/node"; import { resolveVitestBackend } from "./vitestBackend"; +// `configureVitest` runs once per project, but one reporter sees every +// project's modules: a second one would fork every benchmark twice. +const vitestsWithForkReporter = new WeakSet(); + // get this file's directory path from import.meta.url const __dirname = new URL(".", import.meta.url).pathname; const isFileInTs = import.meta.url.endsWith(".ts"); @@ -21,7 +27,8 @@ function resolveFile(name: string): string { return join(__dirname, `${name}.${fileExtension}`); } -export default function codspeedPlugin(): Plugin { +/** `options.forks`: see {@link WalltimeOptions}; isolation requires Vitest 5. */ +export default function codspeedPlugin(options: WalltimeOptions = {}): Plugin { return { name: "codspeed:vitest", apply(_, { mode }) { @@ -37,19 +44,32 @@ export default function codspeedPlugin(): Plugin { return true; }, enforce: "post", - config(userConfig): ViteUserConfig { + config(userConfig: ViteUserConfig): ViteUserConfig { + const backend = resolveVitestBackend(); + backend.configureIsolation(userConfig); return { test: { pool: "forks", globalSetup: [resolveFile("globalSetup")], - ...resolveVitestBackend().getBenchmarkTestConfig( + ...backend.getBenchmarkTestConfig( getV8Flags(), resolveFile, - (userConfig as ViteUserConfig).test, + userConfig.test, ), }, }; }, + configureVitest({ project, vitest }) { + const backend = resolveVitestBackend(); + backend.configureProject(project); + if (vitestsWithForkReporter.has(vitest)) return; + vitestsWithForkReporter.add(vitest); + const reporter = backend.forkReporter(options); + // Vitest builds its reporters from the config after this hook runs. + if (reporter) { + vitest.config.reporters.push(reporter); + } + }, }; } diff --git a/packages/vitest-plugin/src/instrument.ts b/packages/vitest-plugin/src/instrument.ts index 24039088..e9a7d7fb 100644 --- a/packages/vitest-plugin/src/instrument.ts +++ b/packages/vitest-plugin/src/instrument.ts @@ -10,6 +10,7 @@ import { type BenchmarkStats, } from "@codspeed/core"; import type * as tinybench from "tinybench"; +import type { TaskMeta } from "vitest"; export type Tinybench = typeof tinybench; @@ -51,6 +52,32 @@ export interface TinybenchOptions { iterations?: number; } +/** A benchmark the main process measures in forks once the run is over. */ +export interface ForkedBenchmark { + uri: string; + name: string; + /** The benchmark's own `forks` option; the plugin option applies otherwise. */ + forks: number | undefined; + options: TinybenchOptions; +} + +// Vitest ships a test's meta to the main process with its result, so the +// worker hands its queued benchmarks to the reporter through it. +const FORKED_BENCHMARKS_META = "codspeedForkedBenchmarks"; + +type ForkMeta = TaskMeta & { [FORKED_BENCHMARKS_META]?: ForkedBenchmark[] }; + +export function setForkedBenchmarks( + meta: TaskMeta, + benchmarks: ForkedBenchmark[], +): void { + (meta as ForkMeta)[FORKED_BENCHMARKS_META] = benchmarks; +} + +export function forkedBenchmarks(meta: TaskMeta): ForkedBenchmark[] { + return (meta as ForkMeta)[FORKED_BENCHMARKS_META] ?? []; +} + /** * Drive the instrumentation window from each bench's run-mode setup/teardown * hooks so it brackets only tinybench's measured loop, excluding the warmup diff --git a/packages/vitest-plugin/src/v5/forkContext.ts b/packages/vitest-plugin/src/v5/forkContext.ts new file mode 100644 index 00000000..0da54bc1 --- /dev/null +++ b/packages/vitest-plugin/src/v5/forkContext.ts @@ -0,0 +1,49 @@ +import { randomUUID } from "crypto"; +import { readFileSync, rmSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { deserialize, serialize } from "v8"; +import type { TestModule, TestProject } from "vitest/node"; + +const FORK_CONTEXT_FILE_ENV = "CODSPEED_VITEST_FORK_CONTEXT_FILE"; + +/** What each project's global setup provided, by project name. */ +type ProvidedContexts = Record>; + +/** + * Hands the modules' provided context to the forks `run` starts, through a + * file the forks inherit by its environment variable (not the environment + * itself: a large provided fixture would exceed its size limit). + */ +export async function withForkContext( + modules: readonly TestModule[], + run: () => Promise, +): Promise { + const contexts: ProvidedContexts = {}; + for (const { project } of modules) { + contexts[project.name] = { ...project.getProvidedContext() }; + } + const file = join(tmpdir(), `codspeed-vitest-context-${randomUUID()}`); + writeFileSync(file, serialize(contexts)); + process.env[FORK_CONTEXT_FILE_ENV] = file; + try { + return await run(); + } finally { + delete process.env[FORK_CONTEXT_FILE_ENV]; + rmSync(file, { force: true }); + } +} + +/** In a fork: provides `project` what the parent's global setup provided it. */ +export function provideParentContext(project: TestProject): void { + const file = process.env[FORK_CONTEXT_FILE_ENV]; + if (file === undefined) { + throw new Error("[CodSpeed] fork started without its parent's context"); + } + const contexts: ProvidedContexts = deserialize(readFileSync(file)); + // `provide` is typed by the user's `ProvidedContext` augmentation, empty here. + const provide = project.provide as (key: string, value: unknown) => void; + for (const [key, value] of Object.entries(contexts[project.name] ?? {})) { + provide(key, value); + } +} diff --git a/packages/vitest-plugin/src/v5/forkReporter.ts b/packages/vitest-plugin/src/v5/forkReporter.ts new file mode 100644 index 00000000..7055eb7b --- /dev/null +++ b/packages/vitest-plugin/src/v5/forkReporter.ts @@ -0,0 +1,57 @@ +import { + resolveWalltimeForks, + runForks, + samplesToResult, + type Benchmark, + type WalltimeOptions, +} from "@codspeed/core"; +import type { Reporter, TestModule } from "vitest/node"; +import { + forkedBenchmarks, + tinybenchTaskToBenchmark, + writeAndLogWalltimeResults, +} from "../instrument"; +import { withForkContext } from "./forkContext"; + +/** + * Measures the benchmarks the workers queued (see `provider.ts`) in fresh + * processes and reports their pooled rounds. Runs in the main process once the + * run is over: every file has torn down, so resources its hooks held (ports, + * databases, ...) are free for the forks. The global setup is still up (it + * tears down on close), so the forks skip it and inherit what it provided. + */ +export function forkReporter(options: WalltimeOptions): Reporter { + return { + async onTestRunEnd(modules, _errors, reason) { + if (reason === "interrupted") return; + const benchmarks = await withForkContext(modules, () => + measureInForks(modules, options), + ); + writeAndLogWalltimeResults(benchmarks); + }, + }; +} + +async function measureInForks( + modules: readonly TestModule[], + options: WalltimeOptions, +): Promise { + const benchmarks: Benchmark[] = []; + for (const module of modules) { + for (const test of module.children.allTests()) { + for (const queued of forkedBenchmarks(test.meta())) { + const forks = resolveWalltimeForks(queued.forks ?? options.forks); + const samples = await runForks(queued.uri, forks); + const benchmark = tinybenchTaskToBenchmark( + { name: queued.name, result: samplesToResult(samples) }, + queued.uri, + queued.options, + ); + if (benchmark) { + benchmarks.push(benchmark); + } + } + } + } + return benchmarks; +} diff --git a/packages/vitest-plugin/src/v5/provider.ts b/packages/vitest-plugin/src/v5/provider.ts index ef38fe0e..bd15ae80 100644 --- a/packages/vitest-plugin/src/v5/provider.ts +++ b/packages/vitest-plugin/src/v5/provider.ts @@ -2,13 +2,16 @@ import { getGitDir, getInstrumentMode, InstrumentHooks, + walltimeIsolation, optimizeFunction, optimizeFunctionSync, setupCore, teardownCore, wrapWithRootFrame, wrapWithRootFrameSync, + writeForkSamples, type Benchmark, + type WalltimeOptions, } from "@codspeed/core"; import { createRequire } from "module"; import path from "path"; @@ -21,6 +24,7 @@ import type { } from "vitest"; import { installInstrumentHooks, + setForkedBenchmarks, tinybenchTaskToBenchmark, writeAndLogWalltimeResults, type TinybenchBench, @@ -221,6 +225,16 @@ async function runWalltime( getUri: (name: string) => string, test: BenchmarkGroup["test"], ): Promise { + const mode = walltimeIsolation(); + if (mode.kind === "Forks") { + await queueForks(registrations, options, getUri, test); + return registrations.map(({ name }) => analysisResult(name)); + } + // A fork measures only its benchmark; Vitest still gets every result. + const measured = registrations.filter( + ({ name }) => mode.kind !== "Fork" || mode.uri === getUri(name), + ); + const { Bench } = await importHostTinybench(); const bench = new Bench({ signal: test.context.signal, @@ -229,7 +243,7 @@ async function runWalltime( retainSamples: true, }); - for (const { name, fn, fnOpts } of registrations) { + for (const { name, fn, fnOpts } of measured) { // tinybench v6 keeps `fn` private, so the root frame is baked in here bench.add( name, @@ -242,9 +256,48 @@ async function runWalltime( const tasks = await bench.run(); throwOnErroredTasks(tasks); - collectWalltimeResults(bench, getUri); + if (mode.kind === "Fork") { + for (const task of tasks) { + writeForkSamples(mode.samplesFile, [ + ...(task.result?.latency.samples ?? []), + ]); + } + } else { + collectWalltimeResults(bench, getUri); + } - return tasks.map(tinybenchTaskToBenchResult); + const results = new Map( + tasks.map((task) => [task.name, tinybenchTaskToBenchResult(task)]), + ); + return registrations.map( + ({ name }) => results.get(name) ?? analysisResult(name), + ); +} + +/** + * Queues every benchmark for measurement in fresh processes once the run is + * over (see `forkReporter.ts`). Vitest gets zeroed results, like in analysis mode. + */ +async function queueForks( + registrations: BenchRegistrationInput[], + options: BenchRunOptions | undefined, + getUri: (name: string) => string, + test: BenchmarkGroup["test"], +): Promise { + const { Bench } = await importHostTinybench(); + // Never run: it resolves tinybench's defaults for the reported config. + const bench = new Bench(options); + setForkedBenchmarks( + test.meta, + registrations.map(({ name, fnOpts }) => ({ + uri: getUri(name), + name, + forks: + (fnOpts as WalltimeOptions | undefined)?.forks ?? + (options as WalltimeOptions | undefined)?.forks, + options: resolvedOptions(bench), + })), + ); } /** Fail the test like Vitest's own provider does, instead of returning empty results. */ @@ -260,11 +313,9 @@ function throwOnErroredTasks(tasks: TinybenchTask[]): void { } } -function collectWalltimeResults( - bench: TinybenchWithTasks, - getUri: (name: string) => string, -): void { - const options: TinybenchOptions = { +/** The options the bench runs with, as reported to CodSpeed. */ +function resolvedOptions(bench: TinybenchWithTasks): TinybenchOptions { + return { time: bench.time, iterations: bench.iterations, // A bench with warmup turned off keeps its warmup timings, so they'd @@ -272,6 +323,13 @@ function collectWalltimeResults( warmupTime: bench.warmup ? bench.warmupTime : 0, warmupIterations: bench.warmup ? bench.warmupIterations : 0, }; +} + +function collectWalltimeResults( + bench: TinybenchWithTasks, + getUri: (name: string) => string, +): void { + const options = resolvedOptions(bench); const benchmarks: Benchmark[] = []; for (const task of bench.tasks) { if (task.result?.state !== "completed") continue; diff --git a/packages/vitest-plugin/src/vitestBackend.ts b/packages/vitest-plugin/src/vitestBackend.ts index 46c171d7..7a5cf3e1 100644 --- a/packages/vitest-plugin/src/vitestBackend.ts +++ b/packages/vitest-plugin/src/vitestBackend.ts @@ -1,11 +1,29 @@ -import { getInstrumentMode } from "@codspeed/core"; +import { + getGitDir, + getInstrumentMode, + walltimeIsolation, + type WalltimeOptions, +} from "@codspeed/core"; import { join } from "path"; import { type ViteUserConfig } from "vitest/config"; -import { version as vitestVersion } from "vitest/node"; +import { + type Reporter, + type TestProject, + version as vitestVersion, +} from "vitest/node"; +import { provideParentContext } from "./v5/forkContext"; +import { forkReporter } from "./v5/forkReporter"; /** The parts of the Vitest integration that depend on the installed major. */ export interface VitestBackend { isActive(mode: string): boolean; + configureIsolation(userConfig: ViteUserConfig): void; + + /** Runs per project, before Vitest initializes its global setup. */ + configureProject(project: TestProject): void; + + /** The reporter that measures the queued benchmarks in forks, if any. */ + forkReporter(options: WalltimeOptions): Reporter | undefined; getBenchmarkTestConfig( v8Flags: string[], @@ -34,6 +52,39 @@ class V5Backend implements VitestBackend { return getInstrumentMode() !== "disabled"; } + configureIsolation(userConfig: ViteUserConfig): void { + if (getInstrumentMode() !== "walltime") return; + + const isolationMode = walltimeIsolation(); + if (isolationMode.kind !== "Fork") return; + restrictToBenchmarkFile(userConfig, isolationMode.uri); + // The parent's global setup is still up while its forks run: forks skip + // it and inherit what it provided (see `configureProject`). + userConfig.test!.globalSetup = []; + } + + configureProject(project: TestProject): void { + if (getInstrumentMode() !== "walltime") return; + if (walltimeIsolation().kind !== "Fork") return; + provideParentContext(project); + } + + /** + * The parent of a forked walltime run measures nothing in its workers: they + * queue the benchmarks, and the main process forks them once the run is over. + */ + forkReporter(options: WalltimeOptions): Reporter | undefined { + if (getInstrumentMode() !== "walltime") return undefined; + const isolationMode = walltimeIsolation(); + switch (isolationMode.kind) { + case "Forks": + return forkReporter(options); + case "SharedProcess": + case "Fork": + return undefined; + } + } + getBenchmarkTestConfig( v8Flags: string[], resolveFile: (name: string) => string, @@ -53,6 +104,24 @@ class V5Backend implements VitestBackend { } } +function restrictToBenchmarkFile( + userConfig: ViteUserConfig, + uri: string, +): void { + const gitDir = getGitDir(userConfig.root ?? process.cwd()); + if (gitDir === undefined) { + throw new Error("Could not find a git repository"); + } + // Replace rather than merge include globs: isolated children must not + // transform and import unrelated benchmark files. + const file = join(gitDir, uri.split("::")[0]) + .replace(/\\/g, "/") + .replace(/[\\*?{}()[\]!+@]/g, "\\$&"); + userConfig.test ??= {}; + userConfig.test.benchmark ??= {}; + userConfig.test.benchmark.include = [file]; +} + /** * Vitest 3/4: `vitest bench` runs in the `"benchmark"` Vite mode, and each * instrument mode has its own `test.runner` (`legacy/analysis`, `legacy/walltime`). @@ -64,6 +133,11 @@ class LegacyBackend implements VitestBackend { return mode === "benchmark"; } + // Vitest 3/4 do not support process isolation. + configureIsolation(): void {} + configureProject(): void {} + forkReporter(): undefined {} + getBenchmarkTestConfig( v8Flags: string[], resolveFile: (name: string) => string, diff --git a/packages/vitest-plugin/tests/fixtures/isolation/globalSetup.ts b/packages/vitest-plugin/tests/fixtures/isolation/globalSetup.ts new file mode 100644 index 00000000..7aedb03c --- /dev/null +++ b/packages/vitest-plugin/tests/fixtures/isolation/globalSetup.ts @@ -0,0 +1,34 @@ +import { createServer, type Server } from "net"; +import type { TestProject } from "vitest/node"; + +// Fixed on purpose: a fork that ran this setup again would fail with EADDRINUSE. +const PORT = 48_124; + +declare module "vitest" { + export interface ProvidedContext { + port: number; + } +} + +// Vitest 5 runs this for the base project and for the `(bench)` project it +// clones from it, in the same process: bind once, and provide to both. +let server: Server | undefined; + +export default async function setup(project: TestProject) { + if (server === undefined) { + const listening = Promise.withResolvers(); + server = createServer((socket) => socket.end()) + .once("error", listening.reject) + .listen(PORT, listening.resolve); + await listening.promise; + } + project.provide("port", PORT); + + return () => { + const closed = Promise.withResolvers(); + if (server === undefined) return closed.resolve(); + server.close(() => closed.resolve()); + server = undefined; + return closed.promise; + }; +} diff --git a/packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts b/packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts new file mode 100644 index 00000000..834da854 --- /dev/null +++ b/packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts @@ -0,0 +1,47 @@ +import { appendFileSync } from "fs"; +import { connect } from "net"; +import { describe, inject, test, type BenchRegistration } from "vitest"; + +// Vitest swallows what workers print, so the PIDs that run each benchmark go +// to a file. `beforeAll` only runs when tinybench measures the benchmark. +// It also connects to the global setup's server: a fork without the provided +// port, or one that re-ran the setup, fails here. +function logPid(name: string) { + return async () => { + const closed = Promise.withResolvers(); + connect(inject("port")) + .once("error", closed.reject) + .once("close", () => closed.resolve()); + await closed.promise; + appendFileSync( + process.env.ISOLATION_LOG_FILE!, + `ran ${process.pid} ${name}\n`, + ); + }; +} + +const firstForks = process.env.ISOLATION_FIXTURE_FIRST_FORKS; + +describe("isolation", () => { + test("benchmarks", async ({ bench }) => { + const benchmarks: BenchRegistration[] = [ + bench( + "first", + { + beforeAll: logPid("first"), + ...(firstForks && { forks: Number(firstForks) }), + }, + () => {}, + ), + bench("second", { beforeAll: logPid("second") }, () => {}), + ]; + if (process.env.ISOLATION_FIXTURE_FAIL !== undefined) { + benchmarks.push( + bench("failing", () => { + throw new Error("benchmark failed"); + }), + ); + } + await bench.compare(...benchmarks, { iterations: 4, time: 1 }); + }); +}); diff --git a/packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts b/packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts new file mode 100644 index 00000000..2adcb374 --- /dev/null +++ b/packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from "vitest/config"; +import codspeedPlugin from "../../../dist/index.mjs"; + +const pluginForks = process.env.ISOLATION_FIXTURE_PLUGIN_FORKS; + +export default defineConfig({ + plugins: [codspeedPlugin(pluginForks ? { forks: Number(pluginForks) } : {})], + define: { + __VERSION__: JSON.stringify("1.0.0"), + }, + test: { + include: ["**/*.bench.ts"], + globalSetup: ["./globalSetup.ts"], + }, +}); diff --git a/packages/vitest-plugin/vitest.config.ts b/packages/vitest-plugin/vitest.config.ts index 672e3a27..e874589b 100644 --- a/packages/vitest-plugin/vitest.config.ts +++ b/packages/vitest-plugin/vitest.config.ts @@ -10,5 +10,8 @@ export default defineConfig({ test: { exclude: ["**/tests/**/*", "**/.rollup.cache/**/*"], mockReset: true, + benchmark: { + exclude: ["**/node_modules/**", "**/tests/**/*", "**/.rollup.cache/**/*"], + }, }, }); From eae555ce9c7eedc35b2b8bec08b14af4ea7406f3 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 4 Nov 2026 17:46:53 +0100 Subject: [PATCH 04/13] fixup! feat(tinybench): run walltime benchmarks in forks --- packages/tinybench-plugin/src/walltime.ts | 22 ++++--------------- .../tests/fixtures/isolation.mjs | 8 +------ .../tests/isolation.integ.test.ts | 21 ++++-------------- 3 files changed, 9 insertions(+), 42 deletions(-) diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index 7168cb0c..acf8837b 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -103,7 +103,6 @@ class WalltimeBenchRunner extends BaseBenchRunner { const { isolationMode } = this; if (isolationMode.kind === "Forks") { const samples = await runForks(uri, isolationMode.forks); - setTaskResult(task, samples); return this.registerCodspeedBenchmark(task, samplesToResult(samples)); } @@ -123,7 +122,6 @@ class WalltimeBenchRunner extends BaseBenchRunner { const { isolationMode } = this; if (isolationMode.kind === "Forks") { const samples = runForksSync(uri, isolationMode.forks); - setTaskResult(task, samples); return this.registerCodspeedBenchmark(task, samplesToResult(samples)); } @@ -153,9 +151,10 @@ class WalltimeBenchRunner extends BaseBenchRunner { } // A fork hands its samples to the parent, which reports the benchmark. if (this.isolationMode.kind === "Fork") { - writeForkSamples(this.isolationMode.samplesFile, [ - ...task.result.latency.samples, - ]); + writeForkSamples( + this.isolationMode.samplesFile, + task.result.latency.samples, + ); return; } this.registerCodspeedBenchmark(task, task.result); @@ -238,16 +237,3 @@ function convertTinybenchResultToBenchmarkStats( warmup_iters: warmupIterations, }; } - -/** - * Fills `task.result` from the pooled fork samples so `bench.table()` and - * `bench.results` work. Only tinybench 4/5 expose a way in; v6 keeps the - * result `#private`. - */ -function setTaskResult(task: Task, samples: number[]): void { - const { processRunResult } = task as unknown as { - processRunResult?: (run: { latencySamples: number[] }) => void; - }; - // tinybench sorts the samples in place. - processRunResult?.call(task, { latencySamples: [...samples] }); -} diff --git a/packages/tinybench-plugin/tests/fixtures/isolation.mjs b/packages/tinybench-plugin/tests/fixtures/isolation.mjs index e347110d..118c5b07 100644 --- a/packages/tinybench-plugin/tests/fixtures/isolation.mjs +++ b/packages/tinybench-plugin/tests/fixtures/isolation.mjs @@ -1,5 +1,5 @@ // Run by `isolation.integ.test.ts`. Prints the PID of every process that runs a -// benchmark, and the sample count of every `task.result` the parent ends with. +// benchmark. import { Bench } from "tinybench"; import { withCodSpeed } from "../../dist/index.es5.js"; @@ -24,9 +24,3 @@ if (process.env.ISOLATION_FIXTURE_SYNC !== undefined) { } else { await bench.run(); } - -if (process.env.CODSPEED_ISOLATED_BENCHMARK === undefined) { - for (const task of bench.tasks) { - console.log(`result ${task.name} ${task.result?.samples.length ?? 0}`); - } -} diff --git a/packages/tinybench-plugin/tests/isolation.integ.test.ts b/packages/tinybench-plugin/tests/isolation.integ.test.ts index b78a5db4..2f2a2962 100644 --- a/packages/tinybench-plugin/tests/isolation.integ.test.ts +++ b/packages/tinybench-plugin/tests/isolation.integ.test.ts @@ -19,7 +19,6 @@ const MIN_ROUNDS_PER_FORK = 4; /** Fails a run whose benchmark process got stuck instead of blocking the suite. */ const RUN_TIMEOUT_MS = 60_000; const RAN_LINE = /^ran (\d+) (.+)$/; -const RESULT_LINE = /^result (.+) (\d+)$/; interface FixtureRun { status: number | null; @@ -29,8 +28,6 @@ interface FixtureRun { parentPid: number; /** Rounds of each benchmark in the results file. */ roundsByName: Map; - /** Samples in the parent's `task.result` of each benchmark. */ - parentSamplesByName: Map; } interface FixtureOptions { @@ -83,19 +80,12 @@ async function runFixture({ const [status] = (await once(child, "close")) as [number | null]; const pidsByName = new Map>(); - const parentSamplesByName = new Map(); for (const line of stdout.split("\n")) { const ran = RAN_LINE.exec(line); - if (ran) { - const pids = pidsByName.get(ran[2]) ?? new Set(); - pids.add(Number(ran[1])); - pidsByName.set(ran[2], pids); - continue; - } - const result = RESULT_LINE.exec(line); - if (result) { - parentSamplesByName.set(result[1], Number(result[2])); - } + if (!ran) continue; + const pids = pidsByName.get(ran[2]) ?? new Set(); + pids.add(Number(ran[1])); + pidsByName.set(ran[2], pids); } const roundsByName = new Map(); @@ -111,7 +101,6 @@ async function runFixture({ pidsByName, parentPid: child.pid!, roundsByName, - parentSamplesByName, }; } finally { rmSync(profileFolder, { recursive: true, force: true }); @@ -141,8 +130,6 @@ describe("walltime forks", () => { expect(rounds).toBeGreaterThanOrEqual( DEFAULT_FORKS * MIN_ROUNDS_PER_FORK, ); - // The parent reports the pooled rounds as its own `task.result`. - expect(run.parentSamplesByName.get(name)).toBe(rounds); } // Forks of different benchmarks are different processes too. const all = new Set( From 6c9d38d033c81bdf7869cb9e221a93906527dc03 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 4 Nov 2026 17:47:39 +0100 Subject: [PATCH 05/13] fixup! feat(vitest): run walltime benchmarks in forks --- packages/vitest-plugin/src/index.ts | 15 +------ packages/vitest-plugin/src/vitestBackend.ts | 46 +++++++++++---------- 2 files changed, 26 insertions(+), 35 deletions(-) diff --git a/packages/vitest-plugin/src/index.ts b/packages/vitest-plugin/src/index.ts index 0b7d7854..fdfbc7c1 100644 --- a/packages/vitest-plugin/src/index.ts +++ b/packages/vitest-plugin/src/index.ts @@ -10,13 +10,8 @@ import { import { join } from "path"; import { Plugin } from "vite"; import { type ViteUserConfig } from "vitest/config"; -import { type Vitest } from "vitest/node"; import { resolveVitestBackend } from "./vitestBackend"; -// `configureVitest` runs once per project, but one reporter sees every -// project's modules: a second one would fork every benchmark twice. -const vitestsWithForkReporter = new WeakSet(); - // get this file's directory path from import.meta.url const __dirname = new URL(".", import.meta.url).pathname; const isFileInTs = import.meta.url.endsWith(".ts"); @@ -60,15 +55,7 @@ export default function codspeedPlugin(options: WalltimeOptions = {}): Plugin { }; }, configureVitest({ project, vitest }) { - const backend = resolveVitestBackend(); - backend.configureProject(project); - if (vitestsWithForkReporter.has(vitest)) return; - vitestsWithForkReporter.add(vitest); - const reporter = backend.forkReporter(options); - // Vitest builds its reporters from the config after this hook runs. - if (reporter) { - vitest.config.reporters.push(reporter); - } + resolveVitestBackend().configureVitest(project, vitest, options); }, }; } diff --git a/packages/vitest-plugin/src/vitestBackend.ts b/packages/vitest-plugin/src/vitestBackend.ts index 7a5cf3e1..d1867bfa 100644 --- a/packages/vitest-plugin/src/vitestBackend.ts +++ b/packages/vitest-plugin/src/vitestBackend.ts @@ -7,8 +7,8 @@ import { import { join } from "path"; import { type ViteUserConfig } from "vitest/config"; import { - type Reporter, type TestProject, + type Vitest, version as vitestVersion, } from "vitest/node"; import { provideParentContext } from "./v5/forkContext"; @@ -20,10 +20,11 @@ export interface VitestBackend { configureIsolation(userConfig: ViteUserConfig): void; /** Runs per project, before Vitest initializes its global setup. */ - configureProject(project: TestProject): void; - - /** The reporter that measures the queued benchmarks in forks, if any. */ - forkReporter(options: WalltimeOptions): Reporter | undefined; + configureVitest( + project: TestProject, + vitest: Vitest, + options: WalltimeOptions, + ): void; getBenchmarkTestConfig( v8Flags: string[], @@ -41,6 +42,10 @@ export function resolveVitestBackend(): VitestBackend { return major >= 5 ? new V5Backend() : new LegacyBackend(major); } +// `configureVitest` runs once per project, but one reporter sees every +// project's modules: a second one would fork every benchmark twice. +const vitestsWithForkReporter = new WeakSet(); + /** Vitest 5+: benchmarks run through a `benchmark.provider` (`v5/provider.ts`). */ class V5Backend implements VitestBackend { /** @@ -63,26 +68,26 @@ class V5Backend implements VitestBackend { userConfig.test!.globalSetup = []; } - configureProject(project: TestProject): void { - if (getInstrumentMode() !== "walltime") return; - if (walltimeIsolation().kind !== "Fork") return; - provideParentContext(project); - } - /** * The parent of a forked walltime run measures nothing in its workers: they * queue the benchmarks, and the main process forks them once the run is over. */ - forkReporter(options: WalltimeOptions): Reporter | undefined { - if (getInstrumentMode() !== "walltime") return undefined; + configureVitest( + project: TestProject, + vitest: Vitest, + options: WalltimeOptions, + ): void { + if (getInstrumentMode() !== "walltime") return; const isolationMode = walltimeIsolation(); - switch (isolationMode.kind) { - case "Forks": - return forkReporter(options); - case "SharedProcess": - case "Fork": - return undefined; + if (isolationMode.kind === "Fork") { + provideParentContext(project); + return; } + if (isolationMode.kind !== "Forks") return; + if (vitestsWithForkReporter.has(vitest)) return; + vitestsWithForkReporter.add(vitest); + // Vitest builds its reporters from the config after this hook runs. + vitest.config.reporters.push(forkReporter(options)); } getBenchmarkTestConfig( @@ -135,8 +140,7 @@ class LegacyBackend implements VitestBackend { // Vitest 3/4 do not support process isolation. configureIsolation(): void {} - configureProject(): void {} - forkReporter(): undefined {} + configureVitest(): void {} getBenchmarkTestConfig( v8Flags: string[], From 409385ce5c0e9bbafbc5c21decc394a5b6324871 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 4 Nov 2026 17:47:55 +0100 Subject: [PATCH 06/13] fixup! feat(tinybench): run walltime benchmarks in forks --- packages/tinybench-plugin/src/analysis.ts | 4 +--- packages/tinybench-plugin/src/shared.ts | 25 +++++++++++------------ 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/packages/tinybench-plugin/src/analysis.ts b/packages/tinybench-plugin/src/analysis.ts index 54c8d975..65bd6183 100644 --- a/packages/tinybench-plugin/src/analysis.ts +++ b/packages/tinybench-plugin/src/analysis.ts @@ -14,9 +14,7 @@ export function setupCodspeedAnalysisBench( bench: Bench, rootCallingFile: string, ): void { - const runner = new AnalysisBenchRunner(bench, rootCallingFile, { - kind: "SharedProcess", - }); + const runner = new AnalysisBenchRunner(bench, rootCallingFile); runner.setupBenchMethods(); } diff --git a/packages/tinybench-plugin/src/shared.ts b/packages/tinybench-plugin/src/shared.ts index 1e10b4bf..2d86eced 100644 --- a/packages/tinybench-plugin/src/shared.ts +++ b/packages/tinybench-plugin/src/shared.ts @@ -8,17 +8,15 @@ declare const __VERSION__: string; export abstract class BaseBenchRunner { protected bench: Bench; protected rootCallingFile: string; - /** Walltime runners pass the isolation of this process; analysis shares it. */ - protected readonly isolationMode: IsolationMode; constructor( bench: Bench, rootCallingFile: string, - isolationMode: IsolationMode, + /** Only walltime runs are isolated. */ + protected readonly isolationMode: IsolationMode = { kind: "SharedProcess" }, ) { this.bench = bench; this.rootCallingFile = rootCallingFile; - this.isolationMode = isolationMode; } protected get isChild(): boolean { @@ -85,7 +83,8 @@ export abstract class BaseBenchRunner { } this.setupBenchRun(); - for (const { task, uri } of tasks) { + for (const task of tasks) { + const uri = this.getTaskUri(task); await this.runTaskAsync(task, uri); } @@ -99,7 +98,8 @@ export abstract class BaseBenchRunner { } this.setupBenchRun(); - for (const { task, uri } of tasks) { + for (const task of tasks) { + const uri = this.getTaskUri(task); this.runTaskSync(task, uri); } @@ -108,12 +108,11 @@ export abstract class BaseBenchRunner { } /** The tasks this process measures: all of them, or the isolated one. */ - private selectTasks(): { task: Task; uri: string }[] { - return this.bench.tasks - .map((task) => ({ task, uri: this.getTaskUri(task) })) - .filter( - ({ uri }) => - this.isolationMode.kind !== "Fork" || this.isolationMode.uri === uri, - ); + private selectTasks(): Task[] { + const { isolationMode } = this; + if (isolationMode.kind !== "Fork") return this.bench.tasks; + return this.bench.tasks.filter( + (task) => this.getTaskUri(task) === isolationMode.uri, + ); } } From ff3bce0f5eae1ef1555d2960acbe8ccc8ddb2668 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 4 Nov 2026 17:48:04 +0100 Subject: [PATCH 07/13] fixup! feat(vitest): run walltime benchmarks in forks --- .../tinybench-plugin/tests/isolation.integ.test.ts | 13 ------------- .../vitest-plugin/src/__tests__/isolation.test.ts | 12 ------------ 2 files changed, 25 deletions(-) diff --git a/packages/tinybench-plugin/tests/isolation.integ.test.ts b/packages/tinybench-plugin/tests/isolation.integ.test.ts index 2f2a2962..f7ec64cd 100644 --- a/packages/tinybench-plugin/tests/isolation.integ.test.ts +++ b/packages/tinybench-plugin/tests/isolation.integ.test.ts @@ -153,19 +153,6 @@ describe("walltime forks", () => { RUN_TIMEOUT_MS, ); - it( - "takes the fork count from the environment over the option", - async () => { - const run = await runFixture({ forksOption: 2, forksEnv: 1 }); - - expect(run.status).toBe(0); - for (const name of BENCHMARKS) { - expect(forkPids(run, name).size).toBe(1); - } - }, - RUN_TIMEOUT_MS, - ); - it( "runs every benchmark in the parent process when isolation is disabled", async () => { diff --git a/packages/vitest-plugin/src/__tests__/isolation.test.ts b/packages/vitest-plugin/src/__tests__/isolation.test.ts index 0169373e..e5b0f92a 100644 --- a/packages/vitest-plugin/src/__tests__/isolation.test.ts +++ b/packages/vitest-plugin/src/__tests__/isolation.test.ts @@ -121,18 +121,6 @@ describe("walltime forks", () => { RUN_TIMEOUT_MS, ); - it( - "takes the environment over the benchmark's option", - async () => { - const run = await runFixture({ firstForks: 2, forksEnv: 1 }); - - expect(run.status).toBe(0); - expect(run.pidsByName.get("first")?.size).toBe(1); - expect(run.pidsByName.get("second")?.size).toBe(1); - }, - RUN_TIMEOUT_MS, - ); - it( "runs every benchmark in one process when isolation is disabled", async () => { From 979fac59bedde665822c44e0e5aaba5ba2f90e7d Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 4 Nov 2026 17:48:05 +0100 Subject: [PATCH 08/13] fixup! feat(vitest): run walltime benchmarks in forks --- .../benches/exclusive-resource.bench.ts | 28 ------------------- 1 file changed, 28 deletions(-) delete mode 100644 packages/vitest-plugin/benches/exclusive-resource.bench.ts diff --git a/packages/vitest-plugin/benches/exclusive-resource.bench.ts b/packages/vitest-plugin/benches/exclusive-resource.bench.ts deleted file mode 100644 index a6d45b55..00000000 --- a/packages/vitest-plugin/benches/exclusive-resource.bench.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { connect, createServer, type Server } from "node:net"; -import { afterAll, beforeAll, test } from "vitest"; - -// Fixed on purpose: a second process holding it must fail with EADDRINUSE. -const PORT = 48_123; -let server: Server; - -beforeAll(() => { - const { promise, resolve, reject } = Promise.withResolvers(); - server = createServer((socket) => socket.end()) - .once("error", reject) - .listen(PORT, resolve); - return promise; -}); - -afterAll(() => { - const { promise, resolve } = Promise.withResolvers(); - server.close(() => resolve()); - return promise; -}); - -test("exclusive port", async ({ bench }) => { - await bench("connect", async () => { - const { promise, resolve } = Promise.withResolvers(); - connect(PORT).once("close", resolve); - return promise; - }).run(); -}); From a591f57bcf4302ddb4a46438f58b2eabca1f994a Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 4 Nov 2026 18:28:11 +0100 Subject: [PATCH 09/13] fixup! feat(tinybench): run walltime benchmarks in forks --- packages/tinybench-plugin/src/walltime.ts | 22 +++++++++++++++---- .../tests/fixtures/isolation.mjs | 8 ++++++- .../tests/isolation.integ.test.ts | 21 ++++++++++++++---- 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index acf8837b..7168cb0c 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -103,6 +103,7 @@ class WalltimeBenchRunner extends BaseBenchRunner { const { isolationMode } = this; if (isolationMode.kind === "Forks") { const samples = await runForks(uri, isolationMode.forks); + setTaskResult(task, samples); return this.registerCodspeedBenchmark(task, samplesToResult(samples)); } @@ -122,6 +123,7 @@ class WalltimeBenchRunner extends BaseBenchRunner { const { isolationMode } = this; if (isolationMode.kind === "Forks") { const samples = runForksSync(uri, isolationMode.forks); + setTaskResult(task, samples); return this.registerCodspeedBenchmark(task, samplesToResult(samples)); } @@ -151,10 +153,9 @@ class WalltimeBenchRunner extends BaseBenchRunner { } // A fork hands its samples to the parent, which reports the benchmark. if (this.isolationMode.kind === "Fork") { - writeForkSamples( - this.isolationMode.samplesFile, - task.result.latency.samples, - ); + writeForkSamples(this.isolationMode.samplesFile, [ + ...task.result.latency.samples, + ]); return; } this.registerCodspeedBenchmark(task, task.result); @@ -237,3 +238,16 @@ function convertTinybenchResultToBenchmarkStats( warmup_iters: warmupIterations, }; } + +/** + * Fills `task.result` from the pooled fork samples so `bench.table()` and + * `bench.results` work. Only tinybench 4/5 expose a way in; v6 keeps the + * result `#private`. + */ +function setTaskResult(task: Task, samples: number[]): void { + const { processRunResult } = task as unknown as { + processRunResult?: (run: { latencySamples: number[] }) => void; + }; + // tinybench sorts the samples in place. + processRunResult?.call(task, { latencySamples: [...samples] }); +} diff --git a/packages/tinybench-plugin/tests/fixtures/isolation.mjs b/packages/tinybench-plugin/tests/fixtures/isolation.mjs index 118c5b07..e347110d 100644 --- a/packages/tinybench-plugin/tests/fixtures/isolation.mjs +++ b/packages/tinybench-plugin/tests/fixtures/isolation.mjs @@ -1,5 +1,5 @@ // Run by `isolation.integ.test.ts`. Prints the PID of every process that runs a -// benchmark. +// benchmark, and the sample count of every `task.result` the parent ends with. import { Bench } from "tinybench"; import { withCodSpeed } from "../../dist/index.es5.js"; @@ -24,3 +24,9 @@ if (process.env.ISOLATION_FIXTURE_SYNC !== undefined) { } else { await bench.run(); } + +if (process.env.CODSPEED_ISOLATED_BENCHMARK === undefined) { + for (const task of bench.tasks) { + console.log(`result ${task.name} ${task.result?.samples.length ?? 0}`); + } +} diff --git a/packages/tinybench-plugin/tests/isolation.integ.test.ts b/packages/tinybench-plugin/tests/isolation.integ.test.ts index f7ec64cd..5977a192 100644 --- a/packages/tinybench-plugin/tests/isolation.integ.test.ts +++ b/packages/tinybench-plugin/tests/isolation.integ.test.ts @@ -19,6 +19,7 @@ const MIN_ROUNDS_PER_FORK = 4; /** Fails a run whose benchmark process got stuck instead of blocking the suite. */ const RUN_TIMEOUT_MS = 60_000; const RAN_LINE = /^ran (\d+) (.+)$/; +const RESULT_LINE = /^result (.+) (\d+)$/; interface FixtureRun { status: number | null; @@ -28,6 +29,8 @@ interface FixtureRun { parentPid: number; /** Rounds of each benchmark in the results file. */ roundsByName: Map; + /** Samples in the parent's `task.result` of each benchmark. */ + parentSamplesByName: Map; } interface FixtureOptions { @@ -80,12 +83,19 @@ async function runFixture({ const [status] = (await once(child, "close")) as [number | null]; const pidsByName = new Map>(); + const parentSamplesByName = new Map(); for (const line of stdout.split("\n")) { const ran = RAN_LINE.exec(line); - if (!ran) continue; - const pids = pidsByName.get(ran[2]) ?? new Set(); - pids.add(Number(ran[1])); - pidsByName.set(ran[2], pids); + if (ran) { + const pids = pidsByName.get(ran[2]) ?? new Set(); + pids.add(Number(ran[1])); + pidsByName.set(ran[2], pids); + continue; + } + const result = RESULT_LINE.exec(line); + if (result) { + parentSamplesByName.set(result[1], Number(result[2])); + } } const roundsByName = new Map(); @@ -101,6 +111,7 @@ async function runFixture({ pidsByName, parentPid: child.pid!, roundsByName, + parentSamplesByName, }; } finally { rmSync(profileFolder, { recursive: true, force: true }); @@ -130,6 +141,8 @@ describe("walltime forks", () => { expect(rounds).toBeGreaterThanOrEqual( DEFAULT_FORKS * MIN_ROUNDS_PER_FORK, ); + // The parent reports the pooled rounds as its own `task.result`. + expect(run.parentSamplesByName.get(name)).toBe(rounds); } // Forks of different benchmarks are different processes too. const all = new Set( From 2e8744995c9b5fd47f988da3cb75146f73a3512f Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 4 Nov 2026 19:28:45 +0100 Subject: [PATCH 10/13] fixup! feat(tinybench): run walltime benchmarks in forks --- packages/tinybench-plugin/src/walltime.ts | 22 ++++--------------- .../tests/fixtures/isolation.mjs | 8 +------ .../tests/isolation.integ.test.ts | 21 ++++-------------- 3 files changed, 9 insertions(+), 42 deletions(-) diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index 7168cb0c..acf8837b 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -103,7 +103,6 @@ class WalltimeBenchRunner extends BaseBenchRunner { const { isolationMode } = this; if (isolationMode.kind === "Forks") { const samples = await runForks(uri, isolationMode.forks); - setTaskResult(task, samples); return this.registerCodspeedBenchmark(task, samplesToResult(samples)); } @@ -123,7 +122,6 @@ class WalltimeBenchRunner extends BaseBenchRunner { const { isolationMode } = this; if (isolationMode.kind === "Forks") { const samples = runForksSync(uri, isolationMode.forks); - setTaskResult(task, samples); return this.registerCodspeedBenchmark(task, samplesToResult(samples)); } @@ -153,9 +151,10 @@ class WalltimeBenchRunner extends BaseBenchRunner { } // A fork hands its samples to the parent, which reports the benchmark. if (this.isolationMode.kind === "Fork") { - writeForkSamples(this.isolationMode.samplesFile, [ - ...task.result.latency.samples, - ]); + writeForkSamples( + this.isolationMode.samplesFile, + task.result.latency.samples, + ); return; } this.registerCodspeedBenchmark(task, task.result); @@ -238,16 +237,3 @@ function convertTinybenchResultToBenchmarkStats( warmup_iters: warmupIterations, }; } - -/** - * Fills `task.result` from the pooled fork samples so `bench.table()` and - * `bench.results` work. Only tinybench 4/5 expose a way in; v6 keeps the - * result `#private`. - */ -function setTaskResult(task: Task, samples: number[]): void { - const { processRunResult } = task as unknown as { - processRunResult?: (run: { latencySamples: number[] }) => void; - }; - // tinybench sorts the samples in place. - processRunResult?.call(task, { latencySamples: [...samples] }); -} diff --git a/packages/tinybench-plugin/tests/fixtures/isolation.mjs b/packages/tinybench-plugin/tests/fixtures/isolation.mjs index e347110d..118c5b07 100644 --- a/packages/tinybench-plugin/tests/fixtures/isolation.mjs +++ b/packages/tinybench-plugin/tests/fixtures/isolation.mjs @@ -1,5 +1,5 @@ // Run by `isolation.integ.test.ts`. Prints the PID of every process that runs a -// benchmark, and the sample count of every `task.result` the parent ends with. +// benchmark. import { Bench } from "tinybench"; import { withCodSpeed } from "../../dist/index.es5.js"; @@ -24,9 +24,3 @@ if (process.env.ISOLATION_FIXTURE_SYNC !== undefined) { } else { await bench.run(); } - -if (process.env.CODSPEED_ISOLATED_BENCHMARK === undefined) { - for (const task of bench.tasks) { - console.log(`result ${task.name} ${task.result?.samples.length ?? 0}`); - } -} diff --git a/packages/tinybench-plugin/tests/isolation.integ.test.ts b/packages/tinybench-plugin/tests/isolation.integ.test.ts index 5977a192..f7ec64cd 100644 --- a/packages/tinybench-plugin/tests/isolation.integ.test.ts +++ b/packages/tinybench-plugin/tests/isolation.integ.test.ts @@ -19,7 +19,6 @@ const MIN_ROUNDS_PER_FORK = 4; /** Fails a run whose benchmark process got stuck instead of blocking the suite. */ const RUN_TIMEOUT_MS = 60_000; const RAN_LINE = /^ran (\d+) (.+)$/; -const RESULT_LINE = /^result (.+) (\d+)$/; interface FixtureRun { status: number | null; @@ -29,8 +28,6 @@ interface FixtureRun { parentPid: number; /** Rounds of each benchmark in the results file. */ roundsByName: Map; - /** Samples in the parent's `task.result` of each benchmark. */ - parentSamplesByName: Map; } interface FixtureOptions { @@ -83,19 +80,12 @@ async function runFixture({ const [status] = (await once(child, "close")) as [number | null]; const pidsByName = new Map>(); - const parentSamplesByName = new Map(); for (const line of stdout.split("\n")) { const ran = RAN_LINE.exec(line); - if (ran) { - const pids = pidsByName.get(ran[2]) ?? new Set(); - pids.add(Number(ran[1])); - pidsByName.set(ran[2], pids); - continue; - } - const result = RESULT_LINE.exec(line); - if (result) { - parentSamplesByName.set(result[1], Number(result[2])); - } + if (!ran) continue; + const pids = pidsByName.get(ran[2]) ?? new Set(); + pids.add(Number(ran[1])); + pidsByName.set(ran[2], pids); } const roundsByName = new Map(); @@ -111,7 +101,6 @@ async function runFixture({ pidsByName, parentPid: child.pid!, roundsByName, - parentSamplesByName, }; } finally { rmSync(profileFolder, { recursive: true, force: true }); @@ -141,8 +130,6 @@ describe("walltime forks", () => { expect(rounds).toBeGreaterThanOrEqual( DEFAULT_FORKS * MIN_ROUNDS_PER_FORK, ); - // The parent reports the pooled rounds as its own `task.result`. - expect(run.parentSamplesByName.get(name)).toBe(rounds); } // Forks of different benchmarks are different processes too. const all = new Set( From 42c80d162ec421b25015bf20c7694e006677cd0d Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 5 Nov 2026 16:19:32 +0100 Subject: [PATCH 11/13] fixup! feat(core): run walltime benchmarks in forks --- packages/core/src/isolation.ts | 50 ++++++++++++++++++------- packages/core/src/walltime/quantiles.ts | 40 +++++++++++++------- packages/core/src/walltime/utils.ts | 29 ++++++++++---- 3 files changed, 85 insertions(+), 34 deletions(-) diff --git a/packages/core/src/isolation.ts b/packages/core/src/isolation.ts index 6f0f1d36..e40dd902 100644 --- a/packages/core/src/isolation.ts +++ b/packages/core/src/isolation.ts @@ -110,20 +110,37 @@ function currentCommand(): string[] { * fork would only copy the calling thread. Call it from the process whose * command runs the benchmarks: Vitest's main process, not a worker. */ -export async function runForks(uri: string, forks: number): Promise { - let samples: number[] = []; +export async function runForks( + uri: string, + forks: number, +): Promise { + const samples: Float64Array[] = []; for (let fork = 0; fork < forks; fork++) { - // Not push(...): spreading a time-based run's samples overflows the stack. - samples = samples.concat(await runFork(uri)); + samples.push(await runFork(uri)); } - return samples; + return concatSamples(samples); } /** {@link runForks} for synchronous runs, which block anyway. */ -export function runForksSync(uri: string, forks: number): number[] { - let samples: number[] = []; +export function runForksSync(uri: string, forks: number): Float64Array { + const samples: Float64Array[] = []; for (let fork = 0; fork < forks; fork++) { - samples = samples.concat(runForkSync(uri)); + samples.push(runForkSync(uri)); + } + return concatSamples(samples); +} + +/** + * Samples stay in typed arrays: a fast benchmark records tens of millions of + * them per fork, which as JS numbers exceed the V8 heap limit of the parent. + */ +function concatSamples(chunks: Float64Array[]): Float64Array { + const length = chunks.reduce((sum, chunk) => sum + chunk.length, 0); + const samples = new Float64Array(length); + let offset = 0; + for (const chunk of chunks) { + samples.set(chunk, offset); + offset += chunk.length; } return samples; } @@ -137,7 +154,7 @@ function newSamplesFile(): string { } /** Runs one fork of `uri` and returns the samples it wrote. */ -async function runFork(uri: string): Promise { +async function runFork(uri: string): Promise { const samplesFile = newSamplesFile(); const [executable, ...args] = currentCommand(); const child = spawn(executable, args, { @@ -152,7 +169,7 @@ async function runFork(uri: string): Promise { return forkSamples(uri, samplesFile, status, signal); } -function runForkSync(uri: string): number[] { +function runForkSync(uri: string): Float64Array { const samplesFile = newSamplesFile(); const [executable, ...args] = currentCommand(); const result = spawnSync(executable, args, { @@ -185,7 +202,7 @@ function forkSamples( samplesFile: string, status: number | null, signal: NodeJS.Signals | null, -): number[] { +): Float64Array { try { if (status !== 0) { throw new Error( @@ -193,7 +210,7 @@ function forkSamples( ); } try { - return deserialize(gunzipSync(readFileSync(samplesFile))) as number[]; + return deserialize(gunzipSync(readFileSync(samplesFile))) as Float64Array; } catch (error) { throw new Error( `[CodSpeed] the process for ${uri} did not report its samples: ${(error as Error).message}`, @@ -205,9 +222,14 @@ function forkSamples( } /** Hands round samples (milliseconds) to the parent as gzip-compressed V8 data. */ -export function writeForkSamples(samplesFile: string, samples: number[]) { +export function writeForkSamples( + samplesFile: string, + samples: ArrayLike, +) { writeFileSync( samplesFile, - gzipSync(serialize(samples), { level: constants.Z_BEST_SPEED }), + gzipSync(serialize(Float64Array.from(samples)), { + level: constants.Z_BEST_SPEED, + }), ); } diff --git a/packages/core/src/walltime/quantiles.ts b/packages/core/src/walltime/quantiles.ts index 3f893a38..0809c263 100644 --- a/packages/core/src/walltime/quantiles.ts +++ b/packages/core/src/walltime/quantiles.ts @@ -3,7 +3,7 @@ export function calculateQuantiles({ stdevNs, meanNs, }: { - sortedTimesNs: number[]; + sortedTimesNs: ArrayLike; stdevNs: number; meanNs: number; }): { @@ -41,17 +41,16 @@ export function calculateQuantiles({ const iqr_ns = q3_ns - q1_ns; // Calculate outliers using same algorithm as pytest-codspeed - const iqr_outlier_rounds = sortedTimesNs.filter( - (t) => - t < q1_ns - IQR_OUTLIER_FACTOR * iqr_ns || - t > q3_ns + IQR_OUTLIER_FACTOR * iqr_ns, - ).length; - - const stdev_outlier_rounds = sortedTimesNs.filter( - (t) => - t < meanNs - STDEV_OUTLIER_FACTOR * stdevNs || - t > meanNs + STDEV_OUTLIER_FACTOR * stdevNs, - ).length; + const iqr_outlier_rounds = countOutside( + sortedTimesNs, + q1_ns - IQR_OUTLIER_FACTOR * iqr_ns, + q3_ns + IQR_OUTLIER_FACTOR * iqr_ns, + ); + const stdev_outlier_rounds = countOutside( + sortedTimesNs, + meanNs - STDEV_OUTLIER_FACTOR * stdevNs, + meanNs + STDEV_OUTLIER_FACTOR * stdevNs, + ); return { q1_ns, @@ -62,7 +61,22 @@ export function calculateQuantiles({ }; } -function interpolateQuantile(sortedArray: number[], index: number): number { +function countOutside( + times: ArrayLike, + low: number, + high: number, +): number { + let count = 0; + for (let i = 0; i < times.length; i++) { + if (times[i] < low || times[i] > high) count++; + } + return count; +} + +function interpolateQuantile( + sortedArray: ArrayLike, + index: number, +): number { const lowerIndex = Math.floor(index); const upperIndex = Math.ceil(index); diff --git a/packages/core/src/walltime/utils.ts b/packages/core/src/walltime/utils.ts index baf97181..3e21e143 100644 --- a/packages/core/src/walltime/utils.ts +++ b/packages/core/src/walltime/utils.ts @@ -19,6 +19,19 @@ export const msToNs = (ms: number) => ms * 1e6; */ export const msToS = (ms: number) => ms / 1e3; +/** + * The samples (milliseconds) in nanoseconds, sorted. A typed array keeps them + * off the V8 heap: a fast benchmark records tens of millions of samples. + */ +export function sortedSamplesNs(samples: ArrayLike): Float64Array { + const timesNs = new Float64Array(samples.length); + for (let i = 0; i < samples.length; i++) { + timesNs[i] = msToNs(samples[i]); + } + // Typed arrays sort numerically by default. + return timesNs.sort(); +} + /** tinybench-style statistics of round samples, in milliseconds. */ export interface RoundsResult { totalTime: number; @@ -27,7 +40,7 @@ export interface RoundsResult { max: number; mean: number; sd: number; - samples: number[]; + samples: ArrayLike; }; } @@ -35,7 +48,7 @@ export interface RoundsResult { * The result of nonempty round samples in milliseconds, e.g. the samples of * several forks of one benchmark. Throws if no samples were recorded. */ -export function samplesToResult(samples: number[]): RoundsResult { +export function samplesToResult(samples: ArrayLike): RoundsResult { const n = samples.length; if (n === 0) { throw new Error("[CodSpeed] Cannot compute statistics from empty samples"); @@ -43,16 +56,18 @@ export function samplesToResult(samples: number[]): RoundsResult { let totalTime = 0; let min = Infinity; let max = -Infinity; - for (const ms of samples) { + for (let i = 0; i < n; i++) { + const ms = samples[i]; totalTime += ms; if (ms < min) min = ms; if (ms > max) max = ms; } const mean = totalTime / n; - const variance = - n > 1 - ? samples.reduce((sum, ms) => sum + (ms - mean) ** 2, 0) / (n - 1) - : 0; + let squaredDeviations = 0; + for (let i = 0; i < n; i++) { + squaredDeviations += (samples[i] - mean) ** 2; + } + const variance = n > 1 ? squaredDeviations / (n - 1) : 0; return { totalTime, latency: { From d2265fd1fb6680caba6ce7055bb018fb43cf1883 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 5 Nov 2026 16:19:33 +0100 Subject: [PATCH 12/13] fixup! feat(tinybench): run walltime benchmarks in forks --- packages/tinybench-plugin/src/walltime.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index acf8837b..bfd04e2b 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -9,6 +9,7 @@ import { runForks, runForksSync, samplesToResult, + sortedSamplesNs, walltimeIsolation, writeForkSamples, writeWalltimeResults, @@ -213,8 +214,7 @@ function convertTinybenchResultToBenchmarkStats( ): BenchmarkStats { const { min, max, mean, sd, samples } = result.latency; - // Get individual sample times in nanoseconds and sort them - const sortedTimesNs = samples.map(msToNs).sort((a, b) => a - b); + const sortedTimesNs = sortedSamplesNs(samples); const meanNs = msToNs(mean); const stdevNs = msToNs(sd); From e3f548d80c9acdc172656b269548e749b678aca0 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 5 Nov 2026 16:19:33 +0100 Subject: [PATCH 13/13] fixup! feat(vitest): run walltime benchmarks in forks --- packages/vitest-plugin/src/instrument.ts | 5 +++-- packages/vitest-plugin/src/v5/provider.ts | 6 ++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/vitest-plugin/src/instrument.ts b/packages/vitest-plugin/src/instrument.ts index e9a7d7fb..abd6c7ef 100644 --- a/packages/vitest-plugin/src/instrument.ts +++ b/packages/vitest-plugin/src/instrument.ts @@ -5,6 +5,7 @@ import { MARKER_TYPE_BENCHMARK_START, msToNs, msToS, + sortedSamplesNs, writeWalltimeResults, type Benchmark, type BenchmarkStats, @@ -34,7 +35,7 @@ interface TinybenchStatistics { max: number; mean: number; sd: number; - samples: number[] | undefined; + samples: ArrayLike | undefined; } interface TinybenchTaskResult { @@ -169,7 +170,7 @@ function tinybenchResultToStats( const { totalTime, latency } = result; const { min, max, mean, sd, samples } = latency; - const sortedTimesNs = (samples ?? []).map(msToNs).sort((a, b) => a - b); + const sortedTimesNs = sortedSamplesNs(samples ?? []); const meanNs = msToNs(mean); const stdevNs = msToNs(sd); diff --git a/packages/vitest-plugin/src/v5/provider.ts b/packages/vitest-plugin/src/v5/provider.ts index bd15ae80..0ba973ae 100644 --- a/packages/vitest-plugin/src/v5/provider.ts +++ b/packages/vitest-plugin/src/v5/provider.ts @@ -2,11 +2,11 @@ import { getGitDir, getInstrumentMode, InstrumentHooks, - walltimeIsolation, optimizeFunction, optimizeFunctionSync, setupCore, teardownCore, + walltimeIsolation, wrapWithRootFrame, wrapWithRootFrameSync, writeForkSamples, @@ -258,9 +258,7 @@ async function runWalltime( throwOnErroredTasks(tasks); if (mode.kind === "Fork") { for (const task of tasks) { - writeForkSamples(mode.samplesFile, [ - ...(task.result?.latency.samples ?? []), - ]); + writeForkSamples(mode.samplesFile, task.result?.latency.samples ?? []); } } else { collectWalltimeResults(bench, getUri);