From 200b6b1fc0b6e59165e95b03fcdb5c60c5b4bab8 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Wed, 1 Jul 2026 10:56:43 +0200 Subject: [PATCH 1/3] feat(vitest-plugin): support vitest 5 alongside 3 and 4 Vitest 5 reworked the benchmark backend: the dedicated `NodeBenchmarkRunner` and the `vitest/runners` / `vitest/suite` entrypoints are gone, benchmarks now run inside `test()` through the unified `TestRunner`, and tinybench moved to v6 (stats moved from `result.benchmark` to `task.result.latency`, `includeSamples` became `retainSamples`). Detect the installed Vitest generation and select the integration seam behind a `VitestBackend` abstraction so the rest of the plugin never inspects the version: - v3/4 keep the custom benchmark runner per instrument mode (analysis/walltime). - v5 installs instrumentation from a setup file that patches the shared `TestRunner.runBenchmarks` static. A setup file (rather than a custom `test.runner`) leaves the runner untouched for non-benchmark tests and also applies to the browser pool. --- packages/vitest-plugin/package.json | 4 +- packages/vitest-plugin/rollup.config.ts | 16 +- .../vitest-plugin/src/__tests__/index.test.ts | 138 +++++++--- .../src/__tests__/instrumented.test.ts | 10 +- packages/vitest-plugin/src/index.ts | 74 ++--- packages/vitest-plugin/src/instrument.ts | 258 ++++++++++++++++++ .../src/{ => legacy}/analysis.ts | 12 +- .../vitest-plugin/src/{ => legacy}/common.ts | 7 +- .../src/legacy/vitest-legacy.d.ts | 46 ++++ .../src/legacy/walltime-utils.ts | 112 ++++++++ packages/vitest-plugin/src/legacy/walltime.ts | 121 ++++++++ packages/vitest-plugin/src/runner.ts | 3 - packages/vitest-plugin/src/v5/setup.ts | 131 +++++++++ packages/vitest-plugin/src/vitestBackend.ts | 152 +++++++++++ packages/vitest-plugin/src/walltime/index.ts | 216 --------------- packages/vitest-plugin/src/walltime/utils.ts | 130 --------- packages/vitest-plugin/vitest.config.ts | 14 +- pnpm-lock.yaml | 231 +++++++++++----- 18 files changed, 1159 insertions(+), 516 deletions(-) create mode 100644 packages/vitest-plugin/src/instrument.ts rename packages/vitest-plugin/src/{ => legacy}/analysis.ts (83%) rename packages/vitest-plugin/src/{ => legacy}/common.ts (76%) create mode 100644 packages/vitest-plugin/src/legacy/vitest-legacy.d.ts create mode 100644 packages/vitest-plugin/src/legacy/walltime-utils.ts create mode 100644 packages/vitest-plugin/src/legacy/walltime.ts delete mode 100644 packages/vitest-plugin/src/runner.ts create mode 100644 packages/vitest-plugin/src/v5/setup.ts create mode 100644 packages/vitest-plugin/src/vitestBackend.ts delete mode 100644 packages/vitest-plugin/src/walltime/index.ts delete mode 100644 packages/vitest-plugin/src/walltime/utils.ts diff --git a/packages/vitest-plugin/package.json b/packages/vitest-plugin/package.json index b319ac29..a994dfcd 100644 --- a/packages/vitest-plugin/package.json +++ b/packages/vitest-plugin/package.json @@ -41,13 +41,13 @@ "peerDependencies": { "tinybench": ">=2.9.0", "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0", - "vitest": "^3.2 || ^4" + "vitest": "^3.2 || ^4 || ^5.0.0-beta" }, "devDependencies": { "@total-typescript/shoehorn": "^0.1.1", "execa": "^8.0.1", "tinybench": "^2.9.0", "vite": "^7.0.0", - "vitest": "^4.0.18" + "vitest": "5.0.0-beta.5" } } diff --git a/packages/vitest-plugin/rollup.config.ts b/packages/vitest-plugin/rollup.config.ts index fe9ff81a..e199b397 100644 --- a/packages/vitest-plugin/rollup.config.ts +++ b/packages/vitest-plugin/rollup.config.ts @@ -20,16 +20,24 @@ export default defineConfig([ plugins: jsPlugins(pkg.version), external: ["@codspeed/core", /^vitest/], }, + // The built layout mirrors the source layout (dist/legacy/*, dist/v5/*) so the + // plugin resolves the seam files with one path rule in both dev and prod. { - input: "src/analysis.ts", - output: { file: "dist/analysis.mjs", format: "es" }, + input: "src/legacy/analysis.ts", + output: { file: "dist/legacy/analysis.mjs", format: "es" }, plugins: jsPlugins(pkg.version), external: ["@codspeed/core", /^vitest/], }, { - input: "src/walltime/index.ts", - output: { file: "dist/walltime.mjs", format: "es" }, + input: "src/legacy/walltime.ts", + output: { file: "dist/legacy/walltime.mjs", format: "es" }, plugins: jsPlugins(pkg.version), external: ["@codspeed/core", /^vitest/], }, + { + input: "src/v5/setup.ts", + output: { file: "dist/v5/setup.mjs", format: "es" }, + plugins: jsPlugins(pkg.version), + external: ["@codspeed/core", /^vitest/, "tinybench"], + }, ]); diff --git a/packages/vitest-plugin/src/__tests__/index.test.ts b/packages/vitest-plugin/src/__tests__/index.test.ts index 98bb38e0..09c953f6 100644 --- a/packages/vitest-plugin/src/__tests__/index.test.ts +++ b/packages/vitest-plugin/src/__tests__/index.test.ts @@ -43,6 +43,19 @@ vi.mock("fs", () => { console.warn = vi.fn(); +const EXPECTED_EXEC_ARGV = [ + "--interpreted-frames-native-stack", + "--allow-natives-syntax", + "--hash-seed=1", + "--random-seed=1", + "--no-opt", + "--predictable", + "--predictable-gc-schedule", + "--expose-gc", + "--no-concurrent-sweeping", + "--max-old-space-size=4096", +]; + describe("codSpeedPlugin", () => { beforeAll(() => { // Set environment variables to trigger instrumented mode @@ -54,6 +67,7 @@ describe("codSpeedPlugin", () => { // Clean up environment variables delete process.env.CODSPEED_ENV; delete process.env.CODSPEED_RUNNER_MODE; + fsMocks.setMockVersion("4.0.18"); }); it("should have a name", async () => { @@ -65,7 +79,9 @@ describe("codSpeedPlugin", () => { }); describe("apply", () => { - it("should not apply the plugin when the mode is not benchmark", async () => { + it("should not apply the plugin when the mode is not benchmark (v3/v4)", async () => { + fsMocks.setMockVersion("4.0.18"); + const applyPlugin = applyPluginFunction( {}, fromPartial({ mode: "test" }), @@ -74,7 +90,8 @@ describe("codSpeedPlugin", () => { expect(applyPlugin).toBe(false); }); - it("should apply the plugin when there is no instrumentation", async () => { + it("should apply the plugin when there is no instrumentation (v3/v4)", async () => { + fsMocks.setMockVersion("4.0.18"); coreMocks.InstrumentHooks.isInstrumented.mockReturnValue(false); const applyPlugin = applyPluginFunction( @@ -88,7 +105,8 @@ describe("codSpeedPlugin", () => { expect(applyPlugin).toBe(true); }); - it("should apply the plugin when there is instrumentation", async () => { + it("should apply the plugin when there is instrumentation (v3/v4)", async () => { + fsMocks.setMockVersion("4.0.18"); coreMocks.InstrumentHooks.isInstrumented.mockReturnValue(true); const applyPlugin = applyPluginFunction( @@ -98,14 +116,32 @@ describe("codSpeedPlugin", () => { expect(applyPlugin).toBe(true); }); + + it("should stay active regardless of mode on v5 (benchmark gating happens in config)", async () => { + fsMocks.setMockVersion("5.0.0-beta.5"); + coreMocks.InstrumentHooks.isInstrumented.mockReturnValue(true); + + const applyPlugin = applyPluginFunction( + {}, + fromPartial({ mode: "test" }), + ); + + expect(applyPlugin).toBe(true); + fsMocks.setMockVersion("4.0.18"); + }); }); it("should apply the codspeed config for v4", () => { + fsMocks.setMockVersion("4.0.18"); const config = resolvedCodSpeedPlugin.config; if (typeof config !== "function") throw new Error("config is not a function"); - const result = config.call({} as never, {}, fromPartial({})); + const result = config.call( + {} as never, + {}, + fromPartial({ mode: "benchmark" }), + ); expect(result).toStrictEqual({ test: { @@ -113,36 +149,27 @@ describe("codSpeedPlugin", () => { expect.stringContaining("packages/vitest-plugin/src/globalSetup.ts"), ], pool: "forks", - execArgv: [ - "--interpreted-frames-native-stack", - "--allow-natives-syntax", - "--hash-seed=1", - "--random-seed=1", - "--no-opt", - "--predictable", - "--predictable-gc-schedule", - "--expose-gc", - "--no-concurrent-sweeping", - "--max-old-space-size=4096", - ], + execArgv: EXPECTED_EXEC_ARGV, runner: expect.stringContaining( - "packages/vitest-plugin/src/analysis.ts", + "packages/vitest-plugin/src/legacy/analysis.ts", ), }, }); }); it("should apply the codspeed config for v3 with poolOptions", () => { - // Set mock version to v3 fsMocks.setMockVersion("3.2.0"); - // Create a new plugin instance to pick up the mocked version const v3Plugin = codspeedPlugin(); const config = v3Plugin.config; if (typeof config !== "function") throw new Error("config is not a function"); - const result = config.call({} as never, {}, fromPartial({})); + const result = config.call( + {} as never, + {}, + fromPartial({ mode: "benchmark" }), + ); expect(result).toStrictEqual({ test: { @@ -152,27 +179,70 @@ describe("codSpeedPlugin", () => { pool: "forks", poolOptions: { forks: { - execArgv: [ - "--interpreted-frames-native-stack", - "--allow-natives-syntax", - "--hash-seed=1", - "--random-seed=1", - "--no-opt", - "--predictable", - "--predictable-gc-schedule", - "--expose-gc", - "--no-concurrent-sweeping", - "--max-old-space-size=4096", - ], + execArgv: EXPECTED_EXEC_ARGV, }, }, runner: expect.stringContaining( - "packages/vitest-plugin/src/analysis.ts", + "packages/vitest-plugin/src/legacy/analysis.ts", ), }, }); - // Reset mock version back to v4 fsMocks.setMockVersion("4.0.18"); }); + + describe("v5 config", () => { + it("should not inject config when benchmarks are not enabled", () => { + fsMocks.setMockVersion("5.0.0-beta.5"); + const v5Plugin = codspeedPlugin(); + const config = v5Plugin.config; + if (typeof config !== "function") + throw new Error("config is not a function"); + + const result = config.call( + {} as never, + {}, + fromPartial({ mode: "test" }), + ); + + expect(result).toBeUndefined(); + fsMocks.setMockVersion("4.0.18"); + }); + + it("should inject the v5 setup file (not a runner) when benchmarks are enabled", () => { + fsMocks.setMockVersion("5.0.0-beta.5"); + const v5Plugin = codspeedPlugin(); + const config = v5Plugin.config; + if (typeof config !== "function") + throw new Error("config is not a function"); + + const result = config.call( + {} as never, + // `benchmark.enabled` is a Vitest 5 config field the v3/4 typings (which + // this file may be compiled against) don't expose. + { test: { benchmark: { enabled: true } } } as never, + fromPartial({ mode: "test" }), + ); + + expect(result).toStrictEqual({ + test: { + globalSetup: [ + expect.stringContaining( + "packages/vitest-plugin/src/globalSetup.ts", + ), + ], + pool: "forks", + execArgv: EXPECTED_EXEC_ARGV, + setupFiles: [ + expect.stringContaining("packages/vitest-plugin/src/v5/setup.ts"), + ], + }, + }); + // The v5 path must not set a custom runner. + expect( + (result as { test?: { runner?: unknown } })?.test?.runner, + ).toBeUndefined(); + fsMocks.setMockVersion("4.0.18"); + }); + }); }); diff --git a/packages/vitest-plugin/src/__tests__/instrumented.test.ts b/packages/vitest-plugin/src/__tests__/instrumented.test.ts index ee880797..ee3b6238 100644 --- a/packages/vitest-plugin/src/__tests__/instrumented.test.ts +++ b/packages/vitest-plugin/src/__tests__/instrumented.test.ts @@ -1,7 +1,15 @@ import { fromPartial } from "@total-typescript/shoehorn"; import { describe, expect, it, vi, type RunnerTestSuite } from "vitest"; +// `vitest/suite` only exists on Vitest 3/4; this file is excluded from the test +// run under v5+ (see vitest.config.ts). +// eslint-disable-next-line import/no-unresolved import { getBenchFn } from "vitest/suite"; -import { AnalysisRunner as CodSpeedRunner } from "../analysis"; +import { AnalysisRunner as CodSpeedRunner } from "../legacy/analysis"; + +// The legacy AnalysisRunner targets the Vitest 3/4 benchmark backend +// (`NodeBenchmarkRunner`, `vitest/suite`), which Vitest 5 removed. This whole +// file is excluded from the test run under v5+ (see vitest.config.ts); the v5 +// path is covered separately. const coreMocks = vi.hoisted(() => { return { diff --git a/packages/vitest-plugin/src/index.ts b/packages/vitest-plugin/src/index.ts index 2bdf59ed..c3216cab 100644 --- a/packages/vitest-plugin/src/index.ts +++ b/packages/vitest-plugin/src/index.ts @@ -7,49 +7,33 @@ import { SetupInstrumentsRequestBody, SetupInstrumentsResponse, } from "@codspeed/core"; -import { readFileSync } from "fs"; -import { createRequire } from "module"; import { join } from "path"; import { Plugin } from "vite"; import { type ViteUserConfig } from "vitest/config"; +import { resolveVitestBackend } from "./vitestBackend"; // get this file's directory path from import.meta.url const __dirname = new URL(".", import.meta.url).pathname; const isFileInTs = import.meta.url.endsWith(".ts"); -function getCodSpeedFileFromName(name: string) { +/** + * Resolve a plugin-owned file (globalSetup, seam entry points) shipped alongside + * this module. Source (`.ts`) and built (`.mjs`) layouts are kept identical (see + * rollup.config.ts), so the same relative `name` works in both. + */ +function resolveFile(name: string): string { const fileExtension = isFileInTs ? "ts" : "mjs"; - return join(__dirname, `${name}.${fileExtension}`); } -function getVitestMajorVersion(): number | null { - try { - // Resolve vitest from the project's perspective (cwd), not from the plugin's location - // This ensures we detect the vitest version the user has installed - const require = createRequire(join(process.cwd(), "package.json")); - const vitestPkgPath = require.resolve("vitest/package.json"); - const vitestPkg = JSON.parse(readFileSync(vitestPkgPath, "utf-8")); - return parseInt(vitestPkg.version.split(".")[0], 10); - } catch { - return null; - } -} - -function getRunnerFile(): string | undefined { - const instrumentMode = getInstrumentMode(); - if (instrumentMode === "disabled") { - return undefined; - } - - return getCodSpeedFileFromName(instrumentMode); -} - export default function codspeedPlugin(): Plugin { + // Resolved lazily on each hook rather than once here: the installed Vitest + // version is detected from the project's cwd, which isn't reliably knowable at + // plugin-construction time (and tests swap it between construction and use). return { name: "codspeed:vitest", apply(_, { mode }) { - if (mode !== "benchmark") { + if (!resolveVitestBackend().isActiveForViteMode(mode)) { return false; } if ( @@ -61,37 +45,19 @@ export default function codspeedPlugin(): Plugin { return true; }, enforce: "post", - config(): ViteUserConfig { - const runnerFile = getRunnerFile(); - const runnerMode = getCodspeedRunnerMode(); - const v8Flags = getV8Flags(); - const vitestMajorVersion = getVitestMajorVersion(); - // by default, assume Vitest v4 or higher - const isVitestV4OrHigher = (vitestMajorVersion ?? 4) >= 4; + config(incomingConfig, { mode }): ViteUserConfig | undefined { + const backend = resolveVitestBackend(); + if (!backend.isBenchmarkRun(incomingConfig, mode)) { + return undefined; + } const config: ViteUserConfig = { test: { pool: "forks", - ...(isVitestV4OrHigher - ? { execArgv: v8Flags } - : { - // Compat with Vitest v3 - // See: https://vitest.dev/guide/migration.html#pool-rework - // poolOptions only exists in Vitest v3 - poolOptions: { - forks: { - execArgv: v8Flags, - }, - }, - }), - globalSetup: [getCodSpeedFileFromName("globalSetup")], - ...(runnerFile && { - runner: runnerFile, - }), - ...(runnerMode === "walltime" && { - benchmark: { - includeSamples: true, - }, + globalSetup: [resolveFile("globalSetup")], + ...backend.getBenchmarkTestConfig(getV8Flags(), resolveFile), + ...(getCodspeedRunnerMode() === "walltime" && { + benchmark: backend.getWalltimeBenchmarkConfig(), }), }, }; diff --git a/packages/vitest-plugin/src/instrument.ts b/packages/vitest-plugin/src/instrument.ts new file mode 100644 index 00000000..b880ca89 --- /dev/null +++ b/packages/vitest-plugin/src/instrument.ts @@ -0,0 +1,258 @@ +import { + calculateQuantiles, + InstrumentHooks, + MARKER_TYPE_BENCHMARK_END, + MARKER_TYPE_BENCHMARK_START, + msToNs, + msToS, + wrapWithRootFrame, + writeWalltimeResults, + type Benchmark, + type BenchmarkStats, +} from "@codspeed/core"; +import type * as tinybench from "tinybench"; + +export type Tinybench = typeof tinybench; + +/** A tinybench task, exposing the `fn` the runner wraps with the root frame. */ +export interface TinybenchTask { + name: string; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + fn: (...args: any[]) => any; + result?: TinybenchTaskResult; +} + +/** tinybench's per-task setup/teardown hook signature. */ +export type TinybenchHook = ( + task: TinybenchTask, + mode: "run" | "warmup", +) => Promise | void; + +/** The mutable subset of a tinybench Bench the runner reaches into. */ +export interface TinybenchBench { + setup: TinybenchHook; + teardown: TinybenchHook; +} + +/** The minimal task shape `patchTaskRunWithRootFrame` mutates. */ +interface RunnableTask { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + fn: (...args: any[]) => any; +} + +/** The tinybench Task prototype whose `run` we wrap. */ +interface TinybenchTaskClass { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + prototype: { run: (this: any) => Promise }; +} + +/** + * The tinybench statistics shape (latency/throughput) shared across the v2 and + * v6 lines. Only the fields the conversion needs are modeled. + */ +interface TinybenchStatistics { + min: number; + max: number; + mean: number; + sd: number; + samples: number[] | undefined; +} + +interface TinybenchTaskResult { + state?: string; + totalTime: number; + latency: TinybenchStatistics; +} + +/** The subset of tinybench bench options that maps onto a CodSpeed benchmark config. */ +export interface TinybenchOptions { + time?: number; + warmupTime?: number; + warmupIterations?: number; + iterations?: number; +} + +/** Timestamp marking the open edge of a task's measured loop. */ +interface InstrumentWindow { + runStart: bigint | null; +} + +let isTaskPatched = false; + +/** + * The window bracketing the currently running task's measured loop, driven by + * the setup/teardown hooks below. Tasks run strictly sequentially within a + * worker, so one shared value suffices. + */ +const instrumentWindow: InstrumentWindow = { runStart: null }; + +/** + * Wrap every task's fn with the root frame so collected stacks are attributed to + * a benchmark. Idempotent: patching the shared `Task.prototype.run` in place hits + * every Bench instance, so repeat calls are no-ops. + * + * `TaskClass` must be the exact prototype the host constructed its tasks against + * (taken from a live task, not imported) so the patch applies even when multiple + * copies of tinybench are installed. + */ +export function patchTaskRunOnce(TaskClass: TinybenchTaskClass): void { + if (isTaskPatched) { + return; + } + isTaskPatched = true; + + const originalRun = TaskClass.prototype.run; + TaskClass.prototype.run = async function (this: RunnableTask) { + const originalFn = this.fn; + this.fn = wrapWithRootFrame(() => originalFn.call(this)); + + try { + return await originalRun.call(this); + } finally { + this.fn = originalFn; + } + }; +} + +/** + * Drive the instrumentation window from each bench's run-mode setup/teardown + * hooks so it brackets only tinybench's measured loop, excluding the warmup + * that runs beforehand and the statistics computation tinybench performs after + * the loop. Wrapping the whole `Task.run()` would otherwise fold all of that + * framework overhead into the recorded sample. + * + * User-provided hooks are preserved and keep their order relative to the work + * under test. + */ +export function installInstrumentHooks( + bench: TinybenchBench, + getUri: (taskName: string) => string, +): void { + const userSetup = bench.setup; + const userTeardown = bench.teardown; + + bench.setup = async (task, mode) => { + await userSetup(task, mode); + if (mode === "run") { + InstrumentHooks.startBenchmark(); + instrumentWindow.runStart = InstrumentHooks.currentTimestamp(); + } + }; + + bench.teardown = async (task, mode) => { + if (mode === "run") { + closeInstrumentWindow(getUri(task.name)); + } + await userTeardown(task, mode); + }; +} + +function closeInstrumentWindow(uri: string): void { + const runEnd = InstrumentHooks.currentTimestamp(); + const pid = process.pid; + + // Benchmark markers must land inside the sample window opened by + // startBenchmark(), so they have to be emitted before stopBenchmark() + // closes it. The runner consumes the FIFO stream in order, so a marker + // sent after StopBenchmark falls outside the sample and breaks the + // expected SampleStart > BenchmarkStart > BenchmarkEnd > SampleEnd nesting. + InstrumentHooks.addMarker( + pid, + MARKER_TYPE_BENCHMARK_START, + instrumentWindow.runStart!, + ); + InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_END, runEnd); + + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(pid, uri); + instrumentWindow.runStart = null; +} + +/** + * Persist collected walltime benchmarks, if any. The per-seam result traversal + * differs (legacy walks the Vitest suite tree, v5 iterates the live tinybench + * tasks) because each generation exposes results differently, but the write and + * the summary log are identical. + */ +export function writeAndLogWalltimeResults(benchmarks: Benchmark[]): void { + if (benchmarks.length === 0) { + return; + } + writeWalltimeResults(benchmarks); + console.log( + `[CodSpeed] Done collecting walltime data for ${benchmarks.length} benches.`, + ); +} + +/** + * Convert a completed tinybench task into a CodSpeed walltime benchmark. Returns + * null when the task produced no samples (e.g. fully optimized out), in which + * case there is nothing to record. + */ +export function tinybenchTaskToBenchmark( + task: TinybenchTask, + uri: string, + options: TinybenchOptions, +): Benchmark | null { + const stats = tinybenchResultToStats(task.result, options); + if (stats === null) { + return null; + } + + return { + name: task.name, + uri, + config: { + max_rounds: options.iterations ?? null, + max_time_ns: options.time ? msToNs(options.time) : null, + min_round_time_ns: null, // tinybench does not have an option for this + warmup_time_ns: + options.warmupIterations !== 0 && options.warmupTime + ? msToNs(options.warmupTime) + : null, + }, + stats, + }; +} + +function tinybenchResultToStats( + result: TinybenchTaskResult | undefined, + options: TinybenchOptions, +): BenchmarkStats | null { + if (!result) { + throw new Error("No benchmark data available in result"); + } + + const { totalTime, latency } = result; + const { min, max, mean, sd, samples } = latency; + + const sortedTimesNs = (samples ?? []).map(msToNs).sort((a, b) => a - b); + const meanNs = msToNs(mean); + const stdevNs = msToNs(sd); + + if (sortedTimesNs.length == 0) { + // Sometimes the benchmarks can be completely optimized out and not even + // run, but their beforeEach and afterEach hooks are still executed, and the + // task is still considered a success. + return null; + } + + const { q1_ns, q3_ns, median_ns, iqr_outlier_rounds, stdev_outlier_rounds } = + calculateQuantiles({ meanNs, stdevNs, sortedTimesNs }); + + return { + min_ns: msToNs(min), + max_ns: msToNs(max), + mean_ns: meanNs, + stdev_ns: stdevNs, + q1_ns, + median_ns, + q3_ns, + total_time: msToS(totalTime), + iter_per_round: 1, // tinybench runs one iteration per round + rounds: sortedTimesNs.length, + iqr_outlier_rounds, + stdev_outlier_rounds, + warmup_iters: options.warmupIterations ?? 0, + }; +} diff --git a/packages/vitest-plugin/src/analysis.ts b/packages/vitest-plugin/src/legacy/analysis.ts similarity index 83% rename from packages/vitest-plugin/src/analysis.ts rename to packages/vitest-plugin/src/legacy/analysis.ts index 424e1a70..e2ed4855 100644 --- a/packages/vitest-plugin/src/analysis.ts +++ b/packages/vitest-plugin/src/legacy/analysis.ts @@ -8,7 +8,11 @@ import { wrapWithRootFrame, } from "@codspeed/core"; import { Benchmark, type RunnerTestSuite } from "vitest"; +// `vitest/runners` and `vitest/suite` only exist on Vitest 3/4; this runner is +// loaded only there. +// eslint-disable-next-line import/no-unresolved import { NodeBenchmarkRunner } from "vitest/runners"; +// eslint-disable-next-line import/no-unresolved import { getBenchFn } from "vitest/suite"; import { callSuiteHook, @@ -36,11 +40,14 @@ async function runAnalysisBench( currentSuiteName: string, ) { const uri = `${currentSuiteName}::${benchmark.name}`; - const fn = getBenchFn(benchmark); + // tinybench's bench fn carries a `this: Bench` requirement on Vitest 3/4 that + // we don't need (the work under test is self-contained); call it as a plain + // parameterless function. The cast also smooths over the typing differences + // across supported Vitest versions. + const fn = getBenchFn(benchmark) as () => unknown; await optimizeFunction(async () => { await callSuiteHook(suite, benchmark, "beforeEach"); - // @ts-expect-error we do not need to bind the function to an instance of tinybench's Bench await fn(); await callSuiteHook(suite, benchmark, "afterEach"); }); @@ -50,7 +57,6 @@ async function runAnalysisBench( global.gc?.(); await wrapWithRootFrame(async () => { InstrumentHooks.startBenchmark(); - // @ts-expect-error we do not need to bind the function to an instance of tinybench's Bench await fn(); InstrumentHooks.stopBenchmark(); InstrumentHooks.setExecutedBenchmark(process.pid, uri); diff --git a/packages/vitest-plugin/src/common.ts b/packages/vitest-plugin/src/legacy/common.ts similarity index 76% rename from packages/vitest-plugin/src/common.ts rename to packages/vitest-plugin/src/legacy/common.ts index 9ef422a7..5f552286 100644 --- a/packages/vitest-plugin/src/common.ts +++ b/packages/vitest-plugin/src/legacy/common.ts @@ -1,6 +1,8 @@ import { getGitDir } from "@codspeed/core"; import path from "path"; import { Benchmark, type RunnerTask, type RunnerTestSuite } from "vitest"; +// `vitest/suite` only exists on Vitest 3/4; this module is used only there. +// eslint-disable-next-line import/no-unresolved import { getHooks } from "vitest/suite"; type SuiteHooks = ReturnType; @@ -19,8 +21,9 @@ export async function callSuiteHook( const hooks = getSuiteHooks(suite, name); - // @ts-expect-error TODO: add support for hooks parameters - await Promise.all(hooks.map((fn) => fn())); + // TODO: add support for hook parameters. The hook signature differs across + // supported Vitest versions, so we call them through a parameterless cast. + await Promise.all((hooks as Array<() => unknown>).map((fn) => fn())); if (name === "afterEach" && suite?.suite) { await callSuiteHook(suite.suite, currentTask, name); diff --git a/packages/vitest-plugin/src/legacy/vitest-legacy.d.ts b/packages/vitest-plugin/src/legacy/vitest-legacy.d.ts new file mode 100644 index 00000000..274db403 --- /dev/null +++ b/packages/vitest-plugin/src/legacy/vitest-legacy.d.ts @@ -0,0 +1,46 @@ +// Vitest 3/4 exposed benchmark internals through `vitest/runners` and +// `vitest/suite`, and surfaced the `Benchmark` task type from `vitest`. Vitest 5 +// removed all of these (benchmarks now run through the unified `TestRunner`). +// +// The legacy runner (`analysis.ts`, `walltime/`) still imports them and is only +// loaded when the user runs Vitest 3/4, but the plugin is type-checked against +// whichever Vitest is installed — including 5, where these are gone. These +// ambient declarations keep the legacy code compiling there without affecting +// runtime: the modules are never imported under v5. + +import type * as tinybench from "tinybench"; + +declare module "vitest" { + import type { RunnerTestCase } from "vitest"; + + // In v3/4 a benchmark is a test case carrying tinybench output on its result. + interface Benchmark extends RunnerTestCase { + meta: RunnerTestCase["meta"] & { benchmark?: boolean }; + } +} + +declare module "vitest/runners" { + import type { RunnerTestSuite } from "vitest"; + + export class NodeBenchmarkRunner { + constructor(config?: unknown); + config: unknown; + runSuite(suite: RunnerTestSuite): Promise; + importTinybench(): Promise; + } +} + +declare module "vitest/suite" { + import type { Benchmark } from "vitest"; + + export function getBenchFn(benchmark: Benchmark): () => unknown; + export function getBenchOptions(benchmark: Benchmark): { + time?: number; + warmupTime?: number; + warmupIterations?: number; + iterations?: number; + }; + export function getHooks( + suite: unknown, + ): Record unknown>>; +} diff --git a/packages/vitest-plugin/src/legacy/walltime-utils.ts b/packages/vitest-plugin/src/legacy/walltime-utils.ts new file mode 100644 index 00000000..c4dd2179 --- /dev/null +++ b/packages/vitest-plugin/src/legacy/walltime-utils.ts @@ -0,0 +1,112 @@ +import { type Benchmark } from "@codspeed/core"; +import { + type RunnerTaskResult, + type RunnerTestSuite, + type Benchmark as VitestBenchmark, +} from "vitest"; +// `vitest/suite` only exists on Vitest 3/4; this module is used only there. +// eslint-disable-next-line import/no-unresolved +import { getBenchOptions } from "vitest/suite"; +import { + tinybenchTaskToBenchmark, + type TinybenchOptions, + type TinybenchTask, +} from "../instrument"; +import { isVitestTaskBenchmark } from "./common"; + +export async function extractBenchmarkResults( + suite: RunnerTestSuite, + parentPath = "", +): Promise { + const benchmarks: Benchmark[] = []; + const currentPath = parentPath ? `${parentPath}::${suite.name}` : suite.name; + + for (const task of suite.tasks) { + if (isVitestTaskBenchmark(task) && task.result?.state === "pass") { + const benchmark = processBenchmarkTask(task, currentPath); + if (benchmark) { + benchmarks.push(benchmark); + } + } else if (task.type === "suite") { + const nestedBenchmarks = await extractBenchmarkResults(task, currentPath); + benchmarks.push(...nestedBenchmarks); + } + } + + return benchmarks; +} + +function processBenchmarkTask( + task: VitestBenchmark, + suitePath: string, +): Benchmark | null { + const uri = `${suitePath}::${task.name}`; + + const result = task.result; + if (!result) { + console.warn(` ⚠ No result data available for ${uri}`); + return null; + } + + try { + const benchOptions = getBenchOptions(task); + const benchmark = tinybenchTaskToBenchmark( + adaptLegacyResult(task.name, result), + uri, + benchOptions as TinybenchOptions, + ); + + if (benchmark === null) { + console.log(` ✔ No walltime data to collect for ${uri}`); + return null; + } + + console.log(` ✔ Collected walltime data for ${uri}`); + return benchmark; + } catch (error) { + console.warn(` ⚠ Failed to process benchmark result for ${uri}:`, error); + return null; + } +} + +/** + * Vitest 3/4 attaches the raw tinybench v2 result under `result.benchmark`, + * whose statistics are a flat object ({ totalTime, min, max, mean, sd, samples }). + * Reshape it into the `{ result: { totalTime, latency } }` form the shared + * converter expects (tinybench v6 nests statistics under `latency`). + */ +interface LegacyBenchmarkStats { + totalTime: number; + min: number; + max: number; + mean: number; + sd: number; + samples: number[]; +} + +function adaptLegacyResult( + name: string, + result: RunnerTaskResult, +): TinybenchTask { + // `result.benchmark` only exists on the Vitest 3/4 task result; the v5 typings + // (compiled against here) dropped it. + const benchmark = (result as { benchmark?: LegacyBenchmarkStats }).benchmark; + if (!benchmark) { + throw new Error("No benchmark data available in result"); + } + + return { + name, + fn: () => undefined, + result: { + totalTime: benchmark.totalTime, + latency: { + min: benchmark.min, + max: benchmark.max, + mean: benchmark.mean, + sd: benchmark.sd, + samples: benchmark.samples, + }, + }, + }; +} diff --git a/packages/vitest-plugin/src/legacy/walltime.ts b/packages/vitest-plugin/src/legacy/walltime.ts new file mode 100644 index 00000000..1ac5569b --- /dev/null +++ b/packages/vitest-plugin/src/legacy/walltime.ts @@ -0,0 +1,121 @@ +import { setupCore } from "@codspeed/core"; +import type * as tinybench from "tinybench"; +import { + RunnerTaskEventPack, + RunnerTaskResultPack, + type RunnerTestSuite, +} from "vitest"; +// `vitest/runners` only exists on Vitest 3/4; this runner is loaded only there. +// eslint-disable-next-line import/no-unresolved +import { NodeBenchmarkRunner } from "vitest/runners"; +import { + installInstrumentHooks, + patchTaskRunOnce, + writeAndLogWalltimeResults, + type TinybenchBench, +} from "../instrument"; +import { patchRootSuiteWithFullFilePath } from "./common"; +import { extractBenchmarkResults } from "./walltime-utils"; + +type Tinybench = typeof tinybench; + +/** + * Lets tinybench run the benches through Vitest's default benchmark execution, + * instrumenting each measured loop, then extracts the results from the suite + * tree afterwards. (The v5 seam instruments the same way but reads results off + * the live tinybench tasks instead — see `v5/setup.ts`.) + */ +export class WalltimeRunner extends NodeBenchmarkRunner { + private suiteUris = new Map(); + /// Suite ID of the currently running suite, to allow constructing the URI in the context of tinybench tasks + private currentSuiteId: string | null = null; + + async runSuite(suite: RunnerTestSuite): Promise { + patchRootSuiteWithFullFilePath(suite); + this.populateBenchmarkUris(suite); + + setupCore(); + + await super.runSuite(suite); + + const benchmarks = await extractBenchmarkResults(suite); + if (benchmarks.length === 0) { + console.warn( + `[CodSpeed] No benchmark results found after suite execution`, + ); + return; + } + writeAndLogWalltimeResults(benchmarks); + } + + private populateBenchmarkUris(suite: RunnerTestSuite, parentPath = ""): void { + const currentPath = + parentPath !== "" ? `${parentPath}::${suite.name}` : suite.name; + + for (const task of suite.tasks) { + if (task.type === "suite") { + this.suiteUris.set(task.id, `${currentPath}::${task.name}`); + this.populateBenchmarkUris(task, currentPath); + } + } + } + + private getBenchmarkUri(taskName: string): string { + if (this.currentSuiteId === null) { + throw new Error("currentSuiteId is null - something went wrong"); + } + const suiteUri = this.suiteUris.get(this.currentSuiteId) || ""; + return `${suiteUri}::${taskName}`; + } + + async importTinybench(): Promise { + const tinybench = await super.importTinybench(); + + // `tinybench` is a frozen ES module namespace, so the `Bench` export cannot + // be reassigned. The shared `Task.prototype` is patched in place; the + // instrumented `Bench` is handed back through a fresh module-shaped object + // that Vitest destructures from. + patchTaskRunOnce(tinybench.Task); + + return { + ...tinybench, + Bench: this.createInstrumentedBench(tinybench), + }; + } + + private createInstrumentedBench( + tinybench: Tinybench, + ): typeof tinybench.Bench { + // eslint-disable-next-line @typescript-eslint/no-this-alias + const runner = this; + const OriginalBench = tinybench.Bench; + + class InstrumentedBench extends OriginalBench { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + constructor(...benchArgs: any[]) { + super(...benchArgs); + installInstrumentHooks(this as unknown as TinybenchBench, (taskName) => + runner.getBenchmarkUri(taskName), + ); + } + } + + return InstrumentedBench; + } + + // Allow tinybench to retrieve the path to the currently running suite + async onTaskUpdate( + _: RunnerTaskResultPack[], + events: RunnerTaskEventPack[], + ): Promise { + events.map((event) => { + const [id, eventName] = event; + + if (eventName === "suite-prepare") { + this.currentSuiteId = id; + } + }); + } +} + +export default WalltimeRunner; diff --git a/packages/vitest-plugin/src/runner.ts b/packages/vitest-plugin/src/runner.ts deleted file mode 100644 index 60a627a6..00000000 --- a/packages/vitest-plugin/src/runner.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { AnalysisRunner } from "./analysis"; - -export default AnalysisRunner; diff --git a/packages/vitest-plugin/src/v5/setup.ts b/packages/vitest-plugin/src/v5/setup.ts new file mode 100644 index 00000000..6a851463 --- /dev/null +++ b/packages/vitest-plugin/src/v5/setup.ts @@ -0,0 +1,131 @@ +import { + getGitDir, + getInstrumentMode, + setupCore, + type Benchmark, +} from "@codspeed/core"; +import path from "path"; +// `TestRunner` is the unified runner Vitest 5 introduced; it replaces the +// `NodeBenchmarkRunner` the legacy seam subclasses, and is re-exported from the +// package root. It is read off the namespace (rather than a named import) so the +// plugin still type-checks against Vitest 3/4, which don't export it; this +// module only ever runs under v5. +import * as vitest from "vitest"; +import { + installInstrumentHooks, + patchTaskRunOnce, + tinybenchTaskToBenchmark, + writeAndLogWalltimeResults, + type TinybenchBench, + type TinybenchOptions, + type TinybenchTask, +} from "../instrument"; + +const TestRunner = (vitest as unknown as { TestRunner: unknown }).TestRunner; + +/** + * Vitest 5 runs benchmarks inside `test()` through `TestRunner`, calling the + * static `TestRunner.runBenchmarks(tinybench)` with a fully built tinybench + * instance. That static is referenced directly (not through `this`), so a + * runner subclass cannot intercept it — we patch the static in place instead. + * + * This module is registered as a Vitest `setupFile`, which runs in the worker + * before any test file is collected. Importing it installs the patch as a side + * effect. A setup file (rather than a custom `test.runner`) is used because it + * leaves the runner untouched for non-benchmark tests and also applies to the + * browser pool, where a Node runner file would not. + */ + +interface TinybenchWithTasks extends TinybenchBench { + name: string; + tasks: TinybenchTask[]; + run(): Promise; + // tinybench exposes the resolved options on the instance + opts?: TinybenchOptions; +} + +/** Minimal shape of the current Vitest test task we read for URI construction. */ +interface CurrentTest { + fullTestName?: string; + file?: { filepath: string }; +} + +function getCurrentTest(): CurrentTest | undefined { + // `getCurrentTest` is a static on the runner; typings don't surface it. + const getter = (TestRunner as unknown as { getCurrentTest?: () => unknown }) + .getCurrentTest; + return getter ? (getter() as CurrentTest | undefined) : undefined; +} + +/** + * Build the benchmark URI from the running test and the tinybench task name. + * Matches the legacy convention: git-relative file, then the suite/test path, + * then the bench name, all `::`-separated. + */ +function buildUri(taskName: string): string { + const test = getCurrentTest(); + const filepath = test?.file?.filepath; + if (!filepath) { + throw new Error("[CodSpeed] could not resolve the running benchmark file"); + } + const gitDir = getGitDir(filepath); + if (gitDir === undefined) { + throw new Error("Could not find a git repository"); + } + const relativeFile = path.relative(gitDir, filepath); + // `fullTestName` uses " > " between suite levels; normalize to "::". + const testPath = (test?.fullTestName ?? "").split(" > ").join("::"); + return [relativeFile, testPath, taskName].filter(Boolean).join("::"); +} + +function collectWalltimeResults(tinybench: TinybenchWithTasks): void { + const options: TinybenchOptions = tinybench.opts ?? {}; + const benchmarks: Benchmark[] = []; + + for (const task of tinybench.tasks) { + if (task.result?.state && task.result.state !== "completed") continue; + const benchmark = tinybenchTaskToBenchmark( + task, + buildUri(task.name), + options, + ); + if (benchmark) { + benchmarks.push(benchmark); + } + } + + writeAndLogWalltimeResults(benchmarks); +} + +function patchRunBenchmarks(): void { + const Runner = TestRunner as unknown as { + runBenchmarks: (tinybench: TinybenchWithTasks) => Promise; + }; + const originalRunBenchmarks = Runner.runBenchmarks.bind(TestRunner); + const isWalltime = getInstrumentMode() === "walltime"; + + Runner.runBenchmarks = async (tinybench: TinybenchWithTasks) => { + setupCore(); + + // tinybench's `Task` class isn't exported from the bench instance, so we + // reach it through a constructed task (Vitest adds them before running). + const TaskClass = tinybench.tasks[0]?.constructor as + | { prototype: { run: (this: unknown) => Promise } } + | undefined; + if (TaskClass) { + patchTaskRunOnce(TaskClass); + } + + installInstrumentHooks(tinybench, buildUri); + + const result = await originalRunBenchmarks(tinybench); + + if (isWalltime) { + collectWalltimeResults(tinybench); + } + + return result; + }; +} + +patchRunBenchmarks(); diff --git a/packages/vitest-plugin/src/vitestBackend.ts b/packages/vitest-plugin/src/vitestBackend.ts new file mode 100644 index 00000000..db8c0d26 --- /dev/null +++ b/packages/vitest-plugin/src/vitestBackend.ts @@ -0,0 +1,152 @@ +import { getInstrumentMode } from "@codspeed/core"; +import { readFileSync } from "fs"; +import { createRequire } from "module"; +import { join } from "path"; +import { type ViteUserConfig } from "vitest/config"; + +/** + * Everything about integrating with Vitest that depends on which Vitest + * generation the user installed, resolved once so the rest of the plugin reads a + * `VitestBackend` and never inspects the version itself. + */ +export interface VitestBackend { + /** + * Whether the plugin should stay active for this Vite `mode`. When false the + * plugin's `apply` returns false and it is dropped entirely. + */ + isActiveForViteMode(mode: string): boolean; + + /** + * Whether the current invocation is running benchmarks (as opposed to tests), + * given the incoming config and Vite `mode`. + */ + isBenchmarkRun(config: ViteUserConfig, mode: string): boolean; + + /** + * The `test` config fragment that wires the benchmark instrumentation into + * Vitest: the V8 exec args (whose placement moved across versions) plus the + * integration seam (a custom runner on legacy, a setup file on v5). + */ + getBenchmarkTestConfig( + v8Flags: string[], + resolveFile: (name: string) => string, + ): ViteUserConfig["test"]; + + /** + * The `test.benchmark` fragment asking tinybench to retain per-iteration + * samples so the walltime runner can compute quantiles. Only used in walltime + * mode. The option was renamed (`includeSamples` → `retainSamples`) when + * Vitest 5 moved to tinybench v6. + */ + getWalltimeBenchmarkConfig(): Record; +} + +/** + * Vitest 5 reworked the benchmark backend: the dedicated `NodeBenchmarkRunner` + * and the `vitest/runners` / `vitest/suite` entrypoints are gone, benchmarks run + * inside `test()` through the unified `TestRunner`, and tinybench moved to v6. + * The integration seam therefore differs fundamentally (a `TestRunner` patch + * installed from a setup file vs. a runner subclass per mode), which is why the + * two backends are separate implementations rather than a pile of inline + * ternaries. + * + * When the version cannot be detected we assume the latest supported major. + */ +export function resolveVitestBackend(): VitestBackend { + const major = getVitestMajorVersion() ?? 5; + return major >= 5 ? new V5Backend() : new LegacyBackend(major); +} + +/** + * Resolve the major version of the Vitest the *user's project* depends on, not + * the one bundled alongside this plugin. Returns null when it cannot be found, + * letting `resolveVitestBackend` fall back to the latest supported major. + */ +function getVitestMajorVersion(): number | null { + try { + const require = createRequire(join(process.cwd(), "package.json")); + const vitestPkgPath = require.resolve("vitest/package.json"); + const vitestPkg = JSON.parse(readFileSync(vitestPkgPath, "utf-8")); + return parseInt(vitestPkg.version.split(".")[0], 10); + } catch { + return null; + } +} + +/** + * Vitest 5+. `vitest bench` runs under the `"test"` mode with + * `test.benchmark.enabled` flipped (there is no dedicated benchmark mode), so + * the plugin stays active for every mode and gates on the config instead. + * Instrumentation is installed from a setup file that patches the shared + * `TestRunner` (see `v5/setup.ts`). + */ +class V5Backend implements VitestBackend { + isActiveForViteMode(): boolean { + return true; + } + + isBenchmarkRun(config: ViteUserConfig): boolean { + // `benchmark.enabled` only exists on the Vitest 5 config; the v3/4 typings + // we may be compiled against don't know about it. + const benchmark = config.test?.benchmark as + | { enabled?: boolean } + | undefined; + return benchmark?.enabled === true; + } + + getBenchmarkTestConfig( + v8Flags: string[], + resolveFile: (name: string) => string, + ): ViteUserConfig["test"] { + return { + execArgv: v8Flags, + setupFiles: [resolveFile("v5/setup")], + }; + } + + getWalltimeBenchmarkConfig(): Record { + return { retainSamples: true }; + } +} + +/** + * Vitest 3/4. `vitest bench` runs under a dedicated `"benchmark"` Vite mode, and + * instrumentation is installed through a custom `test.runner` subclass of + * `NodeBenchmarkRunner`, one per instrument mode (`analysis` / `walltime`). + */ +class LegacyBackend implements VitestBackend { + constructor(private readonly major: number) {} + + isActiveForViteMode(mode: string): boolean { + return mode === "benchmark"; + } + + isBenchmarkRun(_config: ViteUserConfig, mode: string): boolean { + return mode === "benchmark"; + } + + getBenchmarkTestConfig( + v8Flags: string[], + resolveFile: (name: string) => string, + ): ViteUserConfig["test"] { + const instrumentMode = getInstrumentMode(); + const runner = + instrumentMode === "disabled" + ? undefined + : resolveFile(join("legacy", instrumentMode)); + + return { + // Vitest 3 nests exec args under `poolOptions.forks`; v4 moved them to a + // top-level `test.execArgv`. + // See: https://vitest.dev/guide/migration.html#pool-rework + ...(this.major >= 4 + ? { execArgv: v8Flags } + : { poolOptions: { forks: { execArgv: v8Flags } } }), + ...(runner && { runner }), + }; + } + + getWalltimeBenchmarkConfig(): Record { + return { includeSamples: true }; + } +} diff --git a/packages/vitest-plugin/src/walltime/index.ts b/packages/vitest-plugin/src/walltime/index.ts deleted file mode 100644 index 18c71215..00000000 --- a/packages/vitest-plugin/src/walltime/index.ts +++ /dev/null @@ -1,216 +0,0 @@ -import { - InstrumentHooks, - MARKER_TYPE_BENCHMARK_END, - MARKER_TYPE_BENCHMARK_START, - setupCore, - wrapWithRootFrame, - writeWalltimeResults, -} from "@codspeed/core"; -import type * as tinybench from "tinybench"; -import { - RunnerTaskEventPack, - RunnerTaskResultPack, - type RunnerTestSuite, -} from "vitest"; -import { NodeBenchmarkRunner } from "vitest/runners"; -import { patchRootSuiteWithFullFilePath } from "../common"; -import { extractBenchmarkResults } from "./utils"; - -type Tinybench = typeof tinybench; - -/** A tinybench task, exposing the `fn` the runner wraps with the root frame. */ -interface TinybenchTask { - name: string; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - fn: (...args: any[]) => any; -} - -/** tinybench's per-task setup/teardown hook signature. */ -type TinybenchHook = ( - task: TinybenchTask, - mode: "run" | "warmup", -) => Promise | void; - -/** The mutable subset of a tinybench Bench the runner reaches into. */ -interface TinybenchBench { - setup: TinybenchHook; - teardown: TinybenchHook; -} - -/** - * WalltimeRunner uses Vitest's default benchmark execution - * and extracts results from the suite after completion - */ -export class WalltimeRunner extends NodeBenchmarkRunner { - private isTinybenchHookedWithCodspeed = false; - private suiteUris = new Map(); - /// Suite ID of the currently running suite, to allow constructing the URI in the context of tinybench tasks - private currentSuiteId: string | null = null; - // Carries the window start timestamp from the setup hook to the teardown - // hook. Tasks run strictly sequentially, so a single field is enough. - private runStart: bigint | null = null; - - async runSuite(suite: RunnerTestSuite): Promise { - patchRootSuiteWithFullFilePath(suite); - this.populateBenchmarkUris(suite); - - setupCore(); - - await super.runSuite(suite); - - const benchmarks = await extractBenchmarkResults(suite); - - if (benchmarks.length > 0) { - writeWalltimeResults(benchmarks); - console.log( - `[CodSpeed] Done collecting walltime data for ${benchmarks.length} benches.`, - ); - } else { - console.warn( - `[CodSpeed] No benchmark results found after suite execution`, - ); - } - } - - private populateBenchmarkUris(suite: RunnerTestSuite, parentPath = ""): void { - const currentPath = - parentPath !== "" ? `${parentPath}::${suite.name}` : suite.name; - - for (const task of suite.tasks) { - if (task.type === "suite") { - this.suiteUris.set(task.id, `${currentPath}::${task.name}`); - this.populateBenchmarkUris(task, currentPath); - } - } - } - - private getBenchmarkUri(taskName: string): string { - if (this.currentSuiteId === null) { - throw new Error("currentSuiteId is null - something went wrong"); - } - const suiteUri = this.suiteUris.get(this.currentSuiteId) || ""; - return `${suiteUri}::${taskName}`; - } - - async importTinybench(): Promise { - const tinybench = await super.importTinybench(); - - // `tinybench` is a frozen ES module namespace, so the `Bench` export cannot - // be reassigned. Mutating the shared `Task.prototype` in place is allowed - // and only needs to happen once; the instrumented `Bench` is handed back - // through a fresh module-shaped object that Vitest destructures from. - if (!this.isTinybenchHookedWithCodspeed) { - this.isTinybenchHookedWithCodspeed = true; - this.patchTaskWithRootFrame(tinybench); - } - - return { - ...tinybench, - Bench: this.createInstrumentedBench(tinybench), - }; - } - - /** - * Wrap each task's function with the root frame so collected stacks can be - * attributed to a benchmark. The window itself is driven by the bench's - * setup/teardown hooks (see createInstrumentedBench). - */ - private patchTaskWithRootFrame(tinybench: Tinybench): void { - const originalRun = tinybench.Task.prototype.run; - - tinybench.Task.prototype.run = async function () { - const task = this as unknown as TinybenchTask; - const originalFn = task.fn; - task.fn = wrapWithRootFrame(() => originalFn.call(task)); - - try { - await originalRun.call(this); - } finally { - task.fn = originalFn; - } - - return this; - }; - } - - /** - * Drive the instrumentation window from each bench's run-mode setup/teardown - * hooks so it brackets only tinybench's measured loop, excluding the warmup - * that Vitest runs beforehand and the statistics computation tinybench - * performs after the loop. Wrapping the whole `Task.run()` would otherwise - * fold all of that framework overhead into the recorded sample. - * - * User-provided hooks are preserved and keep their order relative to the work - * under test. - */ - private createInstrumentedBench( - tinybench: Tinybench, - ): typeof tinybench.Bench { - // eslint-disable-next-line @typescript-eslint/no-this-alias - const runner = this; - const OriginalBench = tinybench.Bench; - - class InstrumentedBench extends OriginalBench { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - constructor(...benchArgs: any[]) { - super(...benchArgs); - runner.installInstrumentHooks(this as unknown as TinybenchBench); - } - } - - return InstrumentedBench; - } - - private installInstrumentHooks(bench: TinybenchBench): void { - const userSetup = bench.setup; - const userTeardown = bench.teardown; - - bench.setup = async (task, mode) => { - await userSetup(task, mode); - if (mode === "run") { - InstrumentHooks.startBenchmark(); - this.runStart = InstrumentHooks.currentTimestamp(); - } - }; - - bench.teardown = async (task, mode) => { - if (mode === "run") { - this.closeInstrumentWindow(this.getBenchmarkUri(task.name)); - } - await userTeardown(task, mode); - }; - } - - private closeInstrumentWindow(uri: string): void { - const runEnd = InstrumentHooks.currentTimestamp(); - const pid = process.pid; - - // Benchmark markers must land inside the sample window opened by - // startBenchmark(), so they have to be emitted before stopBenchmark() - // closes it. The runner consumes the FIFO stream in order, so a marker - // sent after StopBenchmark falls outside the sample and breaks the - // expected SampleStart > BenchmarkStart > BenchmarkEnd > SampleEnd nesting. - InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_START, this.runStart!); - InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_END, runEnd); - - InstrumentHooks.stopBenchmark(); - InstrumentHooks.setExecutedBenchmark(pid, uri); - this.runStart = null; - } - - // Allow tinybench to retrieve the path to the currently running suite - async onTaskUpdate( - _: RunnerTaskResultPack[], - events: RunnerTaskEventPack[], - ): Promise { - events.map((event) => { - const [id, eventName] = event; - - if (eventName === "suite-prepare") { - this.currentSuiteId = id; - } - }); - } -} - -export default WalltimeRunner; diff --git a/packages/vitest-plugin/src/walltime/utils.ts b/packages/vitest-plugin/src/walltime/utils.ts deleted file mode 100644 index 2b3c1f33..00000000 --- a/packages/vitest-plugin/src/walltime/utils.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { - calculateQuantiles, - msToNs, - msToS, - type Benchmark, - type BenchmarkStats, -} from "@codspeed/core"; -import { - type RunnerTaskResult, - type RunnerTestSuite, - type Benchmark as VitestBenchmark, -} from "vitest"; -import { getBenchOptions } from "vitest/suite"; -import { isVitestTaskBenchmark } from "../common"; - -export async function extractBenchmarkResults( - suite: RunnerTestSuite, - parentPath = "", -): Promise { - const benchmarks: Benchmark[] = []; - const currentPath = parentPath ? `${parentPath}::${suite.name}` : suite.name; - - for (const task of suite.tasks) { - if (isVitestTaskBenchmark(task) && task.result?.state === "pass") { - const benchmark = await processBenchmarkTask(task, currentPath); - if (benchmark) { - benchmarks.push(benchmark); - } - } else if (task.type === "suite") { - const nestedBenchmarks = await extractBenchmarkResults(task, currentPath); - benchmarks.push(...nestedBenchmarks); - } - } - - return benchmarks; -} - -async function processBenchmarkTask( - task: VitestBenchmark, - suitePath: string, -): Promise { - const uri = `${suitePath}::${task.name}`; - - const result = task.result; - if (!result) { - console.warn(` ⚠ No result data available for ${uri}`); - return null; - } - - try { - // Get tinybench configuration options from vitest - const benchOptions = getBenchOptions(task); - - const stats = convertVitestResultToBenchmarkStats(result, benchOptions); - - if (stats === null) { - console.log(` ✔ No walltime data to collect for ${uri}`); - return null; - } - - const coreBenchmark: Benchmark = { - name: task.name, - uri, - config: { - max_rounds: benchOptions.iterations ?? null, - max_time_ns: benchOptions.time ? msToNs(benchOptions.time) : null, - min_round_time_ns: null, // tinybench does not have an option for this - warmup_time_ns: - benchOptions.warmupIterations !== 0 && benchOptions.warmupTime - ? msToNs(benchOptions.warmupTime) - : null, - }, - stats, - }; - - console.log(` ✔ Collected walltime data for ${uri}`); - return coreBenchmark; - } catch (error) { - console.warn(` ⚠ Failed to process benchmark result for ${uri}:`, error); - return null; - } -} - -function convertVitestResultToBenchmarkStats( - result: RunnerTaskResult, - benchOptions: { - time?: number; - warmupTime?: number; - warmupIterations?: number; - iterations?: number; - }, -): BenchmarkStats | null { - const benchmark = result.benchmark; - - if (!benchmark) { - throw new Error("No benchmark data available in result"); - } - - const { totalTime, min, max, mean, sd, samples } = benchmark; - - // Get individual sample times in nanoseconds and sort them - const sortedTimesNs = samples.map(msToNs).sort((a, b) => a - b); - const meanNs = msToNs(mean); - const stdevNs = msToNs(sd); - - if (sortedTimesNs.length == 0) { - // Sometimes the benchmarks can be completely optimized out and not even run, but its beforeEach and afterEach hooks are still executed, and the task is still considered a success. - // This is the case for the hooks.bench.ts example in this package - return null; - } - - const { q1_ns, q3_ns, median_ns, iqr_outlier_rounds, stdev_outlier_rounds } = - calculateQuantiles({ meanNs, stdevNs, sortedTimesNs }); - - return { - min_ns: msToNs(min), - max_ns: msToNs(max), - mean_ns: meanNs, - stdev_ns: stdevNs, - q1_ns, - median_ns, - q3_ns, - total_time: msToS(totalTime), - iter_per_round: 1, // as there is only one round in tinybench, we define that there were n rounds of 1 iteration - rounds: sortedTimesNs.length, - iqr_outlier_rounds, - stdev_outlier_rounds, - warmup_iters: benchOptions.warmupIterations ?? 0, - }; -} diff --git a/packages/vitest-plugin/vitest.config.ts b/packages/vitest-plugin/vitest.config.ts index 672e3a27..444869bd 100644 --- a/packages/vitest-plugin/vitest.config.ts +++ b/packages/vitest-plugin/vitest.config.ts @@ -1,6 +1,18 @@ +import { createRequire } from "module"; import { defineConfig } from "vitest/config"; import codspeedPlugin from "./dist/index.mjs"; +// The legacy runner tests exercise the Vitest 3/4 benchmark backend +// (`vitest/suite`, `NodeBenchmarkRunner`), which Vitest 5 removed. Exclude them +// when running under v5+ so the file's static imports don't fail to resolve. +const require = createRequire(import.meta.url); +const vitestMajor = parseInt( + (require("vitest/package.json").version as string).split(".")[0], + 10, +); +const legacyOnlyTests = + vitestMajor >= 5 ? ["**/__tests__/instrumented.test.ts"] : []; + export default defineConfig({ // @ts-expect-error - TODO: investigate why importing from '.' wants to import only "main" field and thus fail plugins: [codspeedPlugin()], @@ -8,7 +20,7 @@ export default defineConfig({ __VERSION__: JSON.stringify("1.0.0"), }, test: { - exclude: ["**/tests/**/*", "**/.rollup.cache/**/*"], + exclude: ["**/tests/**/*", "**/.rollup.cache/**/*", ...legacyOnlyTests], mockReset: true, }, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 497f37e2..e4ac74fb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -297,7 +297,7 @@ importers: devDependencies: '@types/find-up': specifier: ^4.0.0 - version: 4.0.0 + version: 4.0.2 '@types/stack-trace': specifier: ^0.0.30 version: 0.0.30 @@ -365,8 +365,8 @@ importers: specifier: ^7.0.0 version: 7.1.3(@types/node@20.19.11)(yaml@2.9.0) vitest: - specifier: ^4.0.18 - version: 4.0.18(@types/node@20.19.11)(yaml@2.9.0) + specifier: 5.0.0-beta.5 + version: 5.0.0-beta.5(@types/node@20.19.11)(vite@7.1.3(@types/node@20.19.11)(yaml@2.9.0)) packages: @@ -1641,14 +1641,6 @@ packages: resolution: {integrity: sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==} engines: {node: '>=6.0.0'} - '@jridgewell/resolve-uri@3.1.0': - resolution: {integrity: sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==} - engines: {node: '>=6.0.0'} - - '@jridgewell/resolve-uri@3.1.1': - resolution: {integrity: sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==} - engines: {node: '>=6.0.0'} - '@jridgewell/resolve-uri@3.1.2': resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} engines: {node: '>=6.0.0'} @@ -1657,9 +1649,6 @@ packages: resolution: {integrity: sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==} engines: {node: '>=6.0.0'} - '@jridgewell/sourcemap-codec@1.4.14': - resolution: {integrity: sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==} - '@jridgewell/sourcemap-codec@1.4.15': resolution: {integrity: sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==} @@ -1669,11 +1658,8 @@ packages: '@jridgewell/sourcemap-codec@1.5.5': resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} - '@jridgewell/trace-mapping@0.3.18': - resolution: {integrity: sha512-w+niJYzMHdd7USdiH2U6869nqhD2nbfZXND5Yp93qIbEmnDNk7PD48o+YchRVpzMU7M6jVCbenTR7PA1FLQ9pA==} - - '@jridgewell/trace-mapping@0.3.29': - resolution: {integrity: sha512-uw6guiW/gcAGPDhLmd77/6lW8QLeiV5RUTsAX46Db6oLhGaVj4lhnPwb184s1bkc8kdVg/+h988dro8GRDpmYQ==} + '@jridgewell/trace-mapping@0.3.31': + resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} '@jridgewell/trace-mapping@0.3.9': resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} @@ -2214,8 +2200,8 @@ packages: '@types/estree@1.0.8': resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} - '@types/find-up@4.0.0': - resolution: {integrity: sha512-QlRNKeOPFWKisbNtKVOOGXw3AeLbkw8UmT/EyEGM6brfqpYffKBcch7f1y40NYN9O90aK2+K0xBMDJfOAsg2qg==} + '@types/find-up@4.0.2': + resolution: {integrity: sha512-AcxsY79jbVkizdp5BPS9jsMntFD9GTbQJp+HgSvzFvDIL0dgfy1TRHpyQxZkPkgp5oIK0lnaJ8m+qSZhBMIqbw==} deprecated: This is a stub types definition. find-up provides its own type definitions, so you do not need this installed. '@types/graceful-fs@4.1.6': @@ -2493,12 +2479,26 @@ packages: vite: optional: true + '@vitest/mocker@5.0.0-beta.5': + resolution: {integrity: sha512-NZCB4PeGl+YqWBxk4lAsH1oS8EvBG3b/hDaOCgEmsoZSda5q9jUxU8P46HyczFp34tcnxp92KVlNDiKcDAE59w==} + peerDependencies: + msw: ^2.4.9 + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + '@vitest/pretty-format@3.2.4': resolution: {integrity: sha512-IVNZik8IVRJRTr9fxlitMKeJeXFFFN0JaB9PHPGQ8NKQbGpfjlTx9zO4RefN8gp7eqjNy8nyK3NZmBzOPeIxtA==} '@vitest/pretty-format@4.0.18': resolution: {integrity: sha512-P24GK3GulZWC5tz87ux0m8OADrQIUVDPIjjj65vBXYG17ZeU3qD7r+MNZ1RNv4l8CGU2vtTRqixrOi9fYk/yKw==} + '@vitest/pretty-format@5.0.0-beta.5': + resolution: {integrity: sha512-eFj80bS7sN1aOOV7Ibi/sDYhhzs1fj2S8+/Y2mjOw2POpSW/6Esjj7FIdj0cD2/cdsKFumEokh+ijVijisd9+w==} + '@vitest/runner@3.2.4': resolution: {integrity: sha512-oukfKT9Mk41LreEW09vt45f8wx7DordoWUZMYdY/cyAk7w5TWkTRCNZYF7sX7n2wB7jyGAl74OxgwhPgKaqDMQ==} @@ -2517,12 +2517,18 @@ packages: '@vitest/spy@4.0.18': resolution: {integrity: sha512-cbQt3PTSD7P2OARdVW3qWER5EGq7PHlvE+QfzSC0lbwO+xnt7+XH06ZzFjFRgzUX//JmpxrCu92VdwvEPlWSNw==} + '@vitest/spy@5.0.0-beta.5': + resolution: {integrity: sha512-AF4gJhHwopexCGrdUnt8Y3+3eVvgMie4tHmlIJi2i3DH6TQlPN2V2/wJfrQAC5ZAgqSM278qmkJCYGC4aOfwSg==} + '@vitest/utils@3.2.4': resolution: {integrity: sha512-fB2V0JFrQSMsCo9HiSq3Ezpdv4iYaXRG1Sx8edX3MwxfyNn83mKiGzOcH+Fkxt4MHxr3y42fQi1oeAInqgX2QA==} '@vitest/utils@4.0.18': resolution: {integrity: sha512-msMRKLMVLWygpK3u2Hybgi4MNjcYJvwTb0Ru09+fOyCXIgT5raYP041DRRdiJiI3k/2U6SEbAETB3YtBrUkCFA==} + '@vitest/utils@5.0.0-beta.5': + resolution: {integrity: sha512-M+DZy1Q7v6//AbTYjVy86ChuF4tgSdGySVpH49kSkES16IRL5QRSmtAPdZVQ19S6Eke+NzAqWn1PUZNKljftfg==} + '@yarnpkg/lockfile@1.1.0': resolution: {integrity: sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ==} @@ -3344,6 +3350,9 @@ packages: es-module-lexer@1.7.0: resolution: {integrity: sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==} + es-module-lexer@2.1.0: + resolution: {integrity: sha512-n27zTYMjYu1aj4MjCWzSP7G9r75utsaoc8m61weK+W8JMBGGQybd43GstCXZ3WNmSFtGT9wi59qQTW6mhTR5LQ==} + es-object-atoms@1.1.1: resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} engines: {node: '>= 0.4'} @@ -3517,6 +3526,10 @@ packages: resolution: {integrity: sha512-JhFGDVJ7tmDJItKhYgJCGLOWjuK9vPxiXoUFLwLDc99NlmklilbiQJwoctZtt13+xMw91MCk/REan6MWHqDjyA==} engines: {node: '>=12.0.0'} + expect-type@1.3.0: + resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} + engines: {node: '>=12.0.0'} + expect@29.5.0: resolution: {integrity: sha512-yM7xqUrCO2JdpFo4XpM82t+PJBFybdqoQuJLDGeDX2ij8NZzqRHyu3Hp188/JX7SWqud+7t4MUdvcgGBICMHZg==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} @@ -5862,6 +5875,9 @@ packages: std-env@3.9.0: resolution: {integrity: sha512-UGvjygr6F6tpH7o2qyqR6QYpwraIjKSdtzyBdyytFOHmPZY917kwdwLG0RbOjWOnKmnm3PeHjaoLLMie7kPLQw==} + std-env@4.1.0: + resolution: {integrity: sha512-Rq7ybcX2RuC55r9oaPVEW7/xu3tj8u4GeBYHBWCychFtzMIr86A7e3PPEBPT37sHStKX3+TiX/Fr/ACmJLVlLQ==} + stop-iteration-iterator@1.1.0: resolution: {integrity: sha512-eLoXW/DHyl62zxY4SCaIgnRhuMr6ri4juEYARS8E6sCEqzKpOiE521Ucofdx+KnDZl5xmvGYaaKCk5FEOxJCoQ==} engines: {node: '>= 0.4'} @@ -6025,8 +6041,8 @@ packages: resolution: {integrity: sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==} engines: {node: '>=14.0.0'} - tinyrainbow@3.0.3: - resolution: {integrity: sha512-PSkbLUoxOFRzJYjjxHJt9xro7D+iilgMX/C9lawzVuYiIdcihh9DXmVibBe8lmcFrRi/VzlPjBxbN7rH24q8/Q==} + tinyrainbow@3.1.0: + resolution: {integrity: sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==} engines: {node: '>=14.0.0'} tinyspy@4.0.3: @@ -6389,6 +6405,47 @@ packages: jsdom: optional: true + vitest@5.0.0-beta.5: + resolution: {integrity: sha512-Gi7moR+KBro+LGC1jLGalpv4Ujb83dBLOH9o1f9VnWrvl0tRp4p1njh3Gikcexic0xe4foueQvgImq/hKET+bw==} + engines: {node: ^22.12.0 || ^24.0.0 || >=26.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@opentelemetry/api': ^1.9.0 + '@types/node': ^22.0.0 || >=24.0.0 + '@vitest/browser-playwright': 5.0.0-beta.5 + '@vitest/browser-preview': 5.0.0-beta.5 + '@vitest/browser-webdriverio': 5.0.0-beta.5 + '@vitest/coverage-istanbul': 5.0.0-beta.5 + '@vitest/coverage-v8': 5.0.0-beta.5 + '@vitest/ui': 5.0.0-beta.5 + happy-dom: '*' + jsdom: '*' + vite: ^6.4.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@opentelemetry/api': + optional: true + '@types/node': + optional: true + '@vitest/browser-playwright': + optional: true + '@vitest/browser-preview': + optional: true + '@vitest/browser-webdriverio': + optional: true + '@vitest/coverage-istanbul': + optional: true + '@vitest/coverage-v8': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + walk-up-path@3.0.1: resolution: {integrity: sha512-9YlCL/ynK3CTlrSRrDxZvUauLzAswPCrsaCgilqFevUYpeEW0/3ScEjaa3kbW/T0ghhkEr7mv+fpjqn1Y1YuTA==} @@ -6531,8 +6588,8 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} - yocto-queue@1.0.0: - resolution: {integrity: sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==} + yocto-queue@1.2.2: + resolution: {integrity: sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==} engines: {node: '>=12.20'} snapshots: @@ -6540,12 +6597,12 @@ snapshots: '@ampproject/remapping@2.2.1': dependencies: '@jridgewell/gen-mapping': 0.3.3 - '@jridgewell/trace-mapping': 0.3.18 + '@jridgewell/trace-mapping': 0.3.31 '@ampproject/remapping@2.3.0': dependencies: '@jridgewell/gen-mapping': 0.3.12 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 '@apidevtools/json-schema-ref-parser@9.0.9': dependencies: @@ -6608,14 +6665,14 @@ snapshots: dependencies: '@babel/types': 7.22.5 '@jridgewell/gen-mapping': 0.3.3 - '@jridgewell/trace-mapping': 0.3.18 + '@jridgewell/trace-mapping': 0.3.31 jsesc: 2.5.2 '@babel/generator@7.22.5': dependencies: '@babel/types': 7.22.5 '@jridgewell/gen-mapping': 0.3.3 - '@jridgewell/trace-mapping': 0.3.18 + '@jridgewell/trace-mapping': 0.3.31 jsesc: 2.5.2 '@babel/generator@7.28.0': @@ -6623,7 +6680,7 @@ snapshots: '@babel/parser': 7.28.0 '@babel/types': 7.28.1 '@jridgewell/gen-mapping': 0.3.12 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 jsesc: 3.1.0 '@babel/helper-annotate-as-pure@7.22.5': @@ -8065,7 +8122,7 @@ snapshots: '@jest/test-result': 29.5.0 '@jest/transform': 29.5.0 '@jest/types': 29.5.0 - '@jridgewell/trace-mapping': 0.3.18 + '@jridgewell/trace-mapping': 0.3.31 '@types/node': 20.19.11 chalk: 4.1.2 collect-v8-coverage: 1.0.1 @@ -8094,7 +8151,7 @@ snapshots: '@jest/test-result': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 '@types/node': 20.19.11 chalk: 4.1.2 collect-v8-coverage: 1.0.2 @@ -8126,13 +8183,13 @@ snapshots: '@jest/source-map@29.4.3': dependencies: - '@jridgewell/trace-mapping': 0.3.18 + '@jridgewell/trace-mapping': 0.3.31 callsites: 3.1.0 graceful-fs: 4.2.11 '@jest/source-map@29.6.3': dependencies: - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 callsites: 3.1.0 graceful-fs: 4.2.11 @@ -8168,7 +8225,7 @@ snapshots: dependencies: '@babel/core': 7.21.4 '@jest/types': 29.5.0 - '@jridgewell/trace-mapping': 0.3.18 + '@jridgewell/trace-mapping': 0.3.31 babel-plugin-istanbul: 6.1.1 chalk: 4.1.2 convert-source-map: 2.0.0 @@ -8188,7 +8245,7 @@ snapshots: dependencies: '@babel/core': 7.28.0 '@jest/types': 29.6.3 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 babel-plugin-istanbul: 6.1.1 chalk: 4.1.2 convert-source-map: 2.0.0 @@ -8225,44 +8282,33 @@ snapshots: '@jridgewell/gen-mapping@0.3.12': dependencies: '@jridgewell/sourcemap-codec': 1.5.5 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 '@jridgewell/gen-mapping@0.3.3': dependencies: '@jridgewell/set-array': 1.1.2 - '@jridgewell/sourcemap-codec': 1.5.4 - '@jridgewell/trace-mapping': 0.3.18 - - '@jridgewell/resolve-uri@3.1.0': {} - - '@jridgewell/resolve-uri@3.1.1': {} + '@jridgewell/sourcemap-codec': 1.5.5 + '@jridgewell/trace-mapping': 0.3.31 '@jridgewell/resolve-uri@3.1.2': {} '@jridgewell/set-array@1.1.2': {} - '@jridgewell/sourcemap-codec@1.4.14': {} - '@jridgewell/sourcemap-codec@1.4.15': {} '@jridgewell/sourcemap-codec@1.5.4': {} '@jridgewell/sourcemap-codec@1.5.5': {} - '@jridgewell/trace-mapping@0.3.18': - dependencies: - '@jridgewell/resolve-uri': 3.1.0 - '@jridgewell/sourcemap-codec': 1.4.14 - - '@jridgewell/trace-mapping@0.3.29': + '@jridgewell/trace-mapping@0.3.31': dependencies: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 '@jridgewell/trace-mapping@0.3.9': dependencies: - '@jridgewell/resolve-uri': 3.1.1 - '@jridgewell/sourcemap-codec': 1.5.4 + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.5.5 '@jsdevtools/ono@7.1.3': {} @@ -8902,7 +8948,7 @@ snapshots: '@types/estree@1.0.8': {} - '@types/find-up@4.0.0': + '@types/find-up@4.0.2': dependencies: find-up: 6.3.0 @@ -9155,13 +9201,13 @@ snapshots: '@vitest/spy': 4.0.18 '@vitest/utils': 4.0.18 chai: 6.2.2 - tinyrainbow: 3.0.3 + tinyrainbow: 3.1.0 '@vitest/mocker@3.2.4(vite@7.1.3(@types/node@20.19.11)(yaml@2.9.0))': dependencies: '@vitest/spy': 3.2.4 estree-walker: 3.0.3 - magic-string: 0.30.17 + magic-string: 0.30.21 optionalDependencies: vite: 7.1.3(@types/node@20.19.11)(yaml@2.9.0) @@ -9173,13 +9219,26 @@ snapshots: optionalDependencies: vite: 7.1.3(@types/node@20.19.11)(yaml@2.9.0) + '@vitest/mocker@5.0.0-beta.5(vite@7.1.3(@types/node@20.19.11)(yaml@2.9.0))': + dependencies: + '@jridgewell/trace-mapping': 0.3.31 + '@vitest/spy': 5.0.0-beta.5 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 7.1.3(@types/node@20.19.11)(yaml@2.9.0) + '@vitest/pretty-format@3.2.4': dependencies: tinyrainbow: 2.0.0 '@vitest/pretty-format@4.0.18': dependencies: - tinyrainbow: 3.0.3 + tinyrainbow: 3.1.0 + + '@vitest/pretty-format@5.0.0-beta.5': + dependencies: + tinyrainbow: 3.1.0 '@vitest/runner@3.2.4': dependencies: @@ -9195,7 +9254,7 @@ snapshots: '@vitest/snapshot@3.2.4': dependencies: '@vitest/pretty-format': 3.2.4 - magic-string: 0.30.17 + magic-string: 0.30.21 pathe: 2.0.3 '@vitest/snapshot@4.0.18': @@ -9210,6 +9269,8 @@ snapshots: '@vitest/spy@4.0.18': {} + '@vitest/spy@5.0.0-beta.5': {} + '@vitest/utils@3.2.4': dependencies: '@vitest/pretty-format': 3.2.4 @@ -9219,7 +9280,13 @@ snapshots: '@vitest/utils@4.0.18': dependencies: '@vitest/pretty-format': 4.0.18 - tinyrainbow: 3.0.3 + tinyrainbow: 3.1.0 + + '@vitest/utils@5.0.0-beta.5': + dependencies: + '@vitest/pretty-format': 5.0.0-beta.5 + convert-source-map: 2.0.0 + tinyrainbow: 3.1.0 '@yarnpkg/lockfile@1.1.0': {} @@ -10184,6 +10251,8 @@ snapshots: es-module-lexer@1.7.0: {} + es-module-lexer@2.1.0: {} + es-object-atoms@1.1.1: dependencies: es-errors: 1.3.0 @@ -10470,6 +10539,8 @@ snapshots: expect-type@1.2.2: {} + expect-type@1.3.0: {} + expect@29.5.0: dependencies: '@jest/expect-utils': 29.5.0 @@ -12699,7 +12770,7 @@ snapshots: p-limit@4.0.0: dependencies: - yocto-queue: 1.0.0 + yocto-queue: 1.2.2 p-locate@2.0.0: dependencies: @@ -13385,6 +13456,8 @@ snapshots: std-env@3.9.0: {} + std-env@4.1.0: {} + stop-iteration-iterator@1.1.0: dependencies: es-errors: 1.3.0 @@ -13560,7 +13633,7 @@ snapshots: tinyrainbow@2.0.0: {} - tinyrainbow@3.0.3: {} + tinyrainbow@3.1.0: {} tinyspy@4.0.3: {} @@ -13809,13 +13882,13 @@ snapshots: v8-to-istanbul@9.1.0: dependencies: - '@jridgewell/trace-mapping': 0.3.18 + '@jridgewell/trace-mapping': 0.3.31 '@types/istanbul-lib-coverage': 2.0.4 convert-source-map: 1.9.0 v8-to-istanbul@9.3.0: dependencies: - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 '@types/istanbul-lib-coverage': 2.0.6 convert-source-map: 2.0.0 @@ -13911,7 +13984,7 @@ snapshots: '@vitest/spy': 4.0.18 '@vitest/utils': 4.0.18 es-module-lexer: 1.7.0 - expect-type: 1.2.2 + expect-type: 1.3.0 magic-string: 0.30.21 obug: 2.1.1 pathe: 2.0.3 @@ -13920,7 +13993,7 @@ snapshots: tinybench: 2.9.0 tinyexec: 1.0.2 tinyglobby: 0.2.15 - tinyrainbow: 3.0.3 + tinyrainbow: 3.1.0 vite: 7.1.3(@types/node@20.19.11)(yaml@2.9.0) why-is-node-running: 2.3.0 optionalDependencies: @@ -13938,6 +14011,32 @@ snapshots: - tsx - yaml + vitest@5.0.0-beta.5(@types/node@20.19.11)(vite@7.1.3(@types/node@20.19.11)(yaml@2.9.0)): + dependencies: + '@types/chai': 5.2.2 + '@vitest/mocker': 5.0.0-beta.5(vite@7.1.3(@types/node@20.19.11)(yaml@2.9.0)) + '@vitest/pretty-format': 5.0.0-beta.5 + '@vitest/spy': 5.0.0-beta.5 + '@vitest/utils': 5.0.0-beta.5 + chai: 6.2.2 + es-module-lexer: 2.1.0 + expect-type: 1.3.0 + magic-string: 0.30.21 + obug: 2.1.1 + pathe: 2.0.3 + picomatch: 4.0.3 + std-env: 4.1.0 + tinybench: 6.0.2 + tinyexec: 1.0.2 + tinyglobby: 0.2.15 + tinyrainbow: 3.1.0 + vite: 7.1.3(@types/node@20.19.11)(yaml@2.9.0) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 20.19.11 + transitivePeerDependencies: + - msw + walk-up-path@3.0.1: {} walker@1.0.8: @@ -14116,4 +14215,4 @@ snapshots: yocto-queue@0.1.0: {} - yocto-queue@1.0.0: {} + yocto-queue@1.2.2: {} From a52a820dd540088d1bd052ed7e820c54e6234374 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Wed, 1 Jul 2026 11:04:44 +0200 Subject: [PATCH 2/3] test(vitest-plugin): add with-vitest-v4 example Now that the plugin's own dev dependency tracks Vitest 5, add a dedicated Vitest 4 example so the legacy (v3/4) benchmark seam keeps explicit coverage alongside the existing with-vitest-v3 example. Mirrors that example, pinning vitest ^4.1.9. Refs COD-2931 Co-Authored-By: Claude --- examples/with-vitest-v4/package.json | 13 ++ .../with-vitest-v4/src/fibonacci.bench.ts | 20 +++ examples/with-vitest-v4/src/fibonacci.ts | 17 ++ examples/with-vitest-v4/tsconfig.json | 13 ++ examples/with-vitest-v4/vitest.config.ts | 6 + pnpm-lock.yaml | 150 ++++++++++++++++++ 6 files changed, 219 insertions(+) create mode 100644 examples/with-vitest-v4/package.json create mode 100644 examples/with-vitest-v4/src/fibonacci.bench.ts create mode 100644 examples/with-vitest-v4/src/fibonacci.ts create mode 100644 examples/with-vitest-v4/tsconfig.json create mode 100644 examples/with-vitest-v4/vitest.config.ts diff --git a/examples/with-vitest-v4/package.json b/examples/with-vitest-v4/package.json new file mode 100644 index 00000000..8a67c417 --- /dev/null +++ b/examples/with-vitest-v4/package.json @@ -0,0 +1,13 @@ +{ + "name": "with-vitest-v4", + "private": true, + "type": "module", + "scripts": { + "bench-vitest": "vitest bench --run" + }, + "devDependencies": { + "@codspeed/vitest-plugin": "workspace:*", + "typescript": "^5.1.3", + "vitest": "^4.1.9" + } +} diff --git a/examples/with-vitest-v4/src/fibonacci.bench.ts b/examples/with-vitest-v4/src/fibonacci.bench.ts new file mode 100644 index 00000000..227a6e67 --- /dev/null +++ b/examples/with-vitest-v4/src/fibonacci.bench.ts @@ -0,0 +1,20 @@ +import { bench, describe } from "vitest"; +import { iterativeFibonacci, recursiveFibonacci } from "./fibonacci"; + +describe("fibonacci", () => { + bench("recursive fibo 15", () => { + recursiveFibonacci(15); + }); + + bench("recursive fibo 20", () => { + recursiveFibonacci(20); + }); + + bench("iterative fibo 15", () => { + iterativeFibonacci(15); + }); + + bench("iterative fibo 20", () => { + iterativeFibonacci(20); + }); +}); diff --git a/examples/with-vitest-v4/src/fibonacci.ts b/examples/with-vitest-v4/src/fibonacci.ts new file mode 100644 index 00000000..94796660 --- /dev/null +++ b/examples/with-vitest-v4/src/fibonacci.ts @@ -0,0 +1,17 @@ +export function recursiveFibonacci(n: number): number { + if (n < 2) { + return n; + } + return recursiveFibonacci(n - 1) + recursiveFibonacci(n - 2); +} + +export function iterativeFibonacci(n: number): number { + let a = 0; + let b = 1; + for (let i = 0; i < n; i++) { + const temp = a + b; + a = b; + b = temp; + } + return a; +} diff --git a/examples/with-vitest-v4/tsconfig.json b/examples/with-vitest-v4/tsconfig.json new file mode 100644 index 00000000..ace1de03 --- /dev/null +++ b/examples/with-vitest-v4/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "lib": ["es2023"], + "module": "ESNext", + "verbatimModuleSyntax": true, + "target": "es2022", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "moduleResolution": "Node" + } +} diff --git a/examples/with-vitest-v4/vitest.config.ts b/examples/with-vitest-v4/vitest.config.ts new file mode 100644 index 00000000..4b1290c1 --- /dev/null +++ b/examples/with-vitest-v4/vitest.config.ts @@ -0,0 +1,6 @@ +import codspeedPlugin from "@codspeed/vitest-plugin"; +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + plugins: [codspeedPlugin()], +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e4ac74fb..0570ca4a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -252,6 +252,18 @@ importers: specifier: ^3.2.4 version: 3.2.4(@types/node@20.19.11)(yaml@2.9.0) + examples/with-vitest-v4: + devDependencies: + '@codspeed/vitest-plugin': + specifier: workspace:* + version: link:../../packages/vitest-plugin + typescript: + specifier: ^5.1.3 + version: 5.8.3 + vitest: + specifier: ^4.1.9 + version: 4.1.9(@types/node@20.19.11)(vite@7.1.3(@types/node@20.19.11)(yaml@2.9.0)) + packages/benchmark.js-plugin: dependencies: '@codspeed/core': @@ -2457,6 +2469,9 @@ packages: '@vitest/expect@4.0.18': resolution: {integrity: sha512-8sCWUyckXXYvx4opfzVY03EOiYVxyNrHS5QxX3DAIi5dpJAAkyJezHCP77VMX4HKA2LDT/Jpfo8i2r5BE3GnQQ==} + '@vitest/expect@4.1.9': + resolution: {integrity: sha512-vl/rYsUKcBr3SnQn166+XR5ZQcgMx3DQhFWdfli/cWpLnLUmbxZvyrJZotLFUryib+LtArYMSTJ5RbQ57ZqrlA==} + '@vitest/mocker@3.2.4': resolution: {integrity: sha512-46ryTE9RZO/rfDd7pEqFl7etuyzekzEhUbTW3BvmeO/BcCMEgq59BKhek3dXDWgAj4oMK6OZi+vRr1wPW6qjEQ==} peerDependencies: @@ -2479,6 +2494,17 @@ packages: vite: optional: true + '@vitest/mocker@4.1.9': + resolution: {integrity: sha512-EVkXzBjrPGM+cK8/ANWgBrkUCfJfb38/EfTSO8h7pWvKkyPkpWxvR7BkD2MyItMF62C97zAEoqdpUixwR/e+Rw==} + peerDependencies: + msw: ^2.4.9 + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + '@vitest/mocker@5.0.0-beta.5': resolution: {integrity: sha512-NZCB4PeGl+YqWBxk4lAsH1oS8EvBG3b/hDaOCgEmsoZSda5q9jUxU8P46HyczFp34tcnxp92KVlNDiKcDAE59w==} peerDependencies: @@ -2496,6 +2522,9 @@ packages: '@vitest/pretty-format@4.0.18': resolution: {integrity: sha512-P24GK3GulZWC5tz87ux0m8OADrQIUVDPIjjj65vBXYG17ZeU3qD7r+MNZ1RNv4l8CGU2vtTRqixrOi9fYk/yKw==} + '@vitest/pretty-format@4.1.9': + resolution: {integrity: sha512-s0iufns3iIFitdgm+YR7g1whCAaGtXz459VS9/PqyKDEEFgYIhsHOQmXgIgDuYCt7DeQmiZT0Qe2OA2p4ZPu5A==} + '@vitest/pretty-format@5.0.0-beta.5': resolution: {integrity: sha512-eFj80bS7sN1aOOV7Ibi/sDYhhzs1fj2S8+/Y2mjOw2POpSW/6Esjj7FIdj0cD2/cdsKFumEokh+ijVijisd9+w==} @@ -2505,18 +2534,27 @@ packages: '@vitest/runner@4.0.18': resolution: {integrity: sha512-rpk9y12PGa22Jg6g5M3UVVnTS7+zycIGk9ZNGN+m6tZHKQb7jrP7/77WfZy13Y/EUDd52NDsLRQhYKtv7XfPQw==} + '@vitest/runner@4.1.9': + resolution: {integrity: sha512-KXLMDtc7oe70+3mJfGrPUWPesswH+3sTxAMAMl8DG7I8IUQT4XW718dY5ID3vPUcmlu27CcKfY4P3h3I29SLJg==} + '@vitest/snapshot@3.2.4': resolution: {integrity: sha512-dEYtS7qQP2CjU27QBC5oUOxLE/v5eLkGqPE0ZKEIDGMs4vKWe7IjgLOeauHsR0D5YuuycGRO5oSRXnwnmA78fQ==} '@vitest/snapshot@4.0.18': resolution: {integrity: sha512-PCiV0rcl7jKQjbgYqjtakly6T1uwv/5BQ9SwBLekVg/EaYeQFPiXcgrC2Y7vDMA8dM1SUEAEV82kgSQIlXNMvA==} + '@vitest/snapshot@4.1.9': + resolution: {integrity: sha512-Jc7RKGNBo8Z28WYIm0Niej4xdSPByRf6mU58VpHQkd6Zh05rlnA+twjbK5HyeIGHxrzsc3mJgS43uM0CZKzaIA==} + '@vitest/spy@3.2.4': resolution: {integrity: sha512-vAfasCOe6AIK70iP5UD11Ac4siNUNJ9i/9PZ3NKx07sG6sUxeag1LWdNrMWeKKYBLlzuK+Gn65Yd5nyL6ds+nw==} '@vitest/spy@4.0.18': resolution: {integrity: sha512-cbQt3PTSD7P2OARdVW3qWER5EGq7PHlvE+QfzSC0lbwO+xnt7+XH06ZzFjFRgzUX//JmpxrCu92VdwvEPlWSNw==} + '@vitest/spy@4.1.9': + resolution: {integrity: sha512-fHpsS6mIi+PiEW+vcRVOMkX1oSaPKne3VOclSFICPcGOmfKgXPU5iAah+wcNcj2xPrCCmfq99IDGf+EojhhvhA==} + '@vitest/spy@5.0.0-beta.5': resolution: {integrity: sha512-AF4gJhHwopexCGrdUnt8Y3+3eVvgMie4tHmlIJi2i3DH6TQlPN2V2/wJfrQAC5ZAgqSM278qmkJCYGC4aOfwSg==} @@ -2526,6 +2564,9 @@ packages: '@vitest/utils@4.0.18': resolution: {integrity: sha512-msMRKLMVLWygpK3u2Hybgi4MNjcYJvwTb0Ru09+fOyCXIgT5raYP041DRRdiJiI3k/2U6SEbAETB3YtBrUkCFA==} + '@vitest/utils@4.1.9': + resolution: {integrity: sha512-A51o8ymO5PpqlWNnBP9ZHPXDIpuMtTLlGSjN7la4US+LJzoUMyhwjA5QXlm39JexgwHKW4Xjs8Z2d3dLCXOeuA==} + '@vitest/utils@5.0.0-beta.5': resolution: {integrity: sha512-M+DZy1Q7v6//AbTYjVy86ChuF4tgSdGySVpH49kSkES16IRL5QRSmtAPdZVQ19S6Eke+NzAqWn1PUZNKljftfg==} @@ -6405,6 +6446,47 @@ packages: jsdom: optional: true + vitest@4.1.9: + resolution: {integrity: sha512-nE3/LEyc0z87uHYLZebqCUOaJr2hdtuPp7BQ4BosVFnfltxgAvMG08NyrSGlPpOUWvR27c5flSmYFTNr78L9GQ==} + engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@opentelemetry/api': ^1.9.0 + '@types/node': ^20.0.0 || ^22.0.0 || >=24.0.0 + '@vitest/browser-playwright': 4.1.9 + '@vitest/browser-preview': 4.1.9 + '@vitest/browser-webdriverio': 4.1.9 + '@vitest/coverage-istanbul': 4.1.9 + '@vitest/coverage-v8': 4.1.9 + '@vitest/ui': 4.1.9 + happy-dom: '*' + jsdom: '*' + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@opentelemetry/api': + optional: true + '@types/node': + optional: true + '@vitest/browser-playwright': + optional: true + '@vitest/browser-preview': + optional: true + '@vitest/browser-webdriverio': + optional: true + '@vitest/coverage-istanbul': + optional: true + '@vitest/coverage-v8': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + vitest@5.0.0-beta.5: resolution: {integrity: sha512-Gi7moR+KBro+LGC1jLGalpv4Ujb83dBLOH9o1f9VnWrvl0tRp4p1njh3Gikcexic0xe4foueQvgImq/hKET+bw==} engines: {node: ^22.12.0 || ^24.0.0 || >=26.0.0} @@ -9203,6 +9285,15 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.0 + '@vitest/expect@4.1.9': + dependencies: + '@standard-schema/spec': 1.1.0 + '@types/chai': 5.2.2 + '@vitest/spy': 4.1.9 + '@vitest/utils': 4.1.9 + chai: 6.2.2 + tinyrainbow: 3.1.0 + '@vitest/mocker@3.2.4(vite@7.1.3(@types/node@20.19.11)(yaml@2.9.0))': dependencies: '@vitest/spy': 3.2.4 @@ -9219,6 +9310,14 @@ snapshots: optionalDependencies: vite: 7.1.3(@types/node@20.19.11)(yaml@2.9.0) + '@vitest/mocker@4.1.9(vite@7.1.3(@types/node@20.19.11)(yaml@2.9.0))': + dependencies: + '@vitest/spy': 4.1.9 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 7.1.3(@types/node@20.19.11)(yaml@2.9.0) + '@vitest/mocker@5.0.0-beta.5(vite@7.1.3(@types/node@20.19.11)(yaml@2.9.0))': dependencies: '@jridgewell/trace-mapping': 0.3.31 @@ -9236,6 +9335,10 @@ snapshots: dependencies: tinyrainbow: 3.1.0 + '@vitest/pretty-format@4.1.9': + dependencies: + tinyrainbow: 3.1.0 + '@vitest/pretty-format@5.0.0-beta.5': dependencies: tinyrainbow: 3.1.0 @@ -9251,6 +9354,11 @@ snapshots: '@vitest/utils': 4.0.18 pathe: 2.0.3 + '@vitest/runner@4.1.9': + dependencies: + '@vitest/utils': 4.1.9 + pathe: 2.0.3 + '@vitest/snapshot@3.2.4': dependencies: '@vitest/pretty-format': 3.2.4 @@ -9263,12 +9371,21 @@ snapshots: magic-string: 0.30.21 pathe: 2.0.3 + '@vitest/snapshot@4.1.9': + dependencies: + '@vitest/pretty-format': 4.1.9 + '@vitest/utils': 4.1.9 + magic-string: 0.30.21 + pathe: 2.0.3 + '@vitest/spy@3.2.4': dependencies: tinyspy: 4.0.3 '@vitest/spy@4.0.18': {} + '@vitest/spy@4.1.9': {} + '@vitest/spy@5.0.0-beta.5': {} '@vitest/utils@3.2.4': @@ -9282,6 +9399,12 @@ snapshots: '@vitest/pretty-format': 4.0.18 tinyrainbow: 3.1.0 + '@vitest/utils@4.1.9': + dependencies: + '@vitest/pretty-format': 4.1.9 + convert-source-map: 2.0.0 + tinyrainbow: 3.1.0 + '@vitest/utils@5.0.0-beta.5': dependencies: '@vitest/pretty-format': 5.0.0-beta.5 @@ -14011,6 +14134,33 @@ snapshots: - tsx - yaml + vitest@4.1.9(@types/node@20.19.11)(vite@7.1.3(@types/node@20.19.11)(yaml@2.9.0)): + dependencies: + '@vitest/expect': 4.1.9 + '@vitest/mocker': 4.1.9(vite@7.1.3(@types/node@20.19.11)(yaml@2.9.0)) + '@vitest/pretty-format': 4.1.9 + '@vitest/runner': 4.1.9 + '@vitest/snapshot': 4.1.9 + '@vitest/spy': 4.1.9 + '@vitest/utils': 4.1.9 + es-module-lexer: 2.1.0 + expect-type: 1.3.0 + magic-string: 0.30.21 + obug: 2.1.1 + pathe: 2.0.3 + picomatch: 4.0.3 + std-env: 4.1.0 + tinybench: 2.9.0 + tinyexec: 1.0.2 + tinyglobby: 0.2.15 + tinyrainbow: 3.1.0 + vite: 7.1.3(@types/node@20.19.11)(yaml@2.9.0) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 20.19.11 + transitivePeerDependencies: + - msw + vitest@5.0.0-beta.5(@types/node@20.19.11)(vite@7.1.3(@types/node@20.19.11)(yaml@2.9.0)): dependencies: '@types/chai': 5.2.2 From c3a356bf4e8436102833062276de51231e641ad3 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Wed, 1 Jul 2026 14:57:03 +0200 Subject: [PATCH 3/3] --wip-- [skip ci] --- .gitignore | 2 + CLAUDE.md | 23 +- packages/vitest-plugin/benches/flat.bench.ts | 26 +- packages/vitest-plugin/benches/hooks.bench.ts | 59 +++-- packages/vitest-plugin/benches/macos.bench.ts | 8 +- .../vitest-plugin/benches/parsePr.bench.ts | 42 +-- .../vitest-plugin/benches/timing.bench.ts | 32 +-- packages/vitest-plugin/src/globalSetup.ts | 13 +- packages/vitest-plugin/src/instrument.ts | 239 +++++++++++++++--- packages/vitest-plugin/src/v5/setup.ts | 161 +++++++++--- packages/vitest-plugin/src/vitestBackend.ts | 7 +- 11 files changed, 472 insertions(+), 140 deletions(-) diff --git a/.gitignore b/.gitignore index 5233f61e..3cfb1201 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,5 @@ packages/app/.env # turbo .turbo/ .rollup.cache/ + +.codspeed diff --git a/CLAUDE.md b/CLAUDE.md index b82671a6..00a76a53 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -94,4 +94,25 @@ Based on the codebase analysis, to add stats access features: - Build: `pnpm turbo run build --filter=` - Typecheck: `pnpm turbo run typecheck --filter=` - Lint: `pnpm turbo run lint --filter=` - - Run a task across all packages by omitting `--filter` (e.g. `pnpm turbo run build`). \ No newline at end of file + - Run a task across all packages by omitting `--filter` (e.g. `pnpm turbo run build`). + +## Testing a plugin during development + +A plugin behaves differently depending on whether CodSpeed is driving the run, +so exercise all three of these when developing or reviewing a plugin change +(build the plugin first — the benches import from `dist`): + +1. **Fallback (not under CodSpeed).** No env vars. The plugin must stay out of + the way and let the framework run its benchmarks normally (no instrumentation, + no hijacked output). e.g. `pnpm turbo run bench --filter=`. +2. **Instrumentation / simulation.** `CODSPEED_ENV=true CODSPEED_RUNNER_MODE=simulation` + (or `instrumentation`). The plugin hijacks the run to do a single instrumented + pass per benchmark and prints `Measured/Checked ` instead of the normal + harness output. +3. **Walltime.** `CODSPEED_ENV=true CODSPEED_RUNNER_MODE=walltime`. The plugin + instruments the framework's real benchmark loop and collects walltime results. + +Running these locally outside the CodSpeed runner is expected to log +`instrument-hooks: failed to write environment.json` and skip actual measurement +writes — the point is to verify the plugin's control flow and output per mode, +not to produce real measurements. \ No newline at end of file diff --git a/packages/vitest-plugin/benches/flat.bench.ts b/packages/vitest-plugin/benches/flat.bench.ts index 74e67b9d..601070e1 100644 --- a/packages/vitest-plugin/benches/flat.bench.ts +++ b/packages/vitest-plugin/benches/flat.bench.ts @@ -1,4 +1,4 @@ -import { bench, describe } from "vitest"; +import { describe, test } from "vitest"; import parsePr from "./parsePr"; const LONG_BODY = @@ -9,12 +9,16 @@ const LONG_BODY = .join("\n") + "fixes #123"; describe("parsePr", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test-1", number: 1 }); + test("short body", async ({ bench }) => { + await bench("short body", () => { + parsePr({ body: "fixes #123", title: "test-1", number: 1 }); + }).run(); }); - bench("long body", () => { - parsePr({ body: LONG_BODY, title: "test-2", number: 2 }); + test("long body", async ({ bench }) => { + await bench("long body", () => { + parsePr({ body: LONG_BODY, title: "test-2", number: 2 }); + }).run(); }); }); @@ -24,10 +28,14 @@ function fibo(n: number): number { } describe("fibo", () => { - bench("fibo 10", () => { - fibo(10); + test("fibo 10", async ({ bench }) => { + await bench("fibo 10", () => { + fibo(10); + }).run(); }); - bench("fibo 15", () => { - fibo(15); + test("fibo 15", async ({ bench }) => { + await bench("fibo 15", () => { + fibo(15); + }).run(); }); }); diff --git a/packages/vitest-plugin/benches/hooks.bench.ts b/packages/vitest-plugin/benches/hooks.bench.ts index d790ca3c..5baab386 100644 --- a/packages/vitest-plugin/benches/hooks.bench.ts +++ b/packages/vitest-plugin/benches/hooks.bench.ts @@ -1,37 +1,42 @@ -import { - afterAll, - afterEach, - beforeAll, - beforeEach, - bench, - describe, - expect, -} from "vitest"; +import { describe, expect, test } from "vitest"; +// Exercises tinybench's per-benchmark hooks, which Vitest 5 exposes through the +// `bench(name, options, fn)` options object (`beforeAll`/`beforeEach`/...). describe("hooks", () => { let count = 0; + describe("run", () => { - beforeAll(() => { - count += 10; - }); - beforeEach(() => { - count += 1; - }); - afterEach(() => { - count -= 1; - }); - afterAll(() => { - count -= 10; - }); + const hooks = { + beforeAll: () => { + count += 10; + }, + beforeEach: () => { + count += 1; + }, + afterEach: () => { + count -= 1; + }, + afterAll: () => { + count -= 10; + }, + }; - bench("one", () => { - expect(count).toBe(11); + test("one", async ({ bench }) => { + await bench("one", hooks, () => { + expect(count).toBe(11); + }).run(); }); - bench("two", () => { - expect(count).toBe(11); + + test("two", async ({ bench }) => { + await bench("two", hooks, () => { + expect(count).toBe(11); + }).run(); }); }); - bench("end", () => { - expect(count).toBe(0); + + test("end", async ({ bench }) => { + await bench("end", () => { + expect(count).toBe(0); + }).run(); }); }); diff --git a/packages/vitest-plugin/benches/macos.bench.ts b/packages/vitest-plugin/benches/macos.bench.ts index e05a3082..bef403d7 100644 --- a/packages/vitest-plugin/benches/macos.bench.ts +++ b/packages/vitest-plugin/benches/macos.bench.ts @@ -1,4 +1,4 @@ -import { bench, describe } from "vitest"; +import { describe, test } from "vitest"; const isMacOS = process.platform === "darwin"; @@ -10,7 +10,9 @@ function fibo(n: number): number { // macOS-only benchmark: skipped on every other platform, so it only runs on // the `codspeed-walltime-macos` CI job (see .github/workflows/codspeed.yml). describe.skipIf(!isMacOS)("macos only", () => { - bench("fibo darwin", () => { - fibo(30); + test("fibo darwin", async ({ bench }) => { + await bench("fibo darwin", () => { + fibo(30); + }).run(); }); }); diff --git a/packages/vitest-plugin/benches/parsePr.bench.ts b/packages/vitest-plugin/benches/parsePr.bench.ts index 91c0adee..c1253ba4 100644 --- a/packages/vitest-plugin/benches/parsePr.bench.ts +++ b/packages/vitest-plugin/benches/parsePr.bench.ts @@ -1,4 +1,4 @@ -import { bench, describe } from "vitest"; +import { describe, test } from "vitest"; import parsePr from "./parsePr"; const LONG_BODY = @@ -8,44 +8,52 @@ const LONG_BODY = ) .join("\n") + "fixes #123"; +function benchShortBody() { + parsePr({ body: "fixes #123", title: "test", number: 124 }); +} + +function benchLongBody() { + parsePr({ body: LONG_BODY, title: "test", number: 124 }); +} + describe("parsePr", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test", number: 124 }); + test("short body", async ({ bench }) => { + await bench("short body", benchShortBody).run(); }); - bench("long body", () => { - parsePr({ body: LONG_BODY, title: "test", number: 124 }); + test("long body", async ({ bench }) => { + await bench("long body", benchLongBody).run(); }); describe("nested suite", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test", number: 124 }); + test("short body", async ({ bench }) => { + await bench("short body", benchShortBody).run(); }); - bench("long body", () => { - parsePr({ body: LONG_BODY, title: "test", number: 124 }); + test("long body", async ({ bench }) => { + await bench("long body", benchLongBody).run(); }); describe("deeply nested suite", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test", number: 124 }); + test("short body", async ({ bench }) => { + await bench("short body", benchShortBody).run(); }); }); }); }); describe("another parsePr", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test", number: 124 }); + test("short body", async ({ bench }) => { + await bench("short body", benchShortBody).run(); }); - bench("long body", () => { - parsePr({ body: LONG_BODY, title: "test", number: 124 }); + test("long body", async ({ bench }) => { + await bench("long body", benchLongBody).run(); }); describe("nested suite", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test", number: 124 }); + test("short body", async ({ bench }) => { + await bench("short body", benchShortBody).run(); }); }); }); diff --git a/packages/vitest-plugin/benches/timing.bench.ts b/packages/vitest-plugin/benches/timing.bench.ts index 83331ad6..74e1f71f 100644 --- a/packages/vitest-plugin/benches/timing.bench.ts +++ b/packages/vitest-plugin/benches/timing.bench.ts @@ -1,4 +1,4 @@ -import { bench, describe, type BenchOptions } from "vitest"; +import { describe, test, type BenchOptions } from "vitest"; const busySleep = (ms: number): void => { const end = performance.now() + ms; @@ -13,27 +13,21 @@ const timingBenchOptions: BenchOptions = { }; describe("timing tests", () => { - bench( - "wait 1ms", - async () => { + test("wait 1ms", async ({ bench }) => { + await bench("wait 1ms", async () => { busySleep(1); - }, - timingBenchOptions, - ); + }).run(timingBenchOptions); + }); - bench( - "wait 500ms", - async () => { + test("wait 500ms", async ({ bench }) => { + await bench("wait 500ms", async () => { busySleep(500); - }, - timingBenchOptions, - ); + }).run(timingBenchOptions); + }); - bench( - "wait 1sec", - async () => { + test("wait 1sec", async ({ bench }) => { + await bench("wait 1sec", async () => { busySleep(1_000); - }, - timingBenchOptions, - ); + }).run(timingBenchOptions); + }); }); diff --git a/packages/vitest-plugin/src/globalSetup.ts b/packages/vitest-plugin/src/globalSetup.ts index 8c5e1a34..5a7eeaf9 100644 --- a/packages/vitest-plugin/src/globalSetup.ts +++ b/packages/vitest-plugin/src/globalSetup.ts @@ -9,13 +9,22 @@ function logCodSpeed(message: string) { console.log(`[CodSpeed] ${message}`); } +let setupHappened = false; let teardownHappened = false; +// TODO: Check if this can be avoided +// Vitest 5 forks a dedicated `(bench)` project from the base one and runs +// globalSetup for both it and the root project, so setup/teardown fire more than +// once against this shared module. Log only the first pass and make the rest +// no-ops rather than treating the repeat as an error. export default function () { - logCodSpeed(`@codspeed/vitest-plugin v${__VERSION__} - setup`); + if (!setupHappened) { + setupHappened = true; + logCodSpeed(`@codspeed/vitest-plugin v${__VERSION__} - setup`); + } return () => { - if (teardownHappened) throw new Error("teardown called twice"); + if (teardownHappened) return; teardownHappened = true; logCodSpeed(`@codspeed/vitest-plugin v${__VERSION__} - teardown`); diff --git a/packages/vitest-plugin/src/instrument.ts b/packages/vitest-plugin/src/instrument.ts index b880ca89..94edb1e1 100644 --- a/packages/vitest-plugin/src/instrument.ts +++ b/packages/vitest-plugin/src/instrument.ts @@ -5,7 +5,10 @@ import { MARKER_TYPE_BENCHMARK_START, msToNs, msToS, + optimizeFunction, + optimizeFunctionSync, wrapWithRootFrame, + wrapWithRootFrameSync, writeWalltimeResults, type Benchmark, type BenchmarkStats, @@ -14,6 +17,21 @@ import type * as tinybench from "tinybench"; export type Tinybench = typeof tinybench; +/** tinybench's per-task lifecycle hooks (a subset of `FnOptions`). */ +export interface TinybenchFnOptions { + beforeAll?: (mode?: "run" | "warmup") => unknown; + beforeEach?: (mode?: "run" | "warmup") => unknown; + afterEach?: (mode?: "run" | "warmup") => unknown; + afterAll?: (mode?: "run" | "warmup") => unknown; +} + +/** The captured registration for a task: its fn and options. */ +export interface CapturedTask { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + fn: (...args: any[]) => any; + fnOpts?: TinybenchFnOptions; +} + /** A tinybench task, exposing the `fn` the runner wraps with the root frame. */ export interface TinybenchTask { name: string; @@ -34,17 +52,6 @@ export interface TinybenchBench { teardown: TinybenchHook; } -/** The minimal task shape `patchTaskRunWithRootFrame` mutates. */ -interface RunnableTask { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - fn: (...args: any[]) => any; -} - -/** The tinybench Task prototype whose `run` we wrap. */ -interface TinybenchTaskClass { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - prototype: { run: (this: any) => Promise }; -} /** * The tinybench statistics shape (latency/throughput) shared across the v2 and @@ -77,7 +84,7 @@ interface InstrumentWindow { runStart: bigint | null; } -let isTaskPatched = false; +let isBenchAddPatched = false; /** * The window bracketing the currently running task's measured loop, driven by @@ -86,14 +93,85 @@ let isTaskPatched = false; */ const instrumentWindow: InstrumentWindow = { runStart: null }; +// tinybench keeps a task's fn and options as `#private` fields (v6+), so we +// capture them ourselves when `Bench.add` runs, keyed by bench then task name. +// The analysis seam needs the raw fn to run it under its own tight window +// instead of tinybench's timing loop. +const capturedTasks = new WeakMap>(); + +/** The minimal tinybench Bench prototype we patch to capture registrations. */ +interface TinybenchBenchClass { + prototype: { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + add: (...args: any[]) => unknown; + }; +} + +/** + * Patch `Bench.prototype.add` to record each task's fn and options, keyed by + * bench then task name. Idempotent, and applied to the prototype so it captures + * registrations on every Bench the host constructs. + * + * `BenchClass` must be the exact class the host instantiates. In tinybench v6 a + * task's fn is a true `#private` field — it cannot be read or replaced on the + * task afterwards — so capturing (and, for walltime, root-frame-wrapping) has to + * happen here, as the fn is registered. + * + * `registerFn` transforms the fn actually handed to tinybench: identity for + * analysis (which runs the captured fn itself), or a root-frame wrap for + * walltime (where tinybench drives the fn and the frame must already be baked + * in). + */ +export function captureBenchAddOnce( + BenchClass: TinybenchBenchClass, + registerFn: (fn: CapturedTask["fn"]) => CapturedTask["fn"], +): void { + if (isBenchAddPatched) { + return; + } + isBenchAddPatched = true; + + const originalAdd = BenchClass.prototype.add; + BenchClass.prototype.add = function ( + this: object, + name: string, + fn: CapturedTask["fn"], + fnOpts?: TinybenchFnOptions, + ) { + let byName = capturedTasks.get(this); + if (!byName) { + byName = new Map(); + capturedTasks.set(this, byName); + } + byName.set(name, { fn, fnOpts }); + return originalAdd.call(this, name, registerFn(fn), fnOpts); + }; +} + +/** Retrieve the fn/options captured for a task on a given bench, if any. */ +export function getCapturedTask( + bench: object, + taskName: string, +): CapturedTask | undefined { + return capturedTasks.get(bench)?.get(taskName); +} + +/** The tinybench Task prototype whose `run` the legacy seam wraps. */ +interface TinybenchTaskClass { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + prototype: { run: (this: any) => Promise }; +} + +let isTaskPatched = false; + /** - * Wrap every task's fn with the root frame so collected stacks are attributed to - * a benchmark. Idempotent: patching the shared `Task.prototype.run` in place hits - * every Bench instance, so repeat calls are no-ops. + * Wrap every task's fn with the root frame by patching `Task.prototype.run` in + * place. Used only by the legacy (Vitest 3/4) walltime seam, which runs on + * tinybench v2 where a task's `fn` is a plain, reassignable property. * - * `TaskClass` must be the exact prototype the host constructed its tasks against - * (taken from a live task, not imported) so the patch applies even when multiple - * copies of tinybench are installed. + * The Vitest 5 seam cannot use this: tinybench v6 made `fn` a true `#private` + * field, so reassigning `task.fn` is a silent no-op there — the frame must be + * baked in at registration time instead (see rootFrameRegisterFn). */ export function patchTaskRunOnce(TaskClass: TinybenchTaskClass): void { if (isTaskPatched) { @@ -102,7 +180,7 @@ export function patchTaskRunOnce(TaskClass: TinybenchTaskClass): void { isTaskPatched = true; const originalRun = TaskClass.prototype.run; - TaskClass.prototype.run = async function (this: RunnableTask) { + TaskClass.prototype.run = async function (this: CapturedTask) { const originalFn = this.fn; this.fn = wrapWithRootFrame(() => originalFn.call(this)); @@ -114,6 +192,98 @@ export function patchTaskRunOnce(TaskClass: TinybenchTaskClass): void { }; } +/** + * The root-frame wrap to hand tinybench at registration time (walltime, v5). + * Post-hoc assignment to a task's `fn` is a no-op on tinybench v6 (private + * field), so the frame must be baked into the registered fn instead. + */ +export function rootFrameRegisterFn( + fn: CapturedTask["fn"], +): CapturedTask["fn"] { + return wrapWithRootFrame(() => fn()); +} + +/** Identity registration: analysis runs the captured fn itself, unwrapped. */ +export function identityRegisterFn(fn: CapturedTask["fn"]): CapturedTask["fn"] { + return fn; +} + +/** + * Run one benchmark under instrumentation, matching the analysis window the + * Vitest 3/4 runner uses exactly: warm the JIT with `optimizeFunction` outside + * the window, run the user hooks around a single measured `fn()`, and bracket + * only that call with `startBenchmark`/`stopBenchmark` under the root frame. The + * measurement comes from the instrument, so no wall-clock markers are emitted + * and tinybench's timing loop is not involved. + * + * Synchronous benchmarks run through a fully synchronous window + * (`wrapWithRootFrameSync`, no `await`): awaiting a sync fn would splice Node's + * promise-hook machinery in above the root frame and pollute the sample. Async + * benchmarks necessarily use the awaited path. + */ +export async function runAnalysisTask( + { fn, fnOpts }: CapturedTask, + uri: string, +): Promise { + if (isAsyncFn(fn)) { + await runAnalysisTaskAsync(fn, fnOpts, uri); + } else { + await runAnalysisTaskSync(fn, fnOpts, uri); + } +} + +function isAsyncFn(fn: CapturedTask["fn"]): boolean { + return fn.constructor?.name === "AsyncFunction"; +} + +async function runAnalysisTaskAsync( + fn: CapturedTask["fn"], + fnOpts: TinybenchFnOptions | undefined, + uri: string, +): Promise { + await fnOpts?.beforeAll?.("run"); + await optimizeFunction(async () => { + await fnOpts?.beforeEach?.("run"); + await fn(); + await fnOpts?.afterEach?.("run"); + }); + + await fnOpts?.beforeEach?.("run"); + global.gc?.(); + await wrapWithRootFrame(async () => { + InstrumentHooks.startBenchmark(); + await fn(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + })(); + await fnOpts?.afterEach?.("run"); + await fnOpts?.afterAll?.("run"); +} + +function runAnalysisTaskSync( + fn: CapturedTask["fn"], + fnOpts: TinybenchFnOptions | undefined, + uri: string, +): void { + fnOpts?.beforeAll?.("run"); + optimizeFunctionSync(() => { + fnOpts?.beforeEach?.("run"); + fn(); + fnOpts?.afterEach?.("run"); + }); + + fnOpts?.beforeEach?.("run"); + global.gc?.(); + wrapWithRootFrameSync(() => { + InstrumentHooks.startBenchmark(); + fn(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + })(); + fnOpts?.afterEach?.("run"); + fnOpts?.afterAll?.("run"); +} + /** * Drive the instrumentation window from each bench's run-mode setup/teardown * hooks so it brackets only tinybench's measured loop, excluding the warmup @@ -148,24 +318,29 @@ export function installInstrumentHooks( } function closeInstrumentWindow(uri: string): void { - const runEnd = InstrumentHooks.currentTimestamp(); + emitBenchmarkWindow(uri, instrumentWindow.runStart!); + instrumentWindow.runStart = null; +} + +/** + * Close the currently open instrumentation window: emit the benchmark markers + * bracketing [start, now], stop the benchmark, and attribute the sample to `uri`. + * + * Benchmark markers must land inside the sample window opened by + * startBenchmark(), so they are emitted before stopBenchmark() closes it. The + * runner consumes the FIFO stream in order, so a marker sent after stopBenchmark + * would fall outside the sample and break the expected + * SampleStart > BenchmarkStart > BenchmarkEnd > SampleEnd nesting. + */ +function emitBenchmarkWindow(uri: string, start: bigint): void { + const end = InstrumentHooks.currentTimestamp(); const pid = process.pid; - // Benchmark markers must land inside the sample window opened by - // startBenchmark(), so they have to be emitted before stopBenchmark() - // closes it. The runner consumes the FIFO stream in order, so a marker - // sent after StopBenchmark falls outside the sample and breaks the - // expected SampleStart > BenchmarkStart > BenchmarkEnd > SampleEnd nesting. - InstrumentHooks.addMarker( - pid, - MARKER_TYPE_BENCHMARK_START, - instrumentWindow.runStart!, - ); - InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_END, runEnd); + InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_START, start); + InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_END, end); InstrumentHooks.stopBenchmark(); InstrumentHooks.setExecutedBenchmark(pid, uri); - instrumentWindow.runStart = null; } /** diff --git a/packages/vitest-plugin/src/v5/setup.ts b/packages/vitest-plugin/src/v5/setup.ts index 6a851463..9040335e 100644 --- a/packages/vitest-plugin/src/v5/setup.ts +++ b/packages/vitest-plugin/src/v5/setup.ts @@ -1,7 +1,9 @@ import { getGitDir, getInstrumentMode, + InstrumentHooks, setupCore, + teardownCore, type Benchmark, } from "@codspeed/core"; import path from "path"; @@ -10,12 +12,18 @@ import path from "path"; // package root. It is read off the namespace (rather than a named import) so the // plugin still type-checks against Vitest 3/4, which don't export it; this // module only ever runs under v5. +import { createRequire } from "module"; import * as vitest from "vitest"; import { + captureBenchAddOnce, + getCapturedTask, + identityRegisterFn, installInstrumentHooks, - patchTaskRunOnce, + rootFrameRegisterFn, + runAnalysisTask, tinybenchTaskToBenchmark, writeAndLogWalltimeResults, + type CapturedTask, type TinybenchBench, type TinybenchOptions, type TinybenchTask, @@ -23,6 +31,14 @@ import { const TestRunner = (vitest as unknown as { TestRunner: unknown }).TestRunner; +/** The tinybench Bench constructor, as reached from a live bench instance. */ +interface TinybenchBenchClass { + prototype: { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + add: (...args: any[]) => unknown; + }; +} + /** * Vitest 5 runs benchmarks inside `test()` through `TestRunner`, calling the * static `TestRunner.runBenchmarks(tinybench)` with a fully built tinybench @@ -39,9 +55,14 @@ const TestRunner = (vitest as unknown as { TestRunner: unknown }).TestRunner; interface TinybenchWithTasks extends TinybenchBench { name: string; tasks: TinybenchTask[]; - run(): Promise; + constructor: TinybenchBenchClass; // tinybench exposes the resolved options on the instance opts?: TinybenchOptions; + // Bench-level run parameters, collapsed to a single pass for the throwaway + // result run in analysis mode. + iterations: number; + time: number; + warmup: boolean; } /** Minimal shape of the current Vitest test task we read for URI construction. */ @@ -58,11 +79,14 @@ function getCurrentTest(): CurrentTest | undefined { } /** - * Build the benchmark URI from the running test and the tinybench task name. - * Matches the legacy convention: git-relative file, then the suite/test path, - * then the bench name, all `::`-separated. + * Build the benchmark URI from the running test: git-relative file then the + * suite/test path, `::`-separated. In Vitest 5 a benchmark is a `bench()` call + * inside a `test()`, so the enclosing test is the benchmark's identity — the + * inner tinybench task name is an implementation detail and not part of the URI. + * This mirrors the legacy `file::suite...::name` shape with the test playing the + * role of the leaf. */ -function buildUri(taskName: string): string { +function buildUri(): string { const test = getCurrentTest(); const filepath = test?.file?.filepath; if (!filepath) { @@ -75,20 +99,19 @@ function buildUri(taskName: string): string { const relativeFile = path.relative(gitDir, filepath); // `fullTestName` uses " > " between suite levels; normalize to "::". const testPath = (test?.fullTestName ?? "").split(" > ").join("::"); - return [relativeFile, testPath, taskName].filter(Boolean).join("::"); + return [relativeFile, testPath].filter(Boolean).join("::"); } -function collectWalltimeResults(tinybench: TinybenchWithTasks): void { +function collectWalltimeResults( + tinybench: TinybenchWithTasks, + uri: string, +): void { const options: TinybenchOptions = tinybench.opts ?? {}; const benchmarks: Benchmark[] = []; for (const task of tinybench.tasks) { if (task.result?.state && task.result.state !== "completed") continue; - const benchmark = tinybenchTaskToBenchmark( - task, - buildUri(task.name), - options, - ); + const benchmark = tinybenchTaskToBenchmark(task, uri, options); if (benchmark) { benchmarks.push(benchmark); } @@ -97,35 +120,115 @@ function collectWalltimeResults(tinybench: TinybenchWithTasks): void { writeAndLogWalltimeResults(benchmarks); } +const isWalltime = getInstrumentMode() === "walltime"; + +// setupCore starts the perf listener (analysis mode), which must run once for +// the worker's lifetime and be stopped so its data is flushed. Vitest 5 runs +// benchmarks per `test()` with no whole-suite bracket in the worker, so we start +// on the first benchmark and stop when the worker process exits, mirroring the +// single setupCore/teardownCore the Vitest 3/4 runner does around a suite. +let isCoreSetup = false; + +function setupCoreOnce(): void { + if (isCoreSetup) { + return; + } + isCoreSetup = true; + setupCore(); + process.once("beforeExit", () => teardownCore()); +} + +/** + * Capture each benchmark's fn as it is registered. tinybench keeps the fn in a + * `#private` field (v6), so it can't be read off the task later — the analysis + * seam needs the raw fn, and the walltime seam needs the root frame baked in at + * registration time. Both are handled here. + * + * The `Bench` class is resolved relative to the installed Vitest so we patch the + * exact class Vitest instantiates, even though the plugin's own tinybench may be + * a different version. Registration happens before `runBenchmarks`, so this must + * be installed at module load, before any test runs. + */ +async function captureBenchRegistrations(): Promise { + try { + const require = createRequire(import.meta.url); + const vitestRequire = createRequire(require.resolve("vitest/package.json")); + // Resolve the path with `require.resolve` but load with `import()`: + // tinybench v6 is ESM, so `require()`-ing it throws. + const tinybench = (await import(vitestRequire.resolve("tinybench"))) as { + Bench: TinybenchBenchClass; + }; + captureBenchAddOnce( + tinybench.Bench, + isWalltime ? rootFrameRegisterFn : identityRegisterFn, + ); + } catch { + // If tinybench can't be resolved the run will surface a clearer error when + // the benchmark actually executes; nothing to instrument here. + } +} + function patchRunBenchmarks(): void { const Runner = TestRunner as unknown as { runBenchmarks: (tinybench: TinybenchWithTasks) => Promise; }; const originalRunBenchmarks = Runner.runBenchmarks.bind(TestRunner); - const isWalltime = getInstrumentMode() === "walltime"; Runner.runBenchmarks = async (tinybench: TinybenchWithTasks) => { - setupCore(); - - // tinybench's `Task` class isn't exported from the bench instance, so we - // reach it through a constructed task (Vitest adds them before running). - const TaskClass = tinybench.tasks[0]?.constructor as - | { prototype: { run: (this: unknown) => Promise } } - | undefined; - if (TaskClass) { - patchTaskRunOnce(TaskClass); - } - - installInstrumentHooks(tinybench, buildUri); + // Ensure the registration capture is installed before we act on the bench. + // The import kicks off at module load (during collection) and is well + // settled by the time the first benchmark runs; awaiting here avoids a + // top-level await while still guaranteeing ordering. + await captureReady; + setupCoreOnce(); - const result = await originalRunBenchmarks(tinybench); + // Resolve the URI up front, outside any measured window. It walks the + // filesystem (git root lookup), which must not land inside the sample. + const uri = buildUri(); if (isWalltime) { - collectWalltimeResults(tinybench); + // tinybench drives the measured loop; bracket it with the instrument + // window via the setup/teardown hooks (the root frame is already baked + // into the registered fn). + installInstrumentHooks(tinybench, () => uri); + const result = await originalRunBenchmarks(tinybench); + collectWalltimeResults(tinybench, uri); + return result; } - return result; + // Analysis (instrumentation/simulation): run the captured fn ourselves under + // the exact window the Vitest 3/4 runner uses, then let tinybench run + // uninstrumented purely to populate the `result` Vitest reads afterwards. + // The real measurement is ours; collapse tinybench's run to a single pass so + // the throwaway result run stays cheap. + await runAnalysisBenchmarks(tinybench, uri); + tinybench.warmup = false; + tinybench.time = 0; + tinybench.iterations = 1; + return originalRunBenchmarks(tinybench); }; } +async function runAnalysisBenchmarks( + tinybench: TinybenchWithTasks, + uri: string, +): Promise { + const label = InstrumentHooks.isInstrumented() ? "Measured" : "Checked"; + for (const task of tinybench.tasks) { + const captured: CapturedTask | undefined = getCapturedTask( + tinybench, + task.name, + ); + if (!captured) { + continue; + } + await runAnalysisTask(captured, uri); + console.log(`[CodSpeed] ${label} ${uri}`); + } +} + +// Kick off the capture install at module load (during collection). `runBenchmarks` +// awaits this before touching the bench, so it is guaranteed settled before any +// benchmark's results are read — well after benchmarks start executing. +const captureReady = captureBenchRegistrations(); patchRunBenchmarks(); diff --git a/packages/vitest-plugin/src/vitestBackend.ts b/packages/vitest-plugin/src/vitestBackend.ts index db8c0d26..3451eefe 100644 --- a/packages/vitest-plugin/src/vitestBackend.ts +++ b/packages/vitest-plugin/src/vitestBackend.ts @@ -98,9 +98,14 @@ class V5Backend implements VitestBackend { v8Flags: string[], resolveFile: (name: string) => string, ): ViteUserConfig["test"] { + // When CodSpeed isn't driving the run, leave Vitest's benchmark execution + // untouched (no instrumentation setup file), matching the legacy backend. + const setupFiles = + getInstrumentMode() === "disabled" ? undefined : [resolveFile("v5/setup")]; + return { execArgv: v8Flags, - setupFiles: [resolveFile("v5/setup")], + ...(setupFiles && { setupFiles }), }; }