Support vitest 5

CLAUDE.md

@@ -94,4 +94,25 @@ Based on the codebase analysis, to add stats access features:
   - Build: `pnpm turbo run build --filter=<package-name>`
   - Typecheck: `pnpm turbo run typecheck --filter=<package-name>`
   - Lint: `pnpm turbo run lint --filter=<package-name>`
-  - Run a task across all packages by omitting `--filter` (e.g. `pnpm turbo run build`).
+  - Run a task across all packages by omitting `--filter` (e.g. `pnpm turbo run build`).
+
+## Testing a plugin during development
+
+A plugin behaves differently depending on whether CodSpeed is driving the run,
+so exercise all three of these when developing or reviewing a plugin change
+(build the plugin first — the benches import from `dist`):
+
+1. **Fallback (not under CodSpeed).** No env vars. The plugin must stay out of
+   the way and let the framework run its benchmarks normally (no instrumentation,
+   no hijacked output). e.g. `pnpm turbo run bench --filter=<package-name>`.
+2. **Instrumentation / simulation.** `CODSPEED_ENV=true CODSPEED_RUNNER_MODE=simulation`
+   (or `instrumentation`). The plugin hijacks the run to do a single instrumented
+   pass per benchmark and prints `Measured/Checked <uri>` instead of the normal
+   harness output.
+3. **Walltime.** `CODSPEED_ENV=true CODSPEED_RUNNER_MODE=walltime`. The plugin
+   instruments the framework's real benchmark loop and collects walltime results.
+
+Running these locally outside the CodSpeed runner is expected to log
+`instrument-hooks: failed to write environment.json` and skip actual measurement
+writes — the point is to verify the plugin's control flow and output per mode,
+not to produce real measurements.

examples/with-vitest-v4/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "with-vitest-v4",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "bench-vitest": "vitest bench --run"
+  },
+  "devDependencies": {
+    "@codspeed/vitest-plugin": "workspace:*",
+    "typescript": "^5.1.3",
+    "vitest": "^4.1.9"
+  }
+}

examples/with-vitest-v4/src/fibonacci.bench.ts

@@ -0,0 +1,20 @@
+import { bench, describe } from "vitest";
+import { iterativeFibonacci, recursiveFibonacci } from "./fibonacci";
+
+describe("fibonacci", () => {
+  bench("recursive fibo 15", () => {
+    recursiveFibonacci(15);
+  });
+
+  bench("recursive fibo 20", () => {
+    recursiveFibonacci(20);
+  });
+
+  bench("iterative fibo 15", () => {
+    iterativeFibonacci(15);
+  });
+
+  bench("iterative fibo 20", () => {
+    iterativeFibonacci(20);
+  });
+});

examples/with-vitest-v4/src/fibonacci.ts

@@ -0,0 +1,17 @@
+export function recursiveFibonacci(n: number): number {
+  if (n < 2) {
+    return n;
+  }
+  return recursiveFibonacci(n - 1) + recursiveFibonacci(n - 2);
+}
+
+export function iterativeFibonacci(n: number): number {
+  let a = 0;
+  let b = 1;
+  for (let i = 0; i < n; i++) {
+    const temp = a + b;
+    a = b;
+    b = temp;
+  }
+  return a;
+}

examples/with-vitest-v4/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "lib": ["es2023"],
+    "module": "ESNext",
+    "verbatimModuleSyntax": true,
+    "target": "es2022",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "moduleResolution": "Node"
+  }
+}

examples/with-vitest-v4/vitest.config.ts

@@ -0,0 +1,6 @@
+import codspeedPlugin from "@codspeed/vitest-plugin";
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  plugins: [codspeedPlugin()],
+});

packages/vitest-plugin/.codspeed/3318153.json

@@ -0,0 +1,87 @@
+{
+  "creator": {
+    "name": "codspeed-node",
+    "version": "5.7.1",
+    "pid": 3318153
+  },
+  "instrument": {
+    "type": "walltime"
+  },
+  "benchmarks": [
+    {
+      "name": "wait 1ms",
+      "uri": "packages/vitest-plugin/benches/timing.bench.ts::timing tests::wait 1ms",
+      "config": {
+        "max_rounds": null,
+        "max_time_ns": null,
+        "min_round_time_ns": null,
+        "warmup_time_ns": null
+      },
+      "stats": {
+        "min_ns": 1000544.011592865,
+        "max_ns": 1269623.9948272705,
+        "mean_ns": 1005246.9399107153,
+        "stdev_ns": 19360.832010972394,
+        "q1_ns": 1001089.0066623688,
+        "median_ns": 1001929.9983978271,
+        "q3_ns": 1003693.5210227966,
+        "total_time": 1.0002207052111625,
+        "iter_per_round": 1,
+        "rounds": 995,
+        "iqr_outlier_rounds": 79,
+        "stdev_outlier_rounds": 17,
+        "warmup_iters": 0
+      }
+    },
+    {
+      "name": "wait 500ms",
+      "uri": "packages/vitest-plugin/benches/timing.bench.ts::timing tests::wait 500ms",
+      "config": {
+        "max_rounds": null,
+        "max_time_ns": null,
+        "min_round_time_ns": null,
+        "warmup_time_ns": null
+      },
+      "stats": {
+        "min_ns": 500015603.9893627,
+        "max_ns": 500020386.0104084,
+        "mean_ns": 500018516.40343666,
+        "stdev_ns": 1852.787897539228,
+        "q1_ns": 500017994.99988556,
+        "median_ns": 500018939.0182495,
+        "q3_ns": 500019657.9992771,
+        "total_time": 2.5000925820171833,
+        "iter_per_round": 1,
+        "rounds": 5,
+        "iqr_outlier_rounds": 0,
+        "stdev_outlier_rounds": 0,
+        "warmup_iters": 0
+      }
+    },
+    {
+      "name": "wait 1sec",
+      "uri": "packages/vitest-plugin/benches/timing.bench.ts::timing tests::wait 1sec",
+      "config": {
+        "max_rounds": null,
+        "max_time_ns": null,
+        "min_round_time_ns": null,
+        "warmup_time_ns": null
+      },
+      "stats": {
+        "min_ns": 1000017313.00354,
+        "max_ns": 1000021275.9971619,
+        "mean_ns": 1000019479.1972637,
+        "stdev_ns": 1536.5843481770053,
+        "q1_ns": 1000018610.0006104,
+        "median_ns": 1000020076.9901276,
+        "q3_ns": 1000020119.9948788,
+        "total_time": 5.000097395986319,
+        "iter_per_round": 1,
+        "rounds": 5,
+        "iqr_outlier_rounds": 0,
+        "stdev_outlier_rounds": 0,
+        "warmup_iters": 0
+      }
+    }
+  ]
+}