From 150df25e78178c4b2d034d399a32f4ec43e19927 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Wed, 30 Sep 2026 10:01:52 +0200 Subject: [PATCH 1/5] docs: document how to exercise a plugin in the three CodSpeed modes A plugin behaves differently depending on whether CodSpeed drives the run, and each mode takes a different path through the plugin, so all three have to be run when developing or reviewing one. Write down the commands and the expected output per mode, and ignore the local `.codspeed` result directory those runs produce. Co-Authored-By: Claude --- .gitignore | 2 ++ CLAUDE.md | 23 ++++++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5233f61e..3cfb1201 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,5 @@ packages/app/.env # turbo .turbo/ .rollup.cache/ + +.codspeed diff --git a/CLAUDE.md b/CLAUDE.md index b82671a6..00a76a53 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -94,4 +94,25 @@ Based on the codebase analysis, to add stats access features: - Build: `pnpm turbo run build --filter=` - Typecheck: `pnpm turbo run typecheck --filter=` - Lint: `pnpm turbo run lint --filter=` - - Run a task across all packages by omitting `--filter` (e.g. `pnpm turbo run build`). \ No newline at end of file + - Run a task across all packages by omitting `--filter` (e.g. `pnpm turbo run build`). + +## Testing a plugin during development + +A plugin behaves differently depending on whether CodSpeed is driving the run, +so exercise all three of these when developing or reviewing a plugin change +(build the plugin first — the benches import from `dist`): + +1. **Fallback (not under CodSpeed).** No env vars. The plugin must stay out of + the way and let the framework run its benchmarks normally (no instrumentation, + no hijacked output). e.g. `pnpm turbo run bench --filter=`. +2. **Instrumentation / simulation.** `CODSPEED_ENV=true CODSPEED_RUNNER_MODE=simulation` + (or `instrumentation`). The plugin hijacks the run to do a single instrumented + pass per benchmark and prints `Measured/Checked ` instead of the normal + harness output. +3. **Walltime.** `CODSPEED_ENV=true CODSPEED_RUNNER_MODE=walltime`. The plugin + instruments the framework's real benchmark loop and collects walltime results. + +Running these locally outside the CodSpeed runner is expected to log +`instrument-hooks: failed to write environment.json` and skip actual measurement +writes — the point is to verify the plugin's control flow and output per mode, +not to produce real measurements. \ No newline at end of file From 4a42934a5856de674711bade2c06236a3c4f8d6d Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Wed, 30 Sep 2026 10:02:12 +0200 Subject: [PATCH 2/5] feat(vitest-plugin): support vitest 5 alongside 3 and 4 Vitest 5 removed the dedicated `NodeBenchmarkRunner` and the `vitest/runners` / `vitest/suite` entrypoints. Benchmarks are now declared through a `bench` test-context fixture and executed by a `benchmark.provider`: a module Vitest hands the registered functions, their options and the declaring test, and whose results it treats as authoritative. Detect the installed Vitest generation and select the integration behind a `VitestBackend` abstraction so the rest of the plugin never inspects the version: - v3/4 keep the custom benchmark runner per instrument mode. Their Vitest lookups move behind `legacy/compat`, which owns the types of the removed benchmark backend so the package type-checks against Vitest 5. - v5 registers CodSpeed as the benchmark provider, which runs the analysis and walltime modes off the registrations it receives. On v5, benchmark URIs carry the registration name after the test path, so a test declaring several benchmarks through `bench.compare()` reports one benchmark per registration. Gate the injection on the instrument mode: Vitest 5 clones its benchmark project after the Vite config hooks ran, so a benchmark run can no longer be detected from the incoming config, and CodSpeed only ever drives benchmark runs. Without CodSpeed the plugin injects nothing and Vitest runs the benchmarks through its own tinybench provider. Vitest 5 also runs `globalSetup` once per project, including the cloned benchmark project, so make setup and teardown idempotent instead of failing the run on the second teardown. Bump the plugin's own dev dependency to Vitest 5 and move its benches to the fixture API, reading imported bindings into a local before the measured loop so the module-runner getters stay out of the samples. Refs COD-2931 Co-Authored-By: Claude --- packages/vitest-plugin/benches/flat.bench.ts | 37 +- packages/vitest-plugin/benches/hooks.bench.ts | 54 +-- packages/vitest-plugin/benches/macos.bench.ts | 8 +- .../vitest-plugin/benches/parsePr.bench.ts | 58 +-- .../vitest-plugin/benches/timing.bench.ts | 41 +-- packages/vitest-plugin/package.json | 4 +- packages/vitest-plugin/rollup.config.mjs | 16 +- .../src/__tests__/globalSetup.test.ts | 12 +- .../vitest-plugin/src/__tests__/index.test.ts | 188 +++++----- .../src/__tests__/instrumented.test.ts | 20 +- packages/vitest-plugin/src/compat.ts | 35 -- packages/vitest-plugin/src/globalSetup.ts | 12 +- packages/vitest-plugin/src/index.ts | 72 +--- packages/vitest-plugin/src/instrument.ts | 192 ++++++++++ .../src/{ => legacy}/analysis.ts | 24 +- .../vitest-plugin/src/{ => legacy}/common.ts | 11 +- packages/vitest-plugin/src/legacy/compat.ts | 93 +++++ .../src/legacy/vitest-legacy.d.ts | 19 + .../src/legacy/walltime-utils.ts | 99 ++++++ .../{walltime/index.ts => legacy/walltime.ts} | 116 ++---- packages/vitest-plugin/src/runner.ts | 3 - packages/vitest-plugin/src/v5/provider.ts | 334 ++++++++++++++++++ packages/vitest-plugin/src/vitestBackend.ts | 128 +++++++ packages/vitest-plugin/src/walltime/utils.ts | 130 ------- pnpm-lock.yaml | 174 +++++---- 25 files changed, 1257 insertions(+), 623 deletions(-) delete mode 100644 packages/vitest-plugin/src/compat.ts create mode 100644 packages/vitest-plugin/src/instrument.ts rename packages/vitest-plugin/src/{ => legacy}/analysis.ts (85%) rename packages/vitest-plugin/src/{ => legacy}/common.ts (74%) create mode 100644 packages/vitest-plugin/src/legacy/compat.ts create mode 100644 packages/vitest-plugin/src/legacy/vitest-legacy.d.ts create mode 100644 packages/vitest-plugin/src/legacy/walltime-utils.ts rename packages/vitest-plugin/src/{walltime/index.ts => legacy/walltime.ts} (50%) delete mode 100644 packages/vitest-plugin/src/runner.ts create mode 100644 packages/vitest-plugin/src/v5/provider.ts create mode 100644 packages/vitest-plugin/src/vitestBackend.ts delete mode 100644 packages/vitest-plugin/src/walltime/utils.ts diff --git a/packages/vitest-plugin/benches/flat.bench.ts b/packages/vitest-plugin/benches/flat.bench.ts index 74e67b9d..b9b494a4 100644 --- a/packages/vitest-plugin/benches/flat.bench.ts +++ b/packages/vitest-plugin/benches/flat.bench.ts @@ -1,5 +1,10 @@ -import { bench, describe } from "vitest"; -import parsePr from "./parsePr"; +import { describe, test } from "vitest"; +import parsePrExport from "./parsePr"; + +// Read the imported binding once: Vite's module runner exposes exports through +// getters, and reading one inside the measured loop adds overhead to every +// iteration (Vitest warns about it). +const parsePr = parsePrExport; const LONG_BODY = new Array(1_000) @@ -9,12 +14,15 @@ const LONG_BODY = .join("\n") + "fixes #123"; describe("parsePr", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test-1", number: 1 }); - }); - - bench("long body", () => { - parsePr({ body: LONG_BODY, title: "test-2", number: 2 }); + test("body size", async ({ bench }) => { + await bench.compare( + bench("short body", () => { + parsePr({ body: "fixes #123", title: "test-1", number: 1 }); + }), + bench("long body", () => { + parsePr({ body: LONG_BODY, title: "test-2", number: 2 }); + }), + ); }); }); @@ -24,10 +32,13 @@ function fibo(n: number): number { } describe("fibo", () => { - bench("fibo 10", () => { - fibo(10); - }); - bench("fibo 15", () => { - fibo(15); + test("depth", async ({ bench }) => { + await bench("fibo 10", () => { + fibo(10); + }).run(); + + await bench("fibo 15", () => { + fibo(15); + }).run(); }); }); diff --git a/packages/vitest-plugin/benches/hooks.bench.ts b/packages/vitest-plugin/benches/hooks.bench.ts index d790ca3c..c706dda8 100644 --- a/packages/vitest-plugin/benches/hooks.bench.ts +++ b/packages/vitest-plugin/benches/hooks.bench.ts @@ -1,37 +1,39 @@ -import { - afterAll, - afterEach, - beforeAll, - beforeEach, - bench, - describe, - expect, -} from "vitest"; +import { describe, expect, test } from "vitest"; +// Exercises tinybench's per-benchmark hooks, which Vitest 5 exposes through the +// `bench(name, options, fn)` options object (`beforeAll`/`beforeEach`/...). describe("hooks", () => { let count = 0; - describe("run", () => { - beforeAll(() => { + + const hooks = { + beforeAll: () => { count += 10; - }); - beforeEach(() => { + }, + beforeEach: () => { count += 1; - }); - afterEach(() => { + }, + afterEach: () => { count -= 1; - }); - afterAll(() => { + }, + afterAll: () => { count -= 10; - }); + }, + }; - bench("one", () => { - expect(count).toBe(11); - }); - bench("two", () => { - expect(count).toBe(11); - }); + test("hooked benches", async ({ bench }) => { + await bench.compare( + bench("one", hooks, () => { + expect(count).toBe(11); + }), + bench("two", hooks, () => { + expect(count).toBe(11); + }), + ); }); - bench("end", () => { - expect(count).toBe(0); + + test("after the hooked benches", async ({ bench }) => { + await bench("count is back to zero", () => { + expect(count).toBe(0); + }).run(); }); }); diff --git a/packages/vitest-plugin/benches/macos.bench.ts b/packages/vitest-plugin/benches/macos.bench.ts index e05a3082..792e9c3f 100644 --- a/packages/vitest-plugin/benches/macos.bench.ts +++ b/packages/vitest-plugin/benches/macos.bench.ts @@ -1,4 +1,4 @@ -import { bench, describe } from "vitest"; +import { describe, test } from "vitest"; const isMacOS = process.platform === "darwin"; @@ -10,7 +10,9 @@ function fibo(n: number): number { // macOS-only benchmark: skipped on every other platform, so it only runs on // the `codspeed-walltime-macos` CI job (see .github/workflows/codspeed.yml). describe.skipIf(!isMacOS)("macos only", () => { - bench("fibo darwin", () => { - fibo(30); + test("fibo", async ({ bench }) => { + await bench("fibo darwin", () => { + fibo(30); + }).run(); }); }); diff --git a/packages/vitest-plugin/benches/parsePr.bench.ts b/packages/vitest-plugin/benches/parsePr.bench.ts index 91c0adee..60829edf 100644 --- a/packages/vitest-plugin/benches/parsePr.bench.ts +++ b/packages/vitest-plugin/benches/parsePr.bench.ts @@ -1,5 +1,10 @@ -import { bench, describe } from "vitest"; -import parsePr from "./parsePr"; +import { describe, test } from "vitest"; +import parsePrExport from "./parsePr"; + +// Read the imported binding once: Vite's module runner exposes exports through +// getters, and reading one inside the measured loop adds overhead to every +// iteration (Vitest warns about it). +const parsePr = parsePrExport; const LONG_BODY = new Array(1_000) @@ -8,44 +13,49 @@ const LONG_BODY = ) .join("\n") + "fixes #123"; -describe("parsePr", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test", number: 124 }); - }); +function benchShortBody() { + parsePr({ body: "fixes #123", title: "test", number: 124 }); +} - bench("long body", () => { - parsePr({ body: LONG_BODY, title: "test", number: 124 }); +function benchLongBody() { + parsePr({ body: LONG_BODY, title: "test", number: 124 }); +} + +describe("parsePr", () => { + test("body size", async ({ bench }) => { + await bench.compare( + bench("short body", benchShortBody), + bench("long body", benchLongBody), + ); }); describe("nested suite", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test", number: 124 }); - }); - - bench("long body", () => { - parsePr({ body: LONG_BODY, title: "test", number: 124 }); + test("body size", async ({ bench }) => { + await bench.compare( + bench("short body", benchShortBody), + bench("long body", benchLongBody), + ); }); describe("deeply nested suite", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test", number: 124 }); + test("body size", async ({ bench }) => { + await bench("short body", benchShortBody).run(); }); }); }); }); describe("another parsePr", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test", number: 124 }); - }); - - bench("long body", () => { - parsePr({ body: LONG_BODY, title: "test", number: 124 }); + test("body size", async ({ bench }) => { + await bench.compare( + bench("short body", benchShortBody), + bench("long body", benchLongBody), + ); }); describe("nested suite", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test", number: 124 }); + test("body size", async ({ bench }) => { + await bench("short body", benchShortBody).run(); }); }); }); diff --git a/packages/vitest-plugin/benches/timing.bench.ts b/packages/vitest-plugin/benches/timing.bench.ts index 83331ad6..fd6aa5f8 100644 --- a/packages/vitest-plugin/benches/timing.bench.ts +++ b/packages/vitest-plugin/benches/timing.bench.ts @@ -1,4 +1,4 @@ -import { bench, describe, type BenchOptions } from "vitest"; +import { describe, test, type BenchCompareOptions } from "vitest"; const busySleep = (ms: number): void => { const end = performance.now() + ms; @@ -7,33 +7,24 @@ const busySleep = (ms: number): void => { } }; -const timingBenchOptions: BenchOptions = { +const timingBenchOptions: BenchCompareOptions = { iterations: 5, warmupIterations: 0, }; describe("timing tests", () => { - bench( - "wait 1ms", - async () => { - busySleep(1); - }, - timingBenchOptions, - ); - - bench( - "wait 500ms", - async () => { - busySleep(500); - }, - timingBenchOptions, - ); - - bench( - "wait 1sec", - async () => { - busySleep(1_000); - }, - timingBenchOptions, - ); + test("busy sleep", async ({ bench }) => { + await bench.compare( + bench("wait 1ms", async () => { + busySleep(1); + }), + bench("wait 500ms", async () => { + busySleep(500); + }), + bench("wait 1sec", async () => { + busySleep(1_000); + }), + timingBenchOptions, + ); + }); }); diff --git a/packages/vitest-plugin/package.json b/packages/vitest-plugin/package.json index 3c935d88..ab23e739 100644 --- a/packages/vitest-plugin/package.json +++ b/packages/vitest-plugin/package.json @@ -41,13 +41,13 @@ "peerDependencies": { "tinybench": ">=2.9.0", "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0", - "vitest": "^3.2 || ^4" + "vitest": "^3.2 || ^4 || ^5" }, "devDependencies": { "@total-typescript/shoehorn": "^0.1.1", "execa": "^8.0.1", "tinybench": "^2.9.0", "vite": "^8.0.0", - "vitest": "^4.1.11" + "vitest": "^5.0.0" } } diff --git a/packages/vitest-plugin/rollup.config.mjs b/packages/vitest-plugin/rollup.config.mjs index 2fda2b6a..b469682d 100644 --- a/packages/vitest-plugin/rollup.config.mjs +++ b/packages/vitest-plugin/rollup.config.mjs @@ -23,18 +23,26 @@ export default defineConfig([ plugins: jsPlugins(pkg.version), external: ["@codspeed/core", /^vitest/], }, + // Vitest imports these by file path (as `test.runner` or `benchmark.provider`), so + // each needs its own output; `resolveFile` expects it at the same path as in `src/`. { - input: "src/analysis.ts", - output: { file: "dist/analysis.mjs", format: "es" }, + input: "src/legacy/analysis.ts", + output: { file: "dist/legacy/analysis.mjs", format: "es" }, // top-level await plugins: jsPlugins(pkg.version, "es2022"), external: ["@codspeed/core", /^vitest/], }, { - input: "src/walltime/index.ts", - output: { file: "dist/walltime.mjs", format: "es" }, + input: "src/legacy/walltime.ts", + output: { file: "dist/legacy/walltime.mjs", format: "es" }, // top-level await plugins: jsPlugins(pkg.version, "es2022"), external: ["@codspeed/core", /^vitest/], }, + { + input: "src/v5/provider.ts", + output: { file: "dist/v5/provider.mjs", format: "es" }, + plugins: jsPlugins(pkg.version), + external: ["@codspeed/core", /^vitest/], + }, ]); diff --git a/packages/vitest-plugin/src/__tests__/globalSetup.test.ts b/packages/vitest-plugin/src/__tests__/globalSetup.test.ts index 178b3ecb..f475f048 100644 --- a/packages/vitest-plugin/src/__tests__/globalSetup.test.ts +++ b/packages/vitest-plugin/src/__tests__/globalSetup.test.ts @@ -4,19 +4,27 @@ import globalSetup from "../globalSetup"; console.log = vi.fn(); describe("globalSetup", () => { - it("should log the correct message on setup and teardown, and fail when teardown is called twice", async () => { + it("should log setup and teardown once, even when Vitest runs them per project", async () => { const teardown = globalSetup(); expect(console.log).toHaveBeenCalledWith( "[CodSpeed] @codspeed/vitest-plugin v1.0.0 - setup", ); + // Vitest 5 runs the same globalSetup for the base project and for the + // benchmark project it clones from it. + globalSetup(); + + expect(console.log).toHaveBeenCalledTimes(1); + teardown(); expect(console.log).toHaveBeenCalledWith( "[CodSpeed] @codspeed/vitest-plugin v1.0.0 - teardown", ); - expect(() => teardown()).toThrowError("teardown called twice"); + teardown(); + + expect(console.log).toHaveBeenCalledTimes(2); }); }); diff --git a/packages/vitest-plugin/src/__tests__/index.test.ts b/packages/vitest-plugin/src/__tests__/index.test.ts index 3aae6e2b..54713449 100644 --- a/packages/vitest-plugin/src/__tests__/index.test.ts +++ b/packages/vitest-plugin/src/__tests__/index.test.ts @@ -1,6 +1,6 @@ import { getV8Flags } from "@codspeed/core"; import { fromPartial } from "@total-typescript/shoehorn"; -import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import codspeedPlugin from "../index"; const coreMocks = vi.hoisted(() => { @@ -12,7 +12,7 @@ const coreMocks = vi.hoisted(() => { }); const fsMocks = vi.hoisted(() => { - let mockVersion = "4.0.18"; // default to v4 + let mockVersion = ""; return { readFileSync: vi.fn((path: string) => { if (path.includes("vitest/package.json")) { @@ -26,11 +26,6 @@ const fsMocks = vi.hoisted(() => { }; }); -const resolvedCodSpeedPlugin = codspeedPlugin(); -const applyPluginFunction = resolvedCodSpeedPlugin.apply; -if (typeof applyPluginFunction !== "function") - throw new Error("applyPluginFunction is not a function"); - vi.mock("@codspeed/core", async (importOriginal) => { const mod = await importOriginal(); return { ...mod, ...coreMocks }; @@ -44,114 +39,131 @@ vi.mock("fs", () => { console.warn = vi.fn(); -describe("codSpeedPlugin", () => { - beforeAll(() => { - // Set environment variables to trigger instrumented mode - process.env.CODSPEED_ENV = "1"; - process.env.CODSPEED_RUNNER_MODE = "instrumentation"; - }); +const plugin = codspeedPlugin(); - afterAll(() => { - // Clean up environment variables - delete process.env.CODSPEED_ENV; - delete process.env.CODSPEED_RUNNER_MODE; +function apply(mode: string) { + if (typeof plugin.apply !== "function") + throw new Error("apply is not a function"); + return plugin.apply({}, fromPartial({ mode })); +} + +function config() { + if (typeof plugin.config !== "function") + throw new Error("config is not a function"); + return plugin.config.call({} as never, {}, fromPartial({})); +} + +describe("codSpeedPlugin", () => { + beforeEach(() => { + vi.stubEnv("CODSPEED_ENV", "1"); + vi.stubEnv("CODSPEED_RUNNER_MODE", "instrumentation"); }); - it("should have a name", async () => { - expect(resolvedCodSpeedPlugin.name).toBe("codspeed:vitest"); + afterEach(() => { + vi.unstubAllEnvs(); }); - it("should enforce to run after the other plugins", async () => { - expect(resolvedCodSpeedPlugin.enforce).toBe("post"); + it("should enforce to run after the other plugins", () => { + expect(plugin.enforce).toBe("post"); }); - describe("apply", () => { - it("should not apply the plugin when the mode is not benchmark", async () => { - const applyPlugin = applyPluginFunction( - {}, - fromPartial({ mode: "test" }), - ); + describe("Vitest 3/4", () => { + beforeEach(() => { + fsMocks.setMockVersion("4.0.18"); + }); - expect(applyPlugin).toBe(false); + it("should not apply the plugin when the mode is not benchmark", () => { + expect(apply("test")).toBe(false); }); - it("should apply the plugin when there is no instrumentation", async () => { + it("should apply the plugin when there is no instrumentation", () => { coreMocks.InstrumentHooks.isInstrumented.mockReturnValue(false); - const applyPlugin = applyPluginFunction( - {}, - fromPartial({ mode: "benchmark" }), - ); - + expect(apply("benchmark")).toBe(true); expect(console.warn).toHaveBeenCalledWith( "[CodSpeed] bench detected but no instrumentation found", ); - expect(applyPlugin).toBe(true); }); - it("should apply the plugin when there is instrumentation", async () => { + it("should apply the plugin when there is instrumentation", () => { coreMocks.InstrumentHooks.isInstrumented.mockReturnValue(true); - const applyPlugin = applyPluginFunction( - {}, - fromPartial({ mode: "benchmark" }), - ); + expect(apply("benchmark")).toBe(true); + }); - expect(applyPlugin).toBe(true); + it("should apply the codspeed config for v4", () => { + expect(config()).toStrictEqual({ + test: { + globalSetup: [ + expect.stringContaining( + "packages/vitest-plugin/src/globalSetup.ts", + ), + ], + pool: "forks", + execArgv: getV8Flags(), + runner: expect.stringContaining( + "packages/vitest-plugin/src/legacy/analysis.ts", + ), + }, + }); }); - }); - it("should apply the codspeed config for v4", () => { - const config = resolvedCodSpeedPlugin.config; - if (typeof config !== "function") - throw new Error("config is not a function"); - - const result = config.call({} as never, {}, fromPartial({})); - - expect(result).toStrictEqual({ - test: { - globalSetup: [ - expect.stringContaining("packages/vitest-plugin/src/globalSetup.ts"), - ], - pool: "forks", - execArgv: getV8Flags(), - runner: expect.stringContaining( - "packages/vitest-plugin/src/analysis.ts", - ), - }, + it("should apply the codspeed config for v3 with poolOptions", () => { + fsMocks.setMockVersion("3.2.0"); + + expect(config()).toStrictEqual({ + test: { + globalSetup: [ + expect.stringContaining( + "packages/vitest-plugin/src/globalSetup.ts", + ), + ], + pool: "forks", + poolOptions: { + forks: { + execArgv: getV8Flags(), + }, + }, + runner: expect.stringContaining( + "packages/vitest-plugin/src/legacy/analysis.ts", + ), + }, + }); }); }); - it("should apply the codspeed config for v3 with poolOptions", () => { - // Set mock version to v3 - fsMocks.setMockVersion("3.2.0"); - - // Create a new plugin instance to pick up the mocked version - const v3Plugin = codspeedPlugin(); - const config = v3Plugin.config; - if (typeof config !== "function") - throw new Error("config is not a function"); - - const result = config.call({} as never, {}, fromPartial({})); - - expect(result).toStrictEqual({ - test: { - globalSetup: [ - expect.stringContaining("packages/vitest-plugin/src/globalSetup.ts"), - ], - pool: "forks", - poolOptions: { - forks: { - execArgv: getV8Flags(), + describe("Vitest 5", () => { + beforeEach(() => { + fsMocks.setMockVersion("5.0.0"); + }); + + it("should apply the plugin in any mode when CodSpeed drives the run", () => { + expect(apply("test")).toBe(true); + }); + + it("should not apply the plugin when CodSpeed is not driving the run", () => { + vi.stubEnv("CODSPEED_ENV", undefined); + + expect(apply("test")).toBe(false); + }); + + it("should wire the benchmark provider", () => { + expect(config()).toStrictEqual({ + test: { + globalSetup: [ + expect.stringContaining( + "packages/vitest-plugin/src/globalSetup.ts", + ), + ], + pool: "forks", + execArgv: getV8Flags(), + benchmark: { + provider: expect.stringContaining( + "packages/vitest-plugin/src/v5/provider.ts", + ), }, }, - runner: expect.stringContaining( - "packages/vitest-plugin/src/analysis.ts", - ), - }, + }); }); - - // Reset mock version back to v4 - fsMocks.setMockVersion("4.0.18"); }); }); diff --git a/packages/vitest-plugin/src/__tests__/instrumented.test.ts b/packages/vitest-plugin/src/__tests__/instrumented.test.ts index 443d62f2..c3e9178a 100644 --- a/packages/vitest-plugin/src/__tests__/instrumented.test.ts +++ b/packages/vitest-plugin/src/__tests__/instrumented.test.ts @@ -1,7 +1,11 @@ import { fromPartial } from "@total-typescript/shoehorn"; import { describe, expect, it, vi, type RunnerTestSuite } from "vitest"; -import { AnalysisRunner as CodSpeedRunner } from "../analysis"; -import { getBenchFn, getBenchOptions } from "../compat"; +import { AnalysisRunner as CodSpeedRunner } from "../legacy/analysis"; +import { getBenchFn, getBenchOptions } from "../legacy/compat"; + +// `legacy/compat` resolves the Vitest 3/4 benchmark backend from the installed +// Vitest, so it is mocked entirely: that backend is gone on Vitest 5, which this +// package is developed against. const coreMocks = vi.hoisted(() => { return { @@ -28,10 +32,16 @@ vi.mock("@codspeed/core", async (importOriginal) => { console.log = vi.fn(); -vi.mock("../compat", async (importOriginal) => { - const actual = await importOriginal(); +vi.mock("../legacy/compat", () => { + class NodeBenchmarkRunner { + async importTinybench() { + return import("tinybench"); + } + } + return { - ...actual, + NodeBenchmarkRunner, + getHooks: vi.fn(), getBenchFn: vi.fn(), getBenchOptions: vi.fn(), }; diff --git a/packages/vitest-plugin/src/compat.ts b/packages/vitest-plugin/src/compat.ts deleted file mode 100644 index d27baa71..00000000 --- a/packages/vitest-plugin/src/compat.ts +++ /dev/null @@ -1,35 +0,0 @@ -type VitestExports = typeof import("vitest"); - -/** - * Vitest 4.1 moved the benchmark runner and the suite helpers to the main - * `vitest` entry point and deprecated the `vitest/runners` and `vitest/suite` - * subpaths, which warn on import. - */ -async function resolveVitestApi() { - const { BenchmarkRunner, TestRunner }: Partial = - await import("vitest"); - - if (BenchmarkRunner && TestRunner) { - return { - NodeBenchmarkRunner: BenchmarkRunner, - getHooks: TestRunner.getSuiteHooks, - getBenchFn: TestRunner.getBenchFn, - getBenchOptions: TestRunner.getBenchOptions, - }; - } - - const [runners, suite] = await Promise.all([ - import("vitest/runners"), - import("vitest/suite"), - ]); - - return { - NodeBenchmarkRunner: runners.NodeBenchmarkRunner, - getHooks: suite.getHooks, - getBenchFn: suite.getBenchFn, - getBenchOptions: suite.getBenchOptions, - }; -} - -export const { NodeBenchmarkRunner, getHooks, getBenchFn, getBenchOptions } = - await resolveVitestApi(); diff --git a/packages/vitest-plugin/src/globalSetup.ts b/packages/vitest-plugin/src/globalSetup.ts index 8c5e1a34..143dafea 100644 --- a/packages/vitest-plugin/src/globalSetup.ts +++ b/packages/vitest-plugin/src/globalSetup.ts @@ -9,13 +9,21 @@ function logCodSpeed(message: string) { console.log(`[CodSpeed] ${message}`); } +let setupHappened = false; let teardownHappened = false; +// Vitest 5 clones a dedicated `(bench)` project from the base one and runs the +// same globalSetup module for both, so setup and teardown each fire twice. +// Report only the first pass and make the repeats no-ops; throwing on the second +// teardown fails the whole run during close. export default function () { - logCodSpeed(`@codspeed/vitest-plugin v${__VERSION__} - setup`); + if (!setupHappened) { + setupHappened = true; + logCodSpeed(`@codspeed/vitest-plugin v${__VERSION__} - setup`); + } return () => { - if (teardownHappened) throw new Error("teardown called twice"); + if (teardownHappened) return; teardownHappened = true; logCodSpeed(`@codspeed/vitest-plugin v${__VERSION__} - teardown`); diff --git a/packages/vitest-plugin/src/index.ts b/packages/vitest-plugin/src/index.ts index 2bdf59ed..e5e189ba 100644 --- a/packages/vitest-plugin/src/index.ts +++ b/packages/vitest-plugin/src/index.ts @@ -1,5 +1,4 @@ import { - getCodspeedRunnerMode, getInstrumentMode, getV8Flags, InstrumentHooks, @@ -7,49 +6,29 @@ import { SetupInstrumentsRequestBody, SetupInstrumentsResponse, } from "@codspeed/core"; -import { readFileSync } from "fs"; -import { createRequire } from "module"; import { join } from "path"; import { Plugin } from "vite"; import { type ViteUserConfig } from "vitest/config"; +import { resolveVitestBackend } from "./vitestBackend"; // get this file's directory path from import.meta.url const __dirname = new URL(".", import.meta.url).pathname; const isFileInTs = import.meta.url.endsWith(".ts"); -function getCodSpeedFileFromName(name: string) { +/** + * Resolve a plugin-owned file shipped alongside this module. Source (`.ts`) and + * built (`.mjs`) layouts are identical (see rollup.config.mjs). + */ +function resolveFile(name: string): string { const fileExtension = isFileInTs ? "ts" : "mjs"; - return join(__dirname, `${name}.${fileExtension}`); } -function getVitestMajorVersion(): number | null { - try { - // Resolve vitest from the project's perspective (cwd), not from the plugin's location - // This ensures we detect the vitest version the user has installed - const require = createRequire(join(process.cwd(), "package.json")); - const vitestPkgPath = require.resolve("vitest/package.json"); - const vitestPkg = JSON.parse(readFileSync(vitestPkgPath, "utf-8")); - return parseInt(vitestPkg.version.split(".")[0], 10); - } catch { - return null; - } -} - -function getRunnerFile(): string | undefined { - const instrumentMode = getInstrumentMode(); - if (instrumentMode === "disabled") { - return undefined; - } - - return getCodSpeedFileFromName(instrumentMode); -} - export default function codspeedPlugin(): Plugin { return { name: "codspeed:vitest", apply(_, { mode }) { - if (mode !== "benchmark") { + if (!resolveVitestBackend().isActive(mode)) { return false; } if ( @@ -62,41 +41,16 @@ export default function codspeedPlugin(): Plugin { }, enforce: "post", config(): ViteUserConfig { - const runnerFile = getRunnerFile(); - const runnerMode = getCodspeedRunnerMode(); - const v8Flags = getV8Flags(); - const vitestMajorVersion = getVitestMajorVersion(); - // by default, assume Vitest v4 or higher - const isVitestV4OrHigher = (vitestMajorVersion ?? 4) >= 4; - - const config: ViteUserConfig = { + return { test: { pool: "forks", - ...(isVitestV4OrHigher - ? { execArgv: v8Flags } - : { - // Compat with Vitest v3 - // See: https://vitest.dev/guide/migration.html#pool-rework - // poolOptions only exists in Vitest v3 - poolOptions: { - forks: { - execArgv: v8Flags, - }, - }, - }), - globalSetup: [getCodSpeedFileFromName("globalSetup")], - ...(runnerFile && { - runner: runnerFile, - }), - ...(runnerMode === "walltime" && { - benchmark: { - includeSamples: true, - }, - }), + globalSetup: [resolveFile("globalSetup")], + ...resolveVitestBackend().getBenchmarkTestConfig( + getV8Flags(), + resolveFile, + ), }, }; - - return config; }, }; } diff --git a/packages/vitest-plugin/src/instrument.ts b/packages/vitest-plugin/src/instrument.ts new file mode 100644 index 00000000..f1c97d01 --- /dev/null +++ b/packages/vitest-plugin/src/instrument.ts @@ -0,0 +1,192 @@ +import { + calculateQuantiles, + InstrumentHooks, + MARKER_TYPE_BENCHMARK_END, + MARKER_TYPE_BENCHMARK_START, + msToNs, + msToS, + writeWalltimeResults, + type Benchmark, + type BenchmarkStats, +} from "@codspeed/core"; +import type * as tinybench from "tinybench"; + +export type Tinybench = typeof tinybench; + +/** A tinybench task, as read after its run. */ +export interface TinybenchTask { + name: string; + result?: TinybenchTaskResult; +} + +/** tinybench's per-task setup/teardown hook signature. */ +export type TinybenchHook = ( + task: TinybenchTask, + mode: "run" | "warmup", +) => Promise | void; + +/** The mutable subset of a tinybench Bench the runner reaches into. */ +export interface TinybenchBench { + setup: TinybenchHook; + teardown: TinybenchHook; +} + +/** + * The tinybench statistics shape (latency/throughput) shared across the v2 and + * v6 lines. Only the fields the conversion needs are modeled. + */ +interface TinybenchStatistics { + min: number; + max: number; + mean: number; + sd: number; + samples: number[] | undefined; +} + +interface TinybenchTaskResult { + state?: string; + /** Set by tinybench when `state` is `"errored"`. */ + error?: Error; + totalTime: number; + latency: TinybenchStatistics; +} + +/** The subset of tinybench bench options that maps onto a CodSpeed benchmark config. */ +export interface TinybenchOptions { + time?: number; + warmupTime?: number; + warmupIterations?: number; + iterations?: number; +} + +/** + * Drive the instrumentation window from each bench's run-mode setup/teardown + * hooks so it brackets only tinybench's measured loop, excluding the warmup + * that runs beforehand and the statistics computation tinybench performs after + * the loop. Wrapping the whole `Task.run()` would otherwise fold all of that + * framework overhead into the recorded sample. + * + * User-provided hooks are preserved and keep their order relative to the work + * under test. + */ +export function installInstrumentHooks( + bench: TinybenchBench, + getUri: (taskName: string) => string, +): void { + const userSetup = bench.setup; + const userTeardown = bench.teardown; + + // Tasks of a bench run sequentially, so one window is open at a time. + let runStart: bigint | null = null; + + bench.setup = async (task, mode) => { + await userSetup(task, mode); + if (mode === "run") { + InstrumentHooks.startBenchmark(); + runStart = InstrumentHooks.currentTimestamp(); + } + }; + + bench.teardown = async (task, mode) => { + if (mode === "run") { + const pid = process.pid; + // The markers must land inside the sample window, so they go out before + // stopBenchmark(): one sent after it breaks the expected + // SampleStart > BenchmarkStart > BenchmarkEnd > SampleEnd nesting. + InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_START, runStart!); + InstrumentHooks.addMarker( + pid, + MARKER_TYPE_BENCHMARK_END, + InstrumentHooks.currentTimestamp(), + ); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(pid, getUri(task.name)); + runStart = null; + } + await userTeardown(task, mode); + }; +} + +/** Persist collected walltime benchmarks, if any, and log a summary. */ +export function writeAndLogWalltimeResults(benchmarks: Benchmark[]): void { + if (benchmarks.length === 0) { + return; + } + writeWalltimeResults(benchmarks); + console.log( + `[CodSpeed] Done collecting walltime data for ${benchmarks.length} benches.`, + ); +} + +/** + * Convert a completed tinybench task into a CodSpeed walltime benchmark. Returns + * null when the task produced no samples (e.g. fully optimized out), in which + * case there is nothing to record. + */ +export function tinybenchTaskToBenchmark( + task: TinybenchTask, + uri: string, + options: TinybenchOptions, +): Benchmark | null { + const stats = tinybenchResultToStats(task.result, options); + if (stats === null) { + return null; + } + + return { + name: task.name, + uri, + config: { + max_rounds: options.iterations ?? null, + max_time_ns: options.time ? msToNs(options.time) : null, + min_round_time_ns: null, // tinybench does not have an option for this + warmup_time_ns: + options.warmupIterations !== 0 && options.warmupTime + ? msToNs(options.warmupTime) + : null, + }, + stats, + }; +} + +function tinybenchResultToStats( + result: TinybenchTaskResult | undefined, + options: TinybenchOptions, +): BenchmarkStats | null { + if (!result) { + throw new Error("No benchmark data available in result"); + } + + const { totalTime, latency } = result; + const { min, max, mean, sd, samples } = latency; + + const sortedTimesNs = (samples ?? []).map(msToNs).sort((a, b) => a - b); + const meanNs = msToNs(mean); + const stdevNs = msToNs(sd); + + if (sortedTimesNs.length == 0) { + // Sometimes the benchmarks can be completely optimized out and not even + // run, but their beforeEach and afterEach hooks are still executed, and the + // task is still considered a success. + return null; + } + + const { q1_ns, q3_ns, median_ns, iqr_outlier_rounds, stdev_outlier_rounds } = + calculateQuantiles({ meanNs, stdevNs, sortedTimesNs }); + + return { + min_ns: msToNs(min), + max_ns: msToNs(max), + mean_ns: meanNs, + stdev_ns: stdevNs, + q1_ns, + median_ns, + q3_ns, + total_time: msToS(totalTime), + iter_per_round: 1, // tinybench runs one iteration per round + rounds: sortedTimesNs.length, + iqr_outlier_rounds, + stdev_outlier_rounds, + warmup_iters: options.warmupIterations ?? 0, + }; +} diff --git a/packages/vitest-plugin/src/analysis.ts b/packages/vitest-plugin/src/legacy/analysis.ts similarity index 85% rename from packages/vitest-plugin/src/analysis.ts rename to packages/vitest-plugin/src/legacy/analysis.ts index dc50a5e2..f71f0c0a 100644 --- a/packages/vitest-plugin/src/analysis.ts +++ b/packages/vitest-plugin/src/legacy/analysis.ts @@ -7,16 +7,20 @@ import { teardownCore, wrapWithRootFrame, } from "@codspeed/core"; -import type * as tinybench from "tinybench"; -import { Benchmark, type RunnerTestSuite } from "vitest"; +import { type RunnerTestSuite } from "vitest"; +import { type Tinybench } from "../instrument"; + import { callSuiteHook, isVitestTaskBenchmark, patchRootSuiteWithFullFilePath, } from "./common"; -import { getBenchFn, getBenchOptions, NodeBenchmarkRunner } from "./compat"; - -type Tinybench = typeof tinybench; +import { + getBenchFn, + getBenchOptions, + NodeBenchmarkRunner, + type BenchmarkTask, +} from "./compat"; const currentFileName = typeof __filename === "string" @@ -33,7 +37,7 @@ function logCodSpeed(message: string) { } async function runAnalysisBench( - benchmark: Benchmark, + benchmark: BenchmarkTask, suite: RunnerTestSuite, currentSuiteName: string, tinybenchModule: Tinybench, @@ -50,7 +54,6 @@ async function runAnalysisBench( await bench.setup(task, "warmup"); await optimizeFunction(async () => { await callSuiteHook(suite, benchmark, "beforeEach"); - // @ts-expect-error we do not need to bind the function to an instance of tinybench's Bench await fn(); await callSuiteHook(suite, benchmark, "afterEach"); }); @@ -62,7 +65,6 @@ async function runAnalysisBench( global.gc?.(); await wrapWithRootFrame(async () => { InstrumentHooks.startBenchmark(); - // @ts-expect-error we do not need to bind the function to an instance of tinybench's Bench await fn(); InstrumentHooks.stopBenchmark(); InstrumentHooks.setExecutedBenchmark(process.pid, uri); @@ -88,10 +90,10 @@ async function runAnalysisBenchmarkSuite( for (const task of suite.tasks) { if (task.mode !== "run") continue; - if (isVitestTaskBenchmark(task)) { - await runAnalysisBench(task, suite, currentSuiteName, tinybenchModule); - } else if (task.type === "suite") { + if (task.type === "suite") { await runAnalysisBenchmarkSuite(task, tinybenchModule, currentSuiteName); + } else if (isVitestTaskBenchmark(task)) { + await runAnalysisBench(task, suite, currentSuiteName, tinybenchModule); } } diff --git a/packages/vitest-plugin/src/common.ts b/packages/vitest-plugin/src/legacy/common.ts similarity index 74% rename from packages/vitest-plugin/src/common.ts rename to packages/vitest-plugin/src/legacy/common.ts index e853c307..0791b963 100644 --- a/packages/vitest-plugin/src/common.ts +++ b/packages/vitest-plugin/src/legacy/common.ts @@ -1,7 +1,7 @@ import { getGitDir } from "@codspeed/core"; import path from "path"; -import { Benchmark, type RunnerTask, type RunnerTestSuite } from "vitest"; -import { getHooks } from "./compat"; +import { type RunnerTask, type RunnerTestSuite } from "vitest"; +import { getHooks, type BenchmarkTask } from "./compat"; type SuiteHooks = ReturnType; function getSuiteHooks(suite: RunnerTestSuite, name: keyof SuiteHooks) { @@ -19,8 +19,9 @@ export async function callSuiteHook( const hooks = getSuiteHooks(suite, name); - // @ts-expect-error TODO: add support for hooks parameters - await Promise.all(hooks.map((fn) => fn())); + // TODO: add support for hook parameters. The hook signature differs across + // supported Vitest versions, so we call them through a parameterless cast. + await Promise.all((hooks as Array<() => unknown>).map((fn) => fn())); if (name === "afterEach" && suite?.suite) { await callSuiteHook(suite.suite, currentTask, name); @@ -35,6 +36,6 @@ export function patchRootSuiteWithFullFilePath(suite: RunnerTestSuite) { suite.name = path.relative(gitDir, suite.file.filepath); } -export function isVitestTaskBenchmark(task: RunnerTask): task is Benchmark { +export function isVitestTaskBenchmark(task: RunnerTask): task is BenchmarkTask { return task.type === "test" && task.meta.benchmark === true; } diff --git a/packages/vitest-plugin/src/legacy/compat.ts b/packages/vitest-plugin/src/legacy/compat.ts new file mode 100644 index 00000000..a5802de7 --- /dev/null +++ b/packages/vitest-plugin/src/legacy/compat.ts @@ -0,0 +1,93 @@ +import type { RunnerTestCase, RunnerTestSuite } from "vitest"; +import type { Tinybench } from "../instrument"; + +/** + * A Vitest 3/4 benchmark task: a test case flagged as a benchmark, carrying the + * raw tinybench output on its result. + */ +export interface BenchmarkTask extends RunnerTestCase { + meta: RunnerTestCase["meta"] & { benchmark?: boolean }; +} + +/** The tinybench options Vitest 3/4 stores alongside a registered benchmark. */ +export interface LegacyBenchOptions { + time?: number; + warmupTime?: number; + warmupIterations?: number; + iterations?: number; + setup?: ( + task: { name: string }, + mode: "run" | "warmup", + ) => void | Promise; + teardown?: ( + task: { name: string }, + mode: "run" | "warmup", + ) => void | Promise; +} + +/** + * The Vitest 3/4 benchmark backend, which Vitest 5 removed: benchmarks now run + * through a `benchmark.provider` (see `v5/provider.ts`) and the runner is gone. + * The shapes are declared here because neither the removed `vitest/runners` and + * `vitest/suite` subpaths nor the Vitest 5 typings this package compiles against + * describe them. + */ +export interface LegacyBenchmarkApi { + NodeBenchmarkRunner: new (config?: unknown) => { + config: unknown; + runSuite(suite: RunnerTestSuite): Promise; + importTinybench(): Promise; + }; + getHooks: (suite: unknown) => Record unknown>>; + getBenchFn: (benchmark: BenchmarkTask) => () => unknown; + getBenchOptions: (benchmark: BenchmarkTask) => LegacyBenchOptions; +} + +/** + * Vitest 4.1 moved the benchmark runner and the suite helpers to the main + * `vitest` entry point and deprecated the `vitest/runners` and `vitest/suite` + * subpaths, which warn on import. Both lookups have to be dynamic: which of the + * two shapes exists depends on the Vitest the user installed, and the subpaths + * don't resolve at all on Vitest 5. + */ +async function resolveVitestApi(): Promise { + // Vitest 5 exports an unrelated `TestRunner`, so the namespace has to be + // re-typed from scratch rather than narrowed. + const { BenchmarkRunner, TestRunner } = + (await import("vitest")) as unknown as { + BenchmarkRunner?: LegacyBenchmarkApi["NodeBenchmarkRunner"]; + TestRunner?: { + getSuiteHooks: LegacyBenchmarkApi["getHooks"]; + getBenchFn: LegacyBenchmarkApi["getBenchFn"]; + getBenchOptions: LegacyBenchmarkApi["getBenchOptions"]; + }; + }; + + if (BenchmarkRunner && TestRunner) { + return { + NodeBenchmarkRunner: BenchmarkRunner, + getHooks: TestRunner.getSuiteHooks, + getBenchFn: TestRunner.getBenchFn, + getBenchOptions: TestRunner.getBenchOptions, + }; + } + + // These subpaths only exist on Vitest 3/4, so they don't resolve when the + // plugin is installed alongside Vitest 5. + const [runners, suite] = await Promise.all([ + // eslint-disable-next-line import/no-unresolved + import("vitest/runners"), + // eslint-disable-next-line import/no-unresolved + import("vitest/suite"), + ]); + + return { + NodeBenchmarkRunner: runners.NodeBenchmarkRunner, + getHooks: suite.getHooks, + getBenchFn: suite.getBenchFn, + getBenchOptions: suite.getBenchOptions, + }; +} + +export const { NodeBenchmarkRunner, getHooks, getBenchFn, getBenchOptions } = + await resolveVitestApi(); diff --git a/packages/vitest-plugin/src/legacy/vitest-legacy.d.ts b/packages/vitest-plugin/src/legacy/vitest-legacy.d.ts new file mode 100644 index 00000000..be55606f --- /dev/null +++ b/packages/vitest-plugin/src/legacy/vitest-legacy.d.ts @@ -0,0 +1,19 @@ +// Vitest 3/4 exposed the benchmark internals through the `vitest/runners` and +// `vitest/suite` subpaths, which Vitest 5 removed. `legacy/compat.ts` imports +// them only when the installed Vitest doesn't expose them from its main entry +// point, i.e. only on Vitest 3/4 — but the package is type-checked against +// whichever Vitest is installed, including 5, where the subpaths don't resolve. +// These declarations keep that code compiling; nothing imports them under v5. +// +// This file must stay a script (no top-level imports): a `declare module` inside +// a module is an augmentation, which requires the module to resolve. + +declare module "vitest/runners" { + export const NodeBenchmarkRunner: import("./compat").LegacyBenchmarkApi["NodeBenchmarkRunner"]; +} + +declare module "vitest/suite" { + export const getHooks: import("./compat").LegacyBenchmarkApi["getHooks"]; + export const getBenchFn: import("./compat").LegacyBenchmarkApi["getBenchFn"]; + export const getBenchOptions: import("./compat").LegacyBenchmarkApi["getBenchOptions"]; +} diff --git a/packages/vitest-plugin/src/legacy/walltime-utils.ts b/packages/vitest-plugin/src/legacy/walltime-utils.ts new file mode 100644 index 00000000..fe8e042a --- /dev/null +++ b/packages/vitest-plugin/src/legacy/walltime-utils.ts @@ -0,0 +1,99 @@ +import { type Benchmark } from "@codspeed/core"; +import { type RunnerTaskResult, type RunnerTestSuite } from "vitest"; +import { tinybenchTaskToBenchmark, type TinybenchTask } from "../instrument"; +import { isVitestTaskBenchmark } from "./common"; +import { getBenchOptions, type BenchmarkTask } from "./compat"; + +export async function extractBenchmarkResults( + suite: RunnerTestSuite, + parentPath = "", +): Promise { + const benchmarks: Benchmark[] = []; + const currentPath = parentPath ? `${parentPath}::${suite.name}` : suite.name; + + for (const task of suite.tasks) { + if (task.type === "suite") { + const nestedBenchmarks = await extractBenchmarkResults(task, currentPath); + benchmarks.push(...nestedBenchmarks); + } else if (isVitestTaskBenchmark(task) && task.result?.state === "pass") { + const benchmark = processBenchmarkTask(task, currentPath); + if (benchmark) { + benchmarks.push(benchmark); + } + } + } + + return benchmarks; +} + +function processBenchmarkTask( + task: BenchmarkTask, + suitePath: string, +): Benchmark | null { + const uri = `${suitePath}::${task.name}`; + + const result = task.result; + if (!result) { + console.warn(` ⚠ No result data available for ${uri}`); + return null; + } + + try { + const benchmark = tinybenchTaskToBenchmark( + adaptLegacyResult(task.name, result), + uri, + getBenchOptions(task), + ); + + if (benchmark === null) { + console.log(` ✔ No walltime data to collect for ${uri}`); + return null; + } + + console.log(` ✔ Collected walltime data for ${uri}`); + return benchmark; + } catch (error) { + console.warn(` ⚠ Failed to process benchmark result for ${uri}:`, error); + return null; + } +} + +/** tinybench v2's flat statistics, which Vitest 3/4 stores under `result.benchmark`. */ +interface LegacyBenchmarkStats { + totalTime: number; + min: number; + max: number; + mean: number; + sd: number; + samples: number[]; +} + +/** + * Reshape a Vitest 3/4 result into the tinybench v6 form the shared converter + * expects, which nests the statistics under `latency`. + */ +function adaptLegacyResult( + name: string, + result: RunnerTaskResult, +): TinybenchTask { + // `result.benchmark` only exists on the Vitest 3/4 task result; the v5 typings + // (compiled against here) dropped it. + const benchmark = (result as { benchmark?: LegacyBenchmarkStats }).benchmark; + if (!benchmark) { + throw new Error("No benchmark data available in result"); + } + + return { + name, + result: { + totalTime: benchmark.totalTime, + latency: { + min: benchmark.min, + max: benchmark.max, + mean: benchmark.mean, + sd: benchmark.sd, + samples: benchmark.samples, + }, + }, + }; +} diff --git a/packages/vitest-plugin/src/walltime/index.ts b/packages/vitest-plugin/src/legacy/walltime.ts similarity index 50% rename from packages/vitest-plugin/src/walltime/index.ts rename to packages/vitest-plugin/src/legacy/walltime.ts index 880d9279..09271df8 100644 --- a/packages/vitest-plugin/src/walltime/index.ts +++ b/packages/vitest-plugin/src/legacy/walltime.ts @@ -1,54 +1,34 @@ -import { - InstrumentHooks, - MARKER_TYPE_BENCHMARK_END, - MARKER_TYPE_BENCHMARK_START, - setupCore, - wrapWithRootFrame, - writeWalltimeResults, -} from "@codspeed/core"; -import type * as tinybench from "tinybench"; +import { setupCore, wrapWithRootFrame } from "@codspeed/core"; import { RunnerTaskEventPack, RunnerTaskResultPack, type RunnerTestSuite, } from "vitest"; -import { patchRootSuiteWithFullFilePath } from "../common"; -import { NodeBenchmarkRunner } from "../compat"; -import { extractBenchmarkResults } from "./utils"; - -type Tinybench = typeof tinybench; - -/** A tinybench task, exposing the `fn` the runner wraps with the root frame. */ +import { + installInstrumentHooks, + writeAndLogWalltimeResults, + type Tinybench, + type TinybenchBench, +} from "../instrument"; +import { patchRootSuiteWithFullFilePath } from "./common"; +import { NodeBenchmarkRunner } from "./compat"; +import { extractBenchmarkResults } from "./walltime-utils"; + +/** A tinybench v2 task, whose `fn` is a plain, reassignable property. */ interface TinybenchTask { - name: string; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - fn: (...args: any[]) => any; -} - -/** tinybench's per-task setup/teardown hook signature. */ -type TinybenchHook = ( - task: TinybenchTask, - mode: "run" | "warmup", -) => Promise | void; - -/** The mutable subset of a tinybench Bench the runner reaches into. */ -interface TinybenchBench { - setup: TinybenchHook; - teardown: TinybenchHook; + fn: () => unknown; } /** - * WalltimeRunner uses Vitest's default benchmark execution - * and extracts results from the suite after completion + * Lets tinybench run the benches through Vitest's default benchmark execution, + * instrumenting each measured loop, then extracts the results from the suite + * tree afterwards. */ export class WalltimeRunner extends NodeBenchmarkRunner { private isTinybenchHookedWithCodspeed = false; private suiteUris = new Map(); /// Suite ID of the currently running suite, to allow constructing the URI in the context of tinybench tasks private currentSuiteId: string | null = null; - // Carries the window start timestamp from the setup hook to the teardown - // hook. Tasks run strictly sequentially, so a single field is enough. - private runStart: bigint | null = null; async runSuite(suite: RunnerTestSuite): Promise { patchRootSuiteWithFullFilePath(suite); @@ -59,17 +39,13 @@ export class WalltimeRunner extends NodeBenchmarkRunner { await super.runSuite(suite); const benchmarks = await extractBenchmarkResults(suite); - - if (benchmarks.length > 0) { - writeWalltimeResults(benchmarks); - console.log( - `[CodSpeed] Done collecting walltime data for ${benchmarks.length} benches.`, - ); - } else { + if (benchmarks.length === 0) { console.warn( `[CodSpeed] No benchmark results found after suite execution`, ); + return; } + writeAndLogWalltimeResults(benchmarks); } private populateBenchmarkUris(suite: RunnerTestSuite, parentPath = ""): void { @@ -112,8 +88,7 @@ export class WalltimeRunner extends NodeBenchmarkRunner { /** * Wrap each task's function with the root frame so collected stacks can be - * attributed to a benchmark. The window itself is driven by the bench's - * setup/teardown hooks (see createInstrumentedBench). + * attributed to a benchmark. */ private patchTaskWithRootFrame(tinybench: Tinybench): void { const originalRun = tinybench.Task.prototype.run; @@ -133,16 +108,6 @@ export class WalltimeRunner extends NodeBenchmarkRunner { }; } - /** - * Drive the instrumentation window from each bench's run-mode setup/teardown - * hooks so it brackets only tinybench's measured loop, excluding the warmup - * that Vitest runs beforehand and the statistics computation tinybench - * performs after the loop. Wrapping the whole `Task.run()` would otherwise - * fold all of that framework overhead into the recorded sample. - * - * User-provided hooks are preserved and keep their order relative to the work - * under test. - */ private createInstrumentedBench( tinybench: Tinybench, ): typeof tinybench.Bench { @@ -154,50 +119,15 @@ export class WalltimeRunner extends NodeBenchmarkRunner { // eslint-disable-next-line @typescript-eslint/no-explicit-any constructor(...benchArgs: any[]) { super(...benchArgs); - runner.installInstrumentHooks(this as unknown as TinybenchBench); + installInstrumentHooks(this as unknown as TinybenchBench, (taskName) => + runner.getBenchmarkUri(taskName), + ); } } return InstrumentedBench; } - private installInstrumentHooks(bench: TinybenchBench): void { - const userSetup = bench.setup; - const userTeardown = bench.teardown; - - bench.setup = async (task, mode) => { - await userSetup(task, mode); - if (mode === "run") { - InstrumentHooks.startBenchmark(); - this.runStart = InstrumentHooks.currentTimestamp(); - } - }; - - bench.teardown = async (task, mode) => { - if (mode === "run") { - this.closeInstrumentWindow(this.getBenchmarkUri(task.name)); - } - await userTeardown(task, mode); - }; - } - - private closeInstrumentWindow(uri: string): void { - const runEnd = InstrumentHooks.currentTimestamp(); - const pid = process.pid; - - // Benchmark markers must land inside the sample window opened by - // startBenchmark(), so they have to be emitted before stopBenchmark() - // closes it. The runner consumes the FIFO stream in order, so a marker - // sent after StopBenchmark falls outside the sample and breaks the - // expected SampleStart > BenchmarkStart > BenchmarkEnd > SampleEnd nesting. - InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_START, this.runStart!); - InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_END, runEnd); - - InstrumentHooks.stopBenchmark(); - InstrumentHooks.setExecutedBenchmark(pid, uri); - this.runStart = null; - } - // Allow tinybench to retrieve the path to the currently running suite async onTaskUpdate( _: RunnerTaskResultPack[], diff --git a/packages/vitest-plugin/src/runner.ts b/packages/vitest-plugin/src/runner.ts deleted file mode 100644 index 60a627a6..00000000 --- a/packages/vitest-plugin/src/runner.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { AnalysisRunner } from "./analysis"; - -export default AnalysisRunner; diff --git a/packages/vitest-plugin/src/v5/provider.ts b/packages/vitest-plugin/src/v5/provider.ts new file mode 100644 index 00000000..d46da02a --- /dev/null +++ b/packages/vitest-plugin/src/v5/provider.ts @@ -0,0 +1,334 @@ +import { + getGitDir, + getInstrumentMode, + InstrumentHooks, + optimizeFunction, + optimizeFunctionSync, + setupCore, + teardownCore, + wrapWithRootFrame, + wrapWithRootFrameSync, + type Benchmark, +} from "@codspeed/core"; +import { createRequire } from "module"; +import path from "path"; +import type { + BenchmarkGroup, + BenchmarkProvider, + BenchRegistrationInput, + BenchResult, + BenchRunOptions, +} from "vitest"; +import { + installInstrumentHooks, + tinybenchTaskToBenchmark, + writeAndLogWalltimeResults, + type TinybenchBench, + type TinybenchOptions, + type TinybenchTask, +} from "../instrument"; + +/** tinybench's per-task lifecycle hooks (a subset of `FnOptions`). */ +interface TinybenchFnOptions { + beforeAll?: (mode?: "run" | "warmup") => unknown; + beforeEach?: (mode?: "run" | "warmup") => unknown; + afterEach?: (mode?: "run" | "warmup") => unknown; + afterAll?: (mode?: "run" | "warmup") => unknown; +} + +type BenchFn = BenchRegistrationInput["fn"]; + +/** tinybench's statistics for one dimension (latency or throughput). */ +type BenchStatistics = BenchResult["latency"]; + +/** + * The subset of the host tinybench Bench the walltime path drives. The timing + * fields are the *resolved* options (tinybench exposes them as readonly + * instance properties), which is what the recorded benchmark config reports. + */ +interface TinybenchWithTasks extends TinybenchBench { + add: ( + name: string, + fn: BenchRegistrationInput["fn"], + fnOpts?: TinybenchFnOptions, + ) => unknown; + run: () => Promise; + tasks: TinybenchTask[]; + readonly iterations: number; + readonly time: number; + readonly warmup: boolean; + readonly warmupIterations: number; + readonly warmupTime: number; +} + +/** + * The host tinybench module, modeled structurally: the plugin's own tinybench + * dependency may be a different major, so its types can't describe the module + * resolved from the installed Vitest. `BenchRunOptions` is Vitest's re-export of + * the tinybench options it accepts, hence of the version it depends on. + */ +interface HostTinybench { + Bench: new (options?: BenchRunOptions) => TinybenchWithTasks; +} + +const isWalltime = getInstrumentMode() === "walltime"; + +// Vitest imports a provider module at most once per worker, and the plugin only +// wires this one up when CodSpeed drives the run (see `V5Backend`), so this is +// the per-worker setup. It must not be redone per benchmark: `setupCore()` +// truncates the process' perf map, which has to cover the whole worker. +setupCore(); +process.once("beforeExit", () => teardownCore()); + +/** + * Build the URI prefix shared by every benchmark of a group: the git-relative + * file path followed by the suite/test path, `::`-separated (e.g. + * `src/a.bench.ts::my suite::my test`). Each registration name is appended to + * it, so a group registering several benchmarks (`bench.compare()`) reports one + * URI per benchmark. + */ +function buildGroupUri(test: BenchmarkGroup["test"]): string { + const filepath = test.file?.filepath; + if (!filepath) { + throw new Error("[CodSpeed] could not resolve the running benchmark file"); + } + const gitDir = getGitDir(filepath); + if (gitDir === undefined) { + throw new Error("Could not find a git repository"); + } + const relativeFile = path.relative(gitDir, filepath); + // `fullTestName` uses " > " between suite levels; normalize to "::". + const testPath = test.fullTestName.split(" > ").join("::"); + return [relativeFile, testPath].filter(Boolean).join("::"); +} + +/** + * Resolve the tinybench the *host* Vitest uses so walltime mode returns results + * in the exact shape Vitest serializes. The plugin's own tinybench may be a + * different major, so it is resolved relative to the installed Vitest. + */ +async function importHostTinybench(): Promise { + const require = createRequire(import.meta.url); + const vitestRequire = createRequire(require.resolve("vitest/package.json")); + return import(vitestRequire.resolve("tinybench")); +} + +function analysisResult(name: string): BenchResult { + // Zeroed statistics: enough to satisfy Vitest's reporter and its serializer. + const statistics: BenchStatistics = { + aad: 0, + critical: 0, + df: 0, + mad: 0, + max: 0, + mean: 0, + min: 0, + moe: 0, + p50: 0, + p75: 0, + p99: 0, + p995: 0, + p999: 0, + rme: 0, + samples: undefined, + samplesCount: 0, + sd: 0, + sem: 0, + variance: 0, + }; + + return { + name, + state: "completed", + latency: { ...statistics }, + throughput: { ...statistics }, + period: 0, + totalTime: 0, + runtime: "node", + runtimeVersion: process.versions.node, + timestampProviderName: "codspeed", + }; +} + +/** + * The CodSpeed benchmark provider. Two modes: + * - analysis (instrumentation/simulation): CodSpeed runs each fn itself under a + * tight instrument window and returns zeroed-but-valid results — the real + * measurement is captured by the instrument, not returned here. + * - walltime: tinybench drives the measured loop; CodSpeed brackets it with the + * instrument window and converts each task's stats into a result. + */ +const provider: BenchmarkProvider = { + async run({ test, registrations, options }): Promise { + // Resolve the URI up front, outside any measured window — it walks the + // filesystem (git root lookup), which must not land inside a sample. + const groupUri = buildGroupUri(test); + const getUri = (name: string) => `${groupUri}::${name}`; + + if (isWalltime) { + return runWalltime(registrations, options, getUri, test); + } + return runAnalysis(registrations, getUri); + }, +}; + +async function runAnalysis( + registrations: BenchRegistrationInput[], + getUri: (name: string) => string, +): Promise { + const label = InstrumentHooks.isInstrumented() ? "Measured" : "Checked"; + const results: BenchResult[] = []; + for (const { name, fn, fnOpts } of registrations) { + const uri = getUri(name); + await runAnalysisTask(fn, fnOpts, uri); + console.log(`[CodSpeed] ${label} ${uri}`); + results.push(analysisResult(name)); + } + return results; +} + +/** + * Run one benchmark under instrumentation: warm the JIT outside the window, then + * bracket a single `fn()` call with `startBenchmark`/`stopBenchmark` under the + * root frame, with the user hooks around it. + * + * Synchronous benchmarks get a fully synchronous window: awaiting a sync fn + * would splice Node's promise-hook machinery above the root frame. + */ +async function runAnalysisTask( + fn: BenchFn, + fnOpts: TinybenchFnOptions | undefined, + uri: string, +): Promise { + if (fn.constructor?.name === "AsyncFunction") { + await fnOpts?.beforeAll?.("run"); + await optimizeFunction(async () => { + await fnOpts?.beforeEach?.("run"); + await fn(); + await fnOpts?.afterEach?.("run"); + }); + + await fnOpts?.beforeEach?.("run"); + global.gc?.(); + await wrapWithRootFrame(async () => { + InstrumentHooks.startBenchmark(); + await fn(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + })(); + await fnOpts?.afterEach?.("run"); + await fnOpts?.afterAll?.("run"); + return; + } + + fnOpts?.beforeAll?.("run"); + optimizeFunctionSync(() => { + fnOpts?.beforeEach?.("run"); + fn(); + fnOpts?.afterEach?.("run"); + }); + + fnOpts?.beforeEach?.("run"); + global.gc?.(); + wrapWithRootFrameSync(() => { + InstrumentHooks.startBenchmark(); + fn(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + })(); + fnOpts?.afterEach?.("run"); + fnOpts?.afterAll?.("run"); +} + +async function runWalltime( + registrations: BenchRegistrationInput[], + options: BenchRunOptions | undefined, + getUri: (name: string) => string, + test: BenchmarkGroup["test"], +): Promise { + const { Bench } = await importHostTinybench(); + const bench = new Bench({ + signal: test.context.signal, + ...options, + // walltime needs per-iteration samples to compute quantiles + retainSamples: true, + }); + + for (const { name, fn, fnOpts } of registrations) { + // tinybench v6 keeps `fn` private, so the root frame is baked in here + bench.add( + name, + wrapWithRootFrame(() => fn()), + fnOpts, + ); + } + + installInstrumentHooks(bench, getUri); + const tasks = await bench.run(); + + throwOnErroredTasks(tasks); + collectWalltimeResults(bench, getUri); + + return tasks.map(tinybenchTaskToBenchResult); +} + +/** + * Surface benchmark failures the way Vitest's own provider does: a single error + * is rethrown as is, several are aggregated, and the test fails instead of + * reporting empty results. + */ +function throwOnErroredTasks(tasks: TinybenchTask[]): void { + const errors = tasks + .filter((task) => task.result?.state === "errored") + .map((task) => task.result?.error); + if (errors.length === 1) { + throw errors[0]; + } + if (errors.length > 1) { + throw new AggregateError(errors, "Some benchmarks failed"); + } +} + +function collectWalltimeResults( + bench: TinybenchWithTasks, + getUri: (name: string) => string, +): void { + const options: TinybenchOptions = { + time: bench.time, + iterations: bench.iterations, + // A bench with warmup turned off keeps its warmup timings, so they'd + // otherwise be reported as if a warmup had run. + warmupTime: bench.warmup ? bench.warmupTime : 0, + warmupIterations: bench.warmup ? bench.warmupIterations : 0, + }; + const benchmarks: Benchmark[] = []; + for (const task of bench.tasks) { + if (task.result?.state !== "completed") continue; + const benchmark = tinybenchTaskToBenchmark( + task, + getUri(task.name), + options, + ); + if (benchmark) { + benchmarks.push(benchmark); + } + } + writeAndLogWalltimeResults(benchmarks); +} + +function tinybenchTaskToBenchResult(task: TinybenchTask): BenchResult { + const result = task.result; + if (result?.state !== "completed") { + throw new Error( + `[CodSpeed] benchmark "${task.name}" did not complete: received "${result?.state ?? "no result"}"`, + ); + } + // tinybench's completed result already carries the full statistics surface + // Vitest serializes; pass it through, keyed by the task name. + return { + ...(result as unknown as Omit), + name: task.name, + }; +} + +export default provider; diff --git a/packages/vitest-plugin/src/vitestBackend.ts b/packages/vitest-plugin/src/vitestBackend.ts new file mode 100644 index 00000000..441912dc --- /dev/null +++ b/packages/vitest-plugin/src/vitestBackend.ts @@ -0,0 +1,128 @@ +import { getInstrumentMode } from "@codspeed/core"; +import { readFileSync } from "fs"; +import { createRequire } from "module"; +import { join } from "path"; +import { type ViteUserConfig } from "vitest/config"; + +/** + * Everything about integrating with Vitest that depends on which Vitest + * generation the user installed, resolved once so the rest of the plugin reads a + * `VitestBackend` and never inspects the version itself. + */ +export interface VitestBackend { + /** + * Whether the plugin applies for this Vite `mode`; when false it is dropped + * entirely and injects nothing. + */ + isActive(mode: string): boolean; + + /** + * The `test` config fragment that wires the benchmark instrumentation into + * Vitest: the V8 exec args (whose placement moved across versions) plus the + * integration seam. Legacy wires a custom runner subclass (and, in walltime + * mode, asks tinybench to retain samples); v5 wires a `benchmark.provider` + * that owns execution and sample retention entirely. + */ + getBenchmarkTestConfig( + v8Flags: string[], + resolveFile: (name: string) => string, + ): ViteUserConfig["test"]; +} + +/** + * The integration seam differs across Vitest generations: on 3/4 benchmarks run + * through a `NodeBenchmarkRunner` subclass (via `vitest/runners` / + * `vitest/suite`), while 5+ exposes a `benchmark.provider` API and dropped those + * entrypoints. + * + * When the version cannot be detected we assume the latest supported major. + */ +export function resolveVitestBackend(): VitestBackend { + const major = getVitestMajorVersion() ?? 5; + return major >= 5 ? new V5Backend() : new LegacyBackend(major); +} + +/** + * Resolve the major version of the Vitest the *user's project* depends on, not + * the one bundled alongside this plugin. Returns null when it cannot be found, + * letting `resolveVitestBackend` fall back to the latest supported major. + */ +function getVitestMajorVersion(): number | null { + try { + const require = createRequire(join(process.cwd(), "package.json")); + const vitestPkgPath = require.resolve("vitest/package.json"); + const vitestPkg = JSON.parse(readFileSync(vitestPkgPath, "utf-8")); + return parseInt(vitestPkg.version.split(".")[0], 10); + } catch { + return null; + } +} + +/** + * Vitest 5+. Instrumentation is installed through a `benchmark.provider` that + * owns benchmark execution (see `v5/provider.ts`). + */ +class V5Backend implements VitestBackend { + /** + * `vitest bench` no longer has a dedicated Vite mode, and Vitest clones its + * benchmark project *after* the config hooks ran, so a benchmark run can't be + * read off the config. CodSpeed only drives benchmark runs, so gate on the + * instrument mode: without CodSpeed Vitest runs its own tinybench provider. + */ + isActive(): boolean { + return getInstrumentMode() !== "disabled"; + } + + getBenchmarkTestConfig( + v8Flags: string[], + resolveFile: (name: string) => string, + ): ViteUserConfig["test"] { + return { + execArgv: v8Flags, + // The provider owns benchmark execution: it runs the registered functions + // under instrumentation (analysis) or drives tinybench itself (walltime). + benchmark: { provider: resolveFile("v5/provider") }, + }; + } +} + +/** + * Vitest 3/4. `vitest bench` runs under a dedicated `"benchmark"` Vite mode, and + * instrumentation is installed through a custom `test.runner` subclass of + * `NodeBenchmarkRunner`, one per instrument mode (`analysis` / `walltime`). + */ +class LegacyBackend implements VitestBackend { + constructor(private readonly major: number) {} + + isActive(mode: string): boolean { + return mode === "benchmark"; + } + + getBenchmarkTestConfig( + v8Flags: string[], + resolveFile: (name: string) => string, + ): ViteUserConfig["test"] { + const instrumentMode = getInstrumentMode(); + const runner = + instrumentMode === "disabled" + ? undefined + : resolveFile(join("legacy", instrumentMode)); + + // Walltime asks tinybench to retain per-iteration samples so the runner can + // compute quantiles. On tinybench v2 (Vitest 3/4) the option is + // `includeSamples` (renamed `retainSamples` in v6). + const benchmark = + instrumentMode === "walltime" ? { includeSamples: true } : undefined; + + return { + // Vitest 3 nests exec args under `poolOptions.forks`; v4 moved them to a + // top-level `test.execArgv`. + // See: https://vitest.dev/guide/migration.html#pool-rework + ...(this.major >= 4 + ? { execArgv: v8Flags } + : { poolOptions: { forks: { execArgv: v8Flags } } }), + ...(runner && { runner }), + ...(benchmark && { benchmark }), + } as ViteUserConfig["test"]; + } +} diff --git a/packages/vitest-plugin/src/walltime/utils.ts b/packages/vitest-plugin/src/walltime/utils.ts deleted file mode 100644 index e0e3313e..00000000 --- a/packages/vitest-plugin/src/walltime/utils.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { - calculateQuantiles, - msToNs, - msToS, - type Benchmark, - type BenchmarkStats, -} from "@codspeed/core"; -import { - type RunnerTaskResult, - type RunnerTestSuite, - type Benchmark as VitestBenchmark, -} from "vitest"; -import { isVitestTaskBenchmark } from "../common"; -import { getBenchOptions } from "../compat"; - -export async function extractBenchmarkResults( - suite: RunnerTestSuite, - parentPath = "", -): Promise { - const benchmarks: Benchmark[] = []; - const currentPath = parentPath ? `${parentPath}::${suite.name}` : suite.name; - - for (const task of suite.tasks) { - if (isVitestTaskBenchmark(task) && task.result?.state === "pass") { - const benchmark = await processBenchmarkTask(task, currentPath); - if (benchmark) { - benchmarks.push(benchmark); - } - } else if (task.type === "suite") { - const nestedBenchmarks = await extractBenchmarkResults(task, currentPath); - benchmarks.push(...nestedBenchmarks); - } - } - - return benchmarks; -} - -async function processBenchmarkTask( - task: VitestBenchmark, - suitePath: string, -): Promise { - const uri = `${suitePath}::${task.name}`; - - const result = task.result; - if (!result) { - console.warn(` ⚠ No result data available for ${uri}`); - return null; - } - - try { - // Get tinybench configuration options from vitest - const benchOptions = getBenchOptions(task); - - const stats = convertVitestResultToBenchmarkStats(result, benchOptions); - - if (stats === null) { - console.log(` ✔ No walltime data to collect for ${uri}`); - return null; - } - - const coreBenchmark: Benchmark = { - name: task.name, - uri, - config: { - max_rounds: benchOptions.iterations ?? null, - max_time_ns: benchOptions.time ? msToNs(benchOptions.time) : null, - min_round_time_ns: null, // tinybench does not have an option for this - warmup_time_ns: - benchOptions.warmupIterations !== 0 && benchOptions.warmupTime - ? msToNs(benchOptions.warmupTime) - : null, - }, - stats, - }; - - console.log(` ✔ Collected walltime data for ${uri}`); - return coreBenchmark; - } catch (error) { - console.warn(` ⚠ Failed to process benchmark result for ${uri}:`, error); - return null; - } -} - -function convertVitestResultToBenchmarkStats( - result: RunnerTaskResult, - benchOptions: { - time?: number; - warmupTime?: number; - warmupIterations?: number; - iterations?: number; - }, -): BenchmarkStats | null { - const benchmark = result.benchmark; - - if (!benchmark) { - throw new Error("No benchmark data available in result"); - } - - const { totalTime, min, max, mean, sd, samples } = benchmark; - - // Get individual sample times in nanoseconds and sort them - const sortedTimesNs = samples.map(msToNs).sort((a, b) => a - b); - const meanNs = msToNs(mean); - const stdevNs = msToNs(sd); - - if (sortedTimesNs.length == 0) { - // Sometimes the benchmarks can be completely optimized out and not even run, but its beforeEach and afterEach hooks are still executed, and the task is still considered a success. - // This is the case for the hooks.bench.ts example in this package - return null; - } - - const { q1_ns, q3_ns, median_ns, iqr_outlier_rounds, stdev_outlier_rounds } = - calculateQuantiles({ meanNs, stdevNs, sortedTimesNs }); - - return { - min_ns: msToNs(min), - max_ns: msToNs(max), - mean_ns: meanNs, - stdev_ns: stdevNs, - q1_ns, - median_ns, - q3_ns, - total_time: msToS(totalTime), - iter_per_round: 1, // as there is only one round in tinybench, we define that there were n rounds of 1 iteration - rounds: sortedTimesNs.length, - iqr_outlier_rounds, - stdev_outlier_rounds, - warmup_iters: benchOptions.warmupIterations ?? 0, - }; -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3e43f6b7..9bf9e0c6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -368,8 +368,8 @@ importers: specifier: ^8.0.0 version: 8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0) vitest: - specifier: ^4.1.11 - version: 4.1.11(@types/node@26.4.1)(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) + specifier: ^5.0.0 + version: 5.0.2(@types/node@26.4.1)(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) packages: @@ -1828,12 +1828,18 @@ packages: '@jridgewell/sourcemap-codec@1.5.5': resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} + '@jridgewell/sourcemap-codec@1.6.0': + resolution: {integrity: sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==} + '@jridgewell/trace-mapping@0.3.18': resolution: {integrity: sha512-w+niJYzMHdd7USdiH2U6869nqhD2nbfZXND5Yp93qIbEmnDNk7PD48o+YchRVpzMU7M6jVCbenTR7PA1FLQ9pA==} '@jridgewell/trace-mapping@0.3.29': resolution: {integrity: sha512-uw6guiW/gcAGPDhLmd77/6lW8QLeiV5RUTsAX46Db6oLhGaVj4lhnPwb184s1bkc8kdVg/+h988dro8GRDpmYQ==} + '@jridgewell/trace-mapping@0.3.31': + resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@jridgewell/trace-mapping@0.3.9': resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} @@ -2726,9 +2732,6 @@ packages: '@vitest/expect@4.0.18': resolution: {integrity: sha512-8sCWUyckXXYvx4opfzVY03EOiYVxyNrHS5QxX3DAIi5dpJAAkyJezHCP77VMX4HKA2LDT/Jpfo8i2r5BE3GnQQ==} - '@vitest/expect@4.1.11': - resolution: {integrity: sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==} - '@vitest/mocker@3.2.4': resolution: {integrity: sha512-46ryTE9RZO/rfDd7pEqFl7etuyzekzEhUbTW3BvmeO/BcCMEgq59BKhek3dXDWgAj4oMK6OZi+vRr1wPW6qjEQ==} peerDependencies: @@ -2751,8 +2754,8 @@ packages: vite: optional: true - '@vitest/mocker@4.1.11': - resolution: {integrity: sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ==} + '@vitest/mocker@5.0.2': + resolution: {integrity: sha512-Z5FS00Q1SJHkB35xATsmWGdQ5WA1/0MV3CDjqyv7GavHv1OfOj145MNfHOlHk7QLes21dKFDHr8EO2zvL+9WGA==} peerDependencies: msw: ^2.4.9 vite: ^6.0.0 || ^7.0.0 || ^8.0.0 @@ -2768,35 +2771,26 @@ packages: '@vitest/pretty-format@4.0.18': resolution: {integrity: sha512-P24GK3GulZWC5tz87ux0m8OADrQIUVDPIjjj65vBXYG17ZeU3qD7r+MNZ1RNv4l8CGU2vtTRqixrOi9fYk/yKw==} - '@vitest/pretty-format@4.1.11': - resolution: {integrity: sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw==} - '@vitest/runner@3.2.4': resolution: {integrity: sha512-oukfKT9Mk41LreEW09vt45f8wx7DordoWUZMYdY/cyAk7w5TWkTRCNZYF7sX7n2wB7jyGAl74OxgwhPgKaqDMQ==} '@vitest/runner@4.0.18': resolution: {integrity: sha512-rpk9y12PGa22Jg6g5M3UVVnTS7+zycIGk9ZNGN+m6tZHKQb7jrP7/77WfZy13Y/EUDd52NDsLRQhYKtv7XfPQw==} - '@vitest/runner@4.1.11': - resolution: {integrity: sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw==} - '@vitest/snapshot@3.2.4': resolution: {integrity: sha512-dEYtS7qQP2CjU27QBC5oUOxLE/v5eLkGqPE0ZKEIDGMs4vKWe7IjgLOeauHsR0D5YuuycGRO5oSRXnwnmA78fQ==} '@vitest/snapshot@4.0.18': resolution: {integrity: sha512-PCiV0rcl7jKQjbgYqjtakly6T1uwv/5BQ9SwBLekVg/EaYeQFPiXcgrC2Y7vDMA8dM1SUEAEV82kgSQIlXNMvA==} - '@vitest/snapshot@4.1.11': - resolution: {integrity: sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog==} - '@vitest/spy@3.2.4': resolution: {integrity: sha512-vAfasCOe6AIK70iP5UD11Ac4siNUNJ9i/9PZ3NKx07sG6sUxeag1LWdNrMWeKKYBLlzuK+Gn65Yd5nyL6ds+nw==} '@vitest/spy@4.0.18': resolution: {integrity: sha512-cbQt3PTSD7P2OARdVW3qWER5EGq7PHlvE+QfzSC0lbwO+xnt7+XH06ZzFjFRgzUX//JmpxrCu92VdwvEPlWSNw==} - '@vitest/spy@4.1.11': - resolution: {integrity: sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==} + '@vitest/spy@5.0.2': + resolution: {integrity: sha512-Ijc7T1nT9efNb5LxvjaBrEqw3f/QwUv5EE0nKqZxgqsaV/FxAAZ8baGylA8X/Z2oS4Lp+K74Jr6dTJsDKxJDeg==} '@vitest/utils@3.2.4': resolution: {integrity: sha512-fB2V0JFrQSMsCo9HiSq3Ezpdv4iYaXRG1Sx8edX3MwxfyNn83mKiGzOcH+Fkxt4MHxr3y42fQi1oeAInqgX2QA==} @@ -2804,9 +2798,6 @@ packages: '@vitest/utils@4.0.18': resolution: {integrity: sha512-msMRKLMVLWygpK3u2Hybgi4MNjcYJvwTb0Ru09+fOyCXIgT5raYP041DRRdiJiI3k/2U6SEbAETB3YtBrUkCFA==} - '@vitest/utils@4.1.11': - resolution: {integrity: sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==} - '@yarnpkg/lockfile@1.1.0': resolution: {integrity: sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ==} @@ -5119,6 +5110,9 @@ packages: resolution: {integrity: sha512-7xlpfBaQaP/T6Vh8MO/EqXSW5En6INHEvEXQiuff7Gku0PWjU3uf6w/j9o7O+SpB5fOAkrI5HeoNgwjEO0pFsA==} engines: {node: '>=12'} + magic-string@1.4.2: + resolution: {integrity: sha512-vG+rjFRj1PqdIBozIxAGMjPlOhaVe+GXpbttY/iSK7rGcJRMlwNJO7dcUwmUqkymsFLJiNGI06t4D7Fr7yRC9g==} + make-dir@2.1.0: resolution: {integrity: sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==} engines: {node: '>=6'} @@ -5498,6 +5492,10 @@ packages: obug@2.1.1: resolution: {integrity: sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==} + obug@2.2.1: + resolution: {integrity: sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q==} + engines: {node: '>=12.20.0'} + once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} @@ -6387,6 +6385,10 @@ packages: resolution: {integrity: sha512-FlHoQpcFvCzeXK5kVPvV7IVgW/hs/B36QWTz876iSdeJguBDfdTSRQmYmaHX+fQNt4hp+gEFB2XXw+8hT4/y8A==} engines: {node: '>=20.0.0'} + tinybench@6.2.0: + resolution: {integrity: sha512-78U2TlB2CnVenajOFzf3BKSm0J6oz5L0NV7g32LCPccvYc0lbWvys4d3uUUCS2B1N8PAf2+aekR8i1KbC3HO7Q==} + engines: {node: '>=20.0.0'} + tinyexec@0.3.2: resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} @@ -6394,6 +6396,10 @@ packages: resolution: {integrity: sha512-W/KYk+NFhkmsYpuHq5JykngiOCnxeVL8v8dFnqxSD8qEEdRfXk1SDM6JzNqcERbcGYj9tMrDQBYV9cjgnunFIg==} engines: {node: '>=18'} + tinyexec@1.3.1: + resolution: {integrity: sha512-GCvB3aoys96IuDFBMcTB46JOR6mdMtAToqwiW8JlWhsoh1mhHi/xn9ss/Dg7N555GiJyEt2qzoG/NHCwM6h1EA==} + engines: {node: '>=18'} + tinyglobby@0.2.12: resolution: {integrity: sha512-qkf4trmKSIiMTs/E63cxH+ojC2unam7rJ0WrauAzpT3ECNTxGRMlaXxVbfxMUC/w0LaYk6jQ4y/nGR9uBO3tww==} engines: {node: '>=12.0.0'} @@ -6422,10 +6428,6 @@ packages: resolution: {integrity: sha512-PSkbLUoxOFRzJYjjxHJt9xro7D+iilgMX/C9lawzVuYiIdcihh9DXmVibBe8lmcFrRi/VzlPjBxbN7rH24q8/Q==} engines: {node: '>=14.0.0'} - tinyrainbow@3.1.1: - resolution: {integrity: sha512-yau8yJdTt989Mm0Bd/236QnzEiPf2xLLTqUZRUJOo/3CB078LSwzei343DgtJVmfJKJE3TMINY1u42SQsP6mXw==} - engines: {node: '>=14.0.0'} - tinyspy@4.0.3: resolution: {integrity: sha512-t2T/WLB2WRgZ9EpE4jgPJ9w+i66UZfDc8wHh0xrwiRNN+UwH98GIJkTeZqX9rg0i0ptwzqW+uYeIF0T4F8LR7A==} engines: {node: '>=14.0.0'} @@ -6831,23 +6833,23 @@ packages: jsdom: optional: true - vitest@4.1.11: - resolution: {integrity: sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw==} - engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} + vitest@5.0.2: + resolution: {integrity: sha512-7MQrx9pDv5aHiUcovIb/70Ys3tgtkUVgCtledvKdCmEO+/1Dicq5ZqoSxOW034m03oqC+oHOKui2dM6qtMLoJg==} + engines: {node: ^22.12.0 || ^24.0.0 || >=26.0.0} hasBin: true peerDependencies: '@edge-runtime/vm': '*' '@opentelemetry/api': ^1.9.0 - '@types/node': ^20.0.0 || ^22.0.0 || >=24.0.0 - '@vitest/browser-playwright': 4.1.11 - '@vitest/browser-preview': 4.1.11 - '@vitest/browser-webdriverio': 4.1.11 - '@vitest/coverage-istanbul': 4.1.11 - '@vitest/coverage-v8': 4.1.11 - '@vitest/ui': 4.1.11 + '@types/node': ^22.0.0 || >=24.0.0 + '@vitest/browser-playwright': 5.0.2 + '@vitest/browser-preview': 5.0.2 + '@vitest/browser-webdriverio': ^5.0.0-beta.5 || >=5.0.0 + '@vitest/coverage-istanbul': 5.0.2 + '@vitest/coverage-v8': 5.0.2 + '@vitest/ui': 5.0.2 happy-dom: '*' jsdom: '*' - vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + vite: ^6.4.0 || ^7.0.0 || ^8.0.0 peerDependenciesMeta: '@edge-runtime/vm': optional: true @@ -6923,6 +6925,11 @@ packages: engines: {node: '>=8'} hasBin: true + why-is-node-running@3.2.2: + resolution: {integrity: sha512-NKUzAelcoCXhXL4dJzKIwXeR8iEVqsA0Lq6Vnd0UXvgaKbzVo4ZTHROF2Jidrv+SgxOQ03fMinnNhzZATxOD3A==} + engines: {node: '>=20.11'} + hasBin: true + wide-align@1.1.5: resolution: {integrity: sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==} @@ -7098,7 +7105,7 @@ snapshots: dependencies: '@babel/types': 7.22.5 '@jridgewell/gen-mapping': 0.3.3 - '@jridgewell/trace-mapping': 0.3.18 + '@jridgewell/trace-mapping': 0.3.29 jsesc: 2.5.2 '@babel/generator@7.28.0': @@ -8687,7 +8694,7 @@ snapshots: '@jest/source-map@29.4.3': dependencies: - '@jridgewell/trace-mapping': 0.3.18 + '@jridgewell/trace-mapping': 0.3.29 callsites: 3.1.0 graceful-fs: 4.2.11 @@ -8792,7 +8799,7 @@ snapshots: dependencies: '@jridgewell/set-array': 1.1.2 '@jridgewell/sourcemap-codec': 1.5.4 - '@jridgewell/trace-mapping': 0.3.18 + '@jridgewell/trace-mapping': 0.3.29 '@jridgewell/resolve-uri@3.1.0': {} @@ -8810,6 +8817,8 @@ snapshots: '@jridgewell/sourcemap-codec@1.5.5': {} + '@jridgewell/sourcemap-codec@1.6.0': {} + '@jridgewell/trace-mapping@0.3.18': dependencies: '@jridgewell/resolve-uri': 3.1.0 @@ -8820,10 +8829,15 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 + '@jridgewell/trace-mapping@0.3.31': + dependencies: + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.5.5 + '@jridgewell/trace-mapping@0.3.9': dependencies: '@jridgewell/resolve-uri': 3.1.1 - '@jridgewell/sourcemap-codec': 1.5.4 + '@jridgewell/sourcemap-codec': 1.5.5 '@jsdevtools/ono@7.1.3': {} @@ -9767,15 +9781,6 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.0.3 - '@vitest/expect@4.1.11': - dependencies: - '@standard-schema/spec': 1.1.0 - '@types/chai': 5.2.2 - '@vitest/spy': 4.1.11 - '@vitest/utils': 4.1.11 - chai: 6.2.2 - tinyrainbow: 3.1.1 - '@vitest/mocker@3.2.4(vite@7.1.3(@types/node@26.4.1)(lightningcss@1.33.0)(tsx@4.23.12)(yaml@2.9.0))': dependencies: '@vitest/spy': 3.2.4 @@ -9792,11 +9797,12 @@ snapshots: optionalDependencies: vite: 7.1.3(@types/node@26.4.1)(lightningcss@1.33.0)(tsx@4.23.12)(yaml@2.9.0) - '@vitest/mocker@4.1.11(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0))': + '@vitest/mocker@5.0.2(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0))': dependencies: - '@vitest/spy': 4.1.11 + '@jridgewell/trace-mapping': 0.3.31 + '@vitest/spy': 5.0.2 estree-walker: 3.0.3 - magic-string: 0.30.21 + magic-string: 1.4.2 optionalDependencies: vite: 8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0) @@ -9808,10 +9814,6 @@ snapshots: dependencies: tinyrainbow: 3.0.3 - '@vitest/pretty-format@4.1.11': - dependencies: - tinyrainbow: 3.1.1 - '@vitest/runner@3.2.4': dependencies: '@vitest/utils': 3.2.4 @@ -9823,11 +9825,6 @@ snapshots: '@vitest/utils': 4.0.18 pathe: 2.0.3 - '@vitest/runner@4.1.11': - dependencies: - '@vitest/utils': 4.1.11 - pathe: 2.0.3 - '@vitest/snapshot@3.2.4': dependencies: '@vitest/pretty-format': 3.2.4 @@ -9840,20 +9837,13 @@ snapshots: magic-string: 0.30.21 pathe: 2.0.3 - '@vitest/snapshot@4.1.11': - dependencies: - '@vitest/pretty-format': 4.1.11 - '@vitest/utils': 4.1.11 - magic-string: 0.30.21 - pathe: 2.0.3 - '@vitest/spy@3.2.4': dependencies: tinyspy: 4.0.3 '@vitest/spy@4.0.18': {} - '@vitest/spy@4.1.11': {} + '@vitest/spy@5.0.2': {} '@vitest/utils@3.2.4': dependencies: @@ -9866,12 +9856,6 @@ snapshots: '@vitest/pretty-format': 4.0.18 tinyrainbow: 3.0.3 - '@vitest/utils@4.1.11': - dependencies: - '@vitest/pretty-format': 4.1.11 - convert-source-map: 2.0.0 - tinyrainbow: 3.1.1 - '@yarnpkg/lockfile@1.1.0': {} '@yarnpkg/parsers@3.0.2': @@ -12887,6 +12871,10 @@ snapshots: dependencies: '@jridgewell/sourcemap-codec': 1.4.15 + magic-string@1.4.2: + dependencies: + '@jridgewell/sourcemap-codec': 1.6.0 + make-dir@2.1.0: dependencies: pify: 4.0.1 @@ -13354,6 +13342,8 @@ snapshots: obug@2.1.1: {} + obug@2.2.1: {} + once@1.4.0: dependencies: wrappy: 1.0.2 @@ -14304,10 +14294,14 @@ snapshots: tinybench@6.0.2: {} + tinybench@6.2.0: {} + tinyexec@0.3.2: {} tinyexec@1.0.2: {} + tinyexec@1.3.1: {} + tinyglobby@0.2.12: dependencies: fdir: 6.5.0(picomatch@4.0.7) @@ -14334,8 +14328,6 @@ snapshots: tinyrainbow@3.0.3: {} - tinyrainbow@3.1.1: {} - tinyspy@4.0.3: {} tmp@0.0.33: @@ -14587,7 +14579,7 @@ snapshots: v8-to-istanbul@9.1.0: dependencies: - '@jridgewell/trace-mapping': 0.3.18 + '@jridgewell/trace-mapping': 0.3.29 '@types/istanbul-lib-coverage': 2.0.4 convert-source-map: 1.9.0 @@ -14732,28 +14724,22 @@ snapshots: - tsx - yaml - vitest@4.1.11(@types/node@26.4.1)(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)): + vitest@5.0.2(@types/node@26.4.1)(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)): dependencies: - '@vitest/expect': 4.1.11 - '@vitest/mocker': 4.1.11(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) - '@vitest/pretty-format': 4.1.11 - '@vitest/runner': 4.1.11 - '@vitest/snapshot': 4.1.11 - '@vitest/spy': 4.1.11 - '@vitest/utils': 4.1.11 + '@types/chai': 5.2.2 + '@vitest/mocker': 5.0.2(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) + chai: 6.2.2 es-module-lexer: 2.3.2 expect-type: 1.4.0 - magic-string: 0.30.21 - obug: 2.1.1 - pathe: 2.0.3 + magic-string: 1.4.2 + obug: 2.2.1 picomatch: 4.0.7 std-env: 4.2.0 - tinybench: 2.9.0 - tinyexec: 1.0.2 + tinybench: 6.2.0 + tinyexec: 1.3.1 tinyglobby: 0.2.17 - tinyrainbow: 3.1.1 vite: 8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0) - why-is-node-running: 2.3.0 + why-is-node-running: 3.2.2 optionalDependencies: '@types/node': 26.4.1 transitivePeerDependencies: @@ -14834,6 +14820,8 @@ snapshots: siginfo: 2.0.0 stackback: 0.0.2 + why-is-node-running@3.2.2: {} + wide-align@1.1.5: dependencies: string-width: 4.2.3 From 494f4cbd7d410c7c6bd995eaa61aa44de3ad23dd Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Wed, 30 Sep 2026 10:02:16 +0200 Subject: [PATCH 3/5] test(vitest-plugin): add with-vitest-v4 example Now that the plugin's own dev dependency tracks Vitest 5, add a dedicated Vitest 4 example so the legacy (v3/4) benchmark seam keeps explicit coverage alongside the existing with-vitest-v3 example. Mirrors that example, pinning vitest ^4.1.9. Refs COD-2931 Co-Authored-By: Claude --- examples/with-vitest-v4/package.json | 13 ++ .../with-vitest-v4/src/fibonacci.bench.ts | 20 ++ examples/with-vitest-v4/src/fibonacci.ts | 17 ++ examples/with-vitest-v4/tsconfig.json | 13 ++ examples/with-vitest-v4/vitest.config.ts | 6 + pnpm-lock.yaml | 172 +++++++++++++++++- 6 files changed, 233 insertions(+), 8 deletions(-) create mode 100644 examples/with-vitest-v4/package.json create mode 100644 examples/with-vitest-v4/src/fibonacci.bench.ts create mode 100644 examples/with-vitest-v4/src/fibonacci.ts create mode 100644 examples/with-vitest-v4/tsconfig.json create mode 100644 examples/with-vitest-v4/vitest.config.ts diff --git a/examples/with-vitest-v4/package.json b/examples/with-vitest-v4/package.json new file mode 100644 index 00000000..8a67c417 --- /dev/null +++ b/examples/with-vitest-v4/package.json @@ -0,0 +1,13 @@ +{ + "name": "with-vitest-v4", + "private": true, + "type": "module", + "scripts": { + "bench-vitest": "vitest bench --run" + }, + "devDependencies": { + "@codspeed/vitest-plugin": "workspace:*", + "typescript": "^5.1.3", + "vitest": "^4.1.9" + } +} diff --git a/examples/with-vitest-v4/src/fibonacci.bench.ts b/examples/with-vitest-v4/src/fibonacci.bench.ts new file mode 100644 index 00000000..227a6e67 --- /dev/null +++ b/examples/with-vitest-v4/src/fibonacci.bench.ts @@ -0,0 +1,20 @@ +import { bench, describe } from "vitest"; +import { iterativeFibonacci, recursiveFibonacci } from "./fibonacci"; + +describe("fibonacci", () => { + bench("recursive fibo 15", () => { + recursiveFibonacci(15); + }); + + bench("recursive fibo 20", () => { + recursiveFibonacci(20); + }); + + bench("iterative fibo 15", () => { + iterativeFibonacci(15); + }); + + bench("iterative fibo 20", () => { + iterativeFibonacci(20); + }); +}); diff --git a/examples/with-vitest-v4/src/fibonacci.ts b/examples/with-vitest-v4/src/fibonacci.ts new file mode 100644 index 00000000..94796660 --- /dev/null +++ b/examples/with-vitest-v4/src/fibonacci.ts @@ -0,0 +1,17 @@ +export function recursiveFibonacci(n: number): number { + if (n < 2) { + return n; + } + return recursiveFibonacci(n - 1) + recursiveFibonacci(n - 2); +} + +export function iterativeFibonacci(n: number): number { + let a = 0; + let b = 1; + for (let i = 0; i < n; i++) { + const temp = a + b; + a = b; + b = temp; + } + return a; +} diff --git a/examples/with-vitest-v4/tsconfig.json b/examples/with-vitest-v4/tsconfig.json new file mode 100644 index 00000000..ace1de03 --- /dev/null +++ b/examples/with-vitest-v4/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "lib": ["es2023"], + "module": "ESNext", + "verbatimModuleSyntax": true, + "target": "es2022", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "moduleResolution": "Node" + } +} diff --git a/examples/with-vitest-v4/vitest.config.ts b/examples/with-vitest-v4/vitest.config.ts new file mode 100644 index 00000000..4b1290c1 --- /dev/null +++ b/examples/with-vitest-v4/vitest.config.ts @@ -0,0 +1,6 @@ +import codspeedPlugin from "@codspeed/vitest-plugin"; +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + plugins: [codspeedPlugin()], +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9bf9e0c6..25eb3ce5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -255,6 +255,18 @@ importers: specifier: ^3.2.4 version: 3.2.4(@types/node@26.4.1)(lightningcss@1.33.0)(tsx@4.23.12)(yaml@2.9.0) + examples/with-vitest-v4: + devDependencies: + '@codspeed/vitest-plugin': + specifier: workspace:* + version: link:../../packages/vitest-plugin + typescript: + specifier: ^5.1.3 + version: 5.8.3 + vitest: + specifier: ^4.1.9 + version: 4.1.11(@types/node@26.4.1)(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) + packages/benchmark.js-plugin: dependencies: '@codspeed/core': @@ -2732,6 +2744,9 @@ packages: '@vitest/expect@4.0.18': resolution: {integrity: sha512-8sCWUyckXXYvx4opfzVY03EOiYVxyNrHS5QxX3DAIi5dpJAAkyJezHCP77VMX4HKA2LDT/Jpfo8i2r5BE3GnQQ==} + '@vitest/expect@4.1.11': + resolution: {integrity: sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==} + '@vitest/mocker@3.2.4': resolution: {integrity: sha512-46ryTE9RZO/rfDd7pEqFl7etuyzekzEhUbTW3BvmeO/BcCMEgq59BKhek3dXDWgAj4oMK6OZi+vRr1wPW6qjEQ==} peerDependencies: @@ -2754,6 +2769,17 @@ packages: vite: optional: true + '@vitest/mocker@4.1.11': + resolution: {integrity: sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ==} + peerDependencies: + msw: ^2.4.9 + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + '@vitest/mocker@5.0.2': resolution: {integrity: sha512-Z5FS00Q1SJHkB35xATsmWGdQ5WA1/0MV3CDjqyv7GavHv1OfOj145MNfHOlHk7QLes21dKFDHr8EO2zvL+9WGA==} peerDependencies: @@ -2771,24 +2797,36 @@ packages: '@vitest/pretty-format@4.0.18': resolution: {integrity: sha512-P24GK3GulZWC5tz87ux0m8OADrQIUVDPIjjj65vBXYG17ZeU3qD7r+MNZ1RNv4l8CGU2vtTRqixrOi9fYk/yKw==} + '@vitest/pretty-format@4.1.11': + resolution: {integrity: sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw==} + '@vitest/runner@3.2.4': resolution: {integrity: sha512-oukfKT9Mk41LreEW09vt45f8wx7DordoWUZMYdY/cyAk7w5TWkTRCNZYF7sX7n2wB7jyGAl74OxgwhPgKaqDMQ==} '@vitest/runner@4.0.18': resolution: {integrity: sha512-rpk9y12PGa22Jg6g5M3UVVnTS7+zycIGk9ZNGN+m6tZHKQb7jrP7/77WfZy13Y/EUDd52NDsLRQhYKtv7XfPQw==} + '@vitest/runner@4.1.11': + resolution: {integrity: sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw==} + '@vitest/snapshot@3.2.4': resolution: {integrity: sha512-dEYtS7qQP2CjU27QBC5oUOxLE/v5eLkGqPE0ZKEIDGMs4vKWe7IjgLOeauHsR0D5YuuycGRO5oSRXnwnmA78fQ==} '@vitest/snapshot@4.0.18': resolution: {integrity: sha512-PCiV0rcl7jKQjbgYqjtakly6T1uwv/5BQ9SwBLekVg/EaYeQFPiXcgrC2Y7vDMA8dM1SUEAEV82kgSQIlXNMvA==} + '@vitest/snapshot@4.1.11': + resolution: {integrity: sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog==} + '@vitest/spy@3.2.4': resolution: {integrity: sha512-vAfasCOe6AIK70iP5UD11Ac4siNUNJ9i/9PZ3NKx07sG6sUxeag1LWdNrMWeKKYBLlzuK+Gn65Yd5nyL6ds+nw==} '@vitest/spy@4.0.18': resolution: {integrity: sha512-cbQt3PTSD7P2OARdVW3qWER5EGq7PHlvE+QfzSC0lbwO+xnt7+XH06ZzFjFRgzUX//JmpxrCu92VdwvEPlWSNw==} + '@vitest/spy@4.1.11': + resolution: {integrity: sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==} + '@vitest/spy@5.0.2': resolution: {integrity: sha512-Ijc7T1nT9efNb5LxvjaBrEqw3f/QwUv5EE0nKqZxgqsaV/FxAAZ8baGylA8X/Z2oS4Lp+K74Jr6dTJsDKxJDeg==} @@ -2798,6 +2836,9 @@ packages: '@vitest/utils@4.0.18': resolution: {integrity: sha512-msMRKLMVLWygpK3u2Hybgi4MNjcYJvwTb0Ru09+fOyCXIgT5raYP041DRRdiJiI3k/2U6SEbAETB3YtBrUkCFA==} + '@vitest/utils@4.1.11': + resolution: {integrity: sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==} + '@yarnpkg/lockfile@1.1.0': resolution: {integrity: sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ==} @@ -6428,6 +6469,10 @@ packages: resolution: {integrity: sha512-PSkbLUoxOFRzJYjjxHJt9xro7D+iilgMX/C9lawzVuYiIdcihh9DXmVibBe8lmcFrRi/VzlPjBxbN7rH24q8/Q==} engines: {node: '>=14.0.0'} + tinyrainbow@3.1.1: + resolution: {integrity: sha512-yau8yJdTt989Mm0Bd/236QnzEiPf2xLLTqUZRUJOo/3CB078LSwzei343DgtJVmfJKJE3TMINY1u42SQsP6mXw==} + engines: {node: '>=14.0.0'} + tinyspy@4.0.3: resolution: {integrity: sha512-t2T/WLB2WRgZ9EpE4jgPJ9w+i66UZfDc8wHh0xrwiRNN+UwH98GIJkTeZqX9rg0i0ptwzqW+uYeIF0T4F8LR7A==} engines: {node: '>=14.0.0'} @@ -6833,6 +6878,47 @@ packages: jsdom: optional: true + vitest@4.1.11: + resolution: {integrity: sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw==} + engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@opentelemetry/api': ^1.9.0 + '@types/node': ^20.0.0 || ^22.0.0 || >=24.0.0 + '@vitest/browser-playwright': 4.1.11 + '@vitest/browser-preview': 4.1.11 + '@vitest/browser-webdriverio': 4.1.11 + '@vitest/coverage-istanbul': 4.1.11 + '@vitest/coverage-v8': 4.1.11 + '@vitest/ui': 4.1.11 + happy-dom: '*' + jsdom: '*' + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@opentelemetry/api': + optional: true + '@types/node': + optional: true + '@vitest/browser-playwright': + optional: true + '@vitest/browser-preview': + optional: true + '@vitest/browser-webdriverio': + optional: true + '@vitest/coverage-istanbul': + optional: true + '@vitest/coverage-v8': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + vitest@5.0.2: resolution: {integrity: sha512-7MQrx9pDv5aHiUcovIb/70Ys3tgtkUVgCtledvKdCmEO+/1Dicq5ZqoSxOW034m03oqC+oHOKui2dM6qtMLoJg==} engines: {node: ^22.12.0 || ^24.0.0 || >=26.0.0} @@ -7035,7 +7121,7 @@ snapshots: '@ampproject/remapping@2.3.0': dependencies: '@jridgewell/gen-mapping': 0.3.12 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 '@apidevtools/json-schema-ref-parser@9.0.9': dependencies: @@ -7113,7 +7199,7 @@ snapshots: '@babel/parser': 7.28.0 '@babel/types': 7.28.1 '@jridgewell/gen-mapping': 0.3.12 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 jsesc: 3.1.0 '@babel/helper-annotate-as-pure@7.22.5': @@ -8662,7 +8748,7 @@ snapshots: '@jest/test-result': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 '@types/node': 26.4.1 chalk: 4.1.2 collect-v8-coverage: 1.0.2 @@ -8700,7 +8786,7 @@ snapshots: '@jest/source-map@29.6.3': dependencies: - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 callsites: 3.1.0 graceful-fs: 4.2.11 @@ -8756,7 +8842,7 @@ snapshots: dependencies: '@babel/core': 7.28.0 '@jest/types': 29.6.3 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 babel-plugin-istanbul: 6.1.1 chalk: 4.1.2 convert-source-map: 2.0.0 @@ -8792,8 +8878,8 @@ snapshots: '@jridgewell/gen-mapping@0.3.12': dependencies: - '@jridgewell/sourcemap-codec': 1.5.5 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/sourcemap-codec': 1.6.0 + '@jridgewell/trace-mapping': 0.3.31 '@jridgewell/gen-mapping@0.3.3': dependencies: @@ -9781,6 +9867,15 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.0.3 + '@vitest/expect@4.1.11': + dependencies: + '@standard-schema/spec': 1.1.0 + '@types/chai': 5.2.2 + '@vitest/spy': 4.1.11 + '@vitest/utils': 4.1.11 + chai: 6.2.2 + tinyrainbow: 3.1.1 + '@vitest/mocker@3.2.4(vite@7.1.3(@types/node@26.4.1)(lightningcss@1.33.0)(tsx@4.23.12)(yaml@2.9.0))': dependencies: '@vitest/spy': 3.2.4 @@ -9797,6 +9892,14 @@ snapshots: optionalDependencies: vite: 7.1.3(@types/node@26.4.1)(lightningcss@1.33.0)(tsx@4.23.12)(yaml@2.9.0) + '@vitest/mocker@4.1.11(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0))': + dependencies: + '@vitest/spy': 4.1.11 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0) + '@vitest/mocker@5.0.2(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0))': dependencies: '@jridgewell/trace-mapping': 0.3.31 @@ -9814,6 +9917,10 @@ snapshots: dependencies: tinyrainbow: 3.0.3 + '@vitest/pretty-format@4.1.11': + dependencies: + tinyrainbow: 3.1.1 + '@vitest/runner@3.2.4': dependencies: '@vitest/utils': 3.2.4 @@ -9825,6 +9932,11 @@ snapshots: '@vitest/utils': 4.0.18 pathe: 2.0.3 + '@vitest/runner@4.1.11': + dependencies: + '@vitest/utils': 4.1.11 + pathe: 2.0.3 + '@vitest/snapshot@3.2.4': dependencies: '@vitest/pretty-format': 3.2.4 @@ -9837,12 +9949,21 @@ snapshots: magic-string: 0.30.21 pathe: 2.0.3 + '@vitest/snapshot@4.1.11': + dependencies: + '@vitest/pretty-format': 4.1.11 + '@vitest/utils': 4.1.11 + magic-string: 0.30.21 + pathe: 2.0.3 + '@vitest/spy@3.2.4': dependencies: tinyspy: 4.0.3 '@vitest/spy@4.0.18': {} + '@vitest/spy@4.1.11': {} + '@vitest/spy@5.0.2': {} '@vitest/utils@3.2.4': @@ -9856,6 +9977,12 @@ snapshots: '@vitest/pretty-format': 4.0.18 tinyrainbow: 3.0.3 + '@vitest/utils@4.1.11': + dependencies: + '@vitest/pretty-format': 4.1.11 + convert-source-map: 2.0.0 + tinyrainbow: 3.1.1 + '@yarnpkg/lockfile@1.1.0': {} '@yarnpkg/parsers@3.0.2': @@ -14328,6 +14455,8 @@ snapshots: tinyrainbow@3.0.3: {} + tinyrainbow@3.1.1: {} + tinyspy@4.0.3: {} tmp@0.0.33: @@ -14585,7 +14714,7 @@ snapshots: v8-to-istanbul@9.3.0: dependencies: - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 '@types/istanbul-lib-coverage': 2.0.6 convert-source-map: 2.0.0 @@ -14724,6 +14853,33 @@ snapshots: - tsx - yaml + vitest@4.1.11(@types/node@26.4.1)(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)): + dependencies: + '@vitest/expect': 4.1.11 + '@vitest/mocker': 4.1.11(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) + '@vitest/pretty-format': 4.1.11 + '@vitest/runner': 4.1.11 + '@vitest/snapshot': 4.1.11 + '@vitest/spy': 4.1.11 + '@vitest/utils': 4.1.11 + es-module-lexer: 2.3.2 + expect-type: 1.4.0 + magic-string: 0.30.21 + obug: 2.2.1 + pathe: 2.0.3 + picomatch: 4.0.7 + std-env: 4.2.0 + tinybench: 2.9.0 + tinyexec: 1.3.1 + tinyglobby: 0.2.17 + tinyrainbow: 3.1.1 + vite: 8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 26.4.1 + transitivePeerDependencies: + - msw + vitest@5.0.2(@types/node@26.4.1)(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)): dependencies: '@types/chai': 5.2.2 From 2a2f565d9c777b9982081ddc93571e4c375c5c62 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Wed, 30 Sep 2026 10:02:20 +0200 Subject: [PATCH 4/5] test(vitest-plugin): add with-vitest-v5 example Benchmark the plugin against Vitest 5 in CI the way the v3 and v4 examples do for their majors, using the `bench` fixture and `bench.compare()`. Refs COD-2931 Co-Authored-By: Claude --- examples/with-vitest-v5/package.json | 13 +++++++ .../with-vitest-v5/src/fibonacci.bench.ts | 38 +++++++++++++++++++ examples/with-vitest-v5/src/fibonacci.ts | 17 +++++++++ examples/with-vitest-v5/tsconfig.json | 13 +++++++ examples/with-vitest-v5/vitest.config.ts | 6 +++ pnpm-lock.yaml | 34 +++++++++-------- 6 files changed, 106 insertions(+), 15 deletions(-) create mode 100644 examples/with-vitest-v5/package.json create mode 100644 examples/with-vitest-v5/src/fibonacci.bench.ts create mode 100644 examples/with-vitest-v5/src/fibonacci.ts create mode 100644 examples/with-vitest-v5/tsconfig.json create mode 100644 examples/with-vitest-v5/vitest.config.ts diff --git a/examples/with-vitest-v5/package.json b/examples/with-vitest-v5/package.json new file mode 100644 index 00000000..440fd445 --- /dev/null +++ b/examples/with-vitest-v5/package.json @@ -0,0 +1,13 @@ +{ + "name": "with-vitest-v5", + "private": true, + "type": "module", + "scripts": { + "bench-vitest": "vitest bench --run" + }, + "devDependencies": { + "@codspeed/vitest-plugin": "workspace:*", + "typescript": "^5.1.3", + "vitest": "^5.0.0" + } +} diff --git a/examples/with-vitest-v5/src/fibonacci.bench.ts b/examples/with-vitest-v5/src/fibonacci.bench.ts new file mode 100644 index 00000000..3d303e3b --- /dev/null +++ b/examples/with-vitest-v5/src/fibonacci.bench.ts @@ -0,0 +1,38 @@ +import { describe, test } from "vitest"; +import { + iterativeFibonacci as iterativeFibonacciExport, + recursiveFibonacci as recursiveFibonacciExport, +} from "./fibonacci"; + +// Read the imported bindings once: Vite's module runner exposes exports through +// getters, and reading one inside the measured loop adds overhead to every +// iteration (Vitest warns about it). +const recursiveFibonacci = recursiveFibonacciExport; +const iterativeFibonacci = iterativeFibonacciExport; + +// Vitest 5 declares benchmarks through the `bench` test-context fixture: each +// `bench()` returns a registration that runs on `.run()`, or as part of a +// `bench.compare()` group. +describe("fibonacci", () => { + test("fibo 15", async ({ bench }) => { + await bench.compare( + bench("recursive", () => { + recursiveFibonacci(15); + }), + bench("iterative", () => { + iterativeFibonacci(15); + }), + ); + }); + + test("fibo 20", async ({ bench }) => { + await bench.compare( + bench("recursive", () => { + recursiveFibonacci(20); + }), + bench("iterative", () => { + iterativeFibonacci(20); + }), + ); + }); +}); diff --git a/examples/with-vitest-v5/src/fibonacci.ts b/examples/with-vitest-v5/src/fibonacci.ts new file mode 100644 index 00000000..94796660 --- /dev/null +++ b/examples/with-vitest-v5/src/fibonacci.ts @@ -0,0 +1,17 @@ +export function recursiveFibonacci(n: number): number { + if (n < 2) { + return n; + } + return recursiveFibonacci(n - 1) + recursiveFibonacci(n - 2); +} + +export function iterativeFibonacci(n: number): number { + let a = 0; + let b = 1; + for (let i = 0; i < n; i++) { + const temp = a + b; + a = b; + b = temp; + } + return a; +} diff --git a/examples/with-vitest-v5/tsconfig.json b/examples/with-vitest-v5/tsconfig.json new file mode 100644 index 00000000..ace1de03 --- /dev/null +++ b/examples/with-vitest-v5/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "lib": ["es2023"], + "module": "ESNext", + "verbatimModuleSyntax": true, + "target": "es2022", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "moduleResolution": "Node" + } +} diff --git a/examples/with-vitest-v5/vitest.config.ts b/examples/with-vitest-v5/vitest.config.ts new file mode 100644 index 00000000..4b1290c1 --- /dev/null +++ b/examples/with-vitest-v5/vitest.config.ts @@ -0,0 +1,6 @@ +import codspeedPlugin from "@codspeed/vitest-plugin"; +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + plugins: [codspeedPlugin()], +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 25eb3ce5..8eb3ec47 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -267,6 +267,18 @@ importers: specifier: ^4.1.9 version: 4.1.11(@types/node@26.4.1)(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) + examples/with-vitest-v5: + devDependencies: + '@codspeed/vitest-plugin': + specifier: workspace:* + version: link:../../packages/vitest-plugin + typescript: + specifier: ^5.1.3 + version: 5.8.3 + vitest: + specifier: ^5.0.0 + version: 5.0.2(@types/node@26.4.1)(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) + packages/benchmark.js-plugin: dependencies: '@codspeed/core': @@ -1846,9 +1858,6 @@ packages: '@jridgewell/trace-mapping@0.3.18': resolution: {integrity: sha512-w+niJYzMHdd7USdiH2U6869nqhD2nbfZXND5Yp93qIbEmnDNk7PD48o+YchRVpzMU7M6jVCbenTR7PA1FLQ9pA==} - '@jridgewell/trace-mapping@0.3.29': - resolution: {integrity: sha512-uw6guiW/gcAGPDhLmd77/6lW8QLeiV5RUTsAX46Db6oLhGaVj4lhnPwb184s1bkc8kdVg/+h988dro8GRDpmYQ==} - '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} @@ -7191,7 +7200,7 @@ snapshots: dependencies: '@babel/types': 7.22.5 '@jridgewell/gen-mapping': 0.3.3 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 jsesc: 2.5.2 '@babel/generator@7.28.0': @@ -8780,7 +8789,7 @@ snapshots: '@jest/source-map@29.4.3': dependencies: - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 callsites: 3.1.0 graceful-fs: 4.2.11 @@ -8884,8 +8893,8 @@ snapshots: '@jridgewell/gen-mapping@0.3.3': dependencies: '@jridgewell/set-array': 1.1.2 - '@jridgewell/sourcemap-codec': 1.5.4 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/sourcemap-codec': 1.6.0 + '@jridgewell/trace-mapping': 0.3.31 '@jridgewell/resolve-uri@3.1.0': {} @@ -8910,20 +8919,15 @@ snapshots: '@jridgewell/resolve-uri': 3.1.0 '@jridgewell/sourcemap-codec': 1.4.14 - '@jridgewell/trace-mapping@0.3.29': - dependencies: - '@jridgewell/resolve-uri': 3.1.2 - '@jridgewell/sourcemap-codec': 1.5.5 - '@jridgewell/trace-mapping@0.3.31': dependencies: '@jridgewell/resolve-uri': 3.1.2 - '@jridgewell/sourcemap-codec': 1.5.5 + '@jridgewell/sourcemap-codec': 1.6.0 '@jridgewell/trace-mapping@0.3.9': dependencies: '@jridgewell/resolve-uri': 3.1.1 - '@jridgewell/sourcemap-codec': 1.5.5 + '@jridgewell/sourcemap-codec': 1.6.0 '@jsdevtools/ono@7.1.3': {} @@ -14708,7 +14712,7 @@ snapshots: v8-to-istanbul@9.1.0: dependencies: - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 '@types/istanbul-lib-coverage': 2.0.4 convert-source-map: 1.9.0 From 9007e6e1c3919569a1974a818ba0c0454ac082f7 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Wed, 30 Sep 2026 10:02:24 +0200 Subject: [PATCH 5/5] docs(vitest-plugin): show the vitest 5 bench fixture API in the readme Benchmarks are declared through the `bench` test-context fixture on Vitest 5; keep the top-level `bench()` form as a note for Vitest 3 and 4. Drop the fallback log line from the sample output, which the plugin no longer prints. Refs COD-2931 Co-Authored-By: Claude --- packages/vitest-plugin/README.md | 34 +++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/packages/vitest-plugin/README.md b/packages/vitest-plugin/README.md index 6cba8821..47021a0f 100644 --- a/packages/vitest-plugin/README.md +++ b/packages/vitest-plugin/README.md @@ -41,10 +41,10 @@ pnpm add --save-dev @codspeed/vitest-plugin vitest ## Usage -Let's create a fibonacci function and benchmark it with `vitest.bench`: +Let's create a fibonacci function and benchmark it with Vitest's `bench` fixture: ```ts title="benches/fibo.bench.ts" -import { describe, bench } from "vitest"; +import { describe, test } from "vitest"; function fibonacci(n: number): number { if (n < 2) { @@ -54,16 +54,33 @@ function fibonacci(n: number): number { } describe("fibonacci", () => { - bench("fibonacci10", () => { - fibonacci(10); - }); - - bench("fibonacci15", () => { - fibonacci(15); + test("depth", async ({ bench }) => { + await bench.compare( + bench("fibonacci10", () => { + fibonacci(10); + }), + bench("fibonacci15", () => { + fibonacci(15); + }), + ); }); }); ``` +> [!NOTE] +> The `bench` fixture is a Vitest 5 API. On Vitest 3 and 4, benchmarks are +> declared with the top-level `bench()` export instead: +> +> ```ts +> import { bench, describe } from "vitest"; +> +> describe("fibonacci", () => { +> bench("fibonacci10", () => { +> fibonacci(10); +> }); +> }); +> ``` + Create or update your `vitest.config.ts` file to use the CodSpeed runner: ```ts title="vitest.config.ts" @@ -80,7 +97,6 @@ Finally, run your benchmarks (here with `pnpm`): ```bash $ pnpm vitest bench --run -[CodSpeed] bench detected but no instrumentation found, falling back to default vitest runner ... Regular `vitest bench` output ```