feat: add memory profiling support

Author: not-matthias

.github/workflows/codspeed.yml

@@ -23,11 +23,22 @@ jobs:
       - run: pnpm install --frozen-lockfile --prefer-offline
       - run: pnpm moon run :build
 
-      - name: Run benchmarks
-        # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v2`
+      - name: Run simulation benchmarks
+        uses: CodSpeedHQ/action@main
+        with:
+          mode: simulation
+          run: |
+            pnpm moon run tinybench-plugin:bench
+            pnpm moon run vitest-plugin:bench
+            pnpm moon run benchmark.js-plugin:bench
+            pnpm --workspace-concurrency 1 -r bench-tinybench
+            pnpm --workspace-concurrency 1 -r bench-benchmark-js
+            pnpm --workspace-concurrency 1 -r bench-vitest
+
+      - name: Run memory benchmarks
         uses: CodSpeedHQ/action@main
         with:
-          mode: instrumentation
+          mode: memory
           run: |
             pnpm moon run tinybench-plugin:bench
             pnpm moon run vitest-plugin:bench

packages/core/binding.gyp

@@ -14,7 +14,9 @@
         "-Wno-unused-variable",
         "-Wno-unused-parameter",
         "-Wno-unused-but-set-variable",
-        "-Wno-type-limits"
+        "-Wno-type-limits",
+        "-Wno-format",
+        "-Wno-format-security"
       ],
       "cflags_cc": [
         "-Wno-maybe-uninitialized",

packages/core/src/index.ts

@@ -10,7 +10,9 @@ export const isBound = native_core.isBound;
 
 export const mongoMeasurement = new MongoMeasurement();
 
-type CodSpeedRunnerMode = "disabled" | "simulation" | "walltime";
+type CodSpeedRunnerMode = "disabled" | "simulation" | "memory" | "walltime";
+
+type InstrumentMode = "disabled" | "analysis" | "walltime";
 
 export function getCodspeedRunnerMode(): CodSpeedRunnerMode {
   const isCodSpeedEnabled = process.env.CODSPEED_ENV !== undefined;
@@ -25,6 +27,8 @@ export function getCodspeedRunnerMode(): CodSpeedRunnerMode {
     codspeedRunnerMode === "simulation"
   ) {
     return "simulation";
+  } else if (codspeedRunnerMode === "memory") {
+    return "memory";
   } else if (codspeedRunnerMode === "walltime") {
     return "walltime";
   }
@@ -35,6 +39,15 @@ export function getCodspeedRunnerMode(): CodSpeedRunnerMode {
   return "disabled";
 }
 
+export function getInstrumentMode(): InstrumentMode {
+  const runnerMode = getCodspeedRunnerMode();
+  // Both "simulation" and "memory" map to "analysis" instrument mode
+  if (runnerMode === "simulation" || runnerMode === "memory") {
+    return "analysis";
+  }
+  return runnerMode; // "disabled" or "walltime"
+}
+
 export const setupCore = () => {
   if (!native_core.isBound) {
     throw new Error(
@@ -59,4 +72,5 @@ export { getV8Flags, tryIntrospect } from "./introspection";
 export { optimizeFunction, optimizeFunctionSync } from "./optimization";
 export * from "./utils";
 export * from "./walltime";
+export type { InstrumentMode };
 export const InstrumentHooks = native_core.InstrumentHooks;

packages/core/src/introspection.ts

@@ -1,15 +1,15 @@
 import { writeFileSync } from "fs";
-import { getCodspeedRunnerMode } from ".";
+import { getInstrumentMode } from ".";
 
 const CUSTOM_INTROSPECTION_EXIT_CODE = 0;
 
 export const getV8Flags = () => {
   const nodeVersionMajor = parseInt(process.version.slice(1).split(".")[0]);
-  const codspeedRunnerMode = getCodspeedRunnerMode();
+  const instrumentMode = getInstrumentMode();
 
   const flags = ["--interpreted-frames-native-stack", "--allow-natives-syntax"];
 
-  if (codspeedRunnerMode === "simulation") {
+  if (instrumentMode === "analysis") {
     flags.push(
       ...[
         "--hash-seed=1",

packages/core/src/native_core/instruments/hooks

@@ -1,22 +1,24 @@
 import {
+  getCodspeedRunnerMode,
   InstrumentHooks,
   mongoMeasurement,
   optimizeFunction,
 } from "@codspeed/core";
 import { Bench, Fn, FnOptions, Task } from "tinybench";
 import { BaseBenchRunner } from "./shared";
 
-export function setupCodspeedSimulationBench(
+export function setupCodspeedAnalysisBench(
   bench: Bench,
   rootCallingFile: string
 ): void {
-  const runner = new SimulationBenchRunner(bench, rootCallingFile);
+  const runner = new AnalysisBenchRunner(bench, rootCallingFile);
   runner.setupBenchMethods();
 }
 
-class SimulationBenchRunner extends BaseBenchRunner {
+class AnalysisBenchRunner extends BaseBenchRunner {
   protected getModeName(): string {
-    return "simulation mode";
+    const runnerMode = getCodspeedRunnerMode();
+    return `${runnerMode} mode`;
   }
 
   private taskCompletionMessage() {

…kages/tinybench-plugin/src/simulation.ts packages/tinybench-plugin/src/analysis.tspackages/tinybench-plugin/src/simulation.ts renamed to packages/tinybench-plugin/src/analysis.ts packages/tinybench-plugin/src/simulation.ts renamed to packages/tinybench-plugin/src/analysis.ts

@@ -1,6 +1,7 @@
 import {
   getCodspeedRunnerMode,
   getGitDir,
+  getInstrumentMode,
   InstrumentHooks,
   mongoMeasurement,
   SetupInstrumentsRequestBody,
@@ -11,7 +12,7 @@ import path from "path";
 import { get as getStackTrace } from "stack-trace";
 import { Bench } from "tinybench";
 import { fileURLToPath } from "url";
-import { setupCodspeedSimulationBench } from "./simulation";
+import { setupCodspeedAnalysisBench } from "./analysis";
 import { getOrCreateUriMap } from "./uri";
 import { setupCodspeedWalltimeBench } from "./walltime";
 
@@ -39,9 +40,10 @@ export function withCodSpeed(bench: Bench): Bench {
     return rawAdd.bind(bench)(name, fn, opts);
   };
 
-  if (codspeedRunnerMode === "simulation") {
-    setupCodspeedSimulationBench(bench, rootCallingFile);
-  } else if (codspeedRunnerMode === "walltime") {
+  const instrumentMode = getInstrumentMode();
+  if (instrumentMode === "analysis") {
+    setupCodspeedAnalysisBench(bench, rootCallingFile);
+  } else if (instrumentMode === "walltime") {
     setupCodspeedWalltimeBench(bench, rootCallingFile);
   }
 

packages/tinybench-plugin/src/index.ts

@@ -2,12 +2,12 @@ import { Bench } from "tinybench";
 import { beforeEach, describe, expect, it, vi } from "vitest";
 import { withCodSpeed } from ".";
 
-const mockSimulation = vi.hoisted(() => ({
-  setupCodspeedSimulationBench: vi.fn(),
+const mockAnalysis = vi.hoisted(() => ({
+  setupCodspeedAnalysisBench: vi.fn(),
 }));
 
-vi.mock("./simulation", () => ({
-  ...mockSimulation,
+vi.mock("./analysis", () => ({
+  ...mockAnalysis,
 }));
 
 const mockWalltime = vi.hoisted(() => ({
@@ -38,23 +38,33 @@ describe("withCodSpeed behavior without different codspeed modes", () => {
     expect(shouldBeCalled.mock.calls.length).toBeGreaterThan(1000);
   });
 
-  it("should run in simulation mode when CODSPEED_RUNNER_MODE=instrumentation", async () => {
+  it("should run in analysis mode when CODSPEED_RUNNER_MODE=instrumentation", async () => {
     process.env.CODSPEED_ENV = "true";
     process.env.CODSPEED_RUNNER_MODE = "instrumentation";
 
     withCodSpeed(new Bench());
 
-    expect(mockSimulation.setupCodspeedSimulationBench).toHaveBeenCalled();
+    expect(mockAnalysis.setupCodspeedAnalysisBench).toHaveBeenCalled();
     expect(mockWalltime.setupCodspeedWalltimeBench).not.toHaveBeenCalled();
   });
 
-  it("should run in simulation mode when CODSPEED_RUNNER_MODE=simulation", async () => {
+  it("should run in analysis mode when CODSPEED_RUNNER_MODE=simulation", async () => {
     process.env.CODSPEED_ENV = "true";
     process.env.CODSPEED_RUNNER_MODE = "simulation";
 
     withCodSpeed(new Bench());
 
-    expect(mockSimulation.setupCodspeedSimulationBench).toHaveBeenCalled();
+    expect(mockAnalysis.setupCodspeedAnalysisBench).toHaveBeenCalled();
+    expect(mockWalltime.setupCodspeedWalltimeBench).not.toHaveBeenCalled();
+  });
+
+  it("should run in analysis mode when CODSPEED_RUNNER_MODE=memory", async () => {
+    process.env.CODSPEED_ENV = "true";
+    process.env.CODSPEED_RUNNER_MODE = "memory";
+
+    withCodSpeed(new Bench());
+
+    expect(mockAnalysis.setupCodspeedAnalysisBench).toHaveBeenCalled();
     expect(mockWalltime.setupCodspeedWalltimeBench).not.toHaveBeenCalled();
   });
 
@@ -64,7 +74,7 @@ describe("withCodSpeed behavior without different codspeed modes", () => {
 
     withCodSpeed(new Bench());
 
-    expect(mockSimulation.setupCodspeedSimulationBench).not.toHaveBeenCalled();
+    expect(mockAnalysis.setupCodspeedAnalysisBench).not.toHaveBeenCalled();
     expect(mockWalltime.setupCodspeedWalltimeBench).toHaveBeenCalled();
   });
 });

packages/tinybench-plugin/src/index.unit.test.ts

@@ -21,8 +21,8 @@ export default defineConfig([
     external: ["@codspeed/core", /^vitest/],
   },
   {
-    input: "src/simulation.ts",
-    output: { file: "dist/simulation.mjs", format: "es" },
+    input: "src/analysis.ts",
+    output: { file: "dist/analysis.mjs", format: "es" },
     plugins: jsPlugins(pkg.version),
     external: ["@codspeed/core", /^vitest/],
   },

packages/vitest-plugin/rollup.config.ts

@@ -126,7 +126,7 @@ describe("codSpeedPlugin", () => {
           "--max-old-space-size=4096",
         ],
         runner: expect.stringContaining(
-          "packages/vitest-plugin/src/simulation.ts"
+          "packages/vitest-plugin/src/analysis.ts"
         ),
       },
     });
@@ -167,7 +167,7 @@ describe("codSpeedPlugin", () => {
           },
         },
         runner: expect.stringContaining(
-          "packages/vitest-plugin/src/simulation.ts"
+          "packages/vitest-plugin/src/analysis.ts"
         ),
       },
     });