support parallel test execution

package.json

@@ -158,6 +158,12 @@
           "default": {},
           "description": "Specify the list of additional environment variables used for running tests."
         },
+        "mesonbuild.testJobs": {
+          "type": "integer",
+          "default": -1,
+          "minimum": -1,
+          "description": "Specify the maximum number of tests executed in parallel. -1 for number of CPUs, 0 for unlimited."
+        },
         "mesonbuild.benchmarkOptions": {
           "type": "array",
           "default": [

src/extension.ts

@@ -16,7 +16,7 @@ import {
 } from "./utils";
 import { MesonDebugConfigurationProvider, DebuggerType } from "./debug/index";
 import { CpptoolsProvider, registerCppToolsProvider } from "./cpptoolsconfigprovider";
-import { testDebugHandler, testRunHandler, rebuildTests } from "./tests";
+import { testDebugHandler, testRunHandler, regenerateTests } from "./tests";
 import { activateLinters } from "./linters";
 import { activateFormatters } from "./formatters";
 import { SettingsKey, TaskQuickPickItem } from "./types";
@@ -136,7 +136,7 @@ export async function activate(ctx: vscode.ExtensionContext) {
   const changeHandler = async () => {
     mesonTasks = null;
     clearCache();
-    await rebuildTests(controller);
+    await regenerateTests(controller);
     await genEnvFile(buildDir);
     explorer.refresh();
   };
@@ -236,7 +236,7 @@ export async function activate(ctx: vscode.ExtensionContext) {
   if (!checkMesonIsConfigured(buildDir)) {
     if (await askConfigureOnOpen()) runFirstTask("reconfigure");
   } else {
-    await rebuildTests(controller);
+    await regenerateTests(controller);
   }
 
   const server = extensionConfiguration(SettingsKey.languageServer);

src/tests.ts

@@ -1,11 +1,92 @@
+import * as os from "os";
 import * as vscode from "vscode";
-import { ExecResult, exec, extensionConfiguration } from "./utils";
-import { Tests, DebugEnvironmentConfiguration } from "./types";
+import { ExecResult, exec, extensionConfiguration, getTargetName } from "./utils";
+import { Target, Targets, Test, Tests, DebugEnvironmentConfiguration } from "./types";
 import { getMesonTests, getMesonTargets } from "./introspection";
 import { workspaceState } from "./extension";
 
-export async function rebuildTests(controller: vscode.TestController) {
-  let tests = await getMesonTests(workspaceState.get<string>("mesonbuild.buildDir")!);
+// This is far from complete, but should suffice for the
+// "test is made of a single executable is made of a single source file" usecase.
+function findSourceOfTest(test: Test, targets: Targets): vscode.Uri | undefined {
+  const testExe = test.cmd.at(0);
+  if (!testExe) {
+    return undefined;
+  }
+
+  // The meson target such that it is of meson type executable()
+  // and produces the binary that the test() executes.
+  const testDependencyTarget = targets.find((target) => {
+    const depend = test.depends.find((depend) => {
+      return depend == target.id && target.type == "executable";
+    });
+    return depend && testExe == target.filename.at(0);
+  });
+
+  // The first source file belonging to the target.
+  const path = testDependencyTarget?.target_sources
+    ?.find((elem) => {
+      return elem.sources;
+    })
+    ?.sources?.at(0);
+  return path ? vscode.Uri.file(path) : undefined;
+}
+
+/**
+ * Ensures that the given test targets are up to date
+ * @param tests Tests to rebuild
+ * @param buildDir Meson buildDir
+ * @returns `ExecResult` of the `meson compile ...` invocation
+ */
+async function rebuildTests(tests: Test[], buildDir: string): Promise<ExecResult> {
+  // We need to ensure that all test dependencies are built before issuing tests,
+  // as otherwise each meson test ... invocation will lead to a rebuild on it's own
+  const dependencies: Set<string> = new Set(
+    tests.flatMap((test) => {
+      return test.depends;
+    }),
+  );
+
+  const mesonTargets = await getMesonTargets(buildDir);
+  const testDependencies: Set<Target> = new Set(
+    mesonTargets.filter((target) => {
+      return dependencies.has(target.id);
+    }),
+  );
+
+  return exec(extensionConfiguration("mesonPath"), [
+    "compile",
+    "-C",
+    buildDir,
+    ...[...testDependencies].map((test) => {
+      // `test.name` is not guaranteed to be the actual name that meson wants
+      // format is hash@@realname@type
+      return /[^@]+@@(.+)@[^@]+/.exec(test.id)![1];
+    }),
+  ]);
+}
+
+/**
+ * Look up the meson tests that correspond to a given set of vscode TestItems
+ * @param vsCodeTests TestItems to look up
+ * @param mesonTests The set of all existing meson tests
+ * @returns Meson tests corresponding to the TestItems
+ */
+function vsCodeToMeson(vsCodeTests: readonly vscode.TestItem[], mesonTests: Tests): Test[] {
+  return vsCodeTests.map((test) => {
+    return mesonTests.find((mesonTest) => {
+      return mesonTest.name == test.id;
+    })!;
+  });
+}
+
+/**
+ * Regenerate the test view in vscode, adding new tests and deleting stale ones
+ * @param controller VSCode test controller
+ */
+export async function regenerateTests(controller: vscode.TestController) {
+  const buildDir = workspaceState.get<string>("mesonbuild.buildDir")!;
+  const tests = await getMesonTests(buildDir);
+  const targets = await getMesonTargets(buildDir);
 
   controller.items.forEach((item) => {
     if (!tests.some((test) => item.id == test.name)) {
@@ -14,7 +95,8 @@ export async function rebuildTests(controller: vscode.TestController) {
   });
 
   for (let testDescr of tests) {
-    let testItem = controller.createTestItem(testDescr.name, testDescr.name);
+    const testSourceFile = findSourceOfTest(testDescr, targets);
+    const testItem = controller.createTestItem(testDescr.name, testDescr.name, testSourceFile);
     controller.items.add(testItem);
   }
 }
@@ -25,39 +107,96 @@ export async function testRunHandler(
   token: vscode.CancellationToken,
 ) {
   const run = controller.createTestRun(request, undefined, false);
-  const queue: vscode.TestItem[] = [];
+  const parallelTests: vscode.TestItem[] = [];
+  const sequentialTests: vscode.TestItem[] = [];
+
+  const buildDir = workspaceState.get<string>("mesonbuild.buildDir")!;
+  const mesonTests = await getMesonTests(buildDir);
 
+  // Look up the meson test for a given vscode test,
+  // put it in the parallel or sequential queue,
+  // and tell vscode about the enqueued test.
+  const testAdder = (test: vscode.TestItem) => {
+    const mesonTest = vsCodeToMeson([test], mesonTests)[0];
+    if (mesonTest.is_parallel) {
+      parallelTests.push(test);
+    } else {
+      sequentialTests.push(test);
+    }
+    // This way the total number of runs shows up from the beginning,
+    // instead of incrementing as individual runs finish
+    run.enqueued(test);
+  };
   if (request.include) {
-    request.include.forEach((test) => queue.push(test));
+    request.include.forEach(testAdder);
   } else {
-    controller.items.forEach((test) => queue.push(test));
+    controller.items.forEach(testAdder);
   }
 
-  const buildDir = workspaceState.get<string>("mesonbuild.buildDir")!;
+  // we need to ensure that all test dependencies are built before issuing tests,
+  // as otherwise each meson test ... invocation will lead to a rebuild on it's own
+  await rebuildTests(vsCodeToMeson(parallelTests.concat(sequentialTests), mesonTests), buildDir).catch((onrejected) => {
+    const execResult = onrejected as ExecResult;
+    vscode.window.showErrorMessage("Failed to build tests:\r\n" + execResult.stdout + "\r\n" + execResult.stderr);
+    run.end();
+  });
 
-  for (let test of queue) {
+  const dispatchTest = (test: vscode.TestItem) => {
     run.started(test);
-    let starttime = Date.now();
-    try {
-      await exec(
-        extensionConfiguration("mesonPath"),
-        ["test", "-C", buildDir, "--print-errorlog", `"${test.id}"`],
-        extensionConfiguration("testEnvironment"),
-      );
-      let duration = Date.now() - starttime;
-      run.passed(test, duration);
-    } catch (e) {
-      const execResult = e as ExecResult;
-
-      run.appendOutput(execResult.stdout);
-      let duration = Date.now() - starttime;
-      if (execResult.error?.code == 125) {
-        vscode.window.showErrorMessage("Failed to build tests. Results will not be updated");
-        run.errored(test, new vscode.TestMessage(execResult.stderr));
-      } else {
-        run.failed(test, new vscode.TestMessage(execResult.stderr), duration);
-      }
+    return exec(
+      extensionConfiguration("mesonPath"),
+      ["test", "-C", buildDir, "--print-errorlog", "--no-rebuild", `"${test.id}"`],
+      extensionConfiguration("testEnvironment"),
+    ).then(
+      (onfulfilled) => {
+        run.passed(test, onfulfilled.timeMs);
+      },
+      (onrejected) => {
+        const execResult = onrejected as ExecResult;
+
+        let stdout = execResult.stdout;
+        if (os.platform() != "win32") {
+          stdout = stdout.replace(/\n/g, "\r\n");
+        }
+        run.appendOutput(stdout, undefined, test);
+        if (execResult.error?.code == 125) {
+          vscode.window.showErrorMessage("Failed to run tests. Results will not be updated");
+          run.errored(test, new vscode.TestMessage(execResult.stderr));
+        } else {
+          run.failed(test, new vscode.TestMessage(execResult.stderr), execResult.timeMs);
+        }
+      },
+    );
+  };
+
+  const runningTests: Promise<void>[] = [];
+  const maxRunning: number = (() => {
+    const jobsConfig = extensionConfiguration("testJobs");
+    switch (jobsConfig) {
+      case -1:
+        return os.cpus().length;
+      case 0:
+        return Number.MAX_SAFE_INTEGER;
+      default:
+        return jobsConfig;
     }
+  })();
+
+  for (const test of parallelTests) {
+    const runningTest = dispatchTest(test).finally(() => {
+      runningTests.splice(runningTests.indexOf(runningTest), 1);
+    });
+
+    runningTests.push(runningTest);
+
+    if (runningTests.length >= maxRunning) {
+      await Promise.race(runningTests);
+    }
+  }
+  await Promise.all(runningTests);
+
+  for (const test of sequentialTests) {
+    await dispatchTest(test);
   }
 
   run.end();
@@ -89,8 +228,8 @@ export async function testDebugHandler(
   );
 
   let args = ["compile", "-C", buildDir];
-  requiredTargets.forEach((target) => {
-    args.push(target.name);
+  requiredTargets.forEach(async (target) => {
+    args.push(await getTargetName(target));
   });
 
   try {

src/types.ts

@@ -72,6 +72,7 @@ export interface ExtensionConfiguration {
   setupOptions: string[];
   testOptions: string[];
   testEnvironment: { [key: string]: string };
+  testJobs: number;
   benchmarkOptions: string[];
   buildFolder: string;
   mesonPath: string;

src/utils.ts

@@ -19,6 +19,7 @@ import { extensionPath, workspaceState } from "./extension";
 export interface ExecResult {
   stdout: string;
   stderr: string;
+  timeMs: number; // Runtime in milliseconds
   error?: cp.ExecFileException;
 }
 
@@ -32,11 +33,13 @@ export async function exec(
     options.env = { ...(options.env ?? process.env), ...extraEnv };
   }
   return new Promise<ExecResult>((resolve, reject) => {
+    const timeStart = Date.now();
     cp.execFile(command, args, options, (error, stdout, stderr) => {
+      const timeMs = Date.now() - timeStart;
       if (error) {
-        reject({ error, stdout, stderr });
+        reject({ stdout, stderr, timeMs, error });
       } else {
-        resolve({ stdout, stderr });
+        resolve({ stdout, stderr, timeMs });
       }
     });
   });
@@ -49,8 +52,10 @@ export async function execFeed(
   stdin: string,
 ) {
   return new Promise<ExecResult>((resolve) => {
+    const timeStart = Date.now();
     const p = cp.execFile(command, args, options, (error, stdout, stderr) => {
-      resolve({ stdout, stderr, error: error ? error : undefined });
+      const timeMs = Date.now() - timeStart;
+      resolve({ stdout, stderr, timeMs, error: error ?? undefined });
     });
 
     p.stdin?.write(stdin);