Introduce a custom node:test reporter

v-next/hardhat-node-test-reporter/.eslintrc.cjs

@@ -0,0 +1,17 @@
+const { createConfig } = require("../../config-v-next/eslint.cjs");
+
+module.exports = createConfig(__filename, ["src/reporter.ts"]);
+
+module.exports.overrides.push({
+  files: ["integration-tests/**/*.ts"],
+  rules: {
+    "import/no-extraneous-dependencies": [
+      "error",
+      {
+        devDependencies: true,
+      },
+    ],
+    // Disabled until this gets resolved https://github.com/nodejs/node/issues/51292
+    "@typescript-eslint/no-floating-promises": "off",
+  },
+});

v-next/hardhat-node-test-reporter/.gitignore

@@ -0,0 +1,5 @@
+# Node modules
+/node_modules
+
+# Compilation output
+/dist

v-next/hardhat-node-test-reporter/.prettierignore

@@ -0,0 +1,3 @@
+/node_modules
+/dist
+CHANGELOG.md

v-next/hardhat-node-test-reporter/LICENSE

@@ -0,0 +1,9 @@
+MIT License
+
+Copyright (c) 2024 Nomic Foundation
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

v-next/hardhat-node-test-reporter/README.md

@@ -0,0 +1,5 @@
+# Hardhat's `node:test` reporter
+
+This package includes Hardhat's `node:test` reporter.
+
+This project is heavily inspired by https://github.com/voxpelli/node-test-pretty-reporter

v-next/hardhat-node-test-reporter/integration-tests/README.md

@@ -0,0 +1,9 @@
+# Integration tests
+
+This folder contains integration tests for the reporter. They don't use `node:test` as driver of the test runs, as you can't run `node:test` within `node:test`.
+
+Instead, the script `index.ts` runs all the tests in each fixture folder, comparing the reporter's results with `result.txt`.
+
+## Running each test manually
+
+You can run each of the fixture test manually from the package root by building it and running `node --import tsx/esm --test --test-reporter=./dist/src/reporter.js integration-tests/fixture-tests/example-test/*.ts`

v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/example-test/result.txt

@@ -0,0 +1,109 @@
+1
+2
+  Foooo
+    ✔ test
+    1) test with cause
+    child
+      2) asdasd
+
+  a
+    aa
+      aaa
+        ✔ aaaa
+
+  top level test
+    ✔ bar
+
+  ✔ top level test
+
+  in describe
+    3) foo
+
+  testing before each
+    nested
+      ✔ neseted foo
+      ✔ neseted foo 2
+
+
+  + todo test
+
+  - skipped test
+
+
+6 passing (104ms)
+2 failing
+1 skipped
+1 todo
+1 cancelled
+
+  1) Foooo
+       test with cause:
+
+   Error: withCause
+       at TestContext.<anonymous> (integration-tests/fixture-tests/example-test/test.ts:1:136)
+       at Test.runInAsyncScope (node:async_hooks:206:9)
+       ... 6 lines matching cause stack trace ...
+       at async Test.processPendingSubtests (node:internal/test_runner/test:533:7) {
+     [cause]: Error: cause
+         at TestContext.<anonymous> (integration-tests/fixture-tests/example-test/test.ts:1:165)
+         at Test.runInAsyncScope (node:async_hooks:206:9)
+         at Test.run (node:internal/test_runner/test:824:25)
+         at Suite.processPendingSubtests (node:internal/test_runner/test:533:18)
+         at Test.postRun (node:internal/test_runner/test:923:19)
+         at Test.run (node:internal/test_runner/test:866:12)
+         at async Promise.all (index 0)
+         at async Suite.run (node:internal/test_runner/test:1183:7)
+         at async Test.processPendingSubtests (node:internal/test_runner/test:533:7)
+   }
+
+  2) Foooo
+       child
+         asdasd:
+
+   Error: Different arrays
+   - Expected
+   + Received
+
+     Array [
+       1,
+   -   2,
+   +   3,
+       3,
+     ]
+
+       at TestContext.<anonymous> (integration-tests/fixture-tests/example-test/test.ts:1:239)
+       at Test.runInAsyncScope (node:async_hooks:206:9)
+       ... 7 lines matching cause stack trace ...
+       at Array.map (<anonymous>) {
+     [cause]: Error: cause
+         at TestContext.<anonymous> (integration-tests/fixture-tests/example-test/test.ts:1:275)
+         at Test.runInAsyncScope (node:async_hooks:206:9)
+         at Test.run (node:internal/test_runner/test:824:25)
+         at Test.start (node:internal/test_runner/test:721:17)
+         at node:internal/test_runner/test:1181:71
+         at node:internal/per_context/primordials:488:82
+         at new Promise (<anonymous>)
+         at new SafePromise (node:internal/per_context/primordials:456:29)
+         at node:internal/per_context/primordials:488:9
+         at Array.map (<anonymous>)
+   }
+
+  3) in describe
+       foo:
+
+   Test cancelled by parent error
+       This test was cancelled due to an error in its parent suite/it or test/it, or in one of its before/beforeEach
+
+  4) in describe:
+
+   Error: before
+       at SuiteContext.<anonymous> (integration-tests/fixture-tests/example-test/test.ts:1:696)
+       at TestHook.runInAsyncScope (node:async_hooks:206:9)
+       at TestHook.run (node:internal/test_runner/test:824:25)
+       at TestHook.run (node:internal/test_runner/test:1073:18)
+       at TestHook.run (node:internal/util:528:20)
+       at node:internal/test_runner/test:744:20
+       at async Suite.runHook (node:internal/test_runner/test:742:7)
+       at async Suite.run (node:internal/test_runner/test:1177:7)
+       at async Test.processPendingSubtests (node:internal/test_runner/test:533:7)
+

v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/example-test/test.ts

@@ -0,0 +1,79 @@
+import { before, beforeEach, describe, it } from "node:test";
+
+describe("Foooo", () => {
+  it("test", async () => {});
+
+  it("test with cause", async () => {
+    throw new Error("withCause", { cause: new Error("cause") });
+  });
+
+  describe("child", () => {
+    it("asdasd", () => {
+      const error = new Error("Different arrays", {
+        cause: new Error("cause"),
+      });
+
+      Object.defineProperty(error, "expected", {
+        configurable: false,
+        enumerable: false,
+        get() {
+          return [1, 2, 3];
+        },
+      });
+
+      Object.defineProperty(error, "actual", {
+        configurable: false,
+        enumerable: false,
+        get() {
+          return [1, 3, 3];
+        },
+      });
+
+      throw error;
+    });
+  });
+});
+
+describe("a", () => {
+  describe("aa", () => {
+    describe("aaa", () => {
+      it("aaaa", () => {});
+    });
+  });
+});
+
+it("top level test", async (t) => {
+  await t.test("bar", () => {});
+});
+
+describe("in describe", () => {
+  before(() => {
+    throw new Error("before");
+  });
+
+  beforeEach(() => {
+    throw new Error("before each");
+  });
+
+  it("foo", async (t) => {
+    await t.test("foo/bar", () => {});
+
+    throw new Error("asd");
+  });
+});
+
+describe("testing before each", () => {
+  describe("nested", () => {
+    it("neseted foo", async () => {
+      console.log("1");
+    });
+
+    it("neseted foo 2", async () => {
+      console.log("2");
+    });
+  });
+});
+
+it.todo("todo test", async () => {});
+
+it.skip("skipped test", async () => {});

v-next/hardhat-node-test-reporter/integration-tests/index.ts

@@ -0,0 +1,62 @@
+import { run } from "node:test";
+
+import {
+  isDirectory,
+  readdir,
+  getAllFilesMatching,
+  readUtf8File,
+} from "@nomicfoundation/hardhat-utils/fs";
+import { diff } from "jest-diff";
+
+import reporter from "../src/reporter.js";
+const SHOW_OUTPUT = process.argv.includes("--show-output");
+
+for (const entry of await readdir(import.meta.dirname + "/fixture-tests")) {
+  const entryPath = import.meta.dirname + "/fixture-tests/" + entry;
+  if (await isDirectory(entryPath)) {
+    console.log("Running integration test: " + entry);
+
+    const testFiles = await getAllFilesMatching(entryPath, (file) =>
+      file.endsWith(".ts"),
+    );
+
+    const outputChunks = [];
+
+    const reporterStream = run({
+      files: testFiles,
+    }).compose(reporter);
+
+    for await (const chunk of reporterStream) {
+      outputChunks.push(chunk);
+    }
+
+    const output = outputChunks.join("");
+    const expectedOutput = await readUtf8File(entryPath + "/result.txt");
+
+    const normalizedOutput = normalizeOutputs(output);
+    const normalizedExpectedOutput = normalizeOutputs(expectedOutput);
+
+    if (normalizedOutput !== normalizedExpectedOutput) {
+      console.log("Normalized outputs differ:");
+      console.log(diff(normalizedExpectedOutput, normalizedOutput));
+      process.exitCode = 1;
+    } else {
+      console.log("   Passed");
+    }
+
+    if (SHOW_OUTPUT) {
+      console.log();
+      console.log();
+      console.log(output);
+    }
+
+    console.log();
+    console.log();
+    console.log();
+    console.log();
+  }
+}
+
+function normalizeOutputs(output: string): string {
+  return output.replace(/\(\d+ms\)/, "(Xms)").replaceAll("\r\n", "\n");
+}