diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a8712e8890..fa35366033 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1793,6 +1793,73 @@ importers: specifier: 7.7.1 version: 7.7.1(eslint@8.57.0)(typescript@5.4.3) + v-next/hardhat-node-test-reporter: + dependencies: + chalk: + specifier: ^5.3.0 + version: 5.3.0 + jest-diff: + specifier: ^29.7.0 + version: 29.7.0 + devDependencies: + '@nomicfoundation/eslint-plugin-hardhat-internal-rules': + specifier: workspace:^ + version: link:../../packages/eslint-plugin-hardhat-internal-rules + '@nomicfoundation/eslint-plugin-slow-imports': + specifier: workspace:^ + version: link:../../packages/eslint-plugin-slow-imports + '@nomicfoundation/hardhat-utils': + specifier: workspace:^3.0.0 + version: link:../hardhat-utils + '@reporters/github': + specifier: ^1.7.0 + version: 1.7.0 + '@types/node': + specifier: ^20.0.0 + version: 20.12.10 + '@typescript-eslint/eslint-plugin': + specifier: ^7.7.1 + version: 7.9.0(@typescript-eslint/parser@7.9.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5) + '@typescript-eslint/parser': + specifier: ^7.7.1 + version: 7.9.0(eslint@8.57.0)(typescript@5.4.5) + eslint: + specifier: 8.57.0 + version: 8.57.0 + eslint-config-prettier: + specifier: 9.1.0 + version: 9.1.0(eslint@8.57.0) + eslint-import-resolver-typescript: + specifier: ^3.6.1 + version: 3.6.1(@typescript-eslint/parser@7.9.0(eslint@8.57.0)(typescript@5.4.5))(eslint-plugin-import@2.29.1)(eslint@8.57.0) + eslint-plugin-import: + specifier: 2.29.1 + version: 2.29.1(@typescript-eslint/parser@7.9.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) + eslint-plugin-no-only-tests: + specifier: 3.1.0 + version: 3.1.0 + expect-type: + specifier: ^0.19.0 + version: 0.19.0 + glob: + specifier: ^10.3.12 + version: 10.3.12 + prettier: + specifier: 3.2.5 + version: 3.2.5 + rimraf: + specifier: ^5.0.5 + version: 5.0.5 + tsx: + specifier: ^4.7.1 + version: 4.7.1 + typescript: + specifier: ~5.4.0 + version: 5.4.5 + typescript-eslint: + specifier: 7.7.1 + version: 7.7.1(eslint@8.57.0)(typescript@5.4.5) + v-next/hardhat-utils: dependencies: fast-equals: diff --git a/v-next/hardhat-node-test-reporter/.eslintrc.cjs b/v-next/hardhat-node-test-reporter/.eslintrc.cjs new file mode 100644 index 0000000000..86d70ad01c --- /dev/null +++ b/v-next/hardhat-node-test-reporter/.eslintrc.cjs @@ -0,0 +1,17 @@ +const { createConfig } = require("../../config-v-next/eslint.cjs"); + +module.exports = createConfig(__filename, ["src/reporter.ts"]); + +module.exports.overrides.push({ + files: ["integration-tests/**/*.ts"], + rules: { + "import/no-extraneous-dependencies": [ + "error", + { + devDependencies: true, + }, + ], + // Disabled until this gets resolved https://github.com/nodejs/node/issues/51292 + "@typescript-eslint/no-floating-promises": "off", + }, +}); diff --git a/v-next/hardhat-node-test-reporter/.gitignore b/v-next/hardhat-node-test-reporter/.gitignore new file mode 100644 index 0000000000..6aa5402c62 --- /dev/null +++ b/v-next/hardhat-node-test-reporter/.gitignore @@ -0,0 +1,5 @@ +# Node modules +/node_modules + +# Compilation output +/dist diff --git a/v-next/hardhat-node-test-reporter/.prettierignore b/v-next/hardhat-node-test-reporter/.prettierignore new file mode 100644 index 0000000000..3760c88cd5 --- /dev/null +++ b/v-next/hardhat-node-test-reporter/.prettierignore @@ -0,0 +1,3 @@ +/node_modules +/dist +CHANGELOG.md diff --git a/v-next/hardhat-node-test-reporter/LICENSE b/v-next/hardhat-node-test-reporter/LICENSE new file mode 100644 index 0000000000..0781b4a819 --- /dev/null +++ b/v-next/hardhat-node-test-reporter/LICENSE @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) 2024 Nomic Foundation + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/v-next/hardhat-node-test-reporter/README.md b/v-next/hardhat-node-test-reporter/README.md new file mode 100644 index 0000000000..919325d8e2 --- /dev/null +++ b/v-next/hardhat-node-test-reporter/README.md @@ -0,0 +1,5 @@ +# Hardhat's `node:test` reporter + +This package includes Hardhat's `node:test` reporter. + +This project is heavily inspired by https://github.com/voxpelli/node-test-pretty-reporter diff --git a/v-next/hardhat-node-test-reporter/integration-tests/README.md b/v-next/hardhat-node-test-reporter/integration-tests/README.md new file mode 100644 index 0000000000..effe33f301 --- /dev/null +++ b/v-next/hardhat-node-test-reporter/integration-tests/README.md @@ -0,0 +1,9 @@ +# Integration tests + +This folder contains integration tests for the reporter. They don't use `node:test` as driver of the test runs, as you can't run `node:test` within `node:test`. + +Instead, the script `index.ts` runs all the tests in each fixture folder, comparing the reporter's results with `result.txt`. + +## Running each test manually + +You can run each of the fixture test manually from the package root by building it and running `node --import tsx/esm --test --test-reporter=./dist/src/reporter.js integration-tests/fixture-tests/example-test/*.ts` diff --git a/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/example-test/result.txt b/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/example-test/result.txt new file mode 100644 index 0000000000..2316041a77 --- /dev/null +++ b/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/example-test/result.txt @@ -0,0 +1,109 @@ +1 +2 + Foooo + ✔ test + 1) test with cause + child + 2) asdasd + + a + aa + aaa + ✔ aaaa + + top level test + ✔ bar + + ✔ top level test + + in describe + 3) foo + + testing before each + nested + ✔ neseted foo + ✔ neseted foo 2 + + + + todo test + + - skipped test + + +6 passing (104ms) +2 failing +1 skipped +1 todo +1 cancelled + + 1) Foooo + test with cause: + + Error: withCause +  at TestContext. (integration-tests/fixture-tests/example-test/test.ts:1:136) +  at Test.runInAsyncScope (node:async_hooks:206:9) +  ... 6 lines matching cause stack trace ... +  at async Test.processPendingSubtests (node:internal/test_runner/test:533:7) { +  [cause]: Error: cause +  at TestContext. (integration-tests/fixture-tests/example-test/test.ts:1:165) +  at Test.runInAsyncScope (node:async_hooks:206:9) +  at Test.run (node:internal/test_runner/test:824:25) +  at Suite.processPendingSubtests (node:internal/test_runner/test:533:18) +  at Test.postRun (node:internal/test_runner/test:923:19) +  at Test.run (node:internal/test_runner/test:866:12) +  at async Promise.all (index 0) +  at async Suite.run (node:internal/test_runner/test:1183:7) +  at async Test.processPendingSubtests (node:internal/test_runner/test:533:7) + } + + 2) Foooo + child + asdasd: + + Error: Different arrays + - Expected + + Received + +  Array [ +  1, + - 2, + + 3, +  3, +  ] + +  at TestContext. (integration-tests/fixture-tests/example-test/test.ts:1:239) +  at Test.runInAsyncScope (node:async_hooks:206:9) +  ... 7 lines matching cause stack trace ... +  at Array.map () { +  [cause]: Error: cause +  at TestContext. (integration-tests/fixture-tests/example-test/test.ts:1:275) +  at Test.runInAsyncScope (node:async_hooks:206:9) +  at Test.run (node:internal/test_runner/test:824:25) +  at Test.start (node:internal/test_runner/test:721:17) +  at node:internal/test_runner/test:1181:71 +  at node:internal/per_context/primordials:488:82 +  at new Promise () +  at new SafePromise (node:internal/per_context/primordials:456:29) +  at node:internal/per_context/primordials:488:9 +  at Array.map () + } + + 3) in describe + foo: + + Test cancelled by parent error +  This test was cancelled due to an error in its parent suite/it or test/it, or in one of its before/beforeEach + + 4) in describe: + + Error: before +  at SuiteContext. (integration-tests/fixture-tests/example-test/test.ts:1:696) +  at TestHook.runInAsyncScope (node:async_hooks:206:9) +  at TestHook.run (node:internal/test_runner/test:824:25) +  at TestHook.run (node:internal/test_runner/test:1073:18) +  at TestHook.run (node:internal/util:528:20) +  at node:internal/test_runner/test:744:20 +  at async Suite.runHook (node:internal/test_runner/test:742:7) +  at async Suite.run (node:internal/test_runner/test:1177:7) +  at async Test.processPendingSubtests (node:internal/test_runner/test:533:7) + diff --git a/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/example-test/test.ts b/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/example-test/test.ts new file mode 100644 index 0000000000..64d1292dcf --- /dev/null +++ b/v-next/hardhat-node-test-reporter/integration-tests/fixture-tests/example-test/test.ts @@ -0,0 +1,79 @@ +import { before, beforeEach, describe, it } from "node:test"; + +describe("Foooo", () => { + it("test", async () => {}); + + it("test with cause", async () => { + throw new Error("withCause", { cause: new Error("cause") }); + }); + + describe("child", () => { + it("asdasd", () => { + const error = new Error("Different arrays", { + cause: new Error("cause"), + }); + + Object.defineProperty(error, "expected", { + configurable: false, + enumerable: false, + get() { + return [1, 2, 3]; + }, + }); + + Object.defineProperty(error, "actual", { + configurable: false, + enumerable: false, + get() { + return [1, 3, 3]; + }, + }); + + throw error; + }); + }); +}); + +describe("a", () => { + describe("aa", () => { + describe("aaa", () => { + it("aaaa", () => {}); + }); + }); +}); + +it("top level test", async (t) => { + await t.test("bar", () => {}); +}); + +describe("in describe", () => { + before(() => { + throw new Error("before"); + }); + + beforeEach(() => { + throw new Error("before each"); + }); + + it("foo", async (t) => { + await t.test("foo/bar", () => {}); + + throw new Error("asd"); + }); +}); + +describe("testing before each", () => { + describe("nested", () => { + it("neseted foo", async () => { + console.log("1"); + }); + + it("neseted foo 2", async () => { + console.log("2"); + }); + }); +}); + +it.todo("todo test", async () => {}); + +it.skip("skipped test", async () => {}); diff --git a/v-next/hardhat-node-test-reporter/integration-tests/index.ts b/v-next/hardhat-node-test-reporter/integration-tests/index.ts new file mode 100644 index 0000000000..664ff07a9d --- /dev/null +++ b/v-next/hardhat-node-test-reporter/integration-tests/index.ts @@ -0,0 +1,62 @@ +import { run } from "node:test"; + +import { + isDirectory, + readdir, + getAllFilesMatching, + readUtf8File, +} from "@nomicfoundation/hardhat-utils/fs"; +import { diff } from "jest-diff"; + +import reporter from "../src/reporter.js"; +const SHOW_OUTPUT = process.argv.includes("--show-output"); + +for (const entry of await readdir(import.meta.dirname + "/fixture-tests")) { + const entryPath = import.meta.dirname + "/fixture-tests/" + entry; + if (await isDirectory(entryPath)) { + console.log("Running integration test: " + entry); + + const testFiles = await getAllFilesMatching(entryPath, (file) => + file.endsWith(".ts"), + ); + + const outputChunks = []; + + const reporterStream = run({ + files: testFiles, + }).compose(reporter); + + for await (const chunk of reporterStream) { + outputChunks.push(chunk); + } + + const output = outputChunks.join(""); + const expectedOutput = await readUtf8File(entryPath + "/result.txt"); + + const normalizedOutput = normalizeOutputs(output); + const normalizedExpectedOutput = normalizeOutputs(expectedOutput); + + if (normalizedOutput !== normalizedExpectedOutput) { + console.log("Normalized outputs differ:"); + console.log(diff(normalizedExpectedOutput, normalizedOutput)); + process.exitCode = 1; + } else { + console.log(" Passed"); + } + + if (SHOW_OUTPUT) { + console.log(); + console.log(); + console.log(output); + } + + console.log(); + console.log(); + console.log(); + console.log(); + } +} + +function normalizeOutputs(output: string): string { + return output.replace(/\(\d+ms\)/, "(Xms)").replaceAll("\r\n", "\n"); +} diff --git a/v-next/hardhat-node-test-reporter/package.json b/v-next/hardhat-node-test-reporter/package.json new file mode 100644 index 0000000000..78c1dbda94 --- /dev/null +++ b/v-next/hardhat-node-test-reporter/package.json @@ -0,0 +1,65 @@ +{ + "name": "@nomicfoundation/hardhat-node-test-reporter", + "version": "3.0.0", + "description": "A node:test reporter", + "homepage": "https://github.com/nomicfoundation/hardhat/tree/v-next/v-next/hardhat-node-test-reporter", + "repository": "github:nomicfoundation/hardhat", + "author": "Nomic Foundation", + "license": "MIT", + "type": "module", + "exports": { + ".": "./dist/src/reporter.js" + }, + "keywords": [ + "ethereum", + "smart-contracts", + "hardhat" + ], + "scripts": { + "lint": "pnpm prettier --check && pnpm eslint", + "lint:fix": "pnpm prettier --write && pnpm eslint --fix", + "eslint": "eslint \"src/**/*.ts\" \"test/**/*.ts\" \"integration-tests/**/*.ts\"", + "prettier": "prettier \"**/*.{ts,js,md,json}\"", + "test": "glob --cmd=\"node --import tsx/esm --test\" \"test/**/*.ts\"", + "test:github": "glob --cmd=\"node --import tsx/esm --test --test-reporter=@reporters/github --test-reporter-destination=stdout --test-reporter=spec --test-reporter-destination=stdout\" \"test/**/*.ts\"", + "test:integration": "node --import tsx/esm integration-tests/index.ts --color", + "posttest": "pnpm test:integration", + "posttest:github": "pnpm test:integration", + "pretest": "pnpm build", + "build": "tsc --build .", + "prepublishOnly": "pnpm build", + "clean": "rimraf dist" + }, + "files": [ + "dist/src/", + "src/", + "CHANGELOG.md", + "LICENSE", + "README.md" + ], + "devDependencies": { + "@nomicfoundation/eslint-plugin-hardhat-internal-rules": "workspace:^", + "@nomicfoundation/eslint-plugin-slow-imports": "workspace:^", + "@nomicfoundation/hardhat-utils": "workspace:^3.0.0", + "@reporters/github": "^1.7.0", + "@types/node": "^20.0.0", + "@typescript-eslint/eslint-plugin": "^7.7.1", + "@typescript-eslint/parser": "^7.7.1", + "eslint": "8.57.0", + "eslint-config-prettier": "9.1.0", + "eslint-import-resolver-typescript": "^3.6.1", + "eslint-plugin-import": "2.29.1", + "eslint-plugin-no-only-tests": "3.1.0", + "expect-type": "^0.19.0", + "glob": "^10.3.12", + "prettier": "3.2.5", + "rimraf": "^5.0.5", + "tsx": "^4.7.1", + "typescript": "~5.4.0", + "typescript-eslint": "7.7.1" + }, + "dependencies": { + "chalk": "^5.3.0", + "jest-diff": "^29.7.0" + } +} diff --git a/v-next/hardhat-node-test-reporter/src/diagnostics.ts b/v-next/hardhat-node-test-reporter/src/diagnostics.ts new file mode 100644 index 0000000000..0aa24ea37f --- /dev/null +++ b/v-next/hardhat-node-test-reporter/src/diagnostics.ts @@ -0,0 +1,69 @@ +import { TestEventData } from "./node-types.js"; + +export interface GlobalDiagnostics { + tests: number; + suites: number; + pass: number; + fail: number; + cancelled: number; + skipped: number; + todo: number; + // eslint-disable-next-line @typescript-eslint/naming-convention -- keeping this alingned with the node:test event + duration_ms: number; +} + +/** + * This function receives all the diagnostics that have been emitted by the test + * run, and tries to parse a set of well-known global diagnostics that node:test + * emits to report the overall status of the test run. + * + * If the diagnostics are not recognized, or can't be parsed effectively, they + * are returned as `unsedDiagnostics`, so that we can print them at the end. + */ +export function processGlobalDiagnostics( + diagnostics: Array, +): { + globalDiagnostics: GlobalDiagnostics; + unusedDiagnostics: Array; +} { + const globalDiagnostics: GlobalDiagnostics = { + tests: 0, + suites: 0, + pass: 0, + fail: 0, + cancelled: 0, + skipped: 0, + todo: 0, + duration_ms: 0, + }; + + const unusedDiagnostics = []; + for (const diagnostic of diagnostics) { + if (diagnostic.nesting !== 0) { + unusedDiagnostics.push(diagnostic); + continue; + } + + const [name, numberString] = diagnostic.message.split(" "); + if (!(name in globalDiagnostics) || numberString === undefined) { + unusedDiagnostics.push(diagnostic); + continue; + } + + /* eslint-disable-next-line @typescript-eslint/consistent-type-assertions -- + We checked that thsi is a key of globalDiagnostics */ + const nameAsKey = name as keyof GlobalDiagnostics; + + try { + const value = parseFloat(numberString); + + globalDiagnostics[nameAsKey] = value; + } catch { + // If this throwed, the format of the diagnostic isn't what we expected, + // so we just print it as an unused diagnostic. + unusedDiagnostics.push(diagnostic); + } + } + + return { globalDiagnostics, unusedDiagnostics }; +} diff --git a/v-next/hardhat-node-test-reporter/src/error-formatting.ts b/v-next/hardhat-node-test-reporter/src/error-formatting.ts new file mode 100644 index 0000000000..90e3e414c7 --- /dev/null +++ b/v-next/hardhat-node-test-reporter/src/error-formatting.ts @@ -0,0 +1,84 @@ +import { pathToFileURL } from "node:url"; +import { inspect } from "node:util"; + +import chalk from "chalk"; +import { diff } from "jest-diff"; + +import { + cleanupTestFailError, + isCancelledByParentError, +} from "./node-test-error-utils.js"; + +// TODO: Clean up the node internal fames from the stack trace +export function formatError(error: Error): string { + if (isCancelledByParentError(error)) { + return ( + chalk.red("Test cancelled by parent error") + + "\n" + + chalk.gray( + " This test was cancelled due to an error in its parent suite/it or test/it, or in one of its before/beforeEach", + ) + ); + } + + error = cleanupTestFailError(error); + + const defaultFormat = inspect(error); + const indexOfMessage = defaultFormat.indexOf(error.message); + + let title: string; + let stack: string; + if (indexOfMessage !== -1) { + title = defaultFormat.substring(0, indexOfMessage + error.message.length); + stack = defaultFormat + .substring(indexOfMessage + error.message.length) + .replace(/^(\r?\n)*/, ""); + } else { + title = error.message; + stack = error.stack ?? ""; + } + + title = chalk.red(title); + stack = replaceFileUrlsWithRelativePaths(stack); + stack = chalk.gray(stack); + + const diffResult = getErrorDiff(error); + + if (diffResult === undefined) { + return `${title} +${stack}`; + } + + return `${title} +${diffResult} + +${stack}`; +} + +// TODO: Do this in a more robust way and that works well with windows +function replaceFileUrlsWithRelativePaths(stack: string): string { + return stack.replaceAll( + "(" + pathToFileURL(process.cwd() + "/").toString(), + "(", + ); +} + +function isDiffableError( + error: Error, +): error is Error & { actual: any; expected: any } { + return ( + "expected" in error && "actual" in error && error.expected !== undefined + ); +} + +function getErrorDiff(error: Error): string | undefined { + if (!isDiffableError(error)) { + return undefined; + } + + if ("showDiff" in error && error.showDiff === false) { + return undefined; + } + + return diff(error.expected, error.actual) ?? undefined; +} diff --git a/v-next/hardhat-node-test-reporter/src/formatting.ts b/v-next/hardhat-node-test-reporter/src/formatting.ts new file mode 100644 index 0000000000..7d690bdf36 --- /dev/null +++ b/v-next/hardhat-node-test-reporter/src/formatting.ts @@ -0,0 +1,129 @@ +import chalk from "chalk"; + +import { GlobalDiagnostics } from "./diagnostics.js"; +import { formatError } from "./error-formatting.js"; +import { TestEventData } from "./node-types.js"; + +export const INFO_SYMBOL = chalk.blue("\u2139"); +export const SUCCESS_SYMBOL = chalk.green("✔"); + +export interface Failure { + index: number; + testFail: TestEventData["test:fail"]; + contextStack: Array; +} + +export function formatTestContext( + contextStack: Array, + prefix = "", + suffix = "", +): string { + const contextFragments: string[] = []; + + const prefixLength = prefix.length; + + for (const [i, parentTest] of contextStack.entries()) { + const indentation = nestingToIndentationLength(parentTest.nesting); + + if (i === 0) { + contextFragments.push(indent(prefix, indentation)); + } else { + contextFragments.push("\n"); + contextFragments.push(indent("", indentation + prefixLength)); + } + + contextFragments.push(parentTest.name); + } + + contextFragments.push(suffix); + + return contextFragments.join(""); +} + +export function formatTestPass(passData: TestEventData["test:pass"]): string { + let msg: string; + + if (passData.skip === true || typeof passData.skip === "string") { + // TODO: show skip reason + msg = chalk.cyan(`- ${passData.name}`); + } else if (passData.todo === true || typeof passData.todo === "string") { + // TODO: show todo reason + msg = chalk.blue(`+ ${passData.name}`); + } else { + msg = chalk.gray(`${SUCCESS_SYMBOL} ${passData.name}`); + } + + return indent(msg, nestingToIndentationLength(passData.nesting)); +} + +export function formatTestFailure(failure: Failure): string { + return indent( + chalk.red(`${formatFailureIndex(failure.index)}) ${failure.testFail.name}`), + nestingToIndentationLength(failure.testFail.nesting), + ); +} + +export function formatFailureReason(failure: Failure): string { + return `${formatTestContext( + failure.contextStack, + `${formatFailureIndex(failure.index)}) `, + ":", + )} + +${indent(formatError(failure.testFail.details.error), 3)}`; +} + +export function formatSlowTestInfo(durationMs: number): string { + return ` ${chalk.red(chalk.italic(`(${Math.floor(durationMs)}ms)`))}`; +} + +export function formatGlobalDiagnostics( + diagnostics: GlobalDiagnostics, +): string { + let result = + chalk.green(`${diagnostics.pass} passing`) + + chalk.gray(` (${Math.floor(diagnostics.duration_ms)}ms)`); + + if (diagnostics.fail > 0) { + result += chalk.red(` +${diagnostics.fail} failing`); + } + + if (diagnostics.skipped > 0) { + result += chalk.cyan(` +${diagnostics.skipped} skipped`); + } + + if (diagnostics.todo > 0) { + result += chalk.blue(` +${diagnostics.todo} todo`); + } + + if (diagnostics.cancelled > 0) { + result += chalk.gray(` +${diagnostics.cancelled} cancelled`); + } + + return result; +} + +export function formatUnusedDiagnostics( + unusedDiagnostics: Array, +): string { + return unusedDiagnostics + .map(({ message }) => `${INFO_SYMBOL} ${message}`) + .join("\n"); +} + +function formatFailureIndex(index: number): string { + return (index + 1).toString(); +} + +function nestingToIndentationLength(nesting: number): number { + return (nesting + 1) * 2; +} + +function indent(str: string, spaces: number): string { + const padding = " ".repeat(spaces); + return str.replace(/^/gm, padding); +} diff --git a/v-next/hardhat-node-test-reporter/src/node-test-error-utils.ts b/v-next/hardhat-node-test-reporter/src/node-test-error-utils.ts new file mode 100644 index 0000000000..edb649cc1f --- /dev/null +++ b/v-next/hardhat-node-test-reporter/src/node-test-error-utils.ts @@ -0,0 +1,41 @@ +/** + * Returns true if the error represents that a test/suite failed because one of + * its subtests failed. + */ +export function isSubtestFailedError(error: Error): boolean { + return ( + "code" in error && + "failureType" in error && + error.code === "ERR_TEST_FAILURE" && + error.failureType === "subtestsFailed" + ); +} + +/** + * Returns true if the error represents that a test was cancelled because its + * parent failed. + */ +export function isCancelledByParentError(error: Error): boolean { + return ( + "code" in error && + "failureType" in error && + error.code === "ERR_TEST_FAILURE" && + error.failureType === "cancelledByParent" + ); +} + +/** + * Cleans the test:fail event error, as it's usually wrapped by a node:test + * error. + */ +export function cleanupTestFailError(error: Error): Error { + if ( + "code" in error && + error.code === "ERR_TEST_FAILURE" && + error.cause instanceof Error + ) { + return error.cause; + } + + return error; +} diff --git a/v-next/hardhat-node-test-reporter/src/node-types.ts b/v-next/hardhat-node-test-reporter/src/node-types.ts new file mode 100644 index 0000000000..8404358db2 --- /dev/null +++ b/v-next/hardhat-node-test-reporter/src/node-types.ts @@ -0,0 +1,101 @@ +import type { TestEvent } from "node:test/reporters"; + +/** + * This is missing from `@types/node` 20, so we define our own version of it, + * based on the Node 22 docs. + */ +export interface TestCompletedEventData { + column?: number; + details: { + passed: boolean; + /* eslint-disable-next-line @typescript-eslint/naming-convention -- Keeping + this aligned with the node:test event */ + duration_ms: number; + error?: Error; + type?: string; + }; + file?: string; + line?: number; + name: string; + nesting: number; + testNumber: number; + todo?: string | boolean; + skip?: string | boolean; +} + +/** + * This is missing from `@types/node` 20, so we define our own version of it, + * based on the Node 22 docs. + */ +export interface TestCoverageEventData { + summary: { + files: Array<{ + path: string; + totalLineCount: number; + totalBranchCount: number; + totalFunctionCount: number; + coveredLineCount: number; + coveredBranchCount: number; + coveredFunctionCount: number; + coveredLinePercent: number; + coveredBranchPercent: number; + coveredFunctionPercent: number; + functions: Array<{ + name: string; + line: number; + count: number; + }>; + branches: Array<{ + line: number; + count: number; + }>; + lines: Array<{ + line: number; + count: number; + }>; + }>; + totals: { + totalLineCount: number; + totalBranchCount: number; + totalFunctionCount: number; + coveredLineCount: number; + coveredBranchCount: number; + coveredFunctionCount: number; + coveredLinePercent: number; + coveredBranchPercent: number; + coveredFunctionPercent: number; + }; + workingDirectory: string; + }; + nesting: number; +} + +/** + * This is a fixed version of `@types/node`, as that one is incomplete, at least + * in its version 20. + */ +export type CorrectedTestEvent = + | TestEvent + | { type: "test:complete"; data: TestCompletedEventData } + | { type: "test:coverage"; data: TestCoverageEventData }; + +/** + * A map from event type to its data type. + */ +export type TestEventData = UnionToObject; + +type UnionToObject = { + [K in T as K["type"]]: K extends { type: K["type"]; data: infer D } + ? D + : never; +}; + +/** + * The type of the event source that the reporter will receive. + */ +export type TestEventSource = AsyncGenerator; + +/** + * The type of the result of the reporter. + */ +export type TestReporterResult = AsyncGenerator; diff --git a/v-next/hardhat-node-test-reporter/src/reporter.ts b/v-next/hardhat-node-test-reporter/src/reporter.ts new file mode 100644 index 0000000000..bda26f3b71 --- /dev/null +++ b/v-next/hardhat-node-test-reporter/src/reporter.ts @@ -0,0 +1,247 @@ +import chalk from "chalk"; + +import { processGlobalDiagnostics } from "./diagnostics.js"; +import { + formatFailureReason, + formatGlobalDiagnostics, + formatUnusedDiagnostics, + formatTestContext, + formatTestFailure, + formatTestPass, + formatSlowTestInfo, + Failure, +} from "./formatting.js"; +import { isSubtestFailedError } from "./node-test-error-utils.js"; +import { + TestEventData, + TestEventSource, + TestReporterResult, +} from "./node-types.js"; + +export const SLOW_TEST_THRESHOLD = 75; + +/** + * This is a node:test reporter that tries to mimic Mocha's default reporter, as + * close as possible. + * + * It is designed to output information about the test runs as soon as possible + * and in test defintion order. + * + * Once the test run ends, it will output global information about it, based on + * the diagnostics emitted by node:test, and any custom or unrecognized + * diagnostics message. + * + * Finally, it will output the failure reasons for all the failed tests. + * + * @param source + */ +export default async function* customReporter( + source: TestEventSource, +): TestReporterResult { + /** + * The test reporter works by keeping a stack of the currently executing[1] + * tests and suites. We use it to keep track of the context of a test, so that + * when it passes or fails, we can print that context (if necessary), before + * its results. + * + * Printing the context of a test will normally imply printing all the + * describes where it is nested. + * + * As the context may be shared by more than one test, and we don't want to + * repeate it, we keep track of the last printed context element. We do this + * by keeping track of its index in the stack. + * + * We also keep track of any diagnostic message that its reported by node:test + * and at the end we try to parse the global diagnostics to gather information + * about the test run. If during this parsing we don't recognize or can't + * properly parse one of this diagnostics, we will print it at the end. + * + * Whenever a test fails, we pre-format its failure reason, so that don't need + * to keep the failure event in memory, and we can still print the failure + * reason at the end. + * + * This code is structed in the following way: + * - We use an async generator to process the events as they come, printing + * the information as soon as possible. + * - Instead of printing, we yield string. + * - Any formatting that needs to be done is done in the formatting module. + * - The formatting module exports functions that generate strings for the + * different parts of the test run's output. They do not print anything, and + * they never end in a newline. Any newline between different parts of the + * output is added by the generator. + * - The generaor drives the high-level format of the output, and only uses + * the formatting functions to generate repetitive parts of it. + * + * [1] As reporter by node:test, in defintion order, which may differ from + * actual execution order. + */ + + const stack: Array = []; + + let lastPrintedIndex: number | undefined; + + const diagnostics: Array = []; + + const preFormattedFailureReasons: string[] = []; + + for await (const event of source) { + switch (event.type) { + case "test:diagnostic": { + diagnostics.push(event.data); + break; + } + case "test:start": { + stack.push(event.data); + break; + } + case "test:pass": + case "test:fail": { + if (event.data.details.type === "suite") { + // If a suite failed for a reason other than a subtest failing, we + // want to print its failure. + if (event.type === "test:fail") { + if (!isSubtestFailedError(event.data.details.error)) { + preFormattedFailureReasons.push( + formatFailureReason({ + index: preFormattedFailureReasons.length, + testFail: event.data, + contextStack: stack, + }), + ); + } + } + + // If a suite/describe was already printed, we need to descrease + // the lastPrintedIndex, as we are removing it from the stack. + // + // If its nesting was 0, we print an empty line to separate top-level + // describes. + if (event.data.nesting === 0) { + lastPrintedIndex = undefined; + yield "\n"; + } else { + if (lastPrintedIndex !== undefined) { + lastPrintedIndex = lastPrintedIndex - 1; + + if (lastPrintedIndex < 0) { + lastPrintedIndex = undefined; + } + } + } + + // Remove the current test from the stack, as it was just processed + stack.pop(); + continue; + } + + // If we have printed everything except the current element in the stack + // all of it's context/hierarchy has been printed (e.g. its describes). + // + // Otherwise, we print all the unprinted elements in the stack, except + // for the last one, which is the current test. + if (lastPrintedIndex !== stack.length - 2) { + yield formatTestContext( + stack.slice( + lastPrintedIndex !== undefined ? lastPrintedIndex + 1 : 0, + -1, + ), + ); + yield "\n"; + lastPrintedIndex = stack.length - 2; + } + + if (event.type === "test:pass") { + yield formatTestPass(event.data); + } else { + const failure: Failure = { + index: preFormattedFailureReasons.length, + testFail: event.data, + contextStack: stack, + }; + + // We format the failure reason and store it in an array, so that we + // can output it at the end. + preFormattedFailureReasons.push(formatFailureReason(failure)); + + yield formatTestFailure(failure); + } + + // If the test was slow, we print a message about it + if (event.data.details.duration_ms > SLOW_TEST_THRESHOLD) { + yield formatSlowTestInfo(event.data.details.duration_ms); + } + + yield "\n"; + + // Top-level tests are separated by an empty line + if (event.data.nesting === 0) { + yield "\n"; + } + + // Remove the current test from the stack, as it was just processed it + stack.pop(); + break; + } + case "test:stderr": { + yield event.data.message; + break; + } + case "test:stdout": { + yield event.data.message; + break; + } + case "test:plan": { + // Do nothing + break; + } + case "test:enqueue": { + // Do nothing + break; + } + case "test:dequeue": { + // Do nothing + break; + } + case "test:watch:drained": { + // Do nothing + break; + } + case "test:complete": { + // Do nothing + break; + } + case "test:coverage": { + yield chalk.red("\nTest coverage not supported by this reporter\n"); + break; + } + /* eslint-disable-next-line @typescript-eslint/switch-exhaustiveness-check -- + We have this extra check here becase we know the @types/node type is + unreliable */ + default: { + const _isNever: never = event; + void _isNever; + + yield chalk.red(`Unsuported node:test event:`, event); + break; + } + } + } + + const { globalDiagnostics, unusedDiagnostics } = + processGlobalDiagnostics(diagnostics); + + yield "\n"; + yield formatGlobalDiagnostics(globalDiagnostics); + + if (unusedDiagnostics.length > 0) { + yield "\n"; + yield formatUnusedDiagnostics(unusedDiagnostics); + } + + yield "\n\n"; + + for (const reason of preFormattedFailureReasons) { + yield reason; + yield "\n\n"; + } +} diff --git a/v-next/hardhat-node-test-reporter/test/node-types.ts b/v-next/hardhat-node-test-reporter/test/node-types.ts new file mode 100644 index 0000000000..f88283485a --- /dev/null +++ b/v-next/hardhat-node-test-reporter/test/node-types.ts @@ -0,0 +1,19 @@ +import type { TestEvent } from "node:test/reporters"; + +import { describe, it } from "node:test"; + +describe("Missing @types/node definitions", () => { + it("Should miss test:coverage from TestEvent", () => { + const testCoverageEventIsNotTyped: "test:coverage" extends TestEvent["type"] + ? true + : false = false; + void testCoverageEventIsNotTyped; + }); + + it("Should miss test:completed from TestEvent", () => { + const testCompletedEventIsNotTyped: "test:completed" extends TestEvent["type"] + ? true + : false = false; + void testCompletedEventIsNotTyped; + }); +}); diff --git a/v-next/hardhat-node-test-reporter/tsconfig.json b/v-next/hardhat-node-test-reporter/tsconfig.json new file mode 100644 index 0000000000..31433ab439 --- /dev/null +++ b/v-next/hardhat-node-test-reporter/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../config-v-next/tsconfig.json", + "compilerOptions": { + "outDir": "./dist", + "composite": true, + "incremental": true + }, + "exclude": ["./dist", "./node_modules"], + "references": [ + { + "path": "../hardhat-utils" + } + ] +}