fix: handle large JSON files with compiler outputs as streams

Author: galargh

.changeset/proud-walls-pump.md

@@ -0,0 +1,6 @@
+---
+"@nomicfoundation/hardhat-utils": patch
+"hardhat": patch
+---
+
+Started using streams when handling the solc compiler outputs to support compilation of very large codebases where the compilation outputs might exceed the maximum buffer size/string lenght

pnpm-lock.yaml

@@ -80,10 +80,12 @@
     "typescript-eslint": "7.7.1"
   },
   "dependencies": {
+    "@streamparser/json-node": "^0.0.22",
     "debug": "^4.3.2",
     "env-paths": "^2.2.0",
     "ethereum-cryptography": "^2.2.1",
     "fast-equals": "^5.0.1",
+    "json-stream-stringify": "^3.1.6",
     "rfdc": "^1.3.1",
     "undici": "^6.16.1"
   }

v-next/hardhat-utils/package.json

@@ -1,5 +1,12 @@
+import type { JsonTypes, ParsedElementInfo } from "@streamparser/json-node";
+import type { FileHandle } from "node:fs/promises";
+
 import fsPromises from "node:fs/promises";
 import path from "node:path";
+import { pipeline } from "node:stream/promises";
+
+import { JSONParser } from "@streamparser/json-node";
+import { JsonStreamStringify } from "json-stream-stringify";
 
 import { ensureError } from "./error.js";
 import {
@@ -187,6 +194,76 @@ export async function readJsonFile<T>(absolutePathToFile: string): Promise<T> {
   }
 }
 
+/**
+ * Reads a JSON file as a stream and parses it. The encoding used is "utf8".
+ * This function should be used when parsing very large JSON files.
+ *
+ * @param absolutePathToFile The path to the file.
+ * @returns The parsed JSON object.
+ * @throws FileNotFoundError if the file doesn't exist.
+ * @throws InvalidFileFormatError if the file is not a valid JSON file.
+ * @throws IsDirectoryError if the path is a directory instead of a file.
+ * @throws FileSystemAccessError for any other error.
+ */
+export async function readJsonFileAsStream<T>(
+  absolutePathToFile: string,
+): Promise<T> {
+  let fileHandle: FileHandle | undefined;
+
+  try {
+    fileHandle = await fsPromises.open(absolutePathToFile, "r");
+
+    const fileReadStream = fileHandle.createReadStream();
+
+    // NOTE: We set a separator to disable self-closing to be able to use the parser
+    // in the stream.pipeline context; see https://github.com/juanjoDiaz/streamparser-json/issues/47
+    const jsonParser = new JSONParser({
+      separator: "",
+    });
+
+    const result: T | undefined = await pipeline(
+      fileReadStream,
+      jsonParser,
+      async (
+        elements: AsyncIterable<ParsedElementInfo.ParsedElementInfo>,
+      ): Promise<any | undefined> => {
+        let value: JsonTypes.JsonPrimitive | JsonTypes.JsonStruct | undefined;
+        for await (const element of elements) {
+          value = element.value;
+        }
+        return value;
+      },
+    );
+
+    if (result === undefined) {
+      throw new Error("No data");
+    }
+
+    return result;
+  } catch (e) {
+    ensureError<NodeJS.ErrnoException>(e);
+
+    if (e.code === "ENOENT") {
+      throw new FileNotFoundError(absolutePathToFile, e);
+    }
+
+    if (e.code === "EISDIR") {
+      throw new IsDirectoryError(absolutePathToFile, e);
+    }
+
+    // If the code is defined, we assume the error to be related to the file system
+    if (e.code !== undefined) {
+      throw new FileSystemAccessError(absolutePathToFile, e);
+    }
+
+    // Otherwise, we assume the error to be related to the file formatting
+    throw new InvalidFileFormatError(absolutePathToFile, e);
+  } finally {
+    // Explicitly closing the file handle to fully release the underlying resources
+    await fileHandle?.close();
+  }
+}
+
 /**
  * Writes an object to a JSON file. The encoding used is "utf8" and the file is overwritten.
  * If part of the path doesn't exist, it will be created.
@@ -211,6 +288,59 @@ export async function writeJsonFile<T>(
   await writeUtf8File(absolutePathToFile, content);
 }
 
+/**
+ * Writes an object to a JSON file as stream. The encoding used is "utf8" and the file is overwritten.
+ * If part of the path doesn't exist, it will be created.
+ * This function should be used when stringifying very large JSON objects.
+ *
+ * @param absolutePathToFile The path to the file. If the file exists, it will be overwritten.
+ * @param object The object to write.
+ * @throws JsonSerializationError if the object can't be serialized to JSON.
+ * @throws FileSystemAccessError for any other error.
+ */
+export async function writeJsonFileAsStream<T>(
+  absolutePathToFile: string,
+  object: T,
+): Promise<void> {
+  const dirPath = path.dirname(absolutePathToFile);
+  const dirExists = await exists(dirPath);
+  if (!dirExists) {
+    await mkdir(dirPath);
+  }
+
+  let fileHandle: FileHandle | undefined;
+
+  try {
+    fileHandle = await fsPromises.open(absolutePathToFile, "w");
+
+    const jsonStream = new JsonStreamStringify(object);
+    const fileWriteStream = fileHandle.createWriteStream();
+
+    await pipeline(jsonStream, fileWriteStream);
+  } catch (e) {
+    ensureError<NodeJS.ErrnoException>(e);
+    // if the directory was created, we should remove it
+    if (dirExists === false) {
+      try {
+        await remove(dirPath);
+        // we don't want to override the original error
+      } catch (err) {}
+    }
+
+    // If the code is defined, we assume the error to be related to the file system
+    if (e.code !== undefined) {
+      throw new FileSystemAccessError(e.message, e);
+    }
+
+    // Otherwise, we assume the error to be related to the file formatting
+    throw new JsonSerializationError(absolutePathToFile, e);
+  } finally {
+    // NOTE: Historically, not closing the file handle caused issues on Windows,
+    // for example, when trying to move the file previously written to by this function
+    await fileHandle?.close();
+  }
+}
+
 /**
  * Reads a file and returns its content as a string. The encoding used is "utf8".
  *
@@ -537,7 +667,9 @@ export async function move(source: string, destination: string): Promise<void> {
     // On linux, trying to move a non-empty directory will throw ENOTEMPTY,
     // while on Windows it will throw EPERM.
     if (e.code === "ENOTEMPTY" || e.code === "EPERM") {
-      throw new DirectoryNotEmptyError(destination, e);
+      if (await isDirectory(source)) {
+        throw new DirectoryNotEmptyError(destination, e);
+      }
     }
 
     throw new FileSystemAccessError(e.message, e);