fix: handle large JSON files with compiler outputs as streams

Author: galargh

pnpm-lock.yaml

@@ -80,10 +80,12 @@
     "typescript-eslint": "7.7.1"
   },
   "dependencies": {
+    "@streamparser/json-node": "^0.0.22",
     "debug": "^4.3.2",
     "env-paths": "^2.2.0",
     "ethereum-cryptography": "^2.2.1",
     "fast-equals": "^5.0.1",
+    "json-stream-stringify": "^3.1.6",
     "rfdc": "^1.3.1",
     "undici": "^6.16.1"
   }

v-next/hardhat-utils/package.json

@@ -1,5 +1,12 @@
+import type { ReadStream, WriteStream } from "node:fs";
+
+import { createReadStream, createWriteStream } from "node:fs";
 import fsPromises from "node:fs/promises";
 import path from "node:path";
+import { finished } from "node:stream/promises";
+
+import { JSONParser } from "@streamparser/json-node";
+import { JsonStreamStringify } from "json-stream-stringify";
 
 import { ensureError } from "./error.js";
 import {
@@ -187,6 +194,65 @@ export async function readJsonFile<T>(absolutePathToFile: string): Promise<T> {
   }
 }
 
+/**
+ * Reads a JSON file as a stream and parses it. The encoding used is "utf8".
+ * This function should be used when parsing very large JSON files.
+ *
+ * @param absolutePathToFile The path to the file.
+ * @returns The parsed JSON object.
+ * @throws FileNotFoundError if the file doesn't exist.
+ * @throws InvalidFileFormatError if the file is not a valid JSON file.
+ * @throws IsDirectoryError if the path is a directory instead of a file.
+ * @throws FileSystemAccessError for any other error.
+ */
+export async function readJsonFileAsStream<T>(
+  absolutePathToFile: string,
+): Promise<T> {
+  const parser = new JSONParser();
+
+  let fileContents: ReadStream;
+  try {
+    fileContents = createReadStream(absolutePathToFile, {
+      encoding: "utf8",
+      autoClose: true,
+      flags: "r",
+    });
+  } catch (e) {
+    ensureError<NodeJS.ErrnoException>(e);
+
+    if (e.code === "ENOENT") {
+      throw new FileNotFoundError(absolutePathToFile, e);
+    }
+
+    if (e.code === "EISDIR") {
+      throw new IsDirectoryError(absolutePathToFile, e);
+    }
+
+    throw new FileSystemAccessError(e.message, e);
+  }
+
+  const readPipeline = fileContents.pipe(parser);
+
+  // NOTE: We save only the last result as it contains the fully parsed object
+  let result: T | undefined;
+  readPipeline.on("data", ({ value }) => {
+    result = value;
+  });
+
+  try {
+    await finished(readPipeline);
+  } catch (e) {
+    ensureError(e);
+    throw new InvalidFileFormatError(absolutePathToFile, e);
+  }
+
+  if (result === undefined) {
+    throw new InvalidFileFormatError(absolutePathToFile, new Error("No data"));
+  }
+
+  return result;
+}
+
 /**
  * Writes an object to a JSON file. The encoding used is "utf8" and the file is overwritten.
  * If part of the path doesn't exist, it will be created.
@@ -211,6 +277,65 @@ export async function writeJsonFile<T>(
   await writeUtf8File(absolutePathToFile, content);
 }
 
+/**
+ * Writes an object to a JSON file as stream. The encoding used is "utf8" and the file is overwritten.
+ * If part of the path doesn't exist, it will be created.
+ * This function should be used when stringifying very large JSON objects.
+ *
+ * @param absolutePathToFile The path to the file. If the file exists, it will be overwritten.
+ * @param object The object to write.
+ * @throws JsonSerializationError if the object can't be serialized to JSON.
+ * @throws FileSystemAccessError for any other error.
+ */
+export async function writeJsonFileAsStream<T>(
+  absolutePathToFile: string,
+  object: T,
+): Promise<void> {
+  const dirPath = path.dirname(absolutePathToFile);
+  const dirExists = await exists(dirPath);
+  if (!dirExists) {
+    await mkdir(dirPath);
+  }
+
+  let fileContents: WriteStream;
+  try {
+    fileContents = createWriteStream(absolutePathToFile, {
+      encoding: "utf8",
+      autoClose: true,
+      mode: 0o666,
+      flags: "w",
+    });
+  } catch (e) {
+    ensureError<NodeJS.ErrnoException>(e);
+    // if the directory was created, we should remove it
+    if (dirExists === false) {
+      try {
+        await remove(dirPath);
+        // we don't want to override the original error
+      } catch (err) {}
+    }
+
+    throw new FileSystemAccessError(e.message, e);
+  }
+
+  const jsonStream = new JsonStreamStringify(object);
+  const writePipeline = jsonStream.pipe(fileContents);
+
+  try {
+    await finished(jsonStream);
+  } catch (e) {
+    ensureError(e);
+    throw new JsonSerializationError(absolutePathToFile, e);
+  }
+
+  try {
+    await finished(writePipeline);
+  } catch (e) {
+    ensureError(e);
+    throw new FileSystemAccessError(e.message, e);
+  }
+}
+
 /**
  * Reads a file and returns its content as a string. The encoding used is "utf8".
  *
@@ -537,7 +662,9 @@ export async function move(source: string, destination: string): Promise<void> {
     // On linux, trying to move a non-empty directory will throw ENOTEMPTY,
     // while on Windows it will throw EPERM.
     if (e.code === "ENOTEMPTY" || e.code === "EPERM") {
-      throw new DirectoryNotEmptyError(destination, e);
+      if (await isDirectory(source)) {
+        throw new DirectoryNotEmptyError(destination, e);
+      }
     }
 
     throw new FileSystemAccessError(e.message, e);

v-next/hardhat-utils/src/fs.ts

@@ -27,6 +27,8 @@ import {
   getAllFilesMatching,
   readJsonFile,
   remove,
+  writeJsonFile,
+  writeJsonFileAsStream,
   writeUtf8File,
 } from "@nomicfoundation/hardhat-utils/fs";
 import { shortenPath } from "@nomicfoundation/hardhat-utils/path";
@@ -557,23 +559,17 @@ export class SolidityBuildSystemImplementation implements SolidityBuildSystem {
       (async () => {
         const buildInfo = await getBuildInfo(compilationJob);
 
-        await writeUtf8File(
-          buildInfoPath,
-          // TODO: Maybe formatting the build info is slow, but it's mostly
-          // strings, so it probably shouldn't be a problem.
-          JSON.stringify(buildInfo, undefined, 2),
-        );
+        // TODO: Maybe formatting the build info is slow, but it's mostly
+        // strings, so it probably shouldn't be a problem.
+        await writeJsonFile(buildInfoPath, buildInfo);
       })(),
       (async () => {
         const buildInfoOutput = await getBuildInfoOutput(
           compilationJob,
           compilerOutput,
         );
 
-        await writeUtf8File(
-          buildInfoOutputPath,
-          JSON.stringify(buildInfoOutput),
-        );
+        await writeJsonFileAsStream(buildInfoOutputPath, buildInfoOutput);
       })(),
     ]);
 

v-next/hardhat/src/internal/builtin-plugins/solidity/build-system/build-system.ts

@@ -6,9 +6,9 @@ import {
   getAllFilesMatching,
   getFileSize,
   move,
-  readJsonFile,
+  readJsonFileAsStream,
   remove,
-  writeJsonFile,
+  writeJsonFileAsStream,
 } from "@nomicfoundation/hardhat-utils/fs";
 
 export class ObjectCache<T> {
@@ -34,13 +34,15 @@ export class ObjectCache<T> {
 
     // NOTE: We are writing to a temporary file first because the value might
     // be large and we don't want to end up with corrupted files in the cache.
-    await writeJsonFile(tmpPath, value);
+    await writeJsonFileAsStream(tmpPath, value);
     await move(tmpPath, filePath);
   }
 
   public async get(key: string): Promise<T | undefined> {
     const filePath = path.join(this.#path, `${key}.json`);
-    return (await exists(filePath)) ? readJsonFile<T>(filePath) : undefined;
+    return (await exists(filePath))
+      ? readJsonFileAsStream<T>(filePath)
+      : undefined;
   }
 
   public async clean(maxAgeMs?: number, maxSize?: number): Promise<void> {