fix: handle large JSON files with compiler outputs as streams

Author: galargh

pnpm-lock.yaml

@@ -80,10 +80,12 @@
     "typescript-eslint": "7.7.1"
   },
   "dependencies": {
+    "@streamparser/json-node": "^0.0.22",
     "debug": "^4.3.2",
     "env-paths": "^2.2.0",
     "ethereum-cryptography": "^2.2.1",
     "fast-equals": "^5.0.1",
+    "json-stream-stringify": "^3.1.6",
     "rfdc": "^1.3.1",
     "undici": "^6.16.1"
   }

v-next/hardhat-utils/package.json

@@ -1,5 +1,12 @@
+import type { JsonTypes, ParsedElementInfo } from "@streamparser/json-node";
+import type { FileHandle } from "node:fs/promises";
+
 import fsPromises from "node:fs/promises";
 import path from "node:path";
+import { pipeline } from "node:stream/promises";
+
+import { JSONParser } from "@streamparser/json-node";
+import { JsonStreamStringify } from "json-stream-stringify";
 
 import { ensureError } from "./error.js";
 import {
@@ -187,6 +194,73 @@ export async function readJsonFile<T>(absolutePathToFile: string): Promise<T> {
   }
 }
 
+/**
+ * Reads a JSON file as a stream and parses it. The encoding used is "utf8".
+ * This function should be used when parsing very large JSON files.
+ *
+ * @param absolutePathToFile The path to the file.
+ * @returns The parsed JSON object.
+ * @throws FileNotFoundError if the file doesn't exist.
+ * @throws InvalidFileFormatError if the file is not a valid JSON file.
+ * @throws IsDirectoryError if the path is a directory instead of a file.
+ * @throws FileSystemAccessError for any other error.
+ */
+export async function readJsonFileAsStream<T>(
+  absolutePathToFile: string,
+): Promise<T> {
+  let fileHandle: FileHandle | undefined;
+
+  try {
+    fileHandle = await fsPromises.open(absolutePathToFile, "r");
+
+    const fileReadStream = fileHandle.createReadStream();
+
+    // NOTE: We set a separator to disable self-closing to be able to use the parser
+    // in the stream.pipeline context; see https://github.com/juanjoDiaz/streamparser-json/issues/47
+    const jsonParser = new JSONParser({
+      separator: "",
+    });
+
+    const result: T | undefined = await pipeline(
+      fileReadStream,
+      jsonParser,
+      async (
+        elements: AsyncIterable<ParsedElementInfo.ParsedElementInfo>,
+      ): Promise<any | undefined> => {
+        let value: JsonTypes.JsonPrimitive | JsonTypes.JsonStruct | undefined;
+        for await (const element of elements) {
+          value = element.value;
+        }
+        return value;
+      },
+    );
+
+    if (result === undefined) {
+      throw new Error("No data");
+    }
+
+    return result;
+  } catch (e) {
+    ensureError<NodeJS.ErrnoException>(e);
+
+    if (e.code === "ENOENT") {
+      throw new FileNotFoundError(absolutePathToFile, e);
+    }
+
+    if (e.code === "EISDIR") {
+      throw new IsDirectoryError(absolutePathToFile, e);
+    }
+
+    if (e.code !== undefined) {
+      throw new FileSystemAccessError(absolutePathToFile, e);
+    }
+
+    throw new InvalidFileFormatError(absolutePathToFile, e);
+  } finally {
+    await fileHandle?.close();
+  }
+}
+
 /**
  * Writes an object to a JSON file. The encoding used is "utf8" and the file is overwritten.
  * If part of the path doesn't exist, it will be created.
@@ -211,6 +285,55 @@ export async function writeJsonFile<T>(
   await writeUtf8File(absolutePathToFile, content);
 }
 
+/**
+ * Writes an object to a JSON file as stream. The encoding used is "utf8" and the file is overwritten.
+ * If part of the path doesn't exist, it will be created.
+ * This function should be used when stringifying very large JSON objects.
+ *
+ * @param absolutePathToFile The path to the file. If the file exists, it will be overwritten.
+ * @param object The object to write.
+ * @throws JsonSerializationError if the object can't be serialized to JSON.
+ * @throws FileSystemAccessError for any other error.
+ */
+export async function writeJsonFileAsStream<T>(
+  absolutePathToFile: string,
+  object: T,
+): Promise<void> {
+  const dirPath = path.dirname(absolutePathToFile);
+  const dirExists = await exists(dirPath);
+  if (!dirExists) {
+    await mkdir(dirPath);
+  }
+
+  let fileHandle: FileHandle | undefined;
+
+  try {
+    fileHandle = await fsPromises.open(absolutePathToFile, "w");
+
+    const jsonStream = new JsonStreamStringify(object);
+    const fileWriteStream = fileHandle.createWriteStream();
+
+    await pipeline(jsonStream, fileWriteStream);
+  } catch (e) {
+    ensureError<NodeJS.ErrnoException>(e);
+    // if the directory was created, we should remove it
+    if (dirExists === false) {
+      try {
+        await remove(dirPath);
+        // we don't want to override the original error
+      } catch (err) {}
+    }
+
+    if (e.code !== undefined) {
+      throw new FileSystemAccessError(e.message, e);
+    }
+
+    throw new JsonSerializationError(absolutePathToFile, e);
+  } finally {
+    await fileHandle?.close();
+  }
+}
+
 /**
  * Reads a file and returns its content as a string. The encoding used is "utf8".
  *
@@ -537,7 +660,9 @@ export async function move(source: string, destination: string): Promise<void> {
     // On linux, trying to move a non-empty directory will throw ENOTEMPTY,
     // while on Windows it will throw EPERM.
     if (e.code === "ENOTEMPTY" || e.code === "EPERM") {
-      throw new DirectoryNotEmptyError(destination, e);
+      if (await isDirectory(source)) {
+        throw new DirectoryNotEmptyError(destination, e);
+      }
     }
 
     throw new FileSystemAccessError(e.message, e);

v-next/hardhat-utils/src/fs.ts

@@ -28,6 +28,8 @@ import {
   readBinaryFile,
   getAccessTime,
   getFileSize,
+  readJsonFileAsStream,
+  writeJsonFileAsStream,
 } from "../src/fs.js";
 
 import { useTmpDir } from "./helpers/fs.js";
@@ -464,6 +466,137 @@ describe("File system utils", () => {
     });
   });
 
+  describe("readJsonFileAsStream", () => {
+    it("Should read and parse a JSON file", async () => {
+      const expectedObject = { a: 1, b: 2 };
+      const filePath = path.join(getTmpDir(), "file.json");
+      await writeUtf8File(filePath, JSON.stringify(expectedObject));
+
+      assert.deepEqual(await readJsonFileAsStream(filePath), expectedObject);
+      expectTypeOf(await readJsonFileAsStream(filePath)).toBeUnknown();
+      expectTypeOf(
+        await readJsonFileAsStream<{ a: number; b: number }>(filePath),
+      ).toMatchTypeOf<{ a: number; b: number }>();
+    });
+
+    it("Should throw InvalidFileFormatError if the file is not valid JSON", async () => {
+      const filePath = path.join(getTmpDir(), "file.json");
+      await writeUtf8File(filePath, "not-json");
+
+      await assert.rejects(readJsonFileAsStream(filePath), {
+        name: "InvalidFileFormatError",
+        message: `Invalid file format: ${filePath}`,
+      });
+    });
+
+    it("Should throw InvalidFileFormatError if the file is empty", async () => {
+      const filePath = path.join(getTmpDir(), "file.json");
+      await writeUtf8File(filePath, "");
+
+      await assert.rejects(readJsonFileAsStream(filePath), {
+        name: "InvalidFileFormatError",
+        message: `Invalid file format: ${filePath}`,
+      });
+    });
+
+    it("Should throw FileNotFoundError if the file doesn't exist", async () => {
+      const filePath = path.join(getTmpDir(), "not-exists.json");
+
+      await assert.rejects(readJsonFileAsStream(filePath), {
+        name: "FileNotFoundError",
+        message: `File ${filePath} not found`,
+      });
+    });
+
+    it("Should throw IsDirectoryError if the file is a directory", async () => {
+      const filePath = path.join(getTmpDir());
+
+      await assert.rejects(readJsonFileAsStream(filePath), {
+        name: "IsDirectoryError",
+        message: `Path ${filePath} is a directory`,
+      });
+    });
+
+    it("Should throw FileSystemAccessError if a different error is thrown", async () => {
+      const invalidPath = "\0";
+
+      await assert.rejects(readJsonFileAsStream(invalidPath), {
+        name: "FileSystemAccessError",
+      });
+    });
+  });
+
+  describe("writeJsonFileAsStream", () => {
+    it("Should write an object to a JSON file", async () => {
+      const expectedObject = { a: 1, b: 2 };
+      const filePath = path.join(getTmpDir(), "file.json");
+
+      await writeJsonFileAsStream(filePath, expectedObject);
+
+      assert.deepEqual(
+        JSON.parse(await readUtf8File(filePath)),
+        expectedObject,
+      );
+      expectTypeOf(
+        writeJsonFile<{ a: number; b: number }>(filePath, expectedObject),
+      );
+    });
+
+    it("Should write an object tto a JSON file even if part of the path doesn't exist", async () => {
+      const expectedObject = { a: 1, b: 2 };
+      const filePath = path.join(getTmpDir(), "not-exists", "file.json");
+
+      await writeJsonFileAsStream(filePath, expectedObject);
+
+      assert.deepEqual(
+        JSON.parse(await readUtf8File(filePath)),
+        expectedObject,
+      );
+    });
+
+    it("Should throw JsonSerializationError if the object can't be serialized to JSON", async () => {
+      const filePath = path.join(getTmpDir(), "file.json");
+      // create an object with a circular reference
+      const circularObject: { self?: {} } = {};
+      circularObject.self = circularObject;
+
+      await assert.rejects(writeJsonFileAsStream(filePath, circularObject), {
+        name: "JsonSerializationError",
+        message: `Error serializing JSON file ${filePath}`,
+      });
+    });
+
+    it("Should throw FileSystemAccessError if a different error is thrown", async () => {
+      const filePath = path.join(getTmpDir(), "protected-file.json");
+      await createFile(filePath);
+
+      try {
+        await chmod(filePath, 0o444);
+
+        await assert.rejects(writeJsonFileAsStream(filePath, {}), {
+          name: "FileSystemAccessError",
+        });
+      } finally {
+        await chmod(filePath, 0o666);
+      }
+    });
+
+    it("Should remove the part of the path that didn't exist before if an error is thrown", async () => {
+      const dirPath = path.join(getTmpDir(), "not-exists");
+      const filePath = path.join(dirPath, "protected-file.json");
+      // create an object with a circular reference
+      const circularObject: { self?: {} } = {};
+      circularObject.self = circularObject;
+
+      await assert.rejects(writeJsonFileAsStream(filePath, circularObject), {
+        name: "JsonSerializationError",
+        message: `Error serializing JSON file ${filePath}`,
+      });
+
+      assert.ok(!(await exists(dirPath)), "The directory should not exist");
+    });
+  });
+
   describe("readUtf8File", () => {
     it("Should read a file and return its content as a string", async () => {
       const content = "hello";