Use LMDB cache instead of regular FS cache to speed up cache hits

Author: Shubham Kanodia

.github/workflows/ci.yml

@@ -11,6 +11,7 @@ jobs:
       - uses: actions/setup-node@v1
         with:
           node-version: "*"
+          cache: 'yarn'
       - name: Install dependencies
         run: yarn
       - name: Lint
@@ -20,7 +21,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
-        node-version: [10.x, 12.x, 14.x, 15.x]
+        node-version: [12.x, 14.x, 16.x]
         webpack-version: [latest, '4']
         include:
           - node: 14.x
@@ -33,6 +34,7 @@ jobs:
         uses: actions/setup-node@v1
         with:
           node-version: ${{ matrix.node-version }}
+          cache: 'yarn'
       - name: Install dependencies
         run: yarn
       - name: Install webpack ${{ matrix.webpack-version }}
@@ -48,29 +50,4 @@ jobs:
         if: ${{ matrix.coverage }}
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-  test-legacy:
-    name: Test - ubuntu-latest - Node v8.9, Webpack 4
-    runs-on: ubuntu-latest
-    env:
-      YARN_NODE_LINKER: node-modules
-    steps:
-      - uses: actions/checkout@v2
-      - name: Use Node.js 14.x
-        uses: actions/setup-node@v1
-        with:
-          node-version: 14.x
-      - name: Install dependencies
-        run: yarn
-      - name: Install webpack 4
-        run: yarn add -D webpack@4
-      - name: Build babel-loader
-        run: yarn run build
-        env:
-          BABEL_ENV: test
-      - name: Use Node.js 8.9
-        uses: actions/setup-node@v1
-        with:
-          node-version: '8.9'
-      - name: Run tests for webpack version 4
-        run: node scripts/test-legacy
 

.yarn/releases/yarn-2.3.3.cjs

@@ -1 +1 @@
-yarnPath: .yarn/releases/yarn-2.3.3.cjs
+yarnPath: .yarn/releases/yarn-3.2.0.cjs

.yarn/releases/yarn-3.2.0.cjs

@@ -3,7 +3,7 @@
     ["@babel/preset-env", {
       "loose": true,
       "targets": {
-        "node": "6.9"
+        "node": "12.0.0"
       }
     }]
   ],

.yarnrc.yml

@@ -11,8 +11,8 @@
   },
   "dependencies": {
     "find-cache-dir": "^3.3.1",
+    "lmdb": "^2.2.4",
     "loader-utils": "^1.4.0",
-    "make-dir": "^3.1.0",
     "schema-utils": "^2.6.5"
   },
   "peerDependencies": {
@@ -36,6 +36,7 @@
     "eslint-plugin-prettier": "^3.0.0",
     "husky": "^4.3.0",
     "lint-staged": "^10.5.1",
+    "node-preload": "^0.2.1",
     "nyc": "^15.1.0",
     "pnp-webpack-plugin": "^1.6.4",
     "prettier": "^2.1.2",

babel.config.json

@@ -7,62 +7,48 @@
  * @see https://github.com/babel/babel-loader/issues/34
  * @see https://github.com/babel/babel-loader/pull/41
  */
-const fs = require("fs");
 const os = require("os");
-const path = require("path");
-const zlib = require("zlib");
 const crypto = require("crypto");
 const findCacheDir = require("find-cache-dir");
-const { promisify } = require("util");
+const { open } = require("lmdb");
 
 const transform = require("./transform");
 // Lazily instantiated when needed
 let defaultCacheDirectory = null;
-
-const readFile = promisify(fs.readFile);
-const writeFile = promisify(fs.writeFile);
-const gunzip = promisify(zlib.gunzip);
-const gzip = promisify(zlib.gzip);
-const makeDir = require("make-dir");
+let cacheDB = null;
 
 /**
- * Read the contents from the compressed file.
- *
- * @async
- * @params {String} filename
- * @params {Boolean} compress
+ * Initialize cache
  */
-const read = async function (filename, compress) {
-  const data = await readFile(filename + (compress ? ".gz" : ""));
-  const content = compress ? await gunzip(data) : data;
 
-  return JSON.parse(content.toString());
-};
+async function initCacheDB(cacheDir, cacheCompression) {
+  if (cacheDB) return cacheDB;
+  const fallback = cacheDir !== os.tmpdir();
 
-/**
- * Write contents into a compressed file.
- *
- * @async
- * @params {String} filename
- * @params {Boolean} compress
- * @params {String} result
- */
-const write = async function (filename, compress, result) {
-  const content = JSON.stringify(result);
+  try {
+    cacheDB = open({
+      path: cacheDir,
+      compression: cacheCompression,
+      sharedStructuresKey: Symbol.for(`structures`),
+    });
+  } catch (err) {
+    if (fallback) {
+      cacheDB = initCacheDB(os.tmpdir(), cacheCompression);
+    }
 
-  const data = compress ? await gzip(content) : content;
-  return await writeFile(filename + (compress ? ".gz" : ""), data);
-};
+    throw err;
+  }
+}
 
 /**
- * Build the filename for the cached file
+ * Build the cache key for the cached file
  *
  * @params {String} source  File source code
  * @params {Object} options Options used
  *
  * @return {String}
  */
-const filename = function (source, identifier, options) {
+const fileCacheKey = function (source, identifier, options) {
   // md4 hashing is not supported starting with node v17.0.0
   const majorNodeVersion = parseInt(process.versions.node.split(".")[0], 10);
   let hashType = "md4";
@@ -76,7 +62,7 @@ const filename = function (source, identifier, options) {
 
   hash.update(contents);
 
-  return hash.digest("hex") + ".json";
+  return hash.digest("hex");
 };
 
 /**
@@ -85,51 +71,21 @@ const filename = function (source, identifier, options) {
  * @params {String} directory
  * @params {Object} params
  */
-const handleCache = async function (directory, params) {
-  const {
-    source,
-    options = {},
-    cacheIdentifier,
-    cacheDirectory,
-    cacheCompression,
-  } = params;
+const handleCache = async function (params) {
+  const { source, options = {}, cacheIdentifier } = params;
 
-  const file = path.join(directory, filename(source, cacheIdentifier, options));
-
-  try {
-    // No errors mean that the file was previously cached
-    // we just need to return it
-    return await read(file, cacheCompression);
-  } catch (err) {}
+  const cacheKey = fileCacheKey(source, cacheIdentifier, options);
 
-  const fallback =
-    typeof cacheDirectory !== "string" && directory !== os.tmpdir();
-
-  // Make sure the directory exists.
-  try {
-    await makeDir(directory);
-  } catch (err) {
-    if (fallback) {
-      return handleCache(os.tmpdir(), params);
-    }
-
-    throw err;
+  // Fetch cached result if it exists
+  const cached = await cacheDB.get(cacheKey);
+  if (typeof cached !== "undefined") {
+    return cached;
   }
 
-  // Otherwise just transform the file
+  // Otherwise, just transform the cacheKey
   // return it to the user asap and write it in cache
   const result = await transform(source, options);
-
-  try {
-    await write(file, cacheCompression, result);
-  } catch (err) {
-    if (fallback) {
-      // Fallback to tmpdir if node_modules folder not writable
-      return handleCache(os.tmpdir(), params);
-    }
-
-    throw err;
-  }
+  cacheDB.put(cacheKey, result);
 
   return result;
 };
@@ -173,5 +129,7 @@ module.exports = async function (params) {
     directory = defaultCacheDirectory;
   }
 
-  return await handleCache(directory, params);
+  await initCacheDB(directory, params.cacheCompression);
+
+  return await handleCache(params);
 };