ci: add JS benchmark runner (#4978)

Author: agostbiro

book/src/02_development/01_tools.md

@@ -47,6 +47,8 @@ Only those requests will be collected that can be successfully deserialized.
 4. The scenario file will be written to the current working directory with the desired file name prefix.
 5. Optionally, compress the scenario file `gzip -k <SCENARIO_FILE>`. (The `-k` option preserves the original file, omit it if you want it deleted.)
 
+## Rust runner
+
 ```bash
 
 ### Run scenario
@@ -59,3 +61,7 @@ cargo run --bin tools --release scenario <PATH_TO_SCENARIO_FILE>
 The scenario runner supports both compressed and uncompressed scenario files.
 
 The reported running time excludes reading the requests from disk and parsing them.
+
+## JS runner
+
+Please see the [readme](../../../crates/tools/js/benchmark/README.md) for instructions.

crates/edr_eth/src/signature.rs

@@ -59,6 +59,11 @@ pub fn secret_key_from_str(secret_key: &str) -> Result<SecretKey, SignatureError
     SecretKey::from_bytes(&secret_key).map_err(SignatureError::EllipticCurveError)
 }
 
+/// Converts a secret key to a 0x-prefixed hex string.
+pub fn secret_key_to_str(secret_key: &SecretKey) -> String {
+    format!("0x{}", hex::encode(secret_key.to_bytes().as_slice()))
+}
+
 /// An error involving a signature.
 #[derive(Debug)]
 #[cfg_attr(feature = "std", derive(thiserror::Error))]
@@ -451,4 +456,12 @@ mod tests {
         verify(message, hashed_message);
         verify(hashed_message, hashed_message);
     }
+
+    #[test]
+    fn test_from_str_to_str_secret_key() {
+        let secret_key_str = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80";
+        let secret_key = secret_key_from_str(secret_key_str).unwrap();
+        let secret_key_str_result = secret_key_to_str(&secret_key);
+        assert_eq!(secret_key_str, secret_key_str_result);
+    }
 }

crates/edr_provider/src/config.rs

@@ -87,31 +87,33 @@ pub struct ProviderConfig {
 #[derive(Debug, Clone, Deserialize, Serialize)]
 pub struct AccountConfig {
     /// the secret key of the account
-    #[serde(
-        serialize_with = "serialize_secret_key",
-        deserialize_with = "deserialize_secret_key"
-    )]
+    #[serde(with = "secret_key_serde")]
     pub secret_key: k256::SecretKey,
     /// the balance of the account
     pub balance: U256,
 }
 
-fn serialize_secret_key<S>(secret_key: &k256::SecretKey, serializer: S) -> Result<S::Ok, S::Error>
-where
-    S: serde::Serializer,
-{
-    let s = secret_key
-        .to_sec1_pem(k256::pkcs8::LineEnding::LF)
-        .map_err(serde::ser::Error::custom)?;
-    serializer.serialize_str(&s)
-}
+mod secret_key_serde {
+    use edr_eth::signature::{secret_key_from_str, secret_key_to_str};
+    use serde::Deserialize;
+
+    pub(super) fn serialize<S>(
+        secret_key: &k256::SecretKey,
+        serializer: S,
+    ) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        serializer.serialize_str(&secret_key_to_str(secret_key))
+    }
 
-fn deserialize_secret_key<'de, D>(deserializer: D) -> Result<k256::SecretKey, D::Error>
-where
-    D: serde::Deserializer<'de>,
-{
-    let s = String::deserialize(deserializer)?;
-    k256::SecretKey::from_sec1_pem(&s).map_err(serde::de::Error::custom)
+    pub(super) fn deserialize<'de, D>(deserializer: D) -> Result<k256::SecretKey, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let s = <&str as Deserialize>::deserialize(deserializer)?;
+        secret_key_from_str(s).map_err(serde::de::Error::custom)
+    }
 }
 
 impl Default for MemPoolConfig {

crates/tools/js/benchmark/.gitignore

@@ -0,0 +1 @@
+node_modules

crates/tools/js/benchmark/README.md

@@ -0,0 +1,20 @@
+# JS Benchmark Runner
+
+## Run
+
+To run:
+
+```shell
+pnpm install
+pnpm run benchmark
+```
+
+The measurements will be printed to stdout as machine-readable json and to stderr as human-readable output.
+
+## Grep
+
+It's possible to grep the output to run a specific scenario:
+
+```shell
+npm run benchmark -- --grep seaport
+```

crates/tools/js/benchmark/index.js

@@ -0,0 +1,227 @@
+const path = require("path");
+const fs = require("fs");
+const readline = require("readline");
+const zlib = require("zlib");
+
+const { _ } = require("lodash");
+
+const {
+  createHardhatNetworkProvider,
+} = require("hardhat/internal/hardhat-network/provider/provider");
+
+const SCENARIOS_DIR = "../../scenarios/";
+
+function usage() {
+  console.error("Usage: node index.js [--grep|-g <pattern>]");
+  process.exit(1);
+}
+
+async function main() {
+  const numArgs = process.argv.length;
+
+  if (numArgs !== 2 && numArgs !== 4) {
+    usage();
+  }
+
+  let grep = undefined;
+  if (numArgs === 4) {
+    if (process.argv[2] !== "--grep" && process.argv[2] !== "-g") {
+      usage();
+    }
+
+    grep = process.argv[3];
+  }
+
+  const result = {};
+  const scenariosDir = path.join(__dirname, SCENARIOS_DIR);
+
+  // List files in scenarios directory
+  const scenarioFiles = fs.readdirSync(scenariosDir);
+  scenarioFiles.sort();
+  let totalTime = 0;
+  let totalFailures = 0;
+  for (let scenarioFile of scenarioFiles) {
+    if (grep && !scenarioFile.includes(grep)) {
+      continue;
+    }
+    // Get the filename from the path
+    const scenarioResult = await runScenario(
+      path.join(scenariosDir, scenarioFile)
+    );
+    totalTime += scenarioResult.result.timeMs;
+    totalFailures += scenarioResult.result.failures.length;
+    result[scenarioResult.name] = scenarioResult.result;
+  }
+
+  console.log(JSON.stringify(result));
+
+  // Log info to stderr so that it doesn't pollute stdout where we write the result
+  console.error(
+    `Total time ${
+      Math.round(100 * (totalTime / 1000)) / 100
+    } seconds with ${totalFailures} failures.`
+  );
+
+  process.exit(0);
+}
+
+async function runScenario(scenarioPath) {
+  const { config, requests } = await loadScenario(scenarioPath);
+  const name = path.basename(scenarioPath).split(".")[0];
+  console.error(`Running ${name} scenario`);
+
+  const start = performance.now();
+
+  const provider = await createHardhatNetworkProvider(config.providerConfig, {
+    enabled: config.loggerEnabled,
+  });
+
+  const failures = [];
+
+  for (let i = 0; i < requests.length; i += 1) {
+    try {
+      await provider.request(requests[i]);
+    } catch (e) {
+      failures.push(i);
+    }
+  }
+
+  const timeMs = performance.now() - start;
+
+  console.error(
+    `${name} finished in ${
+      Math.round(100 * (timeMs / 1000)) / 100
+    } seconds with ${failures.length} failures.`
+  );
+
+  return {
+    name,
+    result: {
+      timeMs,
+      failures,
+    },
+  };
+}
+
+async function loadScenario(path) {
+  const result = {
+    requests: [],
+  };
+  let i = 0;
+  for await (const line of readFile(path)) {
+    const parsed = JSON.parse(line);
+    if (i === 0) {
+      result.config = preprocessConfig(parsed);
+    } else {
+      result.requests.push(parsed);
+    }
+    i += 1;
+  }
+  return result;
+}
+
+function preprocessConfig(config) {
+  // From https://stackoverflow.com/a/59771233
+  const camelize = (obj) =>
+    _.transform(obj, (acc, value, key, target) => {
+      const camelKey = _.isArray(target) ? key : _.camelCase(key);
+
+      acc[camelKey] = _.isObject(value) ? camelize(value) : value;
+    });
+  config = camelize(config);
+
+  // EDR serializes None as null to json, but Hardhat expects it to be undefined
+  const removeNull = (obj) =>
+    _.transform(obj, (acc, value, key) => {
+      if (_.isObject(value)) {
+        acc[key] = removeNull(value);
+      } else if (!_.isNull(value)) {
+        acc[key] = value;
+      }
+    });
+  config = removeNull(config);
+
+  config.providerConfig.initialDate = new Date(
+    config.providerConfig.initialDate.secsSinceEpoch * 1000
+  );
+
+  config.providerConfig.hardfork = normalizeHardfork(
+    config.providerConfig.hardfork
+  );
+
+  // "accounts" in EDR are "genesisAccounts" in Hardhat
+  if (Object.keys(config.providerConfig.genesisAccounts).length !== 0) {
+    throw new Error("Genesis accounts are not supported");
+  }
+  config.providerConfig.genesisAccounts = config.providerConfig.accounts.map(
+    ({ balance, secretKey }) => {
+      return { balance, privateKey: secretKey };
+    }
+  );
+  delete config.providerConfig.accounts;
+
+  config.providerConfig.automine = config.providerConfig.mining.autoMine;
+  config.providerConfig.mempoolOrder =
+    config.providerConfig.mining.memPool.order.toLowerCase();
+  config.providerConfig.intervalMining =
+    config.providerConfig.mining.interval ?? 0;
+  delete config.providerConfig.mining;
+
+  config.providerConfig.throwOnCallFailures =
+    config.providerConfig.bailOnCallFailure;
+  delete config.providerConfig.bailOnCallFailure;
+  config.providerConfig.throwOnTransactionFailures =
+    config.providerConfig.bailOnTransactionFailure;
+  delete config.providerConfig.bailOnTransactionFailure;
+
+  let chains = new Map();
+  for (let key of Object.keys(config.providerConfig.chains)) {
+    const hardforkHistory = new Map();
+    const hardforks = config.providerConfig.chains[key].hardforks;
+    for (let [blockNumber, hardfork] of hardforks) {
+      hardforkHistory.set(normalizeHardfork(hardfork), blockNumber);
+    }
+    chains.set(Number(key), { hardforkHistory });
+  }
+  config.providerConfig.chains = chains;
+
+  if (!_.isUndefined(config.providerConfig.fork)) {
+    config.providerConfig.forkConfig = config.providerConfig.fork;
+    delete config.providerConfig.fork;
+  }
+
+  config.providerConfig.minGasPrice = BigInt(config.providerConfig.minGasPrice);
+
+  return config;
+}
+
+function normalizeHardfork(hardfork) {
+  hardfork = _.camelCase(hardfork.toLowerCase());
+  if (hardfork === "frontier") {
+    hardfork = "chainstart";
+  } else if (hardfork === "daoFork") {
+    hardfork = "dao";
+  } else if (hardfork == "tangerine") {
+    hardfork = "tangerineWhistle";
+  }
+  return hardfork;
+}
+
+// From https://stackoverflow.com/a/65015455/2650622
+function readFile(path) {
+  let stream = fs.createReadStream(path);
+
+  if (/\.gz$/i.test(path)) {
+    stream = stream.pipe(zlib.createGunzip());
+  }
+
+  return readline.createInterface({
+    input: stream,
+    crlfDelay: Infinity,
+  });
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exit(1);
+});

crates/tools/js/benchmark/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "benchmark",
+  "version": "1.0.0",
+  "private": true,
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "benchmark": "node index.js",
+    "prebenchmark": "cd ../../../edr_napi/ && pnpm build && cd ../../packages/hardhat-core/ && pnpm build"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "hardhat": "workspace:^",
+    "lodash": "^4.17.11",
+    "tsx": "^4.7.1"
+  }
+}

crates/tools/js/benchmark/tsconfig.json

@@ -0,0 +1,11 @@
+{
+  "compilerOptions": {
+    "target": "es2020",
+    "module": "commonjs",
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "resolveJsonModule": true
+  }
+}