External benchmarks for older solc versions

test/benchmarks/external-setup.sh

@@ -40,74 +40,87 @@ function neutralize_via_ir {
     sed -i '/^via_ir\s*=.*$/d' foundry.toml
 }
 
-mkdir -p "$BENCHMARK_DIR"
-cd "$BENCHMARK_DIR"
+function setup_foundry_project {
+    local subdir="$1"
+    local ref_type="$2"
+    local ref="$3"
+    local repo_url="$4"
+    local install_function="${5:-}"
 
-if [[ ! -e openzeppelin/ ]]; then
-    git clone --depth=1 https://github.com/OpenZeppelin/openzeppelin-contracts openzeppelin/ --branch v5.0.2
-    pushd openzeppelin/
-    forge install
-    neutralize_via_ir
-    popd
-else
-    echo "Skipped openzeppelin/. Already exists."
-fi
-
-if [[ ! -e uniswap-v4/ ]]; then
-    git clone --single-branch https://github.com/Uniswap/v4-core uniswap-v4/
-    pushd uniswap-v4/
-    git checkout ae86975b058d386c9be24e8994236f662affacdb # branch main as of 2024-06-06
-    forge install
-    neutralize_via_ir
-    popd
-else
-    echo "Skipped uniswap-v4/. Already exists."
-fi
-
-if [[ ! -e seaport/ ]]; then
-    git clone --single-branch https://github.com/ProjectOpenSea/seaport
-    pushd seaport/
-    # NOTE: Can't select the tag with `git clone` because a branch of the same name exists.
-    git checkout tags/1.6
-    forge install
-    neutralize_via_ir
+    printf ">>> %-22s | " "$subdir"
+
+    [[ $ref_type == commit || $ref_type == tag ]] || assertFail
+
+    [[ ! -e "$subdir" ]] || { printf "already exists\n"; return; }
+    printf "downloading...\n\n"
+
+    if [[ $ref_type == tag ]]; then
+        git clone --depth=1 "$repo_url" "$subdir" --branch "$ref"
+        pushd "$subdir"
+    else
+        git clone "$repo_url" "$subdir"
+        pushd "$subdir"
+        git checkout "$ref"
+    fi
+    if [[ -z $install_function ]]; then
+        forge install
+    else
+        "$install_function"
+    fi
+
+    [[ ! -e foundry.toml ]] || neutralize_via_ir
     neutralize_version_pragmas
     popd
-else
-    echo "Skipped seaport/. Already exists."
-fi
-
-if [[ ! -e eigenlayer/ ]]; then
-    git clone --depth=1 https://github.com/Layr-Labs/eigenlayer-contracts eigenlayer/ --branch v0.3.0-holesky-rewards
-    pushd eigenlayer/
-    neutralize_via_ir
+    echo
+}
+
+function install_liquity {
+    sed -i -e 's|[email protected]:|https://github.com/|g' .gitmodules
     forge install
-    popd
-else
-    echo "Skipped eigenlayer/. Already exists."
-fi
+}
 
-if [[ ! -e sablier-v2/ ]]; then
-    git clone --depth=1 https://github.com/sablier-labs/v2-core sablier-v2/ --branch v1.1.2
-    pushd sablier-v2/
+function install_old_uniswap {
+    openzeppelin_version=$(sed -n 's|\s\+"@openzeppelin/contracts": "\([0-9.]\+\)"|\1|p' package.json)
+    rm package.json
+    rm yarn.lock
+    npm install "@openzeppelin/contracts@${openzeppelin_version}"
+}
+
+function install_sablier {
     # NOTE: To avoid hard-coding dependency versions here we'd have to install them from npm
     forge install --no-commit \
-        foundry-rs/[email protected] \
-        OpenZeppelin/[email protected] \
-        PaulRBerg/[email protected] \
-        PaulRBerg/[email protected] \
+        foundry-rs/[email protected] \
+        OpenZeppelin/[email protected] \
+        PaulRBerg/[email protected] \
         evmcheb/solarray@a547630 \
-        Vectorized/[email protected].129
+        Vectorized/[email protected].208
    cat <<EOF > remappings.txt
 @openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/
 forge-std/=lib/forge-std/
 @prb/math/=lib/prb-math/
-@prb/test/=lib/prb-test/
 solarray/=lib/solarray/
 solady/=lib/solady/
 EOF
-    neutralize_via_ir
-    popd
-else
-    echo "Skipped sablier-v2/. Already exists."
-fi
+}
+
+mkdir -p "$BENCHMARK_DIR"
+cd "$BENCHMARK_DIR"
+
+setup_foundry_project openzeppelin-5.0.2/ tag v5.0.2 https://github.com/OpenZeppelin/openzeppelin-contracts
+setup_foundry_project openzeppelin-4.9.0/ tag v4.9.0 https://github.com/OpenZeppelin/openzeppelin-contracts
+setup_foundry_project openzeppelin-4.8.0/ tag v4.8.0 https://github.com/OpenZeppelin/openzeppelin-contracts
+setup_foundry_project openzeppelin-4.7.0/ tag v4.7.0 https://github.com/OpenZeppelin/openzeppelin-contracts
+
+setup_foundry_project liquity-2024-10-30/ commit 7f93a3f1781dfce2c4e0b6a7262deddd8a10e45b https://github.com/liquity/V2-gov install_liquity
+
+setup_foundry_project uniswap-v4-2024-06-06/ commit ae86975b058d386c9be24e8994236f662affacdb https://github.com/Uniswap/v4-core
+setup_foundry_project uniswap-v4-2022-06-16/ commit 9aeddf76e1b8646908fbcc7519c882bf458b794d https://github.com/Uniswap/v4-core install_old_uniswap
+
+setup_foundry_project farcaster-3.1.0/ tag v3.1.0 https://github.com/farcasterxyz/contracts
+
+# NOTE: Can't select the tag with `git clone` because a branch of the same name exists.
+setup_foundry_project seaport-1.6/ commit tags/1.6 https://github.com/ProjectOpenSea/seaport
+
+setup_foundry_project eigenlayer-0.3.0/ tag v0.3.0-holesky-rewards https://github.com/Layr-Labs/eigenlayer-contracts
+
+setup_foundry_project sablier-v2-1.2.0/ tag v1.2.0 https://github.com/sablier-labs/v2-core install_sablier

test/benchmarks/external.sh

@@ -65,7 +65,7 @@ function benchmark_project {
         > /dev/null \
         2> "../stderr-${project}-${pipeline}.log" || true
 
-    printf '| %-20s | %8s | %6d s | %9d MiB | %9d |\n' \
+    printf '| %-21s | %8s | %6d s | %9d MiB | %9d |\n' \
         "$project" \
         "$pipeline" \
         "$(jq '(.user + .sys) | round' "$time_file")" \
@@ -76,18 +76,24 @@ function benchmark_project {
 
 benchmarks=(
     # Fastest ones first so that we get *some* output quickly
-    openzeppelin
-    uniswap-v4
-    eigenlayer
-    seaport
-    sablier-v2
+    uniswap-v4-2022-06-16  # compiles via IR with solc >=0.8.12
+    openzeppelin-5.0.2     # compiles via IR with solc >=0.8.26
+    openzeppelin-4.9.0     # compiles via IR with solc 0.8.10-0.8.14 and >=0.8.26
+    liquity-2024-10-30     # compiles via IR with solc >=0.8.24
+    openzeppelin-4.7.0     # compiles via IR with solc >=0.8.10
+    openzeppelin-4.8.0     # compiles via IR with solc >=0.8.10
+    uniswap-v4-2024-06-06  # compiles via IR with solc >=0.8.24
+    eigenlayer-0.3.0       # compiles via IR with solc >=0.8.18
+    sablier-v2-1.2.0       # compiles via IR with solc >=0.8.28 (maybe >=0.8.26)
+    seaport-1.6            # StackTooDeep via IR
+    farcaster-3.1.0        # StackTooDeep via IR
 )
 
 mkdir -p "$BENCHMARK_DIR"
 cd "$BENCHMARK_DIR"
 
-echo "|         File         | Pipeline |   Time   | Memory (peak) | Exit code |"
-echo "|----------------------|----------|---------:|--------------:|----------:|"
+echo "|         File          | Pipeline |   Time   | Memory (peak) | Exit code |"
+echo "|-----------------------|----------|---------:|--------------:|----------:|"
 
 for project in "${benchmarks[@]}"; do
     benchmark_project legacy "$project"