Improved benchmarking (risc0#1621)

Author: flaub

.github/workflows/bench_pr.yml

@@ -27,7 +27,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [Linux, macOS]
-        feature: [default]
+        feature: [prove]
         device: [cpu]
         include:
           - os: Linux
@@ -38,16 +38,11 @@ jobs:
             device: apple_m2_pro
 
     env:
-      CI_BENCH_PR: true
       FEATURE: ${{ matrix.feature }}
       DISABLE_S3: ${{ matrix.device == 'nvidia_rtx_3090_ti' }}
 
     steps:
-      # This is a workaround from: https://github.com/actions/checkout/issues/590#issuecomment-970586842
-      - run: "git checkout -f $(git -c user.name=x -c user.email=x@x commit-tree $(git hash-object -t tree /dev/null) < /dev/null) || :"
-
-      - name: Check out the current PR
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v4
 
       - if: matrix.feature == 'cuda'
         uses: ./.github/actions/cuda
@@ -59,5 +54,24 @@ jobs:
 
       - run: cargo run --bin cargo-risczero --no-default-features -- risczero install --version $RISC0_TOOLCHAIN_VERSION
 
-      - name: Run Benchmarks
-        run: cargo bench --bench divan -F $FEATURE
+      - name: Run benchmarks
+        run: cargo bench --bench fib -F $FEATURE
+
+      - name: Download previous benchmark data
+        uses: actions/cache@v4
+        with:
+          path: ./cache
+          key: fib-${{ runner.os }}-${{ matrix.device }}
+
+      - name: Analyze benchmark results
+        uses: risc0/[email protected]
+        with:
+          name: "${{ matrix.os }}-${{ matrix.device }}"
+          tool: 'customBiggerIsBetter'
+          output-file-path: target/hotbench/fib/benchmark.json
+          external-data-json-path: ./cache/benchmark.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          alert-threshold: '120%'
+          comment-on-alert: true
+          fail-on-alert: true
+          summary-always: true

.github/workflows/bench_trends.yml

@@ -8,6 +8,8 @@ on:
       - 'risc0/**'
       - '.github/workflows/bench_*'
 
+  workflow_dispatch:
+
   # uncomment for only for testing changes to this workflow while in a PR
   # pull_request:
   #   branches: [ main ]
@@ -39,7 +41,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [Linux, macOS]
-        feature: [default]
+        feature: [prove]
         device: [cpu]
         include:
           - os: Linux
@@ -51,29 +53,31 @@ jobs:
 
     env:
       FEATURE: ${{ matrix.feature }}
+      DISABLE_S3: ${{ matrix.device == 'nvidia_rtx_3090_ti' }}
 
     steps:
-      # This is a workaround from: https://github.com/actions/checkout/issues/590#issuecomment-970586842
-      - run: "git checkout -f $(git -c user.name=x -c user.email=x@x commit-tree $(git hash-object -t tree /dev/null) < /dev/null) || :"
       - uses: actions/checkout@v4
+
       - if: matrix.feature == 'cuda'
         uses: ./.github/actions/cuda
       - uses: ./.github/actions/rustup
       - uses: ./.github/actions/sccache
         with:
           key: ${{ matrix.os }}-${{ matrix.feature }}
+          disable_s3: ${{ env.DISABLE_S3 }}
+
       - run: cargo run --bin cargo-risczero --no-default-features -- risczero install --version $RISC0_TOOLCHAIN_VERSION
-      - run: cargo bench -F $FEATURE --bench fib -- --output-format=bencher | tee output.txt
+
+      - name: Run benchmarks
+        run: cargo bench -F $FEATURE --bench fib
+
       - name: Store benchmark result
-        uses: risc0/github-action-benchmark@risc0
+        uses: risc0/github-action-benchmark@v1.19.3
         with:
           name: "${{ matrix.os }}-${{ matrix.device }}"
-          tool: 'cargo'
+          tool: 'customBiggerIsBetter'
           github-token: ${{ secrets.BENCHMARK_TOKEN }}
           gh-repository: 'github.com/risc0/ghpages'
-          gh-repository-dir: '${{ runner.temp }}/benchmark-data-repository'
           gh-pages-branch: main
-          output-file-path: output.txt
-          comment-always: false
-          comment-on-alert: false
+          output-file-path: target/hotbench/fib/benchmark.json
           auto-push: true

Cargo.lock

@@ -19,6 +19,7 @@ members = [
   "risc0/zkvm/methods",
   "risc0/zkvm/platform",
   "risc0/zkvm/receipts",
+  "tools/hotbench",
   "website/doc-test/main",
   "xtask",
 ]
@@ -34,6 +35,7 @@ repository = "https://github.com/risc0/risc0/"
 [workspace.dependencies]
 bonsai-rest-api-mock = { version = "0.8.0-alpha.1", default-features = false, path = "bonsai/rest-api-mock" }
 bonsai-sdk = { version = "0.8.0-alpha.1", default-features = false, path = "bonsai/sdk" }
+hotbench = { path = "tools/hotbench" }
 risc0-binfmt = { version = "0.22.0-alpha.1", default-features = false, path = "risc0/binfmt" }
 risc0-build = { version = "0.22.0-alpha.1", default-features = false, path = "risc0/build" }
 risc0-build-kernel = { version = "0.22.0-alpha.1", default-features = false, path = "risc0/build_kernel" }

Cargo.toml

@@ -11,7 +11,7 @@ repository = { workspace = true }
 anyhow = "1.0"
 # This is a workaround to prevent the guest from pulling in a dependency that requires a newer rustc
 cargo-platform = "=0.1.5"
-cargo_metadata = "0.17"
+cargo_metadata = "0.18"
 docker-generate = "0.1"
 risc0-binfmt = { workspace = true }
 risc0-zkp = { workspace = true, features = ["std"] }

benchmarks/Cargo.lock

@@ -239,7 +239,7 @@ mod test {
         build("../../risc0/zkvm/methods/guest/Cargo.toml");
         compare_image_id(
             "risc0_zkvm_methods_guest/multi_test",
-            "1219f3a9e9f6a6639705070c151a90d10120f81fdf13d50273080567ea80cb6c",
+            "254d1845a847dc4a8df7b9663f9418fa41647966109315da4f3dc8622b22dbff",
         );
     }
 }

bonsai/Cargo.lock

@@ -358,7 +358,7 @@ fn build_staticlib(guest_pkg: &str, features: &[&str]) -> String {
     let mut child = cmd.stdout(Stdio::piped()).spawn().unwrap();
     let reader = std::io::BufReader::new(child.stdout.take().unwrap());
     let mut libs = Vec::new();
-    for message in cargo_metadata::Message::parse_stream(reader) {
+    for message in Message::parse_stream(reader) {
         match message.unwrap() {
             Message::CompilerArtifact(artifact) => {
                 for filename in artifact.filenames {

examples/Cargo.lock

@@ -83,6 +83,7 @@ typetag = { version = "0.2", optional = true }
 clap = { version = "4.4", features = ["derive"] }
 criterion = { version = "0.5", features = ["html_reports"] }
 divan = "0.1"
+hotbench = { workspace = true }
 lazy_static = "1.4.0"
 rand = "0.8"
 tracing-forest = "0.1"