Validate-Benchmarks #7195
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # .github/workflows/run-benchmarks.yml | |
| name: Validate-Benchmarks | |
| # Benchmark validation is opt-in per PR: run-to-run variance on the reference | |
| # machine was flagging noise as drift (e.g. batch_all measured 3.8ms, 34.7ms, | |
| # and 9.9ms across three consecutive runs of identical code), so PRs only | |
| # benchmark when the `run-benchmarks` label is applied. The nightly schedule | |
| # reruns the full suite against the default branch as the drift backstop (on | |
| # failure, apply the bench-patch artifact via the apply-benchmark-patch label | |
| # on a fix PR). The gate lives in a cheap `decide` job on a free GitHub runner | |
| # so skipped PRs never occupy the persistent Benchmarking machine. | |
| # validate-benchmarks is not a required merge check today. If that changes, | |
| # this decide-job shape keeps working (the workflow always triggers and the | |
| # skipped job satisfies branch protection), unlike a `paths:` trigger filter. | |
| on: | |
| pull_request: | |
| types: [opened, synchronize, labeled, unlabeled] | |
| schedule: | |
| - cron: "0 3 * * *" # nightly full-suite drift check on the default branch | |
| workflow_dispatch: | |
| # This workflow only builds the node and validates benchmark weights; it never | |
| # pushes commits or edits the PR. The apply-benchmark-patch workflow is what | |
| # writes results back, so the validate job runs with a read-only token to keep | |
| # the persistent `Benchmarking` runner from holding a write-capable token while | |
| # it compiles PR-controlled code (arbitrary build.rs). | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| concurrency: | |
| group: run-benchmarks-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| trusted-pr: | |
| name: Trusted PR source (non-fork) | |
| if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false | |
| runs-on: ubuntu-latest | |
| steps: | |
| - run: echo "Non-fork PR; self-hosted runners may execute checkout." | |
| decide: | |
| name: decide whether to benchmark | |
| runs-on: ubuntu-latest | |
| outputs: | |
| run: ${{ steps.gate.outputs.run }} | |
| steps: | |
| - name: Gate on event and label | |
| id: gate | |
| run: | | |
| set -euo pipefail | |
| if [[ "$GITHUB_EVENT_NAME" != "pull_request" ]]; then | |
| echo "$GITHUB_EVENT_NAME event: full run" | |
| echo "run=true" >> "$GITHUB_OUTPUT" | |
| elif jq -e '.pull_request.labels[].name | select(. == "run-benchmarks")' "$GITHUB_EVENT_PATH" > /dev/null; then | |
| echo "run-benchmarks label present: running" | |
| echo "run=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "no run-benchmarks label: skipping (nightly run covers drift)" | |
| echo "run=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| validate-benchmarks: | |
| needs: [trusted-pr, decide] | |
| if: needs.decide.outputs.run == 'true' | |
| runs-on: Benchmarking | |
| steps: | |
| - name: Check out PR branch | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| with: | |
| # Fall back to the current repo/ref so workflow_dispatch and | |
| # scheduled runs work too. | |
| repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} | |
| ref: ${{ github.event.pull_request.head.ref || github.ref_name }} | |
| fetch-depth: 0 | |
| - name: Install system dependencies | |
| run: | | |
| sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get update | |
| sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get install -y --no-install-recommends \ | |
| -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" \ | |
| build-essential clang curl libssl-dev llvm libudev-dev protobuf-compiler pkg-config | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable | |
| - name: Shared R2 compiler cache | |
| uses: ./.github/actions/sccache-setup | |
| - name: Cache Cargo registry and git data | |
| uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 | |
| with: | |
| key: bench-${{ hashFiles('**/Cargo.lock') }} | |
| cache-on-failure: true | |
| cache-targets: false | |
| - name: Build node with benchmarks + weight tools | |
| run: | | |
| cargo build --profile production -p node-subtensor --features runtime-benchmarks | |
| cargo build --profile production -p subtensor-weight-tools --bin weight-compare | |
| - name: Run & validate benchmarks | |
| timeout-minutes: 180 | |
| run: | | |
| mkdir -p .bench_patch | |
| chmod +x scripts/benchmark_action.sh | |
| scripts/benchmark_action.sh | |
| - name: Archive bench patch | |
| if: always() | |
| run: | | |
| if [ -d ".bench_patch" ]; then | |
| tar -czf bench-patch.tgz .bench_patch | |
| fi | |
| - name: Upload patch artifact | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: bench-patch | |
| path: bench-patch.tgz | |
| if-no-files-found: warn |