[WIP] ci: automate the release process

.github/actions/proof-params-download/action.yml

@@ -1,9 +1,11 @@
 name: Download proof params
+description: Downloads the proof params from the latest proof-params-generate workflow run
 
 inputs:
   github-token:
     description: 'A secrets.GITHUB_TOKEN'
     required: true
+    default: ${{ github.token }}
 
 runs:
   using: 'composite'
@@ -29,7 +31,7 @@ runs:
       # paramfetch is using `ipget` which currently always times out on CI,
       # hence get this file via HTTP instead.
       #cargo run --release --bin paramfetch -- -a -j srs-inner-product.json
-      run: 'curl https://proofs.filecoin.io/v28-fil-inner-product-v1.srs --output /var/tmp/filecoin-proof-parameters/v28-fil-inner-product-v1.srs'
+      run: 'mkdir -p /var/tmp/filecoin-proof-parameters && curl -fSL https://proofs.filecoin.io/v28-fil-inner-product-v1.srs -o /var/tmp/filecoin-proof-parameters/v28-fil-inner-product-v1.srs'
     - name: List parameter files
       shell: bash
       run: ls -al /var/tmp/filecoin-proof-parameters/

.github/actions/setup-environment/action.yml

@@ -0,0 +1,64 @@
+name: Set up environment
+description: Sets up the environment for running the CI
+
+inputs:
+  github-token:
+    description: 'A secrets.GITHUB_TOKEN'
+    required: true
+    default: ${{ github.token }}
+  gpu:
+    description: 'Whether to enable GPU support'
+    required: false
+    default: 'false'
+  proof-params:
+    description: 'Whether to download the proof params'
+    required: false
+    default: 'false'
+  rust-toolchain:
+    description: 'Whether to install the rust toolchain'
+    required: false
+    default: 'false'
+
+runs:
+  using: 'composite'
+  steps:
+    # TODO: Move the driver installation to the AMI.
+    # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/install-nvidia-driver.html
+    # https://www.nvidia.com/en-us/drivers/
+    - name: Install CUDA drivers
+      if: ${{ inputs.gpu == 'true' }}
+      run: |
+        curl -L -o nvidia-driver-local-repo-ubuntu2404-570.148.08_1.0-1_amd64.deb https://us.download.nvidia.com/tesla/570.148.08/nvidia-driver-local-repo-ubuntu2404-570.148.08_1.0-1_amd64.deb
+        sudo dpkg -i nvidia-driver-local-repo-ubuntu2404-570.148.08_1.0-1_amd64.deb
+        sudo cp /var/nvidia-driver-local-repo-ubuntu2404-570.148.08/nvidia-driver-local-*-keyring.gpg /usr/share/keyrings/
+        sudo apt-get update
+        sudo apt-get install --no-install-recommends --yes cuda-drivers nvidia-cuda-toolkit
+        rm nvidia-driver-local-repo-ubuntu2404-570.148.08_1.0-1_amd64.deb
+      shell: bash
+
+    - name: Load NVIDIA kernel modules and verify GPU
+      if: ${{ inputs.gpu == 'true' }}
+      run: |
+        sudo modprobe nvidia
+        sudo modprobe nvidia-uvm
+        nvidia-smi || echo "WARNING: nvidia-smi failed, GPU may not be available"
+      shell: bash
+
+    - name: Install required packages
+      run: |
+        sudo apt-get update
+        sudo apt-get install --no-install-recommends --yes libhwloc-dev ocl-icd-opencl-dev
+      shell: bash
+
+    - name: Download the proof params
+      if: ${{ inputs.proof-params == 'true' }}
+      uses: ./.github/actions/proof-params-download
+      with:
+        github-token: ${{ inputs.github-token }}
+
+    # TODO: Remove this and other rust installation directives from jobs running
+    # on self-hosted runners once rust is available on these machines by default
+    - uses: dtolnay/rust-toolchain@21dc36fb71dd22e3317045c0c31a3f4249868b17
+      if: ${{ inputs.rust-toolchain == 'true' }}
+      with:
+        toolchain: 1.83

.github/workflows/ci.yml

@@ -1,7 +1,7 @@
 name: CI
 
 on:
-  pull_request:
+  # pull_request:
   push:
     branches:
       - master
@@ -24,11 +24,8 @@ jobs:
     runs-on: ubuntu-24.04
     name: Clippy
     steps:
-      - uses: actions/checkout@v4
-      - name: Install required packages
-        run: |
-          sudo apt-get update
-          sudo apt-get install --no-install-recommends --yes libhwloc-dev ocl-icd-opencl-dev
+      - uses: actions/checkout@v5
+      - uses: ./.github/actions/setup-environment
       - name: Install cargo clippy
         run: rustup component add clippy
       - name: Run cargo clippy
@@ -38,7 +35,7 @@ jobs:
     runs-on: ubuntu-24.04
     name: Checking fmt
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Install cargo fmt
         run: rustup component add rustfmt
       - name: Run cargo fmt
@@ -55,22 +52,11 @@ jobs:
       # Run all tests with multicore-SDR enabled.
       FIL_PROOFS_USE_MULTICORE_SDR: true
     steps:
-      - uses: actions/checkout@v4
-      - name: Install required packages
-        run: |
-          sudo apt-get update
-          sudo apt-get install --no-install-recommends --yes libhwloc-dev ocl-icd-opencl-dev
-
-      - name: Download the proof params
-        uses: ./.github/actions/proof-params-download
+      - uses: actions/checkout@v5
+      - uses: ./.github/actions/setup-environment
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-
-      # TODO: Remove this and other rust installation directives from jobs running 
-      # on self-hosted runners once rust is available on these machines by default
-      - uses: dtolnay/rust-toolchain@21dc36fb71dd22e3317045c0c31a3f4249868b17
-        with:
-          toolchain: 1.83
+          proof-params: true
+          rust-toolchain: true
 
       - name: Run usual tests in release profile
         run: cargo test --verbose --release --workspace --all-targets ${{ matrix.cargo-args }} -- --nocapture
@@ -83,16 +69,10 @@ jobs:
     runs-on: ubuntu-24.04
     name: Test ignored in release mode
     steps:
-      - uses: actions/checkout@v4
-      - name: Install required packages
-        run: |
-          sudo apt-get update
-          sudo apt-get install --no-install-recommends --yes libhwloc-dev ocl-icd-opencl-dev
-
-      - name: Download the proof params
-        uses: ./.github/actions/proof-params-download
+      - uses: actions/checkout@v5
+      - uses: ./.github/actions/setup-environment
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          proof-params: true
 
       - name: Test ignored in release profile
         run: cargo test --release --workspace -- ignored --nocapture
@@ -101,20 +81,11 @@ jobs:
     runs-on: ['self-hosted', 'linux', 'x64', '2xlarge']
     name: Test without default features
     steps:
-      - uses: actions/checkout@v4
-      - name: Install required packages
-        run: |
-          sudo apt-get update
-          sudo apt-get install --no-install-recommends --yes libhwloc-dev ocl-icd-opencl-dev
-
-      - name: Download the proof params
-        uses: ./.github/actions/proof-params-download
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-
-      - uses: dtolnay/rust-toolchain@21dc36fb71dd22e3317045c0c31a3f4249868b17
+      - uses: actions/checkout@v5
+      - uses: ./.github/actions/setup-environment
         with:
-          toolchain: 1.83
+          proof-params: true
+          rust-toolchain: true
 
       - name: Test ignored in release profile
         run: cargo test --release --workspace --no-default-features
@@ -123,15 +94,10 @@ jobs:
     runs-on: ['self-hosted', 'linux', 'x64', '2xlarge']
     name: Build with various GPU support enabled
     steps:
-      - uses: actions/checkout@v4
-      - name: Install required packages
-        run: |
-          sudo apt-get update
-          sudo apt-get install --no-install-recommends --yes libhwloc-dev nvidia-cuda-toolkit ocl-icd-opencl-dev
-
-      - uses: dtolnay/rust-toolchain@21dc36fb71dd22e3317045c0c31a3f4249868b17
+      - uses: actions/checkout@v5
+      - uses: ./.github/actions/setup-environment
         with:
-          toolchain: 1.83
+          rust-toolchain: true
 
       - name: Build with `cuda` and `opencl` features enabled
         run: cargo build --workspace --features cuda,opencl
@@ -151,31 +117,12 @@ jobs:
       BELLMAN_CUDA_NVCC_ARGS: --fatbin --gpu-architecture=sm_75 --generate-code=arch=compute_75,code=sm_75
       NEPTUNE_CUDA_NVCC_ARGS: --fatbin --gpu-architecture=sm_75 --generate-code=arch=compute_75,code=sm_75
     steps:
-      - uses: actions/checkout@v4
-      # TODO: Move the driver installation to the AMI.
-      # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/install-nvidia-driver.html
-      # https://www.nvidia.com/en-us/drivers/
-      - name: Install CUDA drivers
-        run: |
-          curl -L -o nvidia-driver-local-repo-ubuntu2404-570.148.08_1.0-1_amd64.deb https://us.download.nvidia.com/tesla/570.148.08/nvidia-driver-local-repo-ubuntu2404-570.148.08_1.0-1_amd64.deb
-          sudo dpkg -i nvidia-driver-local-repo-ubuntu2404-570.148.08_1.0-1_amd64.deb
-          sudo cp /var/nvidia-driver-local-repo-ubuntu2404-570.148.08/nvidia-driver-local-*-keyring.gpg /usr/share/keyrings/
-          sudo apt-get update
-          sudo apt-get install --no-install-recommends --yes cuda-drivers
-          rm nvidia-driver-local-repo-ubuntu2404-570.148.08_1.0-1_amd64.deb
-      - name: Install required packages
-        run: |
-          sudo apt-get update
-          sudo apt-get install --no-install-recommends --yes libhwloc-dev nvidia-cuda-toolkit ocl-icd-opencl-dev
-
-      - name: Download the proof params
-        uses: ./.github/actions/proof-params-download
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-
-      - uses: dtolnay/rust-toolchain@21dc36fb71dd22e3317045c0c31a3f4249868b17
+      - uses: actions/checkout@v5
+      - uses: ./.github/actions/setup-environment
         with:
-          toolchain: 1.83
+          gpu: true
+          proof-params: true
+          rust-toolchain: true
 
       - name: Test with CUDA
         run: cargo test --verbose --release --workspace --features cuda -- --nocapture ${{ matrix.test-args }}
@@ -192,8 +139,6 @@ jobs:
 
       - name: Download the proof params
         uses: ./.github/actions/proof-params-download
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Run usual tests in release profile
         run: cargo test --verbose --release --workspace -- --nocapture

.github/workflows/release.yml

@@ -0,0 +1,46 @@
+name: Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      level:
+        description: 'Release level'
+        required: true
+        default: 'patch'
+        type: choice
+        options:
+          - patch
+          - minor
+          - major
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    permissions:
+      # The contents write should allow:
+      # 1. Push to the branch of the repository that triggered the workflow.
+      # 2. Create a tag.
+      # 3. Push to crates.io.
+      contents: write
+      # The id-token write should allow the OIDC token exchange
+      id-token: write
+    steps:
+      - uses: actions/checkout@v5
+      - name: Install required packages
+        run: sudo apt install --no-install-recommends --yes libhwloc-dev nvidia-cuda-toolkit ocl-icd-opencl-dev
+      - name: Install cargo release
+        run: cargo install --version 0.25.17 cargo-release
+      - name: Set git user
+        run: |
+          git config --global user.email "${GITHUB_TRIGGERING_ACTOR}@users.noreply.github.com"
+          git config --global user.name "${GITHUB_TRIGGERING_ACTOR}"
+      - uses: rust-lang/crates-io-auth-action@v1
+        id: auth
+      - name: Run cargo release
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }}
+        run: cargo release ${{ github.event.inputs.level }} --no-confirm --execute

.github/workflows/test.yml

@@ -0,0 +1,86 @@
+name: Test
+
+on:
+  workflow_dispatch:
+  push:
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  CARGO_INCREMENTAL: 0
+  # Faster crates.io index checkout.
+  CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
+
+jobs:
+  # basic:
+  #   runs-on: ['self-hosted', 'linux', 'x64', 'xlarge+gpu']
+  #   name: Basic tests (${{ matrix.name }})
+  #   strategy:
+  #     matrix:
+  #       include:
+  #         - name: regular
+  #           command: cargo test --release --all
+  #         - name: ignored
+  #           command: cargo test --release --all -- --ignored
+  #     fail-fast: false
+  #   env:
+  #     FIL_PROOFS_VERIFY_CACHE: 1
+  #     FIL_PROOFS_VERIFY_PRODUCTION_PARAMS: 0
+  #     FIL_PROOFS_USE_GPU_COLUMN_BUILDER: 1
+  #     FIL_PROOFS_USE_GPU_TREE_BUILDER: 1
+  #     FIL_PROOFS_USE_MULTICORE_SDR: 1
+  #     RUST_BACKTRACE: full
+  #     RUST_LOG: debug
+  #     RUSTFLAGS: -C target-cpu=native
+  #   steps:
+  #     - uses: actions/checkout@v5
+  #     - uses: ./.github/actions/setup-environment
+  #       with:
+  #         gpu: true
+  #         proof-params: true
+  #         rust-toolchain: true
+  #     - name: Run basic tests
+  #       run: ${{ matrix.command }}
+
+  extended:
+    # needs: basic
+    runs-on: ['self-hosted', 'linux', 'x64', '16xlarge+gpu']
+    name: Extended tests
+    env:
+      FIL_PROOFS_VERIFY_CACHE: 1
+      FIL_PROOFS_VERIFY_PRODUCTION_PARAMS: 1
+      FIL_PROOFS_USE_GPU_COLUMN_BUILDER: 1
+      FIL_PROOFS_USE_GPU_TREE_BUILDER: 1
+      FIL_PROOFS_USE_MULTICORE_SDR: 1
+      RUST_BACKTRACE: full
+      RUST_LOG: debug
+      RUSTFLAGS: -C target-cpu=native
+    steps:
+      - uses: actions/checkout@v5
+      - uses: ./.github/actions/setup-environment
+        with:
+          gpu: true
+          proof-params: true
+          rust-toolchain: true
+      - name: Test 32GiB seal lifecycle
+        run: cargo test --features big-tests --release test_seal_lifecycle_32gib_porep_id_v1_1_top_8_8_0_api_v1_1 -- --nocapture
+      - name: Test 32GiB seal lifecycle (synth-porep)
+        run: cargo test --features big-tests --release test_seal_lifecycle_32gib_porep_id_v1_2_top_8_8_0_api_v1_2 -- --nocapture
+      - name: Test 32GiB seal lifecycle (ni-porep)
+        run: cargo test --features big-tests --release test_seal_lifecycle_32gib_porep_id_v1_2_ni_top_8_8_0_api_v1_2 -- --nocapture
+      - name: Test 32GiB max seal proof aggregation
+        run: cargo test --features big-tests --release test_max_ni_seal_proof_aggregation_32gib -- --nocapture
+      - name: Test 32GiB seal lifecycle sector upgrade test
+        run: cargo test --features big-tests --release test_seal_lifecycle_upgrade_32gib_top_8_8_0_v1_2 -- --nocapture
+      - name: Test 64GiB seal lifecycle
+        run: cargo test --features big-tests --release test_seal_lifecycle_64gib_porep_id_v1_1_top_8_8_2_api_v1_1 -- --nocapture
+      - name: Test 64GiB seal lifecycle (synth-porep)
+        run: cargo test --features big-tests --release test_seal_lifecycle_64gib_porep_id_v1_2_top_8_8_2_api_v1_2 -- --nocapture
+      - name: Test 64GiB seal lifecycle (ni-porep)
+        run: cargo test --features big-tests --release test_seal_lifecycle_64gib_porep_id_v1_2_ni_top_8_8_2_api_v1_2 -- --nocapture
+      - name: Test 64GiB seal proof aggregation
+        run: cargo test --features big-tests --release test_sector_update_proof_aggregation_3_64gib -- --nocapture
+      - name: Test 64GiB seal lifecycle sector upgrade test
+        run: cargo test --features big-tests --release test_seal_lifecycle_upgrade_64gib_top_8_8_2_v1_2 -- --nocapture

.gitignore

@@ -1,6 +1,5 @@
 /target
 **/*.rs.bk
-Cargo.lock
 .criterion
 **/*.h
 heaptrack*