Track max level more lazily

.github/workflows/benchmark_pr.yml

@@ -38,19 +38,19 @@ jobs:
             - name: Run benchmarks
               run: |
                 mkdir results
-                benchpkg --add https://github.com/LilithHafner/ChairmarksForAirspeedVelocity.jl ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --url=${{ github.event.repository.clone_url }} --bench-on="${{github.event.pull_request.head.sha}}" --output-dir=results/ --tune
+                benchpkg --add https://github.com/LilithHafner/ChairmarksForAirspeedVelocity.jl ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.pull_request.base.sha}},${{github.event.pull_request.head.sha}}" --url=${{ github.event.repository.clone_url }} --bench-on="${{github.event.pull_request.head.sha}}" --output-dir=results/ --tune
             - name: Create plots from benchmarks
               run: |
                 mkdir -p plots
-                benchpkgplot ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --npart=10 --format=png --input-dir=results/ --output-dir=plots/
+                benchpkgplot ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.pull_request.base.sha}},${{github.event.pull_request.head.sha}}" --npart=10 --format=png --input-dir=results/ --output-dir=plots/
             - name: Upload plot as artifact
               uses: actions/upload-artifact@v4
               with:
                 name: plots
                 path: plots
             - name: Create markdown table from benchmarks
               run: |
-                benchpkgtable ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --input-dir=results/ --ratio > table.md
+                benchpkgtable ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.pull_request.base.sha}},${{github.event.pull_request.head.sha}}" --input-dir=results/ --ratio > table.md
                 echo '### Benchmark Results' > body.md
                 echo '' >> body.md
                 echo '' >> body.md

.github/workflows/benchmark_pr_2.yml

@@ -0,0 +1,86 @@
+name: Benchmark a pull request (2)
+# Keep this in sync with benchmark_push.yml
+
+on:
+  pull_request:
+concurrency:
+  # Skip and cancel intermediate builds:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  pull-requests: write
+
+jobs:
+    generate_plots:
+        runs-on: ubuntu-latest
+
+        steps:
+            - uses: actions/checkout@v4
+            - uses: julia-actions/setup-julia@v2
+            - uses: julia-actions/cache@v2
+            - name: Extract Package Name from Project.toml
+              id: extract-package-name
+              run: |
+                PACKAGE_NAME=$(grep "^name" Project.toml | sed 's/^name = "\(.*\)"$/\1/')
+                echo "::set-output name=package_name::$PACKAGE_NAME"
+            - name: Build AirspeedVelocity
+              env:
+                JULIA_NUM_THREADS: 2
+              run: |
+                # TODO: cache this build step and skip it if the cache hits (but still receive updates)
+                # Lightweight build step, as sometimes the runner runs out of memory:
+                julia -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.add(;url="https://github.com/MilesCranmer/AirspeedVelocity.jl.git")'
+                julia -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.build("AirspeedVelocity")'
+            - name: Add ~/.julia/bin to PATH
+              run: |
+                echo "$HOME/.julia/bin" >> $GITHUB_PATH
+            - name: Run benchmarks
+              run: |
+                mkdir results
+                benchpkg --add https://github.com/LilithHafner/ChairmarksForAirspeedVelocity.jl ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.pull_request.base.sha}},${{github.event.pull_request.head.sha}}" --url=${{ github.event.repository.clone_url }} --bench-on="${{github.event.pull_request.head.sha}}" --output-dir=results/ --tune
+            - name: Create plots from benchmarks
+              run: |
+                mkdir -p plots
+                benchpkgplot ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.pull_request.base.sha}},${{github.event.pull_request.head.sha}}" --npart=10 --format=png --input-dir=results/ --output-dir=plots/
+            - name: Upload plot as artifact
+              uses: actions/upload-artifact@v4
+              with:
+                name: plots
+                path: plots
+            - name: Create markdown table from benchmarks
+              run: |
+                benchpkgtable ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.pull_request.base.sha}},${{github.event.pull_request.head.sha}}" --input-dir=results/ --ratio > table.md
+                echo '### Benchmark Results' > body.md
+                echo '' >> body.md
+                echo '' >> body.md
+                cat table.md >> body.md
+                echo '' >> body.md
+                echo '' >> body.md
+                echo '### Benchmark Plots' >> body.md
+                echo 'A plot of the benchmark results have been uploaded as an artifact to the workflow run for this PR.' >> body.md
+                echo 'Go to "Actions"->"Benchmark a pull request"->[the most recent run]->"Artifacts" (at the bottom).' >> body.md
+
+            - name: wait
+              run: sleep 45
+
+            - name: Find Comment
+              uses: peter-evans/find-comment@v3
+              id: fcbenchmark
+              with:
+                issue-number: ${{ github.event.pull_request.number }}
+                comment-author: 'github-actions[bot]'
+                body-includes: Benchmark Results
+
+            - name: Join Tables
+              run: |
+                echo '${{ steps.fcbenchmark.outputs.comment-body }}' >> old_body.md
+                julia .github/workflows/join_table.jl body.md old_body.md merged_body.md
+
+            - name: Comment on PR
+              uses: peter-evans/create-or-update-comment@v4
+              with:
+                comment-id: ${{ steps.fcbenchmark.outputs.comment-id }}
+                issue-number: ${{ github.event.pull_request.number }}
+                body-path: merged_body.md
+                edit-mode: replace

.github/workflows/benchmark_pr_3.yml

@@ -0,0 +1,86 @@
+name: Benchmark a pull request (3)
+# Keep this in sync with benchmark_push.yml
+
+on:
+  pull_request:
+concurrency:
+  # Skip and cancel intermediate builds:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  pull-requests: write
+
+jobs:
+    generate_plots:
+        runs-on: ubuntu-latest
+
+        steps:
+            - uses: actions/checkout@v4
+            - uses: julia-actions/setup-julia@v2
+            - uses: julia-actions/cache@v2
+            - name: Extract Package Name from Project.toml
+              id: extract-package-name
+              run: |
+                PACKAGE_NAME=$(grep "^name" Project.toml | sed 's/^name = "\(.*\)"$/\1/')
+                echo "::set-output name=package_name::$PACKAGE_NAME"
+            - name: Build AirspeedVelocity
+              env:
+                JULIA_NUM_THREADS: 2
+              run: |
+                # TODO: cache this build step and skip it if the cache hits (but still receive updates)
+                # Lightweight build step, as sometimes the runner runs out of memory:
+                julia -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.add(;url="https://github.com/MilesCranmer/AirspeedVelocity.jl.git")'
+                julia -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.build("AirspeedVelocity")'
+            - name: Add ~/.julia/bin to PATH
+              run: |
+                echo "$HOME/.julia/bin" >> $GITHUB_PATH
+            - name: Run benchmarks
+              run: |
+                mkdir results
+                benchpkg --add https://github.com/LilithHafner/ChairmarksForAirspeedVelocity.jl ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.pull_request.base.sha}},${{github.event.pull_request.head.sha}}" --url=${{ github.event.repository.clone_url }} --bench-on="${{github.event.pull_request.head.sha}}" --output-dir=results/ --tune
+            - name: Create plots from benchmarks
+              run: |
+                mkdir -p plots
+                benchpkgplot ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.pull_request.base.sha}},${{github.event.pull_request.head.sha}}" --npart=10 --format=png --input-dir=results/ --output-dir=plots/
+            - name: Upload plot as artifact
+              uses: actions/upload-artifact@v4
+              with:
+                name: plots
+                path: plots
+            - name: Create markdown table from benchmarks
+              run: |
+                benchpkgtable ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.pull_request.base.sha}},${{github.event.pull_request.head.sha}}" --input-dir=results/ --ratio > table.md
+                echo '### Benchmark Results' > body.md
+                echo '' >> body.md
+                echo '' >> body.md
+                cat table.md >> body.md
+                echo '' >> body.md
+                echo '' >> body.md
+                echo '### Benchmark Plots' >> body.md
+                echo 'A plot of the benchmark results have been uploaded as an artifact to the workflow run for this PR.' >> body.md
+                echo 'Go to "Actions"->"Benchmark a pull request"->[the most recent run]->"Artifacts" (at the bottom).' >> body.md
+
+            - name: wait
+              run: sleep 90
+
+            - name: Find Comment
+              uses: peter-evans/find-comment@v3
+              id: fcbenchmark
+              with:
+                issue-number: ${{ github.event.pull_request.number }}
+                comment-author: 'github-actions[bot]'
+                body-includes: Benchmark Results
+
+            - name: Join Tables
+              run: |
+                echo '${{ steps.fcbenchmark.outputs.comment-body }}' >> old_body.md
+                julia .github/workflows/join_table.jl body.md old_body.md merged_body.md
+
+            - name: Comment on PR
+              uses: peter-evans/create-or-update-comment@v4
+              with:
+                comment-id: ${{ steps.fcbenchmark.outputs.comment-id }}
+                issue-number: ${{ github.event.pull_request.number }}
+                body-path: merged_body.md
+                edit-mode: replace

.github/workflows/join_table.jl

@@ -0,0 +1,39 @@
+new = replace(String(read(Sys.ARGS[1])), ".../"=>" / ", "..."=>"")
+old = replace(String(read(Sys.ARGS[2])), ".../"=>" / ", "..."=>"")
+
+function combine((n,o))
+    @show n o # Debug print statement, just in case.
+    if count(==('|'), n) <= 3
+        # @assert n == o
+        return n
+    end
+
+    n_cols = split(n, '|')
+    o_cols = split(o, '|')
+    # @assert length(n_cols) == length(o_cols)
+    # @assert isempty(first(n_cols))
+    # @assert isempty(last(n_cols))
+    # @assert isempty(first(o_cols))
+    # @assert isempty(last(o_cols))
+    # @assert n_cols[2] == o_cols[2]
+
+    if all(isspace, n_cols[2]) || all(∈([':','-']), n_cols[2])
+        # @assert n == o
+        return n
+    end
+
+    o_data = strip(o_cols[end-1])
+    n_data = strip(n_cols[end-1])
+    n_cols[end-1] = if o_data == n_data * "," * n_data # If all three results are the same, only report one
+        n_data
+    else
+        o_data * "," * n_data
+    end
+    join(n_cols, '|')
+end
+
+new2 = join(combine.(zip(split(new, '\n'), split(old, '\n'))), '\n')
+
+open(Sys.ARGS[3], "w") do io
+    write(io, new2)
+end

.gitignore

@@ -4,3 +4,5 @@
 /Manifest.toml
 /docs/Manifest.toml
 /docs/build/
+results_*@*.json
+results_*@*.json.tmp

Project.toml

@@ -4,22 +4,19 @@ authors = ["Lilith Orion Hafner <[email protected]>, Adriano Meligrana <adr
 version = "0.0.0"
 
 [deps]
-Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
-StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 
 [compat]
-Distributions = "0.25"
 Random = "1.10.0"
-StaticArrays = "1.9.8"
 julia = "1.10.0"
 
 [extras]
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
 HypothesisTests = "09f84164-cd44-5f33-b23f-e6b0d136a0d5"
 StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"
 StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
+StatsFuns = "4c63d2b9-4356-54db-8cca-17b64c39e42c"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Aqua", "HypothesisTests", "StableRNGs", "StatsBase", "Test"]
+test = ["Aqua", "HypothesisTests", "StableRNGs", "StatsBase", "StatsFuns", "Test"]