Skip to content

Commit 2314b4c

Browse files
soonumsoonum
committed
chore(ci): add workflow to update documentation benchmark tables
This new workflow can trigger all the required benchmarks needed to populate benchmarks tables in documentation. It also can generate SVG tables and store them as artifacts. Optionally, it can open a pull-request to update the current tables in documentation.
1 parent 4497a16 commit 2314b4c

14 files changed

+497
-99
lines changed

‎.github/workflows/benchmark_cpu.yml‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,14 @@ on:
3434
- smart
3535
- unchecked
3636
- misc
37-
all_precisions:
38-
description: "Run all precisions"
39-
type: boolean
40-
default: false
37+
precisions_set:
38+
description: "Bit precisions set"
39+
type: choice
40+
default: fast
41+
options:
42+
- fast
43+
- all
44+
- documentation
4145
bench_type:
4246
description: "Benchmarks type"
4347
type: choice
@@ -71,7 +75,7 @@ jobs:
7175
op_flavor: ${{ inputs.op_flavor }}
7276
bench_type: ${{ inputs.bench_type }}
7377
params_type: ${{ inputs.params_type }}
74-
all_precisions: ${{ inputs.all_precisions }}
78+
precisions_set: ${{ inputs.precisions_set }}
7579
secrets:
7680
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
7781
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}

‎.github/workflows/benchmark_cpu_common.yml‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,9 @@ on:
1616
params_type:
1717
type: string
1818
default: classical
19-
all_precisions:
20-
type: boolean
21-
default: false
19+
precisions_set:
20+
type: string
21+
default: fast
2222
additional_recipe: # Make recipes to run aside the benchmarks.
2323
type: string # Use comma separated values to generate an array
2424
additional_file_to_parse: # Other files to parse, located under tfhe-benchmark/ directory
@@ -54,7 +54,6 @@ env:
5454
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
5555
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
5656
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
57-
FAST_BENCH: TRUE
5857

5958
permissions: {}
6059

@@ -162,19 +161,15 @@ jobs:
162161
with:
163162
toolchain: nightly
164163

165-
- name: Should run benchmarks with all precisions
166-
if: inputs.all_precisions
167-
run: |
168-
echo "FAST_BENCH=FALSE" >> "${GITHUB_ENV}"
169-
170164
- name: Run benchmarks with AVX512
171165
run: |
172-
make BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"
166+
make BIT_SIZES_SET="${PRECISIONS_SET}" BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"
173167
env:
174168
OP_FLAVOR: ${{ matrix.op_flavor }}
175169
BENCH_TYPE: ${{ matrix.bench_type }}
176170
BENCH_PARAMS_TYPE: ${{ matrix.params_type }}
177171
BENCH_COMMAND: ${{ matrix.command }}
172+
PRECISIONS_SET: ${{ inputs.precisions_set }}
178173

179174
- name: Parse results
180175
run: |

‎.github/workflows/benchmark_cpu_weekly.yml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ jobs:
3030
is_weekly_bench_group_2: ${{ steps.check_bench_group_2.outputs.is_weekly_bench_group_2 }}
3131
is_quarterly_bench: ${{ steps.check_quarterly_bench.outputs.is_quarterly_bench }}
3232
op_flavor: ${{ steps.set_op_flavor.outputs.op_flavor }}
33-
all_precisions: ${{ steps.set_all_precisions.outputs.all_precisions }}
33+
precisions_set: ${{ steps.set_precisions_set.outputs.precisions_set }}
3434
steps:
3535
- name: Check is weekly bench group 1
3636
id: check_bench_group_1
@@ -51,23 +51,23 @@ jobs:
5151
if: steps.check_bench_group_1.outputs.is_weekly_bench_group_1 || steps.check_bench_group_2.outputs.is_weekly_bench_group_2
5252
run: |
5353
echo "OP_FLAVOR=[\"default\"]" >> "${GITHUB_ENV}"
54-
echo "ALL_PRECISIONS=false" >> "${GITHUB_ENV}"
54+
echo "PRECISIONS_SET=false" >> "${GITHUB_ENV}"
5555
5656
- name: Quarterly benchmarks
5757
if: steps.check_quarterly_bench.outputs.is_quarterly_bench
5858
run: |
5959
echo "OP_FLAVOR=[\"default\", \"unchecked\"]" >> "${GITHUB_ENV}"
60-
echo "ALL_PRECISIONS=true" >> "${GITHUB_ENV}"
60+
echo "PRECISIONS_SET=true" >> "${GITHUB_ENV}"
6161
6262
- name: Set operation flavor output
6363
id: set_op_flavor
6464
run: | # zizmor: ignore[template-injection] this env variable is safe
6565
echo "op_flavor=${{ toJSON(env.OP_FLAVOR) }}" >> "${GITHUB_OUTPUT}"
6666
6767
- name: Set bit precisions output
68-
id: set_all_precisions
68+
id: set_precisions_set
6969
run: | # zizmor: ignore[template-injection] this env variable is safe
70-
echo "all_precisions=${{ toJSON(env.ALL_PRECISIONS) }}" >> "${GITHUB_OUTPUT}"
70+
echo "precisions_set=${{ toJSON(env.PRECISIONS_SET) }}" >> "${GITHUB_OUTPUT}"
7171
7272
run-benchmarks-integer:
7373
name: benchmark_gpu_weekly/run-benchmarks-integer
@@ -78,7 +78,7 @@ jobs:
7878
with:
7979
command: integer,signed_integer, integer_compression
8080
op_flavor: ${{ needs.prepare-inputs.outputs.op_flavor }}
81-
all_precisions: ${{ needs.prepare-inputs.outputs.all_precisions }}
81+
precisions_set: ${{ needs.prepare-inputs.outputs.precisions_set }}
8282
secrets:
8383
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
8484
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}

‎.github/workflows/benchmark_documentation.yml‎

Lines changed: 177 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,188 @@
1-
name: benchmark_documentation.yml
1+
# Run all benchmarks displayed in the public documentation.
2+
name: benchmark_documentation
23

34
on:
45
workflow_dispatch:
6+
inputs:
7+
run-benchmarks:
8+
description: "Run benchmarks"
9+
type: boolean
10+
default: true
11+
generate-svgs:
12+
description: "Generate SVG tables"
13+
type: boolean
14+
default: true
15+
open-pr:
16+
description: "Open a PR with the benchmark results"
17+
type: boolean
18+
default: false
519

620
permissions: {}
721

822
# zizmor: ignore[concurrency-limits] only Zama organization members can trigger this workflow
923

1024
jobs:
11-
placeholder:
12-
name: benchmark_documentation/placeholder
13-
runs-on: ubuntu-latest
25+
run-benchmarks-cpu-integer:
26+
name: benchmark_documentation/run-benchmarks-cpu-integer
27+
uses: ./.github/workflows/benchmark_cpu_common.yml
28+
with:
29+
command: integer
30+
op_flavor: fast_default
31+
bench_type: both
32+
precisions_set: documentation
33+
secrets:
34+
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
35+
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
36+
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
37+
REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }}
38+
JOB_SECRET: ${{ secrets.JOB_SECRET }}
39+
SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }}
40+
SLAB_URL: ${{ secrets.SLAB_URL }}
41+
SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }}
42+
43+
run-benchmarks-gpu-integer:
44+
name: benchmark_documentation/run-benchmarks-gpu-integer
45+
uses: ./.github/workflows/benchmark_gpu_common.yml
46+
with:
47+
profile: multi-h100-sxm5
48+
hardware_name: n3-H100-SXM5x8
49+
command: integer_multi_bit
50+
op_flavor: fast_default
51+
bench_type: both
52+
precisions_set: documentation
53+
secrets:
54+
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
55+
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
56+
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
57+
REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }}
58+
JOB_SECRET: ${{ secrets.JOB_SECRET }}
59+
SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }}
60+
SLAB_URL: ${{ secrets.SLAB_URL }}
61+
SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }}
62+
63+
run-benchmarks-hpu-integer:
64+
name: benchmark_documentation/run-benchmarks-hpu-integer
65+
uses: ./.github/workflows/benchmark_hpu_common.yml
66+
with:
67+
command: integer_hpu
68+
op_flavor: default
69+
bench_type: both
70+
precisions_set: documentation
71+
v80_pcie_dev: 24
72+
v80_serial_number: XFL12NWY3ZKG
73+
secrets:
74+
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
75+
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
76+
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
77+
REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }}
78+
JOB_SECRET: ${{ secrets.JOB_SECRET }}
79+
SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }}
80+
SLAB_URL: ${{ secrets.SLAB_URL }}
81+
SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }}
82+
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
83+
84+
run-benchmarks-cpu-core-crypto:
85+
name: benchmark_documentation/run-benchmarks-cpu-core-crypto
86+
uses: ./.github/workflows/benchmark_cpu_common.yml
87+
with:
88+
command: pbs, ks_pbs
89+
bench_type: latency
90+
params_type: classical_documentation + multi_bit_documentation
91+
secrets:
92+
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
93+
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
94+
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
95+
REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }}
96+
JOB_SECRET: ${{ secrets.JOB_SECRET }}
97+
SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }}
98+
SLAB_URL: ${{ secrets.SLAB_URL }}
99+
SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }}
100+
101+
run-benchmarks-gpu-core-crypto:
102+
name: benchmark_gpu_weekly/run-benchmarks-gpu-core-crypto
103+
uses: ./.github/workflows/benchmark_gpu_common.yml
104+
with:
105+
profile: multi-h100-sxm5
106+
hardware_name: n3-H100-SXM5x8
107+
command: pbs, ks_pbs
108+
bench_type: latency
109+
params_type: classical_documentation + multi_bit_documentation
110+
secrets:
111+
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
112+
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
113+
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
114+
REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }}
115+
JOB_SECRET: ${{ secrets.JOB_SECRET }}
116+
SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }}
117+
SLAB_URL: ${{ secrets.SLAB_URL }}
118+
SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }}
119+
120+
generate-svgs-with-benchmarks-run:
121+
name: benchmark-documentation/generate-svgs-with-benchmarks-run
122+
if: inputs.run-benchmarks && inputs.generate-svgs
123+
needs: [
124+
run-benchmarks-cpu-integer, run-benchmarks-gpu-integer, run-benchmarks-hpu-integer,
125+
run-benchmarks-cpu-core-crypto, run-benchmarks-gpu-core-crypto
126+
]
127+
uses: ./.github/workflows/generate_svgs.yml
128+
with:
129+
time_span_days: 1 # This ensures extraction is performed on the last values stored in database/
130+
secrets:
131+
DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
132+
DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
133+
DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}
14134

135+
generate-svgs-without-benchmarks-run:
136+
name: benchmark-documentation/generate-svgs-without-benchmarks-run
137+
if: ${{ !inputs.run-benchmarks && inputs.generate-svgs }}
138+
uses: ./.github/workflows/generate_svgs.yml
139+
with:
140+
time_span_days: 60
141+
secrets:
142+
DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
143+
DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
144+
DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}
145+
146+
open-pr:
147+
name: benchmark-documentation/open-pr
148+
needs: [ generate-svgs-with-benchmarks-run, generate-svgs-without-benchmarks-run ]
149+
if: ${{ always() && inputs.open-pr &&
150+
(needs.generate-svgs-with-benchmarks-run.result == 'success' || needs.generate-svgs-without-benchmarks-run.result == 'success') }}
151+
runs-on: ubuntu-latest
152+
permissions:
153+
contents: write # Needed to create a commit
154+
pull-requests: write # Needed to open a pull-request
155+
env:
156+
PATH_TO_DOC_ASSETS: tfhe/docs/.gitbook/assets
15157
steps:
16-
- run: |
17-
echo "Hello this is a placeholder workflow"
158+
- name: Checkout tfhe-rs
159+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
160+
with:
161+
persist-credentials: 'false'
162+
163+
- name: Download SVG tables
164+
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
165+
with:
166+
path: svg_tables
167+
merge-multiple: 'true'
168+
169+
- name: Copy SVG tables to documentation location
170+
run: |
171+
cp -f svg_tables/*integer-benchmark*.svg "${PATH_TO_DOC_ASSETS}"
172+
cp -f svg_tables/*pbs-benchmark-tuniform*.svg "${PATH_TO_DOC_ASSETS}"
173+
174+
- name: Create pull-request
175+
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
176+
with:
177+
sign-commits: true
178+
token: ${{ secrets.FHE_DIVISION_TOKEN }} # Sign commit as Zama Bot
179+
add-paths: ${{ env.PATH_TO_DOC_ASSETS }}/*.svg
180+
commit-message: |
181+
chore(docs): update benchmark results for all backends
182+
183+
Automated documentation update from tfhe-rs CI pipeline.
184+
title: |
185+
[CI] chore(docs): update benchmark results for all backends
186+
body: |
187+
Documentation update triggered by GitHub workflow*
188+
labels: documentation

‎.github/workflows/benchmark_gpu.yml‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,14 @@ on:
4040
- default
4141
- fast_default
4242
- unchecked
43-
all_precisions:
44-
description: "Run all precisions"
45-
type: boolean
46-
default: false
43+
precisions_set:
44+
description: "Bit precisions set"
45+
type: choice
46+
default: fast
47+
options:
48+
- fast
49+
- all
50+
- documentation
4751
bench_type:
4852
description: "Benchmarks type"
4953
type: choice
@@ -106,7 +110,7 @@ jobs:
106110
op_flavor: ${{ inputs.op_flavor }}
107111
bench_type: ${{ inputs.bench_type }}
108112
params_type: ${{ inputs.params_type }}
109-
all_precisions: ${{ inputs.all_precisions }}
113+
precisions_set: ${{ inputs.precisions_set }}
110114
secrets:
111115
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
112116
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}

‎.github/workflows/benchmark_gpu_4090.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ env:
1111
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
1212
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
1313
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
14-
FAST_BENCH: TRUE
14+
BIT_SIZES_SET: FAST
1515

1616
on:
1717
# Allows you to run this workflow manually from the Actions tab as an alternative.
@@ -53,7 +53,6 @@ jobs:
5353
echo "BENCH_DATE=$(date --iso-8601=seconds)";
5454
echo "COMMIT_DATE=${COMMIT_DATE}";
5555
echo "COMMIT_HASH=$(git describe --tags --dirty)";
56-
echo "FAST_BENCH=TRUE";
5756
} >> "${GITHUB_ENV}"
5857
env:
5958
SHA: ${{ github.sha }}

0 commit comments

Comments
 (0)