Skip to content

Commit 691cce5

Browse files
soonumsoonum
committed
chore(ci): add workflow to update documentation benchmark tables
This new workflow can trigger all the required benchmarks needed to populate benchmarks tables in documentation. It also can generate SVG tables and store them as artifacts. Optionally, it can open a pull-request to update the current tables in documentation.
1 parent 610c811 commit 691cce5

14 files changed

+524
-80
lines changed

‎.github/workflows/benchmark_cpu.yml‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,14 @@ on:
3434
- smart
3535
- unchecked
3636
- misc
37-
all_precisions:
38-
description: "Run all precisions"
39-
type: boolean
40-
default: false
37+
precisions_set:
38+
description: "Bit precisions set"
39+
type: choice
40+
default: fast
41+
options:
42+
- fast
43+
- all
44+
- documentation
4145
bench_type:
4246
description: "Benchmarks type"
4347
type: choice
@@ -69,7 +73,7 @@ jobs:
6973
op_flavor: ${{ inputs.op_flavor }}
7074
bench_type: ${{ inputs.bench_type }}
7175
params_type: ${{ inputs.params_type }}
72-
all_precisions: ${{ inputs.all_precisions }}
76+
precisions_set: ${{ inputs.precisions_set }}
7377
secrets:
7478
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
7579
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}

‎.github/workflows/benchmark_cpu_common.yml‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,9 @@ on:
1616
params_type:
1717
type: string
1818
default: classical
19-
all_precisions:
20-
type: boolean
21-
default: false
19+
precisions_set:
20+
type: string
21+
default: fast
2222
additional_recipe: # Make recipes to run aside the benchmarks.
2323
type: string # Use comma separated values to generate an array
2424
additional_file_to_parse: # Make recipes aditional to run aside the benchamrks. Use comma separated values to generate an array
@@ -54,7 +54,6 @@ env:
5454
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
5555
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
5656
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
57-
FAST_BENCH: TRUE
5857

5958
permissions: {}
6059

@@ -160,19 +159,15 @@ jobs:
160159
with:
161160
toolchain: nightly
162161

163-
- name: Should run benchmarks with all precisions
164-
if: inputs.all_precisions
165-
run: |
166-
echo "FAST_BENCH=FALSE" >> "${GITHUB_ENV}"
167-
168162
- name: Run benchmarks with AVX512
169163
run: |
170-
make BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"
164+
make BIT_SIZES_SET="${PRECISIONS_SET}" BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"
171165
env:
172166
OP_FLAVOR: ${{ matrix.op_flavor }}
173167
BENCH_TYPE: ${{ matrix.bench_type }}
174168
BENCH_PARAMS_TYPE: ${{ matrix.params_type }}
175169
BENCH_COMMAND: ${{ matrix.command }}
170+
PRECISIONS_SET: ${{ inputs.precisions_set }}
176171

177172
- name: Parse results
178173
run: |

‎.github/workflows/benchmark_cpu_weekly.yml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ jobs:
2828
is_weekly_bench_group_2: ${{ steps.check_bench_group_2.outputs.is_weekly_bench_group_2 }}
2929
is_quarterly_bench: ${{ steps.check_quarterly_bench.outputs.is_quarterly_bench }}
3030
op_flavor: ${{ steps.set_op_flavor.outputs.op_flavor }}
31-
all_precisions: ${{ steps.set_all_precisions.outputs.all_precisions }}
31+
precisions_set: ${{ steps.set_precisions_set.outputs.precisions_set }}
3232
steps:
3333
- name: Check is weekly bench group 1
3434
id: check_bench_group_1
@@ -49,23 +49,23 @@ jobs:
4949
if: env.WEEKLY_BENCH_GROUP_1 || env.WEEKLY_BENCH_GROUP_2
5050
run: |
5151
echo "OP_FLAVOR=[\"default\"]" >> "${GITHUB_ENV}"
52-
echo "ALL_PRECISIONS=false" >> "${GITHUB_ENV}"
52+
echo "PRECISIONS_SET=false" >> "${GITHUB_ENV}"
5353
5454
- name: Quarterly benchmarks
5555
if: env.QUARTERLY_BENCH
5656
run: |
5757
echo "OP_FLAVOR=[\"default\", \"unchecked\"]" >> "${GITHUB_ENV}"
58-
echo "ALL_PRECISIONS=true" >> "${GITHUB_ENV}"
58+
echo "PRECISIONS_SET=true" >> "${GITHUB_ENV}"
5959
6060
- name: Set operation flavor output
6161
id: set_op_flavor
6262
run: | # zizmor: ignore[template-injection] this env variable is safe
6363
echo "op_flavor=${{ toJSON(env.OP_FLAVOR) }}" >> "${GITHUB_OUTPUT}"
6464
6565
- name: Set bit precisions output
66-
id: set_all_precisions
66+
id: set_precisions_set
6767
run: | # zizmor: ignore[template-injection] this env variable is safe
68-
echo "all_precisions=${{ toJSON(env.ALL_PRECISIONS) }}" >> "${GITHUB_OUTPUT}"
68+
echo "precisions_set=${{ toJSON(env.PRECISIONS_SET) }}" >> "${GITHUB_OUTPUT}"
6969
7070
run-benchmarks-integer:
7171
name: benchmark_gpu_weekly/run-benchmarks-integer
@@ -76,7 +76,7 @@ jobs:
7676
with:
7777
command: integer,signed_integer, integer_compression
7878
op_flavor: ${{ needs.prepare-inputs.outputs.op_flavor }}
79-
all_precisions: ${{ needs.prepare-inputs.outputs.all_precisions }}
79+
precisions_set: ${{ needs.prepare-inputs.outputs.precisions_set }}
8080
secrets:
8181
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
8282
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}

‎.github/workflows/benchmark_documentation.yml‎

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
# Run all benchmarks displayed in the public documentation.
2+
name: benchmark_documentation
3+
4+
on:
5+
workflow_dispatch:
6+
inputs:
7+
run-benchmarks:
8+
description: "Run benchmarks"
9+
type: boolean
10+
default: true
11+
generate-svgs:
12+
description: "Generate SVG tables"
13+
type: boolean
14+
default: true
15+
open-pr:
16+
description: "Open a PR with the benchmark results"
17+
type: boolean
18+
default: false
19+
20+
permissions: {}
21+
22+
jobs:
23+
run-benchmarks-cpu-integer:
24+
name: benchmark_documentation/run-benchmarks-cpu-integer
25+
uses: ./.github/workflows/benchmark_cpu_common.yml
26+
with:
27+
command: integer
28+
op_flavor: fast_default
29+
bench_type: both
30+
precisions_set: documentation
31+
secrets:
32+
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
33+
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
34+
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
35+
REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }}
36+
JOB_SECRET: ${{ secrets.JOB_SECRET }}
37+
SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }}
38+
SLAB_URL: ${{ secrets.SLAB_URL }}
39+
SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }}
40+
41+
run-benchmarks-gpu-integer:
42+
name: benchmark_documentation/run-benchmarks-gpu-integer
43+
uses: ./.github/workflows/benchmark_gpu_common.yml
44+
with:
45+
profile: multi-h100-sxm5
46+
hardware_name: n3-H100-SXM5x8
47+
command: integer_multi_bit
48+
op_flavor: fast_default
49+
bench_type: both
50+
precisions_set: documentation
51+
secrets:
52+
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
53+
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
54+
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
55+
REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }}
56+
JOB_SECRET: ${{ secrets.JOB_SECRET }}
57+
SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }}
58+
SLAB_URL: ${{ secrets.SLAB_URL }}
59+
SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }}
60+
61+
run-benchmarks-hpu-integer:
62+
name: benchmark_documentation/run-benchmarks-hpu-integer
63+
uses: ./.github/workflows/benchmark_hpu_common.yml
64+
with:
65+
command: integer_hpu
66+
op_flavor: default
67+
bench_type: both
68+
precisions_set: fast
69+
v80_pcie_dev: 24
70+
v80_serial_number: XFL12NWY3ZKG
71+
secrets:
72+
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
73+
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
74+
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
75+
REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }}
76+
JOB_SECRET: ${{ secrets.JOB_SECRET }}
77+
SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }}
78+
SLAB_URL: ${{ secrets.SLAB_URL }}
79+
SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }}
80+
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
81+
82+
run-benchmarks-cpu-core-crypto:
83+
name: benchmark_documentation/run-benchmarks-cpu-core-crypto
84+
uses: ./.github/workflows/benchmark_cpu_common.yml
85+
with:
86+
command: pbs, ks_pbs
87+
bench_type: latency
88+
params_type: classical_documentation + multi_bit_documentation
89+
secrets:
90+
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
91+
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
92+
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
93+
REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }}
94+
JOB_SECRET: ${{ secrets.JOB_SECRET }}
95+
SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }}
96+
SLAB_URL: ${{ secrets.SLAB_URL }}
97+
SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }}
98+
99+
run-benchmarks-gpu-core-crypto:
100+
name: benchmark_gpu_weekly/run-benchmarks-gpu-core-crypto
101+
uses: ./.github/workflows/benchmark_gpu_common.yml
102+
with:
103+
profile: multi-h100-sxm5
104+
hardware_name: n3-H100-SXM5x8
105+
command: pbs, ks_pbs
106+
bench_type: latency
107+
params_type: classical_documentation + multi_bit_documentation
108+
secrets:
109+
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
110+
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
111+
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
112+
REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }}
113+
JOB_SECRET: ${{ secrets.JOB_SECRET }}
114+
SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }}
115+
SLAB_URL: ${{ secrets.SLAB_URL }}
116+
SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }}
117+
118+
generate-svgs-with-benchmarks-run:
119+
name: benchmark-documentation/generate-svgs-with-benchmarks-run
120+
if: inputs.run-benchmarks && inputs.generate-svgs
121+
needs: [
122+
run-benchmarks-cpu-integer, run-benchmarks-gpu-integer, run-benchmarks-hpu-integer,
123+
run-benchmarks-cpu-core-crypto, run-benchmarks-gpu-core-crypto
124+
]
125+
uses: ./.github/workflows/generate_svgs.yml
126+
with:
127+
time_span_days: 1 # This ensures extraction is performed on the last values stored in database/
128+
secrets:
129+
DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
130+
DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
131+
DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}
132+
133+
generate-svgs-without-benchmarks-run:
134+
name: benchmark-documentation/generate-svgs-without-benchmarks-run
135+
if: ${{ !inputs.run-benchmarks && inputs.generate-svgs }}
136+
uses: ./.github/workflows/generate_svgs.yml
137+
with:
138+
time_span_days: 60
139+
secrets:
140+
DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
141+
DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
142+
DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}
143+
144+
open-pr:
145+
name: benchmark-documentation/open-pr
146+
needs: [ generate-svgs-with-benchmarks-run, generate-svgs-without-benchmarks-run ]
147+
if: ${{ always() && inputs.open-pr &&
148+
(needs.generate-svgs-with-benchmarks-run.result == 'success' || needs.generate-svgs-without-benchmarks-run.result == 'success') }}
149+
runs-on: ubuntu-latest
150+
permissions:
151+
contents: write # Needed to create a commit
152+
pull-requests: write # Needed to open a pull-request
153+
env:
154+
PATH_TO_DOC_ASSETS: tfhe/docs/.gitbook/assets
155+
steps:
156+
- name: Checkout tfhe-rs
157+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
158+
with:
159+
persist-credentials: 'false'
160+
161+
- name: Download SVG tables
162+
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
163+
with:
164+
path: svg_tables
165+
merge-multiple: 'true'
166+
167+
- name: Copy SVG tables to documentation location
168+
run: |
169+
cp -f svg_tables/*integer-benchmark*.svg "${PATH_TO_DOC_ASSETS}"
170+
cp -f svg_tables/*pbs-benchmark-tuniform*.svg "${PATH_TO_DOC_ASSETS}"
171+
172+
- name: Create pull-request
173+
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
174+
with:
175+
sign-commits: true
176+
token: ${{ secrets.FHE_DIVISION_TOKEN }} # Sign commit as Zama Bot
177+
add-paths: ${{ env.PATH_TO_DOC_ASSETS }}/*.svg
178+
commit-message: |
179+
chore(docs): update benchmark results for all backends
180+
181+
Automated documentation update from tfhe-rs CI pipeline.
182+
title: |
183+
[CI] chore(docs): update benchmark results for all backends
184+
body: |
185+
Documentation update triggered by GitHub workflow*
186+
labels: documentation

‎.github/workflows/benchmark_gpu.yml‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,14 @@ on:
4040
- default
4141
- fast_default
4242
- unchecked
43-
all_precisions:
44-
description: "Run all precisions"
45-
type: boolean
46-
default: false
43+
precisions_set:
44+
description: "Bit precisions set"
45+
type: choice
46+
default: fast
47+
options:
48+
- fast
49+
- all
50+
- documentation
4751
bench_type:
4852
description: "Benchmarks type"
4953
type: choice
@@ -104,7 +108,7 @@ jobs:
104108
op_flavor: ${{ inputs.op_flavor }}
105109
bench_type: ${{ inputs.bench_type }}
106110
params_type: ${{ inputs.params_type }}
107-
all_precisions: ${{ inputs.all_precisions }}
111+
precisions_set: ${{ inputs.precisions_set }}
108112
secrets:
109113
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
110114
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}

‎.github/workflows/benchmark_gpu_4090.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ env:
1111
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
1212
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
1313
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
14-
FAST_BENCH: TRUE
14+
BIT_SIZES_SET: FAST
1515

1616
on:
1717
# Allows you to run this workflow manually from the Actions tab as an alternative.
@@ -51,7 +51,7 @@ jobs:
5151
echo "BENCH_DATE=$(date --iso-8601=seconds)";
5252
echo "COMMIT_DATE=${COMMIT_DATE}";
5353
echo "COMMIT_HASH=$(git describe --tags --dirty)";
54-
echo "FAST_BENCH=TRUE";
54+
echo "BIT_SIZES_SET=FAST";
5555
} >> "${GITHUB_ENV}"
5656
env:
5757
SHA: ${{ github.sha }}

‎.github/workflows/benchmark_gpu_common.yml‎

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,9 @@ on:
2525
params_type:
2626
type: string
2727
default: multi_bit
28-
all_precisions:
29-
type: boolean
30-
default: false
28+
precisions_set:
29+
type: string
30+
default: fast
3131
secrets:
3232
REPO_CHECKOUT_TOKEN:
3333
required: true
@@ -56,8 +56,6 @@ env:
5656
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
5757
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
5858
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
59-
FAST_BENCH: TRUE
60-
6159

6260
permissions: {}
6361

@@ -273,19 +271,15 @@ jobs:
273271
with:
274272
toolchain: nightly
275273

276-
- name: Should run benchmarks with all precisions
277-
if: inputs.all_precisions
278-
run: |
279-
echo "FAST_BENCH=FALSE" >> "${GITHUB_ENV}"
280-
281274
- name: Run benchmarks
282275
run: |
283-
make BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"_gpu
276+
make BIT_SIZES_SET="${PRECISIONS_SET}" BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"_gpu
284277
env:
285278
OP_FLAVOR: ${{ matrix.op_flavor }}
286279
BENCH_TYPE: ${{ matrix.bench_type }}
287280
BENCH_PARAMS_TYPE: ${{ matrix.params_type }}
288281
BENCH_COMMAND: ${{ matrix.command }}
282+
PRECISIONS_SET: ${{ inputs.precisions_set }}
289283

290284
- name: Parse results
291285
run: |

0 commit comments

Comments
 (0)