Skip to content

Commit 6358e7e

Browse files
committed
WIP: modify benchmark to get more flexibility (to fix-up)
1 parent 38c63a3 commit 6358e7e

File tree

5 files changed

+80
-81
lines changed

5 files changed

+80
-81
lines changed

.github/workflows/benchmark_cpu_common.yml

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -68,9 +68,6 @@ jobs:
6868
op_flavor: ${{ steps.set_matrix_args.outputs.op_flavor }}
6969
bench_type: ${{ steps.set_matrix_args.outputs.bench_type }}
7070
params_type: ${{ steps.set_matrix_args.outputs.params_type }}
71-
env:
72-
INPUTS_COMMAND: ${{ inputs.command }}
73-
INPUTS_OP_FLAVOR: ${{ inputs.op_flavor }}
7471
steps:
7572
- name: Parse user inputs
7673
shell: python

.github/workflows/benchmark_documentation.yml

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,16 @@ name: benchmark_documentation
44
on:
55
workflow_dispatch:
66
inputs:
7-
run-benchmarks:
8-
description: "Run benchmarks"
7+
run-cpu-benchmarks:
8+
description: "Run CPU benchmarks"
9+
type: boolean
10+
default: true
11+
run-gpu-benchmarks:
12+
description: "Run GPU benchmarks"
13+
type: boolean
14+
default: true
15+
run-hpu-benchmarks:
16+
description: "Run HPU benchmarks"
917
type: boolean
1018
default: true
1119
generate-svgs:
@@ -25,6 +33,7 @@ jobs:
2533
run-benchmarks-cpu-integer:
2634
name: benchmark_documentation/run-benchmarks-cpu-integer
2735
uses: ./.github/workflows/benchmark_cpu_common.yml
36+
if: inputs.run-cpu-benchmarks
2837
with:
2938
command: integer
3039
op_flavor: fast_default
@@ -43,6 +52,7 @@ jobs:
4352
run-benchmarks-gpu-integer:
4453
name: benchmark_documentation/run-benchmarks-gpu-integer
4554
uses: ./.github/workflows/benchmark_gpu_common.yml
55+
if: inputs.run-gpu-benchmarks
4656
with:
4757
profile: l40
4858
hardware_name: n3-L40x1
@@ -63,6 +73,7 @@ jobs:
6373
run-benchmarks-hpu-integer:
6474
name: benchmark_documentation/run-benchmarks-hpu-integer
6575
uses: ./.github/workflows/benchmark_hpu_common.yml
76+
if: inputs.run-hpu-benchmarks
6677
with:
6778
command: integer
6879
op_flavor: default
@@ -84,6 +95,7 @@ jobs:
8495
run-benchmarks-cpu-core-crypto:
8596
name: benchmark_documentation/run-benchmarks-cpu-core-crypto
8697
uses: ./.github/workflows/benchmark_cpu_common.yml
98+
if: inputs.run-cpu-benchmarks
8799
with:
88100
command: pbs, ks_pbs
89101
bench_type: latency
@@ -101,6 +113,7 @@ jobs:
101113
run-benchmarks-gpu-core-crypto:
102114
name: benchmark_documentation/run-benchmarks-gpu-core-crypto
103115
uses: ./.github/workflows/benchmark_gpu_common.yml
116+
if: inputs.run-gpu-benchmarks
104117
with:
105118
profile: l40
106119
hardware_name: n3-L40x1
@@ -119,22 +132,28 @@ jobs:
119132

120133
generate-svgs-with-benchmarks-run:
121134
name: benchmark-documentation/generate-svgs-with-benchmarks-run
122-
if: inputs.run-benchmarks && inputs.generate-svgs
135+
if: ${{ always() &&
136+
(inputs.run-cpu-benchmarks || inputs.run-gpu-benchmarks ||inputs.run-hpu-benchmarks) &&
137+
inputs.generate-svgs }}
123138
needs: [
124139
run-benchmarks-cpu-integer, run-benchmarks-gpu-integer, run-benchmarks-hpu-integer,
125140
run-benchmarks-cpu-core-crypto, run-benchmarks-gpu-core-crypto
126141
]
127142
uses: ./.github/workflows/generate_svgs.yml
128143
with:
129-
time_span_days: 1 # This ensures extraction is performed on the last values stored in database/
144+
time_span_days: 5
145+
generate-cpu-svgs: ${{ inputs.run-cpu-benchmarks }}
146+
generate-gpu-svgs: ${{ inputs.run-gpu-benchmarks }}
147+
generate-hpu-svgs: ${{ inputs.run-hpu-benchmarks }}
130148
secrets:
131149
DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
132150
DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
133151
DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}
134152

135153
generate-svgs-without-benchmarks-run:
136154
name: benchmark-documentation/generate-svgs-without-benchmarks-run
137-
if: ${{ !inputs.run-benchmarks && inputs.generate-svgs }}
155+
if: ${{ !(inputs.run-cpu-benchmarks || inputs.run-gpu-benchmarks || inputs.run-hpu-benchmarks) &&
156+
inputs.generate-svgs }}
138157
uses: ./.github/workflows/generate_svgs.yml
139158
with:
140159
time_span_days: 60

.github/workflows/benchmark_gpu_common.yml

Lines changed: 38 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -66,79 +66,45 @@ jobs:
6666
name: benchmark_gpu_common/prepare-matrix
6767
runs-on: ubuntu-latest
6868
outputs:
69-
command: ${{ steps.set_command.outputs.command }}
70-
op_flavor: ${{ steps.set_op_flavor.outputs.op_flavor }}
71-
bench_type: ${{ steps.set_bench_type.outputs.bench_type }}
72-
params_type: ${{ steps.set_params_type.outputs.params_type }}
73-
env:
74-
INPUTS_COMMAND: ${{ inputs.command }}
75-
INPUTS_OP_FLAVOR: ${{ inputs.op_flavor }}
69+
command: ${{ steps.set_matrix_args.outputs.command }}
70+
op_flavor: ${{ steps.set_matrix_args.outputs.op_flavor }}
71+
bench_type: ${{ steps.set_matrix_args.outputs.bench_type }}
72+
params_type: ${{ steps.set_matrix_args.outputs.params_type }}
7673
steps:
77-
- name: Set single command
78-
if: ${{ !contains(inputs.command, ',')}}
79-
run: |
80-
echo "COMMAND=[\"${INPUTS_COMMAND}\"]" >> "${GITHUB_ENV}"
81-
82-
- name: Set multiple commands
83-
if: ${{ contains(inputs.command, ',')}}
84-
run: |
85-
# Use Sed to extract a value from a string, this cannot be done with the ${variable//search/replace} pattern.
86-
# shellcheck disable=SC2001
87-
PARSED_COMMAND=$(echo "${INPUTS_COMMAND}" | sed 's/[[:space:]]*,[[:space:]]*/\", \"/g')
88-
echo "COMMAND=[\"${PARSED_COMMAND}\"]" >> "${GITHUB_ENV}"
89-
90-
- name: Set single operations flavor
91-
if: ${{ !contains(inputs.op_flavor, ',')}}
92-
run: |
93-
echo "OP_FLAVOR=[\"${INPUTS_OP_FLAVOR}\"]" >> "${GITHUB_ENV}"
94-
95-
- name: Set multiple operations flavors
96-
if: ${{ contains(inputs.op_flavor, ',')}}
97-
run: |
98-
# Use Sed to extract a value from a string, this cannot be done with the ${variable//search/replace} pattern.
99-
# shellcheck disable=SC2001
100-
PARSED_OP_FLAVOR=$(echo "${INPUTS_OP_FLAVOR}" | sed 's/[[:space:]]*,[[:space:]]*/", "/g')
101-
echo "OP_FLAVOR=[\"${PARSED_OP_FLAVOR}\"]" >> "${GITHUB_ENV}"
102-
103-
- name: Set benchmark types
104-
run: |
105-
if [[ "${INPUTS_BENCH_TYPE}" == "both" ]]; then
106-
echo "BENCH_TYPE=[\"latency\", \"throughput\"]" >> "${GITHUB_ENV}"
107-
else
108-
echo "BENCH_TYPE=[\"${INPUTS_BENCH_TYPE}\"]" >> "${GITHUB_ENV}"
109-
fi
110-
env:
111-
INPUTS_BENCH_TYPE: ${{ inputs.bench_type }}
112-
113-
- name: Set parameters types
114-
run: |
115-
if [[ "${INPUTS_PARAMS_TYPE}" == "both" ]]; then
116-
echo "PARAMS_TYPE=[\"classical\", \"multi_bit\"]" >> "${GITHUB_ENV}"
117-
else
118-
echo "PARAMS_TYPE=[\"${INPUTS_PARAMS_TYPE}\"]" >> "${GITHUB_ENV}"
119-
fi
120-
env:
121-
INPUTS_PARAMS_TYPE: ${{ inputs.params_type }}
122-
123-
- name: Set command output
124-
id: set_command
125-
run: | # zizmor: ignore[template-injection] this env variable is safe
126-
echo "command=${{ toJSON(env.COMMAND) }}" >> "${GITHUB_OUTPUT}"
127-
128-
- name: Set operation flavor output
129-
id: set_op_flavor
130-
run: | # zizmor: ignore[template-injection] this env variable is safe
131-
echo "op_flavor=${{ toJSON(env.OP_FLAVOR) }}" >> "${GITHUB_OUTPUT}"
132-
133-
- name: Set benchmark types output
134-
id: set_bench_type
135-
run: | # zizmor: ignore[template-injection] this env variable is safe
136-
echo "bench_type=${{ toJSON(env.BENCH_TYPE) }}" >> "${GITHUB_OUTPUT}"
137-
138-
- name: Set parameters types output
139-
id: set_params_type
140-
run: | # zizmor: ignore[template-injection] this env variable is safe
141-
echo "params_type=${{ toJSON(env.PARAMS_TYPE) }}" >> "${GITHUB_OUTPUT}"
74+
- name: Parse user inputs
75+
shell: python
76+
run: | # zizmor: ignore[template-injection] these env variables are safe
77+
split_command = "${{ inputs.command }}".replace(" ", "").split(",")
78+
split_op_flavor = "${{ inputs.op_flavor }}".replace(" ", "").split(",")
79+
80+
if "${{ inputs.bench_type }}" == "both":
81+
bench_type = ["latency", "throughput"]
82+
else:
83+
bench_type = ["${{ inputs.bench_type }}", ]
84+
85+
if "+" in "${{ inputs.params_type }}":
86+
split_params_type= "${{ inputs.params_type }}".replace(" ", "").split("+")
87+
else:
88+
split_params_type = ["${{ inputs.params_type }}", ]
89+
90+
with open("${{ github.env }}", "a") as f:
91+
for env_name, values_to_join in [
92+
("COMMAND", split_command),
93+
("OP_FLAVOR", split_op_flavor),
94+
("BENCH_TYPE", bench_type),
95+
("PARAMS_TYPE", split_params_type),
96+
]:
97+
f.write(f"""{env_name}=["{'", "'.join(values_to_join)}"]\n""")
98+
99+
- name: Set martix arguments outputs
100+
id: set_matrix_args
101+
run: | # zizmor: ignore[template-injection] these env variable are safe
102+
{
103+
echo "command=${{ toJSON(env.COMMAND) }}";
104+
echo "op_flavor=${{ toJSON(env.OP_FLAVOR) }}";
105+
echo "bench_type=${{ toJSON(env.BENCH_TYPE) }}";
106+
echo "params_type=${{ toJSON(env.PARAMS_TYPE) }}";
107+
} >> "${GITHUB_OUTPUT}"
142108
143109
setup-instance:
144110
name: benchmark_gpu_common/setup-instance

.github/workflows/generate_svg_common.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ jobs:
5454
python3 -m pip install -r ci/data_extractor/requirements.txt
5555
python3 ci/data_extractor/src/data_extractor.py "${OUTPUT_FILENAME}" \
5656
--generate-svg \
57-
--branch "${REF_NAME}"\
57+
--branch "${REF_NAME}" \
5858
--backend "${BACKEND}" \
5959
--hardware "${HARDWARE_NAME}" \
6060
--tfhe-rs-layer "${LAYER}" \

.github/workflows/generate_svgs.yml

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,15 @@ on:
77
time_span_days:
88
type: string
99
required: true
10+
generate-cpu-svgs:
11+
type: boolean
12+
default: true
13+
generate-gpu-svgs:
14+
type: boolean
15+
default: true
16+
generate-hpu-svgs:
17+
type: boolean
18+
default: true
1019
secrets:
1120
DATA_EXTRACTOR_DATABASE_USER:
1221
required: true
@@ -27,6 +36,7 @@ jobs:
2736
cpu-integer-latency-table:
2837
name: generate_documentation_svgs/cpu-integer-latency-table
2938
uses: ./.github/workflows/generate_svg_common.yml
39+
if: inputs.generate-cpu-svgs
3040
with:
3141
backend: cpu
3242
hardware_name: hpc7a.96xlarge
@@ -43,6 +53,7 @@ jobs:
4353
cpu-integer-throughput-table:
4454
name: generate_documentation_svgs/cpu-integer-latency-table
4555
uses: ./.github/workflows/generate_svg_common.yml
56+
if: inputs.generate-cpu-svgs
4657
with:
4758
backend: cpu
4859
hardware_name: hpc7a.96xlarge
@@ -59,6 +70,7 @@ jobs:
5970
gpu-integer-latency-table:
6071
name: generate_documentation_svgs/gpu-integer-latency-table
6172
uses: ./.github/workflows/generate_svg_common.yml
73+
if: inputs.generate-gpu-svgs
6274
with:
6375
backend: gpu
6476
hardware_name: n3-L40x1
@@ -76,6 +88,7 @@ jobs:
7688
gpu-integer-throughput-table:
7789
name: generate_documentation_svgs/gpu-integer-throughput-table
7890
uses: ./.github/workflows/generate_svg_common.yml
91+
if: inputs.generate-gpu-svgs
7992
with:
8093
backend: gpu
8194
hardware_name: n3-L40x1
@@ -93,6 +106,7 @@ jobs:
93106
hpu-integer-latency-table:
94107
name: generate_documentation_svgs/hpu-integer-latency-table
95108
uses: ./.github/workflows/generate_svg_common.yml
109+
if: inputs.generate-hpu-svgs
96110
with:
97111
backend: hpu
98112
hardware_name: hpu_x1
@@ -109,6 +123,7 @@ jobs:
109123
hpu-integer-throughput-table:
110124
name: generate_documentation_svgs/hpu-integer-throughput-table
111125
uses: ./.github/workflows/generate_svg_common.yml
126+
if: inputs.generate-hpu-svgs
112127
with:
113128
backend: hpu
114129
hardware_name: hpu_x1
@@ -129,6 +144,7 @@ jobs:
129144
cpu-pbs-tables:
130145
name: generate_documentation_svgs/cpu-pbs-tables
131146
uses: ./.github/workflows/generate_svg_common.yml
147+
if: inputs.generate-cpu-svgs
132148
with:
133149
backend: cpu
134150
hardware_name: hpc7a.96xlarge
@@ -146,6 +162,7 @@ jobs:
146162
gpu-pbs-tables:
147163
name: generate_documentation_svgs/gpu-pbs-tables
148164
uses: ./.github/workflows/generate_svg_common.yml
165+
if: inputs.generate-gpu-svgs
149166
with:
150167
backend: gpu
151168
hardware_name: n3-L40x1

0 commit comments

Comments
 (0)