|
| 1 | +""" |
| 2 | +
|
| 3 | +Generate the graphs for the FROST perfomance blog post. |
| 4 | +
|
| 5 | +Install cargo-criterion: |
| 6 | +
|
| 7 | +cargo install cargo-criterion |
| 8 | +
|
| 9 | +Run the benchmarks with: |
| 10 | +
|
| 11 | +(check out old code) |
| 12 | +
|
| 13 | +cargo criterion --message-format=json 'FROST' | tee > benchmark-verify-all-shares.txt |
| 14 | +
|
| 15 | +(check out new code) |
| 16 | +
|
| 17 | +cargo criterion --message-format=json 'FROST' | tee > benchmark-verify-aggregate.txt |
| 18 | +
|
| 19 | +And then run: |
| 20 | +
|
| 21 | +python3 plot.py |
| 22 | +
|
| 23 | +It will generate the figures (names are partially hardcoded in each functions) |
| 24 | +and will insert/update the tables inside `performance.md` |
| 25 | +""" |
| 26 | + |
| 27 | +import matplotlib.pyplot as plt |
| 28 | +import numpy as np |
| 29 | +import json |
| 30 | + |
| 31 | + |
| 32 | +def load_data(filename): |
| 33 | + ciphersuite_lst = [] |
| 34 | + fn_lst = [] |
| 35 | + size_lst = [] |
| 36 | + data = {} |
| 37 | + with open(filename, 'r') as f: |
| 38 | + for line in f: |
| 39 | + line_data = json.loads(line) |
| 40 | + if line_data['reason'] == 'benchmark-complete': |
| 41 | + ciphersuite, fn, size = line_data['id'].split('/') |
| 42 | + ciphersuite = ciphersuite.replace('FROST Signing ', '') |
| 43 | + size = int(size) |
| 44 | + unit = line_data['typical']['unit'] |
| 45 | + time = line_data['typical']['estimate'] |
| 46 | + assert unit == 'ns' |
| 47 | + if unit == 'ns': |
| 48 | + time = time / 1e6 |
| 49 | + if ciphersuite not in ciphersuite_lst: |
| 50 | + ciphersuite_lst.append(ciphersuite) |
| 51 | + if fn not in fn_lst: |
| 52 | + fn_lst.append(fn) |
| 53 | + if size in (2, 7, 67, 667): |
| 54 | + size = {2: 3, 7: 10, 67: 100, 667: 1000}[size] |
| 55 | + if size not in size_lst: |
| 56 | + size_lst.append(size) |
| 57 | + data.setdefault(ciphersuite, {}).setdefault(fn, {})[size] = time |
| 58 | + return ciphersuite_lst, fn_lst, size_lst, data |
| 59 | + |
| 60 | + |
| 61 | +def plot(title, filename, get_group_value, group_lst, series_lst, fmt, figsize): |
| 62 | + x = np.arange(len(group_lst)) # the label locations |
| 63 | + total_width = 0.8 |
| 64 | + bar_width = total_width / len(series_lst) # the width of the bars |
| 65 | + |
| 66 | + fig, ax = plt.subplots(figsize=figsize) |
| 67 | + |
| 68 | + offsets = [-total_width / 2 + bar_width / 2 + (bar_width * i) for i in range(len(series_lst))] |
| 69 | + rect_lst = [] |
| 70 | + for series_idx, series in enumerate(series_lst): |
| 71 | + values = [get_group_value(series_idx, series, group_idx, group) for group_idx, group in enumerate(group_lst)] |
| 72 | + rect = ax.bar(x + offsets[series_idx], values, bar_width, label=series) |
| 73 | + rect_lst.append(rect) |
| 74 | + |
| 75 | + # Add some text for labels, title and custom x-axis tick labels, etc. |
| 76 | + ax.set_ylabel('Time (ms)') |
| 77 | + ax.set_title(title) |
| 78 | + ax.set_xticks(x, group_lst) |
| 79 | + ax.legend() |
| 80 | + |
| 81 | + for rect in rect_lst: |
| 82 | + ax.bar_label(rect, padding=3, fmt=fmt) |
| 83 | + |
| 84 | + fig.tight_layout() |
| 85 | + |
| 86 | + plt.savefig(filename) |
| 87 | + plt.close() |
| 88 | + |
| 89 | + |
| 90 | +def times_by_size_and_function(data, ciphersuite, fn_lst, size_lst, fmt, suffix): |
| 91 | + group_lst = [str(int((size * 2 + 2) / 3)) + "-of-" + str(size) for size in size_lst] |
| 92 | + series_lst = fn_lst |
| 93 | + title = f'Times by number of signers and functions; {ciphersuite} ciphersuite' |
| 94 | + filename = f'times-by-size-and-function-{ciphersuite}-{suffix}.png' |
| 95 | + |
| 96 | + def get_group_value(series_idx, series, group_idx, group): |
| 97 | + return data[ciphersuite][series][size_lst[group_idx]] |
| 98 | + |
| 99 | + plot(title, filename, get_group_value, group_lst, series_lst, fmt, (8, 6)) |
| 100 | + |
| 101 | + |
| 102 | +def times_by_ciphersuite_and_function(data, ciphersuite_lst, fn_lst, size, fmt): |
| 103 | + ciphersuite_lst = ciphersuite_lst.copy() |
| 104 | + ciphersuite_lst.sort(key=lambda cs: data[cs]['Aggregate'][size]) |
| 105 | + group_lst = fn_lst |
| 106 | + series_lst = ciphersuite_lst |
| 107 | + min_signers = int((size * 2 + 2) / 3) |
| 108 | + title = f'Times by ciphersuite and function; {min_signers}-of-{size}' |
| 109 | + filename = f'times-by-ciphersuite-and-function-{size}.png' |
| 110 | + |
| 111 | + def get_group_value(series_idx, series, group_idx, group): |
| 112 | + return data[series][group][size] |
| 113 | + |
| 114 | + plot(title, filename, get_group_value, group_lst, series_lst, fmt, (12, 6)) |
| 115 | + |
| 116 | + |
| 117 | +def verify_aggregated_vs_all_shares(data_aggregated, data_all_shares, ciphersuite_lst, size, fmt): |
| 118 | + ciphersuite_lst = ciphersuite_lst.copy() |
| 119 | + ciphersuite_lst.sort(key=lambda cs: data_aggregated[cs]['Aggregate'][size]) |
| 120 | + group_lst = ciphersuite_lst |
| 121 | + series_lst = ['Verify all shares', 'Verify aggregated'] |
| 122 | + min_signers = int((size * 2 + 2) / 3) |
| 123 | + title = f'Time comparison for Aggregate function; {min_signers}-of-{size}' |
| 124 | + filename = f'verify-aggregated-vs-all-shares-{size}.png' |
| 125 | + |
| 126 | + def get_group_value(series_idx, series, group_idx, group): |
| 127 | + data = [data_all_shares, data_aggregated][series_idx] |
| 128 | + return data[group]['Aggregate'][size] |
| 129 | + |
| 130 | + plot(title, filename, get_group_value, group_lst, series_lst, fmt, (8, 6)) |
| 131 | + |
| 132 | + |
| 133 | +def generate_table(f, data, ciphersuite_lst, fn_lst, size_lst): |
| 134 | + for ciphersuite in ciphersuite_lst: |
| 135 | + print(f'### {ciphersuite}\n', file=f) |
| 136 | + print('|' + '|'.join([''] + fn_lst) + '|', file=f) |
| 137 | + print('|' + '|'.join([':---'] + ['---:'] * len(fn_lst)) + '|', file=f) |
| 138 | + for size in size_lst: |
| 139 | + min_signers = int((size * 2 + 2) / 3) |
| 140 | + print('|' + '|'.join([f'{min_signers}-of-{size}'] + ['{:.2f}'.format(data[ciphersuite][fn][size]) for fn in fn_lst]) + '|', file=f) |
| 141 | + print('', file=f) |
| 142 | + print('', file=f) |
| 143 | + |
| 144 | + |
| 145 | +if __name__ == '__main__': |
| 146 | + ciphersuite_lst, fn_lst, size_lst, data_aggregated = load_data('benchmark-verify-aggregate.txt') |
| 147 | + _, _, _, data_all_shares = load_data('benchmark-verify-all-shares.txt') |
| 148 | + |
| 149 | + import io |
| 150 | + import re |
| 151 | + with io.StringIO() as f: |
| 152 | + generate_table(f, data_aggregated, ciphersuite_lst, fn_lst, size_lst) |
| 153 | + f.seek(0) |
| 154 | + table = f.read() |
| 155 | + with open('performance.md') as f: |
| 156 | + md = f.read() |
| 157 | + md = re.sub('<!-- Benchmarks -->[^<]*<!-- Benchmarks -->', '<!-- Benchmarks -->\n' + table + '<!-- Benchmarks -->', md, count=1, flags=re.DOTALL) |
| 158 | + with open('performance.md', 'w') as f: |
| 159 | + f.write(md) |
| 160 | + |
| 161 | + size_lst = [10, 100, 1000] |
| 162 | + times_by_size_and_function(data_all_shares, 'ristretto255', fn_lst, size_lst, '%.2f', 'all-shares') |
| 163 | + times_by_size_and_function(data_aggregated, 'ristretto255', fn_lst, size_lst, '%.2f', 'aggregated') |
| 164 | + |
| 165 | + times_by_ciphersuite_and_function(data_aggregated, ciphersuite_lst, fn_lst, 10, '%.2f') |
| 166 | + times_by_ciphersuite_and_function(data_aggregated, ciphersuite_lst, fn_lst, 100, '%.1f') |
| 167 | + times_by_ciphersuite_and_function(data_aggregated, ciphersuite_lst, fn_lst, 1000, '%.0f') |
| 168 | + |
| 169 | + verify_aggregated_vs_all_shares(data_aggregated, data_all_shares, ciphersuite_lst, 10, '%.2f') |
| 170 | + verify_aggregated_vs_all_shares(data_aggregated, data_all_shares, ciphersuite_lst, 100, '%.1f') |
| 171 | + verify_aggregated_vs_all_shares(data_aggregated, data_all_shares, ciphersuite_lst, 1000, '%.0f') |
0 commit comments