Skip to content

Commit 4ae8954

Browse files
rbellenscursoragent
rbellens
and
cursoragent
committed
feat(action): deploy combined compile-type benchmark charts on GitHub Pages
Add a custom dashboard that groups jit/aot/js/wasm on one chart per benchmark, treat legacy unsuffixed names as JIT, and always overwrite gh-pages index.html. Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent 754dce8 commit 4ae8954

3 files changed

Lines changed: 372 additions & 2 deletions

File tree

‎README.md‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -317,8 +317,11 @@ The action runs the benchmark CLI once per compile type, converts the JSONL
317317
results to `github-action-benchmark` custom data, and commits benchmark history
318318
to the `gh-pages` branch. Results are stored as `customBiggerIsBetter`, with
319319
benchmark names suffixed by compile type, for example `parse json [jit]` and
320-
`parse json [aot]`. The action always runs with Dart assertions disabled to keep
321-
CI benchmark numbers representative.
320+
`parse json [aot]`. Regression alerts still compare each compile type separately.
321+
The action always deploys a custom dashboard that plots those series on one chart
322+
per benchmark and overwrites `index.html` on each run (github-action-benchmark
323+
itself never replaces an existing `index.html`). The action always runs with
324+
Dart assertions disabled to keep CI benchmark numbers representative.
322325

323326
For Flutter packages, set `sdk` to `flutter` so the action installs Flutter and
324327
runs `flutter pub get` before invoking the benchmark CLI:

‎action.yml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,3 +157,39 @@ runs:
157157
fail-on-alert: ${{ inputs.fail-on-alert }}
158158
comment-on-alert: ${{ inputs.comment-on-alert }}
159159
alert-threshold: ${{ inputs.alert-threshold }}
160+
161+
- name: Update benchmark dashboard
162+
shell: bash
163+
env:
164+
GH_TOKEN: ${{ inputs.github-token }}
165+
run: |
166+
set -euo pipefail
167+
168+
branch="${{ inputs.gh-pages-branch }}"
169+
data_dir="${{ inputs.benchmark-data-dir-path }}"
170+
dashboard="${{ github.action_path }}/tool/github_action/gh_pages_index.html"
171+
172+
git config user.name "github-actions[bot]"
173+
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
174+
175+
if ! git ls-remote --heads origin "$branch" | grep -q .; then
176+
echo "Branch '$branch' does not exist yet; skipping dashboard update." >&2
177+
exit 0
178+
fi
179+
180+
previous_ref="$(git rev-parse HEAD)"
181+
git fetch origin "$branch"
182+
git checkout "$branch"
183+
184+
mkdir -p "$data_dir"
185+
cp -f "$dashboard" "$data_dir/index.html"
186+
187+
git add -f "$data_dir/index.html"
188+
if git diff --cached --quiet; then
189+
git checkout "$previous_ref"
190+
exit 0
191+
fi
192+
193+
git commit -m "Update benchmark dashboard"
194+
git push "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "HEAD:${branch}"
195+
git checkout "$previous_ref"

‎tool/github_action/gh_pages_index.html‎

Lines changed: 331 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,331 @@
1+
<!DOCTYPE html>
2+
<html lang="en">
3+
<head>
4+
<meta charset="utf-8" />
5+
<meta name="viewport" content="width=device-width, minimum-scale=1, initial-scale=1, user-scalable=yes" />
6+
<title>Benchmarks</title>
7+
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,500,700|Material+Icons" />
8+
<script src="https://cdn.jsdelivr.net/npm/chart.js@2.9.4/dist/Chart.min.js"></script>
9+
<style>
10+
html {
11+
font-family: BlinkMacSystemFont, -apple-system, "Segoe UI", Roboto, Oxygen, Ubuntu, Cantarell,
12+
"Fira Sans", "Droid Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
13+
-webkit-font-smoothing: antialiased;
14+
background-color: #fff;
15+
font-size: 16px;
16+
}
17+
body {
18+
color: #4a4a4a;
19+
margin: 8px;
20+
font-size: 1em;
21+
font-weight: 400;
22+
}
23+
header {
24+
margin-bottom: 8px;
25+
display: flex;
26+
flex-direction: column;
27+
}
28+
main {
29+
width: 100%;
30+
display: flex;
31+
flex-direction: column;
32+
}
33+
a {
34+
color: #3273dc;
35+
cursor: pointer;
36+
text-decoration: none;
37+
}
38+
a:hover {
39+
color: #000;
40+
}
41+
button {
42+
color: #fff;
43+
background-color: #3298dc;
44+
border-color: transparent;
45+
cursor: pointer;
46+
text-align: center;
47+
}
48+
button:hover {
49+
background-color: #2793da;
50+
flex: none;
51+
}
52+
.spacer {
53+
flex: auto;
54+
}
55+
.small {
56+
font-size: 0.75rem;
57+
}
58+
footer {
59+
margin-top: 16px;
60+
display: flex;
61+
align-items: center;
62+
}
63+
.header-label {
64+
margin-right: 4px;
65+
}
66+
.benchmark-set {
67+
margin: 8px 0;
68+
width: 100%;
69+
display: flex;
70+
flex-direction: column;
71+
}
72+
.benchmark-title {
73+
font-size: 3rem;
74+
font-weight: 600;
75+
word-break: break-word;
76+
text-align: center;
77+
}
78+
.benchmark-graphs {
79+
display: flex;
80+
flex-direction: row;
81+
justify-content: space-around;
82+
align-items: center;
83+
flex-wrap: wrap;
84+
width: 100%;
85+
}
86+
.benchmark-chart {
87+
max-width: 1000px;
88+
}
89+
</style>
90+
</head>
91+
<body>
92+
<header id="header">
93+
<div class="spacer"></div>
94+
<div>
95+
<span class="header-label">Last Update:</span>
96+
<span id="last-update"></span>
97+
</div>
98+
<div>
99+
<span class="header-label">Repository:</span>
100+
<a id="repository-link"></a>
101+
</div>
102+
<div class="spacer"></div>
103+
</header>
104+
<main id="main"></main>
105+
<footer>
106+
<button id="dl-button">Download data as JSON</button>
107+
<div class="spacer"></div>
108+
<div class="small">Powered by <a href="https://github.com/benchmark-action/github-action-benchmark">github-action-benchmark</a></div>
109+
</footer>
110+
<script src="data.js"></script>
111+
<script>
112+
'use strict';
113+
(function () {
114+
const compileColors = {
115+
jit: '#00B4AB',
116+
aot: '#01579B',
117+
js: '#f1e05a',
118+
wasm: '#654FF0',
119+
_default: '#333333',
120+
};
121+
122+
function parseBenchName(name) {
123+
const match = /^(.*) \[([^\]]+)\]$/.exec(name);
124+
if (match) {
125+
return { baseName: match[1], compileType: match[2] };
126+
}
127+
// Legacy entries without a compile suffix are treated as JIT.
128+
return { baseName: name, compileType: 'jit' };
129+
}
130+
131+
function collectBenchesGrouped(entries) {
132+
const map = new Map();
133+
for (const entry of entries) {
134+
const { commit, date, tool, benches } = entry;
135+
for (const bench of benches) {
136+
const { baseName, compileType } = parseBenchName(bench.name);
137+
const seriesKey = compileType;
138+
const result = { commit, date, tool, bench };
139+
if (!map.has(baseName)) {
140+
map.set(baseName, new Map());
141+
}
142+
const seriesMap = map.get(baseName);
143+
const arr = seriesMap.get(seriesKey);
144+
if (arr === undefined) {
145+
seriesMap.set(seriesKey, [result]);
146+
} else {
147+
arr.push(result);
148+
}
149+
}
150+
}
151+
return map;
152+
}
153+
154+
function init() {
155+
const data = window.BENCHMARK_DATA;
156+
157+
document.getElementById('last-update').textContent = new Date(data.lastUpdate).toString();
158+
const repoLink = document.getElementById('repository-link');
159+
repoLink.href = data.repoUrl;
160+
repoLink.textContent = data.repoUrl;
161+
162+
document.getElementById('dl-button').onclick = () => {
163+
const dataUrl = 'data:,' + JSON.stringify(data, null, 2);
164+
const a = document.createElement('a');
165+
a.href = dataUrl;
166+
a.download = 'benchmark_data.json';
167+
a.click();
168+
};
169+
170+
return Object.keys(data.entries).map((name) => ({
171+
name,
172+
dataSet: collectBenchesGrouped(data.entries[name]),
173+
}));
174+
}
175+
176+
function renderAllCharts(dataSets) {
177+
function sortedCommits(seriesMap) {
178+
const byId = new Map();
179+
for (const series of seriesMap.values()) {
180+
for (const point of series) {
181+
byId.set(point.commit.id, point);
182+
}
183+
}
184+
return Array.from(byId.values()).sort(
185+
(a, b) => new Date(a.date).getTime() - new Date(b.date).getTime(),
186+
);
187+
}
188+
189+
function renderGroupedGraph(parent, baseName, seriesMap) {
190+
const canvas = document.createElement('canvas');
191+
canvas.className = 'benchmark-chart';
192+
parent.appendChild(canvas);
193+
194+
const commits = sortedCommits(seriesMap);
195+
const labels = commits.map((c) => c.commit.id.slice(0, 7));
196+
const seriesData = [];
197+
198+
for (const [seriesKey, series] of seriesMap.entries()) {
199+
seriesData.push({
200+
seriesKey,
201+
valueByCommit: new Map(series.map((s) => [s.commit.id, s])),
202+
});
203+
}
204+
205+
const datasets = seriesData.map(({ seriesKey, valueByCommit }) => {
206+
const color = compileColors[seriesKey] ?? compileColors._default;
207+
const label = seriesKey;
208+
return {
209+
label,
210+
data: commits.map((c) => {
211+
const point = valueByCommit.get(c.commit.id);
212+
return point ? point.bench.value : null;
213+
}),
214+
borderColor: color,
215+
backgroundColor: color + '60',
216+
spanGaps: true,
217+
};
218+
});
219+
220+
const firstPoint = seriesData
221+
.map(({ valueByCommit }) => valueByCommit.values().next().value)
222+
.find((point) => point !== undefined);
223+
const unit = firstPoint ? firstPoint.bench.unit : '';
224+
225+
const options = {
226+
scales: {
227+
xAxes: [
228+
{
229+
scaleLabel: {
230+
display: true,
231+
labelString: 'commit',
232+
},
233+
},
234+
],
235+
yAxes: [
236+
{
237+
scaleLabel: {
238+
display: true,
239+
labelString: unit,
240+
},
241+
ticks: {
242+
beginAtZero: true,
243+
},
244+
},
245+
],
246+
},
247+
tooltips: {
248+
mode: 'index',
249+
intersect: false,
250+
callbacks: {
251+
afterTitle: (items) => {
252+
const commit = commits[items[0].index];
253+
return (
254+
'\n' +
255+
commit.commit.message +
256+
'\n\n' +
257+
commit.commit.timestamp +
258+
' committed by @' +
259+
commit.commit.committer.username +
260+
'\n'
261+
);
262+
},
263+
label: (item, chartData) => {
264+
const { valueByCommit } = seriesData[item.datasetIndex];
265+
const point = valueByCommit.get(commits[item.index].commit.id);
266+
const seriesLabel = chartData.datasets[item.datasetIndex].label;
267+
if (!point) {
268+
return seriesLabel + ': n/a';
269+
}
270+
let label = item.yLabel;
271+
const { range, unit: benchUnit } = point.bench;
272+
label += ' ' + benchUnit;
273+
if (range) {
274+
label += ' (' + range + ')';
275+
}
276+
return seriesLabel + ': ' + label;
277+
},
278+
afterLabel: (item) => {
279+
const { valueByCommit } = seriesData[item.datasetIndex];
280+
const point = valueByCommit.get(commits[item.index].commit.id);
281+
const extra = point ? point.bench.extra : null;
282+
return extra ? '\n' + extra : '';
283+
},
284+
},
285+
},
286+
onClick: (_mouseEvent, activeElems) => {
287+
if (activeElems.length === 0) {
288+
return;
289+
}
290+
const commit = commits[activeElems[0]._index];
291+
window.open(commit.commit.url, '_blank');
292+
},
293+
};
294+
295+
new Chart(canvas, {
296+
type: 'line',
297+
data: { labels, datasets },
298+
options,
299+
});
300+
}
301+
302+
function renderBenchSet(name, benchSet, main) {
303+
const setElem = document.createElement('div');
304+
setElem.className = 'benchmark-set';
305+
main.appendChild(setElem);
306+
307+
const nameElem = document.createElement('h1');
308+
nameElem.className = 'benchmark-title';
309+
nameElem.textContent = name;
310+
setElem.appendChild(nameElem);
311+
312+
const graphsElem = document.createElement('div');
313+
graphsElem.className = 'benchmark-graphs';
314+
setElem.appendChild(graphsElem);
315+
316+
for (const [baseName, seriesMap] of benchSet.entries()) {
317+
renderGroupedGraph(graphsElem, baseName, seriesMap);
318+
}
319+
}
320+
321+
const main = document.getElementById('main');
322+
for (const { name, dataSet } of dataSets) {
323+
renderBenchSet(name, dataSet, main);
324+
}
325+
}
326+
327+
renderAllCharts(init());
328+
})();
329+
</script>
330+
</body>
331+
</html>

0 commit comments

Comments
 (0)