Skip to content

Commit 62ab497

Browse files
kisernlkisernl
committed
Merge benchmarks/master into superserve-dax-template and resolve conflicts
2 parents 727a7e8 + c28bc2c commit 62ab497

35 files changed

Lines changed: 10568 additions & 46239 deletions

‎.github/workflows/ai-gateway-benchmarks.yml‎

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
name: AI Gateway Benchmark
2+
3+
on:
4+
pull_request:
5+
paths:
6+
- 'src/ai-gateway/**'
7+
- 'src/run.ts'
8+
- 'src/merge-results.ts'
9+
- 'package.json'
10+
- '.github/workflows/ai-gateway-benchmarks.yml'
11+
schedule:
12+
- cron: '0 6 * * 5' # Weekly on Friday at 6am UTC
13+
workflow_dispatch:
14+
inputs:
15+
iterations:
16+
description: 'Cold + warm iterations per gateway'
17+
required: false
18+
default: '20'
19+
provider:
20+
description: 'Gateway to run (leave empty for all five, round-robin)'
21+
required: false
22+
default: ''
23+
type: choice
24+
options:
25+
- ''
26+
- openrouter
27+
- vercel-ai-gateway
28+
- cloudflare-ai-gateway
29+
- llmgateway
30+
- anthropic-direct
31+
32+
concurrency:
33+
group: ai-gateway-benchmarks
34+
cancel-in-progress: true
35+
36+
permissions:
37+
contents: write
38+
pull-requests: write
39+
40+
jobs:
41+
bench:
42+
name: AI Gateway Benchmark
43+
runs-on: namespace-profile-default
44+
# Deliberately a single job, not a matrix-per-provider like the other
45+
# benchmark workflows: the round-robin methodology (see AI_GATEWAYS.md)
46+
# requires every gateway to run interleaved within the same process.
47+
# Splitting gateways into separate matrix jobs would silently degrade
48+
# "round-robin" into plain per-gateway iteration, defeating the point.
49+
timeout-minutes: 30
50+
steps:
51+
- uses: actions/checkout@v4
52+
with:
53+
# Full history so the rebase-on-push retry below has a merge base.
54+
fetch-depth: 0
55+
- uses: actions/setup-node@v4
56+
with:
57+
node-version: 24
58+
cache: 'npm'
59+
- name: Install dependencies
60+
run: |
61+
if [ "${{ github.event_name }}" = "schedule" ]; then
62+
npm update
63+
else
64+
npm ci
65+
fi
66+
- name: Clear stale results from checkout
67+
run: rm -rf results/ai-gateway/
68+
- name: Run AI gateway benchmark
69+
env:
70+
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
71+
VERCEL_AI_GATEWAY_API_KEY: ${{ secrets.VERCEL_AI_GATEWAY_API_KEY }}
72+
CLOUDFLARE_AI_GATEWAY_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_AI_GATEWAY_ACCOUNT_ID }}
73+
CLOUDFLARE_AI_GATEWAY_GATEWAY_ID: ${{ secrets.CLOUDFLARE_AI_GATEWAY_GATEWAY_ID }}
74+
CLOUDFLARE_AI_GATEWAY_TOKEN: ${{ secrets.CLOUDFLARE_AI_GATEWAY_TOKEN }}
75+
LLM_GATEWAY_API_KEY: ${{ secrets.LLM_GATEWAY_API_KEY }}
76+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
77+
run: |
78+
PROVIDER_FLAG=""
79+
if [ -n "${{ github.event.inputs.provider }}" ]; then
80+
PROVIDER_FLAG="--provider ${{ github.event.inputs.provider }}"
81+
fi
82+
npm run bench:ai-gateway -- $PROVIDER_FLAG \
83+
--iterations ${{ (github.event_name == 'pull_request' && '2') || github.event.inputs.iterations || '10' }}
84+
- run: npm run generate-ai-gateway-svg
85+
- name: Upload results and SVG as artifacts
86+
if: always()
87+
uses: actions/upload-artifact@v4
88+
with:
89+
name: ai-gateway-results
90+
path: |
91+
results/ai-gateway/
92+
ai-gateway.svg
93+
if-no-files-found: ignore
94+
retention-days: 7
95+
- name: Post results to PR
96+
if: github.event_name == 'pull_request'
97+
continue-on-error: true
98+
uses: actions/github-script@v7
99+
with:
100+
script: |
101+
const fs = require('fs');
102+
const path = require('path');
103+
104+
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
105+
const latestPath = path.join('results', 'ai-gateway', 'latest.json');
106+
let body = '## AI Gateway Benchmark Results\n\n';
107+
let hasResults = false;
108+
109+
const nameFor = {
110+
'openrouter': 'OpenRouter',
111+
'vercel-ai-gateway': 'Vercel AI Gateway',
112+
'cloudflare-ai-gateway': 'Cloudflare AI Gateway',
113+
'anthropic-direct': 'Anthropic (direct)',
114+
};
115+
116+
if (fs.existsSync(latestPath)) {
117+
const data = JSON.parse(fs.readFileSync(latestPath, 'utf-8'));
118+
const results = data.results
119+
.filter(r => !r.skipped)
120+
.sort((a, b) => (b.compositeScore || 0) - (a.compositeScore || 0));
121+
122+
if (results.length > 0) {
123+
hasResults = true;
124+
body += '| # | Gateway | Score | Cold E2E | Warm TTFT | Tok/sec | Status |\n';
125+
body += '|---|---------|-------|----------|-----------|---------|--------|\n';
126+
results.forEach((r, i) => {
127+
const name = nameFor[r.provider] || r.provider;
128+
const score = r.compositeScore !== undefined ? r.compositeScore.toFixed(1) : '--';
129+
const coldE2e = `${Math.round(r.summary.coldE2eMs.median)}ms`;
130+
const warmTtft = `${Math.round(r.summary.warmTtftMs.median)}ms`;
131+
const tps = r.summary.outputTokensPerSec.median.toFixed(1);
132+
const ok = r.iterations.filter(it => !it.error).length;
133+
const total = r.iterations.length;
134+
body += `| ${i + 1} | ${name} | ${score} | ${coldE2e} | ${warmTtft} | ${tps} | ${ok}/${total} |\n`;
135+
});
136+
}
137+
138+
const skipped = data.results.filter(r => r.skipped);
139+
if (skipped.length > 0) {
140+
body += `\n_Skipped: ${skipped.map(r => `${nameFor[r.provider] || r.provider} (${r.skipReason})`).join(', ')}_\n`;
141+
}
142+
}
143+
144+
if (!hasResults) {
145+
body += '> No AI gateway benchmark results were generated.\n\n';
146+
}
147+
148+
body += `\n---\n*[View full run](${runUrl}) · SVG available as a [build artifact](${runUrl}#artifacts)*`;
149+
150+
const marker = '## AI Gateway Benchmark Results';
151+
const { data: comments } = await github.rest.issues.listComments({
152+
owner: context.repo.owner,
153+
repo: context.repo.repo,
154+
issue_number: context.issue.number,
155+
});
156+
157+
const existing = comments.find(c => c.body.startsWith(marker));
158+
159+
if (existing) {
160+
await github.rest.issues.updateComment({
161+
owner: context.repo.owner,
162+
repo: context.repo.repo,
163+
comment_id: existing.id,
164+
body,
165+
});
166+
} else {
167+
await github.rest.issues.createComment({
168+
owner: context.repo.owner,
169+
repo: context.repo.repo,
170+
issue_number: context.issue.number,
171+
body,
172+
});
173+
}
174+
- name: Commit and push
175+
if: github.event_name != 'pull_request'
176+
run: |
177+
git config user.name "github-actions[bot]"
178+
git config user.email "github-actions[bot]@users.noreply.github.com"
179+
git add ai-gateway.svg results/ai-gateway/
180+
git diff --cached --quiet && echo "No changes to commit" && exit 0
181+
git commit -m "chore: update ai gateway benchmark results [skip ci]"
182+
183+
# Remote master can advance during the run (concurrent benchmark
184+
# workflows push to master too), so a plain push fails non-fast-forward.
185+
# Rebase onto the latest remote and retry a few times before giving up.
186+
branch="${GITHUB_REF#refs/heads/}"
187+
for attempt in 1 2 3 4 5; do
188+
git fetch origin "${branch}"
189+
git rebase "origin/${branch}" || { git rebase --abort; exit 1; }
190+
if git push origin "HEAD:${branch}"; then
191+
echo "Pushed on attempt ${attempt}"
192+
exit 0
193+
fi
194+
echo "Push rejected (attempt ${attempt}); will rebase and retry"
195+
done
196+
echo "Failed to push after multiple attempts" >&2
197+
exit 1

‎.github/workflows/browser-benchmarks.yml‎

Lines changed: 36 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
name: Browser Benchmark
22

33
on:
4-
pull_request:
4+
push:
5+
branches: [master]
56
paths:
67
- 'src/browser/**'
78
- 'src/util/**'
@@ -40,6 +41,7 @@ jobs:
4041
- kernel
4142
- notte
4243
- steel
44+
- tilion
4345
# - anchorbrowser
4446
steps:
4547
- uses: actions/checkout@v4
@@ -65,12 +67,14 @@ jobs:
6567
KERNEL_API_KEY: ${{ secrets.KERNEL_API_KEY }}
6668
NOTTE_API_KEY: ${{ secrets.NOTTE_API_KEY }}
6769
STEEL_API_KEY: ${{ secrets.STEEL_API_KEY }}
70+
TILION_API_KEY: ${{ secrets.TILION_API_KEY }}
71+
TILION_BASE_URL: ${{ secrets.TILION_BASE_URL }}
6872
# ANCHORBROWSER_API_KEY: ${{ secrets.ANCHORBROWSER_API_KEY }}
6973
run: |
7074
npm run bench -- \
7175
--mode browser \
7276
--provider ${{ matrix.provider }} \
73-
--iterations ${{ github.event_name == 'pull_request' && '10' || github.event.inputs.iterations || '100' }}
77+
--iterations ${{ github.event_name == 'push' && '10' || github.event.inputs.iterations || '100' }}
7478
- name: Upload results
7579
if: always()
7680
uses: actions/upload-artifact@v4
@@ -111,23 +115,23 @@ jobs:
111115
- name: Merge results
112116
run: npx tsx src/merge-results.ts --input artifacts --mode browser
113117
- name: Ingest results to platform
114-
if: github.event_name != 'pull_request'
118+
if: github.event_name != 'push'
115119
continue-on-error: true
116120
env:
117121
INGEST_URL: ${{ secrets.INGEST_URL }}
118122
INGEST_SECRET: ${{ secrets.INGEST_SECRET }}
119123
run: npx tsx src/ingest.ts --type browser
120124
- run: npm run generate-browser-svg
121125
- name: Upload SVG as artifact
122-
if: github.event_name == 'pull_request'
126+
if: github.event_name == 'push'
123127
uses: actions/upload-artifact@v4
124128
with:
125129
name: browser-benchmark-svg
126130
path: browser.svg
127131
if-no-files-found: ignore
128132
retention-days: 7
129-
- name: Post results to PR
130-
if: github.event_name == 'pull_request'
133+
- name: Post results to merged PR
134+
if: github.event_name == 'push'
131135
continue-on-error: true
132136
uses: actions/github-script@v7
133137
with:
@@ -173,11 +177,33 @@ jobs:
173177
174178
body += `---\n*[View full run](${runUrl}) · SVG available as [build artifact](${runUrl}#artifacts)*`;
175179
180+
// This push is a merge to master — post results to the PR it closed.
181+
// A commit can be associated with several PRs (backports, merge
182+
// queue), so pick the one merged into the branch we pushed to.
183+
const target = context.ref.replace('refs/heads/', '');
184+
const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
185+
owner: context.repo.owner,
186+
repo: context.repo.repo,
187+
commit_sha: context.sha,
188+
});
189+
const pr = prs.find(p => p.merged_at && p.base.ref === target);
190+
if (!pr) {
191+
// Direct push with no associated PR — fall back to a commit comment.
192+
await github.rest.repos.createCommitComment({
193+
owner: context.repo.owner,
194+
repo: context.repo.repo,
195+
commit_sha: context.sha,
196+
body,
197+
});
198+
return;
199+
}
200+
176201
const marker = '## Browser Benchmark Results';
177-
const { data: comments } = await github.rest.issues.listComments({
202+
const comments = await github.paginate(github.rest.issues.listComments, {
178203
owner: context.repo.owner,
179204
repo: context.repo.repo,
180-
issue_number: context.issue.number,
205+
issue_number: pr.number,
206+
per_page: 100,
181207
});
182208
183209
const existing = comments.find(c => c.body.startsWith(marker));
@@ -193,12 +219,12 @@ jobs:
193219
await github.rest.issues.createComment({
194220
owner: context.repo.owner,
195221
repo: context.repo.repo,
196-
issue_number: context.issue.number,
222+
issue_number: pr.number,
197223
body,
198224
});
199225
}
200226
- name: Commit and push
201-
if: github.event_name != 'pull_request'
227+
if: github.event_name != 'push'
202228
run: |
203229
git config user.name "github-actions[bot]"
204230
git config user.email "github-actions[bot]@users.noreply.github.com"

0 commit comments

Comments
 (0)