David/add concentrate ai gateway #74
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: AI Gateway Benchmark | |
| on: | |
| pull_request: | |
| paths: | |
| - 'benchmarks/ai-gateway/**' | |
| - 'benchmarks/src/run.ts' | |
| - 'benchmarks/src/merge-results.ts' | |
| - 'package.json' | |
| - '.github/workflows/ai-gateway-benchmarks.yml' | |
| schedule: | |
| - cron: '0 6 * * 5' # Weekly on Friday at 6am UTC | |
| workflow_dispatch: | |
| inputs: | |
| iterations: | |
| description: 'Cold + warm iterations per gateway' | |
| required: false | |
| default: '20' | |
| provider: | |
| description: 'Gateway to run (leave empty for all seven, round-robin)' | |
| required: false | |
| default: '' | |
| type: choice | |
| options: | |
| - '' | |
| - openrouter | |
| - vercel-ai-gateway | |
| - cloudflare-ai-gateway | |
| - llmgateway | |
| - pydantic-ai-gateway | |
| - concentrate-ai-gateway | |
| - anthropic-direct | |
| dry_run: | |
| description: 'Run without ingesting or committing results' | |
| required: false | |
| default: false | |
| type: boolean | |
| concurrency: | |
| group: ai-gateway-benchmarks | |
| cancel-in-progress: true | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| jobs: | |
| bench: | |
| name: AI Gateway Benchmark | |
| runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe | |
| # Deliberately a single job, not a matrix-per-provider like the other | |
| # benchmark workflows: the round-robin methodology (see AI_GATEWAYS.md) | |
| # requires every gateway to run interleaved within the same process. | |
| # Splitting gateways into separate matrix jobs would silently degrade | |
| # "round-robin" into plain per-gateway iteration, defeating the point. | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| # Full history so the rebase-on-push retry below has a merge base. | |
| fetch-depth: 0 | |
| - uses: pnpm/action-setup@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 24 | |
| cache: 'pnpm' | |
| - name: Install dependencies | |
| run: | | |
| if [ "${{ github.event_name }}" = "schedule" ]; then | |
| pnpm update | |
| else | |
| pnpm install --frozen-lockfile | |
| fi | |
| - name: Run AI gateway benchmark | |
| # Single job (no matrix); the round-robin methodology (see AI_GATEWAYS.md) | |
| # tests every gateway inside one process, so we load the union of all | |
| # ai-gateway providers' vars — not every mode's vars. | |
| run: | | |
| . benchmarks/scripts/load-vault-secrets.sh '^(ANTHROPIC_API_KEY|CLOUDFLARE_AI_GATEWAY_ACCOUNT_ID|CLOUDFLARE_AI_GATEWAY_GATEWAY_ID|LLM_GATEWAY_API_KEY|OPENROUTER_API_KEY|PYDANTIC_AI_GATEWAY_API_KEY|VERCEL_AI_GATEWAY_API_KEY|CONCENTRATE_AI_GATEWAY_API_KEY)' | |
| PROVIDER_FLAG="" | |
| if [ -n "${{ github.event.inputs.provider }}" ]; then | |
| PROVIDER_FLAG="--provider ${{ github.event.inputs.provider }}" | |
| fi | |
| pnpm run bench:ai-gateway -- $PROVIDER_FLAG \ | |
| --iterations ${{ (github.event_name == 'pull_request' && '2') || github.event.inputs.iterations || '10' }} | |
| - run: pnpm run generate-ai-gateway-svg | |
| - name: Upload results and SVG as artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ai-gateway-results | |
| path: | | |
| results/ai-gateway/ | |
| ai-gateway.svg | |
| if-no-files-found: ignore | |
| retention-days: 7 | |
| - name: Post results to PR | |
| if: github.event_name == 'pull_request' | |
| continue-on-error: true | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const path = require('path'); | |
| const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; | |
| const latestPath = path.join('results', 'ai-gateway', 'latest.json'); | |
| let body = '## AI Gateway Benchmark Results\n\n'; | |
| let hasResults = false; | |
| const nameFor = { | |
| 'openrouter': 'OpenRouter', | |
| 'vercel-ai-gateway': 'Vercel AI Gateway', | |
| 'cloudflare-ai-gateway': 'Cloudflare AI Gateway', | |
| 'llmgateway': 'LLM Gateway', | |
| 'pydantic-ai-gateway': 'Pydantic AI Gateway', | |
| 'concentrate-ai-gateway': 'Concentrate AI', | |
| 'anthropic-direct': 'Anthropic (direct)', | |
| }; | |
| if (fs.existsSync(latestPath)) { | |
| const data = JSON.parse(fs.readFileSync(latestPath, 'utf-8')); | |
| const results = data.results | |
| .filter(r => !r.skipped) | |
| .sort((a, b) => (b.compositeScore || 0) - (a.compositeScore || 0)); | |
| if (results.length > 0) { | |
| hasResults = true; | |
| body += '| # | Gateway | Score | Cold E2E | Warm TTFT | Tok/sec | Status |\n'; | |
| body += '|---|---------|-------|----------|-----------|---------|--------|\n'; | |
| results.forEach((r, i) => { | |
| const name = nameFor[r.provider] || r.provider; | |
| const score = r.compositeScore !== undefined ? r.compositeScore.toFixed(1) : '--'; | |
| const coldE2e = `${Math.round(r.summary.coldE2eMs.median)}ms`; | |
| const warmTtft = `${Math.round(r.summary.warmTtftMs.median)}ms`; | |
| const tps = r.summary.outputTokensPerSec.median.toFixed(1); | |
| const ok = r.iterations.filter(it => !it.error).length; | |
| const total = r.iterations.length; | |
| body += `| ${i + 1} | ${name} | ${score} | ${coldE2e} | ${warmTtft} | ${tps} | ${ok}/${total} |\n`; | |
| }); | |
| } | |
| const skipped = data.results.filter(r => r.skipped); | |
| if (skipped.length > 0) { | |
| body += `\n_Skipped: ${skipped.map(r => `${nameFor[r.provider] || r.provider} (${r.skipReason})`).join(', ')}_\n`; | |
| } | |
| } | |
| if (!hasResults) { | |
| body += '> No AI gateway benchmark results were generated.\n\n'; | |
| } | |
| body += `\n---\n*[View full run](${runUrl}) · SVG available as a [build artifact](${runUrl}#artifacts)*`; | |
| const marker = '## AI Gateway Benchmark Results'; | |
| const { data: comments } = await github.rest.issues.listComments({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| }); | |
| const existing = comments.find(c => c.body.startsWith(marker)); | |
| if (existing) { | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: existing.id, | |
| body, | |
| }); | |
| } else { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body, | |
| }); | |
| } | |
| - name: Commit and push | |
| if: github.event_name != 'pull_request' && github.event.inputs.dry_run != 'true' | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add ai-gateway.svg results/ai-gateway/ | |
| git diff --cached --quiet && echo "No changes to commit" && exit 0 | |
| git commit -m "chore: update ai gateway benchmark results [skip ci]" | |
| # Remote master can advance during the run (concurrent benchmark | |
| # workflows push to master too), so a plain push fails non-fast-forward. | |
| # Rebase onto the latest remote and retry a few times before giving up. | |
| branch="${GITHUB_REF#refs/heads/}" | |
| for attempt in 1 2 3 4 5; do | |
| git fetch origin "${branch}" | |
| git rebase "origin/${branch}" || { git rebase --abort; exit 1; } | |
| if git push origin "HEAD:${branch}"; then | |
| echo "Pushed on attempt ${attempt}" | |
| exit 0 | |
| fi | |
| echo "Push rejected (attempt ${attempt}); will rebase and retry" | |
| done | |
| echo "Failed to push after multiple attempts" >&2 | |
| exit 1 |