Merge pull request #240 from computesdk/david/add-concentrate-ai-gateway #262
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Storage Benchmark | |
| on: | |
| push: | |
| branches: [master] | |
| paths: | |
| - 'benchmarks/storage/**' | |
| - 'benchmarks/src/util/**' | |
| - 'benchmarks/src/run.ts' | |
| - 'benchmarks/src/merge-results.ts' | |
| - 'package.json' | |
| schedule: | |
| - cron: '0 6 * * 5' # Weekly on Friday at 6am UTC | |
| workflow_dispatch: | |
| inputs: | |
| iterations: | |
| description: 'Iterations per provider' | |
| required: false | |
| default: '100' | |
| storage_concurrency: | |
| description: 'Parallel storage iterations per job' | |
| required: false | |
| default: '1' | |
| file_size: | |
| description: 'File size to test (leave empty to run all)' | |
| required: false | |
| default: '' | |
| type: choice | |
| options: | |
| - '' | |
| - 1MB | |
| - 4MB | |
| - 10MB | |
| - 16MB | |
| dry_run: | |
| description: 'Run without ingesting or committing results' | |
| required: false | |
| default: false | |
| type: boolean | |
| concurrency: | |
| group: storage-benchmarks | |
| cancel-in-progress: true | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| jobs: | |
| bench: | |
| name: Bench ${{ matrix.provider }} ${{ matrix.file_size }} | |
| runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe | |
| timeout-minutes: 90 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| provider: | |
| - aws-s3 | |
| - cloudflare-r2 | |
| - tigris | |
| - vercel-blob | |
| - gcs | |
| - azure-blob | |
| file_size: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.file_size != '' && fromJson(format('["{0}"]', github.event.inputs.file_size))) || (github.event_name == 'push' && fromJson('["1MB"]')) || fromJson('["1MB","4MB","10MB","16MB"]') }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: pnpm/action-setup@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 24 | |
| cache: 'pnpm' | |
| - name: Install dependencies | |
| run: | | |
| if [ "${{ github.event_name }}" = "schedule" ]; then | |
| pnpm update | |
| else | |
| pnpm install --frozen-lockfile | |
| fi | |
| - name: Clear stale results from checkout | |
| run: rm -rf results/storage/ | |
| - name: Run storage benchmark | |
| run: | | |
| . benchmarks/scripts/load-vault-secrets.sh '^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_REGION|S3_BUCKET|R2_ACCESS_KEY_ID|R2_SECRET_ACCESS_KEY|R2_BUCKET|R2_ACCOUNT_ID|TIGRIS_STORAGE_ACCESS_KEY_ID|TIGRIS_STORAGE_SECRET_ACCESS_KEY|TIGRIS_STORAGE_BUCKET|BLOB_READ_WRITE_TOKEN|VERCEL_BLOB_BUCKET|GCS_PROJECT_ID|GCS_BUCKET|GCS_CLIENT_EMAIL|GCS_PRIVATE_KEY|AZURE_ACCOUNT_NAME|AZURE_ACCOUNT_KEY|AZURE_CONTAINER)' | |
| FILE_SIZE="${{ matrix.file_size }}" | |
| pnpm run bench -- \ | |
| --mode storage \ | |
| --provider ${{ matrix.provider }} \ | |
| --file-size $FILE_SIZE \ | |
| --storage-concurrency ${{ (github.event_name == 'schedule' && '16') || (github.event_name == 'push' && '8') || github.event.inputs.storage_concurrency || '1' }} \ | |
| --iterations ${{ (github.event_name == 'schedule' && '110') || (github.event_name == 'push' && '10') || github.event.inputs.iterations || '100' }} | |
| - name: Upload results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: storage-results-${{ matrix.provider }}-${{ matrix.file_size }} | |
| path: results/storage/ | |
| if-no-files-found: ignore | |
| retention-days: 7 | |
| collect: | |
| name: Collect Results | |
| runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe | |
| needs: [bench] | |
| if: always() | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| # Full history so the rebase-on-push retry below has a merge base. | |
| # A shallow (depth-1) clone can make `git rebase origin/<branch>` | |
| # fail when the remote has advanced past the shallow boundary. | |
| fetch-depth: 0 | |
| - uses: pnpm/action-setup@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 24 | |
| cache: 'pnpm' | |
| - name: Install dependencies | |
| run: | | |
| if [ "${{ github.event_name }}" = "schedule" ]; then | |
| pnpm update | |
| else | |
| pnpm install --frozen-lockfile | |
| fi | |
| - name: Download all artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: artifacts/ | |
| pattern: storage-results-* | |
| - name: Merge results | |
| run: npx tsx benchmarks/src/merge-results.ts --input artifacts --mode storage | |
| - name: Ingest results to platform | |
| if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' | |
| continue-on-error: true | |
| run: | | |
| . benchmarks/scripts/load-vault-secrets.sh '^(INGEST_URL|INGEST_SECRET)' | |
| npx tsx benchmarks/src/ingest.ts --type storage | |
| - run: pnpm run generate-storage-svg | |
| - name: Upload SVGs as artifacts | |
| if: github.event_name == 'push' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: storage-benchmark-svgs | |
| path: storage_*.svg | |
| if-no-files-found: ignore | |
| retention-days: 7 | |
| - name: Post results to merged PR | |
| if: github.event_name == 'push' | |
| continue-on-error: true | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const path = require('path'); | |
| const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; | |
| // Only render sizes that actually ran in this workflow. The checkout | |
| // carries committed latest.json files for every size (from the weekly | |
| // run), so we derive the size list from this run's downloaded | |
| // artifacts (storage-results-<provider>-<size>) instead of hardcoding. | |
| const sizeOrder = ['1mb', '4mb', '10mb', '16mb']; | |
| const ranSizes = new Set(); | |
| if (fs.existsSync('artifacts')) { | |
| for (const name of fs.readdirSync('artifacts')) { | |
| if (!name.startsWith('storage-results-')) continue; | |
| ranSizes.add(name.split('-').pop().toLowerCase()); | |
| } | |
| } | |
| const sizes = sizeOrder.filter(s => ranSizes.has(s)); | |
| let body = '## Storage Benchmark Results\n\n'; | |
| let hasResults = false; | |
| for (const size of sizes) { | |
| const latestPath = path.join('results', 'storage', size, 'latest.json'); | |
| if (!fs.existsSync(latestPath)) continue; | |
| const data = JSON.parse(fs.readFileSync(latestPath, 'utf-8')); | |
| const results = data.results | |
| .filter(r => !r.skipped) | |
| .sort((a, b) => (b.compositeScore || 0) - (a.compositeScore || 0)); | |
| if (results.length === 0) continue; | |
| hasResults = true; | |
| body += `### ${size.toUpperCase()} Files\n\n`; | |
| body += '| # | Provider | Score | Download | Throughput | Upload | Status |\n'; | |
| body += '|---|----------|-------|----------|------------|--------|--------|\n'; | |
| results.forEach((r, i) => { | |
| const name = r.provider === 'aws-s3' ? 'AWS S3' : r.provider === 'cloudflare-r2' ? 'Cloudflare R2' : r.provider === 'vercel-blob' ? 'Vercel Blob' : r.provider === 'gcs' ? 'Google Cloud Storage' : r.provider === 'azure-blob' ? 'Azure Blob Storage' : r.provider.charAt(0).toUpperCase() + r.provider.slice(1); | |
| const score = r.compositeScore !== undefined ? r.compositeScore.toFixed(1) : '--'; | |
| const dl = (r.summary.downloadMs.median / 1000).toFixed(2) + 's'; | |
| const tp = r.summary.throughputMbps.median.toFixed(1) + ' Mbps'; | |
| const ul = (r.summary.uploadMs.median / 1000).toFixed(2) + 's'; | |
| const ok = r.iterations.filter(it => !it.error).length; | |
| const total = r.iterations.length; | |
| body += `| ${i + 1} | ${name} | ${score} | ${dl} | ${tp} | ${ul} | ${ok}/${total} |\n`; | |
| }); | |
| body += '\n'; | |
| } | |
| if (!hasResults) { | |
| body += '> No storage benchmark results were generated.\n\n'; | |
| } | |
| body += `---\n*[View full run](${runUrl}) · SVGs available as [build artifacts](${runUrl}#artifacts)*`; | |
| // This push is a merge to master — post results to the PR it closed. | |
| // A commit can be associated with several PRs (backports, merge | |
| // queue), so pick the one merged into the branch we pushed to. | |
| const target = context.ref.replace('refs/heads/', ''); | |
| const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| commit_sha: context.sha, | |
| }); | |
| const pr = prs.find(p => p.merged_at && p.base.ref === target); | |
| if (!pr) { | |
| // Direct push with no associated PR — fall back to a commit comment. | |
| await github.rest.repos.createCommitComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| commit_sha: context.sha, | |
| body, | |
| }); | |
| return; | |
| } | |
| // Find and update existing comment or create new one | |
| const marker = '## Storage Benchmark Results'; | |
| const comments = await github.paginate(github.rest.issues.listComments, { | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: pr.number, | |
| per_page: 100, | |
| }); | |
| const existing = comments.find(c => c.body.startsWith(marker)); | |
| if (existing) { | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: existing.id, | |
| body, | |
| }); | |
| } else { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: pr.number, | |
| body, | |
| }); | |
| } | |
| - name: Commit and push | |
| if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add package.json package-lock.json storage_*.svg results/storage/ | |
| git diff --cached --quiet && echo "No changes to commit" && exit 0 | |
| git commit -m "chore: update storage benchmark results [skip ci]" | |
| # Remote master can advance during the run (concurrent benchmark | |
| # workflows push to master too), so a plain push fails non-fast-forward. | |
| # Rebase onto the latest remote and retry a few times before giving up. | |
| branch="${GITHUB_REF#refs/heads/}" | |
| for attempt in 1 2 3 4 5; do | |
| # Rebase before each attempt (including the last) so every rebase | |
| # is followed by a push — otherwise the final iteration's rebase | |
| # would be wasted and a resolvable non-fast-forward could still fail. | |
| git fetch origin "${branch}" | |
| git rebase "origin/${branch}" || { git rebase --abort; exit 1; } | |
| if git push origin "HEAD:${branch}"; then | |
| echo "Pushed on attempt ${attempt}" | |
| exit 0 | |
| fi | |
| echo "Push rejected (attempt ${attempt}); will rebase and retry" | |
| done | |
| echo "Failed to push after multiple attempts" >&2 | |
| exit 1 |