David/add concentrate ai gateway

.github/workflows/ai-gateway-benchmarks.yml

@@ -17,7 +17,7 @@ on:
         required: false
         default: '20'
       provider:
-        description: 'Gateway to run (leave empty for all six, round-robin)'
+        description: 'Gateway to run (leave empty for all seven, round-robin)'
         required: false
         default: ''
         type: choice
@@ -28,6 +28,7 @@ on:
           - cloudflare-ai-gateway
           - llmgateway
           - pydantic-ai-gateway
+          - concentrate-ai-gateway
           - anthropic-direct
       dry_run:
         description: 'Run without ingesting or committing results'
@@ -75,7 +76,7 @@ jobs:
         # tests every gateway inside one process, so we load the union of all
         # ai-gateway providers' vars — not every mode's vars.
         run: |
-          . benchmarks/scripts/load-vault-secrets.sh '^(ANTHROPIC_API_KEY|CLOUDFLARE_AI_GATEWAY_ACCOUNT_ID|CLOUDFLARE_AI_GATEWAY_GATEWAY_ID|LLM_GATEWAY_API_KEY|OPENROUTER_API_KEY|PYDANTIC_AI_GATEWAY_API_KEY|VERCEL_AI_GATEWAY_API_KEY)'
+          . benchmarks/scripts/load-vault-secrets.sh '^(ANTHROPIC_API_KEY|CLOUDFLARE_AI_GATEWAY_ACCOUNT_ID|CLOUDFLARE_AI_GATEWAY_GATEWAY_ID|LLM_GATEWAY_API_KEY|OPENROUTER_API_KEY|PYDANTIC_AI_GATEWAY_API_KEY|VERCEL_AI_GATEWAY_API_KEY|CONCENTRATE_AI_GATEWAY_API_KEY)'
 
           PROVIDER_FLAG=""
           if [ -n "${{ github.event.inputs.provider }}" ]; then
@@ -114,6 +115,7 @@ jobs:
               'cloudflare-ai-gateway': 'Cloudflare AI Gateway',
               'llmgateway': 'LLM Gateway',
               'pydantic-ai-gateway': 'Pydantic AI Gateway',
+              'concentrate-ai-gateway': 'Concentrate AI',
               'anthropic-direct': 'Anthropic (direct)',
             };
 

AI_GATEWAYS.md

@@ -1,6 +1,6 @@
 # AI Gateway Benchmark
 
-This document describes the **AI gateway benchmark** — a phase-by-phase latency, throughput, and reliability comparison of OpenRouter, Vercel AI Gateway, Cloudflare AI Gateway, LLM Gateway, and Pydantic AI Gateway, measured against a direct-to-Anthropic baseline.
+This document describes the **AI gateway benchmark** — a phase-by-phase latency, throughput, and reliability comparison of OpenRouter, Vercel AI Gateway, Cloudflare AI Gateway, LLM Gateway, Pydantic AI Gateway, and Concentrate AI, measured against a direct-to-Anthropic baseline.
 
 > **Where this runs**: scheduled and dispatched runs execute in GitHub Actions on [Namespace](https://namespace.so) runners (`namespace-profile-default`), physically placed in **Northern Virginia, US**. This is a single fixed vantage point, not a global or multi-region measurement — every number in this benchmark reflects network conditions from that one location. Confirmed two ways: Namespace's own runner-instance panel reports "Placement: Northern Virginia, US" for this profile, and independently, `cf-ray` receipts captured in real runs include `IAD` — the airport code Cloudflare uses for its Ashburn/Northern Virginia edge datacenter, exactly consistent with a client physically nearby. See [Vantage-point dependent](#limitations) in Limitations for what this does and doesn't mean for the results.
 
@@ -44,23 +44,23 @@ Only `ttfbMs` and `ttftMs` apply — there is no `dnsMs`/`tcpMs`/`tlsMs`/`coldE2
 
 ## Request configuration (identical across every gateway)
 
-- **Model**: Claude Haiku 4.5 for all six participants — `anthropic/claude-haiku-4.5` via OpenRouter's and Vercel AI Gateway's catalog alias, `anthropic/claude-haiku-4-5` via LLM Gateway's provider-pinned catalog naming, `claude-haiku-4-5-20251001` via Cloudflare's, Anthropic's own, and Pydantic AI Gateway's native model ID (Pydantic proxies Anthropic's native API as-is, no gateway-specific model prefix). Same underlying model, addressed the way each API expects it to be addressed.
+- **Model**: Claude Haiku 4.5 for all seven participants — `anthropic/claude-haiku-4.5` via OpenRouter's and Vercel AI Gateway's catalog alias, `anthropic/claude-haiku-4-5` via LLM Gateway's provider-pinned catalog naming, `anthropic/claude-haiku-4-5-20251001` via Concentrate AI's provider-prefixed naming, `claude-haiku-4-5-20251001` via Cloudflare's, Anthropic's own, and Pydantic AI Gateway's native model ID (Pydantic proxies Anthropic's native API as-is, no gateway-specific model prefix). Same underlying model, addressed the way each API expects it to be addressed.
 - **Prompt**: `"Write a two-sentence description of how distributed systems handle partial failures."` — identical for every request, cold or warm, every gateway.
 - **`max_tokens`**: 200. **`temperature`**: 0. **`stream`**: true (required for TTFT; also used for token-count extraction via `stream_options.include_usage` on the OpenAI-compatible path).
 - **Timeout**: 45 seconds per request.
 
 Two wire formats are in play, handled explicitly per gateway (`AIGatewayProviderConfig.wireFormat` in `benchmarks/ai-gateway/types.ts`):
 
 - **`openai`** (OpenRouter, Vercel AI Gateway, LLM Gateway) — OpenAI-compatible `/chat/completions` shape, `Authorization: Bearer <key>`.
-- **`anthropic`** (Cloudflare AI Gateway, Anthropic direct, Pydantic AI Gateway) — Anthropic's native `/v1/messages` shape. Auth header varies within this group: Cloudflare and Anthropic direct use `x-api-key` + `anthropic-version`; Pydantic AI Gateway uses `Authorization: Bearer <key>` + `anthropic-version` instead — confirmed directly against a real request (its own auth failures return a same-shaped 401 regardless of which of the two header styles is wrong, so this took a few rounds of live testing to pin down precisely).
+- **`anthropic`** (Cloudflare AI Gateway, Anthropic direct, Pydantic AI Gateway, Concentrate AI) — Anthropic's native `/v1/messages` shape. Auth header varies within this group: Cloudflare and Anthropic direct use `x-api-key` + `anthropic-version`; Pydantic AI Gateway and Concentrate AI use `Authorization: Bearer <key>` + `anthropic-version` instead — for Pydantic this was confirmed directly against a real request (its own auth failures return a same-shaped 401 regardless of which of the two header styles is wrong, so this took a few rounds of live testing to pin down precisely). Concentrate AI's `/v1/messages/` endpoint is documented as an "Anthropic Messages API compatibility endpoint" in its published OpenAPI spec (`concentrate.ai/docs/api-reference/openapi.json`), but has **not** been confirmed against a real successful response — see the note in `providers.ts` and in Limitations below.
 
 TTFT detection is format-agnostic by design: a single regex (`"(?:content|text)"\s*:\s*"[^"]`) matches OpenAI's `delta.content` and Anthropic's `delta.text` fields alike, so the first-token timestamp doesn't depend on fully parsing every SSE event on the hot path. Token counts are extracted the same lightweight way (regex over the raw buffer, not a full SSE/JSON parser) — see Limitations.
 
 Knowing when the stream has fully ended (needed for `ttfbMs`/`totalMs` and to safely reuse a warm connection) is handled by Node's own HTTP parser (`res.on('end')`), which understands `Content-Length` and chunked-transfer framing generically for any spec-compliant response. This differs from the reference implementation, which reads raw socket bytes and has to recognize completion itself via hand-matched byte sequences (`data: [DONE]`, `"type":"message_stop"`, the chunked terminator `\r\n0\r\n\r\n`) — a reasonable approach when working with raw sockets in Python, but one that has to be kept in sync with each gateway's exact stream-termination convention. Delegating that to Node's HTTP parser avoids needing to enumerate termination formats per gateway at all.
 
 ### Every gateway is hit directly — no gateway is proxied through another
 
-This is the single most important fairness property of this benchmark, worth stating plainly: **Cloudflare AI Gateway is called via its own direct-to-Anthropic passthrough route** (`/v1/{account}/{gateway}/anthropic/v1/messages`), not routed through OpenRouter or any other intermediary. OpenRouter, Vercel AI Gateway, and LLM Gateway are each called via their own native routing to the same model — LLM Gateway's model id is provider-pinned (`anthropic/claude-haiku-4-5`) so its requests route to Anthropic itself rather than to a different host of the same model; this was confirmed directly against a real request, whose response `metadata` block explicitly reports `used_provider: "anthropic"`, `used_model: "claude-haiku-4-5"`. **Pydantic AI Gateway proxies Anthropic's native API directly** (`/proxy/anthropic/v1/messages`, native model ID `claude-haiku-4-5-20251001`, no gateway-specific routing prefix) — confirmed with a real request returning a genuine Anthropic response (`"model":"claude-haiku-4-5-20251001"`, real `usage`/`cost_estimate` fields from Pydantic's own accounting). `anthropic-direct` calls Anthropic's API with no gateway at all, as the no-gateway control — it isolates how much latency each gateway adds on top of the underlying provider.
+This is the single most important fairness property of this benchmark, worth stating plainly: **Cloudflare AI Gateway is called via its own direct-to-Anthropic passthrough route** (`/v1/{account}/{gateway}/anthropic/v1/messages`), not routed through OpenRouter or any other intermediary. OpenRouter, Vercel AI Gateway, and LLM Gateway are each called via their own native routing to the same model — LLM Gateway's model id is provider-pinned (`anthropic/claude-haiku-4-5`) so its requests route to Anthropic itself rather than to a different host of the same model; this was confirmed directly against a real request, whose response `metadata` block explicitly reports `used_provider: "anthropic"`, `used_model: "claude-haiku-4-5"`. **Pydantic AI Gateway proxies Anthropic's native API directly** (`/proxy/anthropic/v1/messages`, native model ID `claude-haiku-4-5-20251001`, no gateway-specific routing prefix) — confirmed with a real request returning a genuine Anthropic response (`"model":"claude-haiku-4-5-20251001"`, real `usage`/`cost_estimate` fields from Pydantic's own accounting). **Concentrate AI** is called via its own `/v1/messages/` endpoint with the model provider-pinned (`anthropic/claude-haiku-4-5-20251001`, its provider-prefix syntax) so the request routes to Anthropic itself rather than to Bedrock or Vertex — both of which its model catalog also lists as providers for this same model. `anthropic-direct` calls Anthropic's API with no gateway at all, as the no-gateway control — it isolates how much latency each gateway adds on top of the underlying provider.
 
 A gateway that's itself proxied through a second gateway would have that second hop's latency baked into its numbers, misattributed to the outer gateway. That's not happening here — every participant's number reflects that gateway's own overhead only.
 
@@ -71,8 +71,8 @@ A gateway that's itself proxied through a second gateway would have that second
 Iterations run **round-robin across every active gateway**, not sequentially per gateway (`runAIGatewayBenchmarks` in `benchmarks/ai-gateway/benchmark.ts`):
 
 ```
-round 1: openrouter → vercel-ai-gateway → cloudflare-ai-gateway → llmgateway → pydantic-ai-gateway → anthropic-direct
-round 2: openrouter → vercel-ai-gateway → cloudflare-ai-gateway → llmgateway → pydantic-ai-gateway → anthropic-direct
+round 1: openrouter → vercel-ai-gateway → cloudflare-ai-gateway → llmgateway → pydantic-ai-gateway → concentrate-ai-gateway → anthropic-direct
+round 2: openrouter → vercel-ai-gateway → cloudflare-ai-gateway → llmgateway → pydantic-ai-gateway → concentrate-ai-gateway → anthropic-direct
 ...
 ```
 
@@ -107,7 +107,7 @@ Cold E2E and warm TTFT are weighted equally (30% median + 15% p95 each) because
 ## Running it
 
 ```bash
-# All six gateways, default 10 cold + 10 warm iterations each
+# All seven gateways, default 10 cold + 10 warm iterations each
 pnpm run bench:ai-gateway
 
 # One gateway
@@ -116,6 +116,7 @@ pnpm run bench:ai-gateway:vercel
 pnpm run bench:ai-gateway:cloudflare
 pnpm run bench:ai-gateway:llmgateway
 pnpm run bench:ai-gateway:pydantic
+pnpm run bench:ai-gateway:concentrate
 pnpm run bench:ai-gateway:anthropic
 
 # Custom iteration count (applies to both cold and warm)
@@ -125,7 +126,7 @@ pnpm run bench:ai-gateway -- --iterations 20
 npx tsx benchmarks/src/run.ts --mode ai-gateway --ai-gateway-iterations-cold 20 --ai-gateway-iterations-warm 0
 ```
 
-Required environment variables (`benchmarks/.env.example`): `OPENROUTER_API_KEY`, `VERCEL_AI_GATEWAY_API_KEY`, `LLM_GATEWAY_API_KEY`, `PYDANTIC_AI_GATEWAY_API_KEY`, `CLOUDFLARE_AI_GATEWAY_ACCOUNT_ID` + `CLOUDFLARE_AI_GATEWAY_GATEWAY_ID` (+ optional `CLOUDFLARE_AI_GATEWAY_TOKEN` if the gateway has Authenticated Gateway enabled), `ANTHROPIC_API_KEY` (shared by Cloudflare's passthrough and the direct baseline). Missing credentials cause that gateway to be reported as `SKIPPED` rather than failing the run.
+Required environment variables (`benchmarks/.env.example`): `OPENROUTER_API_KEY`, `VERCEL_AI_GATEWAY_API_KEY`, `LLM_GATEWAY_API_KEY`, `PYDANTIC_AI_GATEWAY_API_KEY`, `CONCENTRATE_AI_GATEWAY_API_KEY`, `CLOUDFLARE_AI_GATEWAY_ACCOUNT_ID` + `CLOUDFLARE_AI_GATEWAY_GATEWAY_ID` (+ optional `CLOUDFLARE_AI_GATEWAY_TOKEN` if the gateway has Authenticated Gateway enabled), `ANTHROPIC_API_KEY` (shared by Cloudflare's passthrough and the direct baseline). Missing credentials cause that gateway to be reported as `SKIPPED` rather than failing the run.
 
 ## Output
 

benchmarks/.env.example

@@ -129,4 +129,5 @@ CLOUDFLARE_AI_GATEWAY_GATEWAY_ID=your_cloudflare_gateway_id
 CLOUDFLARE_AI_GATEWAY_TOKEN=your_cloudflare_ai_gateway_token
 LLM_GATEWAY_API_KEY=your_llm_gateway_api_key
 PYDANTIC_AI_GATEWAY_API_KEY=your_pydantic_ai_gateway_api_key
+CONCENTRATE_AI_GATEWAY_API_KEY=your_concentrate_ai_gateway_api_key
 ANTHROPIC_API_KEY=your_anthropic_api_key

benchmarks/ai-gateway/generate-svg.ts

@@ -21,6 +21,7 @@ function formatProviderName(s: string): string {
   if (s === 'cloudflare-ai-gateway') return 'Cloudflare AI Gateway';
   if (s === 'llmgateway') return 'LLM Gateway';
   if (s === 'pydantic-ai-gateway') return 'Pydantic AI Gateway';
+  if (s === 'concentrate-ai-gateway') return 'Concentrate AI';
   if (s === 'anthropic-direct') return 'Anthropic (direct)';
   return s.charAt(0).toUpperCase() + s.slice(1);
 }

benchmarks/ai-gateway/providers.ts

@@ -84,6 +84,27 @@ export const providers: AIGatewayProviderConfig[] = [
       'anthropic-version': '2023-06-01',
     }),
   },
+  {
+    // Concentrate AI exposes an Anthropic-Messages-API-compatible endpoint
+    // (`/v1/messages/`, confirmed against its published OpenAPI spec at
+    // concentrate.ai/docs/api-reference/openapi.json) alongside a separate
+    // OpenAI-compatible `/v1/chat/completions/`. We use the Anthropic-shaped
+    // one so this sits in the same wireFormat group as Cloudflare/Pydantic/
+    // anthropic-direct. `anthropic/` is this gateway's provider-prefix syntax
+    // (same idea as llmgateway's pinning above) to route to Anthropic itself
+    // rather than Bedrock/Vertex, which the model catalog also lists as
+    // providers for this model.
+    name: 'concentrate-ai-gateway',
+    requiredEnvVars: ['CONCENTRATE_AI_GATEWAY_API_KEY'],
+    wireFormat: 'anthropic',
+    model: 'anthropic/claude-haiku-4-5-20251001',
+    host: 'api.concentrate.ai',
+    path: '/v1/messages/',
+    buildHeaders: () => ({
+      Authorization: `Bearer ${process.env.CONCENTRATE_AI_GATEWAY_API_KEY}`,
+      'anthropic-version': '2023-06-01',
+    }),
+  },
   {
     // No-gateway baseline/control.
     name: 'anthropic-direct',

package.json

@@ -84,6 +84,7 @@
     "bench:ai-gateway:cloudflare": "tsx benchmarks/src/run.ts --mode ai-gateway --provider cloudflare-ai-gateway",
     "bench:ai-gateway:llmgateway": "tsx benchmarks/src/run.ts --mode ai-gateway --provider llmgateway",
     "bench:ai-gateway:pydantic": "tsx benchmarks/src/run.ts --mode ai-gateway --provider pydantic-ai-gateway",
+    "bench:ai-gateway:concentrate": "tsx benchmarks/src/run.ts --mode ai-gateway --provider concentrate-ai-gateway",
     "bench:ai-gateway:anthropic": "tsx benchmarks/src/run.ts --mode ai-gateway --provider anthropic-direct",
     "generate-ai-gateway-svg": "tsx benchmarks/ai-gateway/generate-svg.ts"
   },