feat(runner): verbs-only bench run with --shape and --run-key

.changeset/benchsdk-client-run-key.md

@@ -0,0 +1,5 @@
+---
+"@benchsdk/client": minor
+---
+
+`createRun` accepts an optional `runKey`: callers passing the same key (per org + benchmark) get-or-create one shared run instead of each opening its own. `BenchmarkRun.runKey` reports the key a run was created with.

.changeset/benchsdk-runner-shapes-run-key.md

@@ -0,0 +1,11 @@
+---
+"@benchsdk/runner": minor
+---
+
+Verbs-only CLI: `bench run <file.bench.ts>` is the one mutating command. The benchmark is declared in the file and materialized (upserted) as a side effect of running it, and a run is opened as a side effect too — there are no imperative `bench create benchmark` / `bench create run` commands.
+
+`bench run` gains `--shape <name>`: a bench file can declare named `shapes`, each swapping in its own platform identity (`slug`/`name`, optional `kind`) and a stable knob (`staggerDelayMs`) while reusing the same task and participants. This collapses the per-shape slug/name/knob triple that was duplicated across package scripts and CI.
+
+`bench run` gains `--run-key <key>`: sibling processes passing the same key (per org + benchmark) get-or-create one shared run instead of each opening its own, so provider jobs running in parallel land in a single, directly-comparable run. Each process registers only the participants it runs. The key binding is permanent, so callers that need a fresh run (e.g. a CI re-run) vary the key (e.g. include `GITHUB_RUN_ATTEMPT`).
+
+`--slug` remains a working alias for `--benchmark`.

.changeset/participant-sized-runs.md

@@ -0,0 +1,5 @@
+---
+"@benchsdk/client": minor
+---
+
+`createRun` no longer requires `totalTasks`: omit it to open a participant-sized run, whose total is the sum of what its participants declare when they register. `BenchmarkRun.participantSized` reports which kind a run is.

.github/workflows/sandbox-tti-benchmarks.yml

@@ -116,19 +116,26 @@ jobs:
         run: |
           . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUNLOOP_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)'
 
-          # One entrypoint, three launch shapes: the knobs come from the CLI
-          # and --slug/--name pick which platform benchmark each reports to.
+          # One entrypoint, three launch shapes selected with --shape; the scale
+          # knobs come from the CLI. Every provider passes the same --run-key, so
+          # each shape's providers get-or-create one shared run and rank against
+          # each other (each still registers itself and claims its own worker).
+          # The shapes are distinct benchmark slugs, so one key value yields one
+          # run per shape; the run attempt is in the key so a workflow re-run
+          # opens fresh runs instead of rejoining the completed ones.
           BENCH=(npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts)
-          COMMON=(--provider ${{ matrix.provider }} \
-            --iterations ${{ github.event.inputs.iterations || (github.event_name == 'push' && '10') || '100' }})
+          RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          COMMON=(
+            --provider ${{ matrix.provider }}
+            --iterations ${{ github.event.inputs.iterations || (github.event_name == 'push' && '10') || '100' }}
+            --run-key "$RUN_KEY"
+          )
           CONCURRENCY=${{ github.event.inputs.concurrency || (github.event_name == 'push' && '10') || '100' }}
           # Sequential is one-at-a-time by definition, so it never gets
           # --concurrency (the config default pins it to 1).
           SEQUENTIAL=("${BENCH[@]}" "${COMMON[@]}")
-          STAGGERED=("${BENCH[@]}" "${COMMON[@]}" --concurrency "$CONCURRENCY" --stagger-delay-ms 200 \
-            --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)')
-          BURST=("${BENCH[@]}" "${COMMON[@]}" --concurrency "$CONCURRENCY" \
-            --slug sandbox-burst-local --name 'Sandbox burst TTI (local)')
+          STAGGERED=("${BENCH[@]}" "${COMMON[@]}" --shape staggered --concurrency "$CONCURRENCY")
+          BURST=("${BENCH[@]}" "${COMMON[@]}" --shape burst --concurrency "$CONCURRENCY")
           case "${{ github.event.inputs.mode }}" in
             sequential) "${SEQUENTIAL[@]}" ;;
             staggered)  "${STAGGERED[@]}" ;;

benchmarks/sandbox/tti.bench.ts

@@ -3,17 +3,23 @@
  * first command (`node -v`) succeeding, excluding destroy. Declarative —
  * exports `config` + `task`; `bench run` owns the entrypoint.
  *
- * One workload, three launch shapes — the only thing that differs is the
- * framework's own knobs, so they're CLI flags rather than separate files:
- *   sequential  one at a time (the config default: concurrency 1)
- *   burst       --concurrency N: all slots open at once, so this launches that
- *               many real sandboxes simultaneously — raise N deliberately
- *   staggered   --concurrency N --stagger-delay-ms D: task i starts at i * D
- * Each shape reports under its own platform benchmark via `--slug`/`--name`.
+ * One workload, three launch shapes. Each shape is its own platform benchmark
+ * but the same task, so they're declared once in `shapes` and picked with
+ * `--shape`; the scale knobs (`--iterations`/`--concurrency`) stay on the CLI:
+ *   sequential  the base config (concurrency 1) — no `--shape`
+ *   burst       --concurrency N: all slots open at once, launching that many
+ *               real sandboxes simultaneously — raise N deliberately
+ *   staggered   its 200ms delay is baked into the shape; task i starts at i * D
  *
  *   bench run benchmarks/sandbox/tti.bench.ts --iterations 5 --provider e2b,modal
- *   bench run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 10 --concurrency 10
- *   bench run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 10 --concurrency 10 --stagger-delay-ms 200
+ *   bench run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 10 --concurrency 10
+ *   bench run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 10 --concurrency 10
+ *
+ * To rank providers against each other, run each provider with the same
+ * `--run-key`: they get-or-create one shared run and each claims its own worker.
+ *
+ *   bench run benchmarks/sandbox/tti.bench.ts --shape burst --provider e2b   --run-key "$GITHUB_RUN_ID"
+ *   bench run benchmarks/sandbox/tti.bench.ts --shape burst --provider modal --run-key "$GITHUB_RUN_ID"
  */
 import '../src/env.js';
 import path from 'node:path';
@@ -52,6 +58,13 @@ export const config = defineBenchmarkConfig({
   benchmarkKind: 'sandbox',
   iterations: 2,
   concurrency: 1,
+  // The launch shapes: same task, distinct platform identities. `--shape burst`
+  // just swaps the slug/name — the caller brings `--concurrency`. Staggered
+  // also carries its defining 200ms delay so no caller has to remember it.
+  shapes: {
+    burst: { slug: 'sandbox-burst-local', name: 'Sandbox burst TTI (local)' },
+    staggered: { slug: 'sandbox-staggered-local', name: 'Sandbox staggered TTI (local)', staggerDelayMs: 200 },
+  },
   participants: providers,
   onComplete: (outcome) => {
     const { resultsDir, mode } = legacyShape(outcome.config);