diff --git a/.changeset/benchsdk-client-run-key.md b/.changeset/benchsdk-client-run-key.md new file mode 100644 index 00000000..90795488 --- /dev/null +++ b/.changeset/benchsdk-client-run-key.md @@ -0,0 +1,5 @@ +--- +"@benchsdk/client": minor +--- + +`createRun` accepts an optional `runKey`: callers passing the same key (per org + benchmark) get-or-create one shared run instead of each opening its own. `BenchmarkRun.runKey` reports the key a run was created with. diff --git a/.changeset/benchsdk-runner-shapes-run-key.md b/.changeset/benchsdk-runner-shapes-run-key.md new file mode 100644 index 00000000..bf198f3c --- /dev/null +++ b/.changeset/benchsdk-runner-shapes-run-key.md @@ -0,0 +1,11 @@ +--- +"@benchsdk/runner": minor +--- + +Verbs-only CLI: `bench run ` is the one mutating command. The benchmark is declared in the file and materialized (upserted) as a side effect of running it, and a run is opened as a side effect too — there are no imperative `bench create benchmark` / `bench create run` commands. + +`bench run` gains `--shape `: a bench file can declare named `shapes`, each swapping in its own platform identity (`slug`/`name`, optional `kind`) and a stable knob (`staggerDelayMs`) while reusing the same task and participants. This collapses the per-shape slug/name/knob triple that was duplicated across package scripts and CI. + +`bench run` gains `--run-key `: sibling processes passing the same key (per org + benchmark) get-or-create one shared run instead of each opening its own, so provider jobs running in parallel land in a single, directly-comparable run. Each process registers only the participants it runs. The key binding is permanent, so callers that need a fresh run (e.g. a CI re-run) vary the key (e.g. include `GITHUB_RUN_ATTEMPT`). + +`--slug` remains a working alias for `--benchmark`. diff --git a/.changeset/benchsdk-runner-slug-flag.md b/.changeset/benchsdk-runner-slug-flag.md deleted file mode 100644 index f100a1c7..00000000 --- a/.changeset/benchsdk-runner-slug-flag.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@benchsdk/runner": minor ---- - -Add `--slug` and `--name` CLI overrides for `benchmarkSlug`/`benchmarkName`, so one `*.bench.ts` can report under several platform benchmarks (e.g. the sandbox TTI entrypoint reporting sequential/staggered/burst runs of the same workload). diff --git a/.changeset/participant-sized-runs.md b/.changeset/participant-sized-runs.md new file mode 100644 index 00000000..6d8b8121 --- /dev/null +++ b/.changeset/participant-sized-runs.md @@ -0,0 +1,5 @@ +--- +"@benchsdk/client": minor +--- + +`createRun` no longer requires `totalTasks`: omit it to open a participant-sized run, whose total is the sum of what its participants declare when they register. `BenchmarkRun.participantSized` reports which kind a run is. diff --git a/.github/workflows/sandbox-tti-benchmarks.yml b/.github/workflows/sandbox-tti-benchmarks.yml index 19a5bb09..0db92f3d 100644 --- a/.github/workflows/sandbox-tti-benchmarks.yml +++ b/.github/workflows/sandbox-tti-benchmarks.yml @@ -116,19 +116,26 @@ jobs: run: | . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUNLOOP_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' - # One entrypoint, three launch shapes: the knobs come from the CLI - # and --slug/--name pick which platform benchmark each reports to. + # One entrypoint, three launch shapes selected with --shape; the scale + # knobs come from the CLI. Every provider passes the same --run-key, so + # each shape's providers get-or-create one shared run and rank against + # each other (each still registers itself and claims its own worker). + # The shapes are distinct benchmark slugs, so one key value yields one + # run per shape; the run attempt is in the key so a workflow re-run + # opens fresh runs instead of rejoining the completed ones. BENCH=(npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts) - COMMON=(--provider ${{ matrix.provider }} \ - --iterations ${{ github.event.inputs.iterations || (github.event_name == 'push' && '10') || '100' }}) + RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + COMMON=( + --provider ${{ matrix.provider }} + --iterations ${{ github.event.inputs.iterations || (github.event_name == 'push' && '10') || '100' }} + --run-key "$RUN_KEY" + ) CONCURRENCY=${{ github.event.inputs.concurrency || (github.event_name == 'push' && '10') || '100' }} # Sequential is one-at-a-time by definition, so it never gets # --concurrency (the config default pins it to 1). SEQUENTIAL=("${BENCH[@]}" "${COMMON[@]}") - STAGGERED=("${BENCH[@]}" "${COMMON[@]}" --concurrency "$CONCURRENCY" --stagger-delay-ms 200 \ - --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)') - BURST=("${BENCH[@]}" "${COMMON[@]}" --concurrency "$CONCURRENCY" \ - --slug sandbox-burst-local --name 'Sandbox burst TTI (local)') + STAGGERED=("${BENCH[@]}" "${COMMON[@]}" --shape staggered --concurrency "$CONCURRENCY") + BURST=("${BENCH[@]}" "${COMMON[@]}" --shape burst --concurrency "$CONCURRENCY") case "${{ github.event.inputs.mode }}" in sequential) "${SEQUENTIAL[@]}" ;; staggered) "${STAGGERED[@]}" ;; diff --git a/benchmarks/sandbox/tti.bench.ts b/benchmarks/sandbox/tti.bench.ts index bbb7a80c..a1b9aa7f 100644 --- a/benchmarks/sandbox/tti.bench.ts +++ b/benchmarks/sandbox/tti.bench.ts @@ -3,17 +3,23 @@ * first command (`node -v`) succeeding, excluding destroy. Declarative — * exports `config` + `task`; `bench run` owns the entrypoint. * - * One workload, three launch shapes — the only thing that differs is the - * framework's own knobs, so they're CLI flags rather than separate files: - * sequential one at a time (the config default: concurrency 1) - * burst --concurrency N: all slots open at once, so this launches that - * many real sandboxes simultaneously — raise N deliberately - * staggered --concurrency N --stagger-delay-ms D: task i starts at i * D - * Each shape reports under its own platform benchmark via `--slug`/`--name`. + * One workload, three launch shapes. Each shape is its own platform benchmark + * but the same task, so they're declared once in `shapes` and picked with + * `--shape`; the scale knobs (`--iterations`/`--concurrency`) stay on the CLI: + * sequential the base config (concurrency 1) — no `--shape` + * burst --concurrency N: all slots open at once, launching that many + * real sandboxes simultaneously — raise N deliberately + * staggered its 200ms delay is baked into the shape; task i starts at i * D * * bench run benchmarks/sandbox/tti.bench.ts --iterations 5 --provider e2b,modal - * bench run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 10 --concurrency 10 - * bench run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 10 --concurrency 10 --stagger-delay-ms 200 + * bench run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 10 --concurrency 10 + * bench run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 10 --concurrency 10 + * + * To rank providers against each other, run each provider with the same + * `--run-key`: they get-or-create one shared run and each claims its own worker. + * + * bench run benchmarks/sandbox/tti.bench.ts --shape burst --provider e2b --run-key "$GITHUB_RUN_ID" + * bench run benchmarks/sandbox/tti.bench.ts --shape burst --provider modal --run-key "$GITHUB_RUN_ID" */ import '../src/env.js'; import path from 'node:path'; @@ -52,6 +58,13 @@ export const config = defineBenchmarkConfig({ benchmarkKind: 'sandbox', iterations: 2, concurrency: 1, + // The launch shapes: same task, distinct platform identities. `--shape burst` + // just swaps the slug/name — the caller brings `--concurrency`. Staggered + // also carries its defining 200ms delay so no caller has to remember it. + shapes: { + burst: { slug: 'sandbox-burst-local', name: 'Sandbox burst TTI (local)' }, + staggered: { slug: 'sandbox-staggered-local', name: 'Sandbox staggered TTI (local)', staggerDelayMs: 200 }, + }, participants: providers, onComplete: (outcome) => { const { resultsDir, mode } = legacyShape(outcome.config); diff --git a/package.json b/package.json index f0a5d010..e3d0b3f3 100644 --- a/package.json +++ b/package.json @@ -5,29 +5,29 @@ "type": "module", "scripts": { "typecheck": "tsc --noEmit", - "bench": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3", + "bench": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3", "bench:sequential": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts", - "bench:staggered": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200", - "bench:burst": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3", + "bench:staggered": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3", + "bench:burst": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3", "bench:sandbox:dax": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/dax.bench.ts", - "bench:blaxel": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider blaxel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider blaxel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider blaxel", - "bench:codesandbox": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider codesandbox && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider codesandbox && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider codesandbox", - "bench:daytona": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider daytona && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider daytona && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider daytona", - "bench:e2b": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider e2b && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider e2b && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider e2b", - "bench:hopx": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider hopx && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider hopx && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider hopx", - "bench:isorun": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider isorun && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider isorun && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider isorun", - "bench:modal": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider modal && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider modal && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider modal", - "bench:namespace": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider namespace && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider namespace && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider namespace", - "bench:northflank": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider northflank && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider northflank && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider northflank", - "bench:railway": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider railway && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider railway && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider railway", - "bench:render": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider render && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider render && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider render", - "bench:runloop": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider runloop && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider runloop && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider runloop", - "bench:vercel": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider vercel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider vercel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider vercel", - "bench:just-bash": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider just-bash && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider just-bash && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider just-bash", - "bench:sprites": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider sprites && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider sprites && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider sprites", - "bench:sandbox0": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider sandbox0 && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider sandbox0 && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider sandbox0", - "bench:opencomputer": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider opencomputer && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider opencomputer && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider opencomputer", - "bench:tensorlake": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider tensorlake && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider tensorlake && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider tensorlake", + "bench:blaxel": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider blaxel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider blaxel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider blaxel", + "bench:codesandbox": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider codesandbox && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider codesandbox && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider codesandbox", + "bench:daytona": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider daytona && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider daytona && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider daytona", + "bench:e2b": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider e2b && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider e2b && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider e2b", + "bench:hopx": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider hopx && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider hopx && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider hopx", + "bench:isorun": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider isorun && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider isorun && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider isorun", + "bench:modal": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider modal && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider modal && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider modal", + "bench:namespace": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider namespace && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider namespace && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider namespace", + "bench:northflank": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider northflank && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider northflank && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider northflank", + "bench:railway": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider railway && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider railway && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider railway", + "bench:render": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider render && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider render && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider render", + "bench:runloop": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider runloop && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider runloop && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider runloop", + "bench:vercel": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider vercel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider vercel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider vercel", + "bench:just-bash": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider just-bash && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider just-bash && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider just-bash", + "bench:sprites": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider sprites && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider sprites && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider sprites", + "bench:sandbox0": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider sandbox0 && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider sandbox0 && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider sandbox0", + "bench:opencomputer": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider opencomputer && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider opencomputer && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider opencomputer", + "bench:tensorlake": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider tensorlake && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider tensorlake && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider tensorlake", "bench:browser": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/browser.bench.ts", "bench:browser:browserbase": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/browser.bench.ts --provider browserbase", "bench:browser:hyperbrowser": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/browser.bench.ts --provider hyperbrowser", diff --git a/packages/benchsdk-runner/src/__tests__/bench-config.test.ts b/packages/benchsdk-runner/src/__tests__/bench-config.test.ts index d390074b..da7df4c4 100644 --- a/packages/benchsdk-runner/src/__tests__/bench-config.test.ts +++ b/packages/benchsdk-runner/src/__tests__/bench-config.test.ts @@ -85,6 +85,31 @@ describe('defineBenchmarkConfig', () => { defineBenchmarkConfig({ benchmarkSlug: 's', benchmarkName: 'n', participants, phases: [{ name: 'cold', iterations: 1 }, { name: 'cold', iterations: 1 }] }), ).toThrow('duplicate phase name'); }); + + it('accepts a valid shapes map', () => { + const config = defineBenchmarkConfig({ + benchmarkSlug: 's', + benchmarkName: 'n', + participants, + shapes: { + burst: { slug: 'sandbox-burst-local', name: 'Burst' }, + staggered: { slug: 'sandbox-staggered-local', staggerDelayMs: 200 }, + }, + }); + expect(config.shapes?.burst.slug).toBe('sandbox-burst-local'); + }); + + it('rejects a shape without a lowercase slug', () => { + expect(() => + defineBenchmarkConfig({ benchmarkSlug: 's', benchmarkName: 'n', participants, shapes: { burst: { slug: 'Burst' } } }), + ).toThrow('burst'); + }); + + it('rejects a shape with a negative staggerDelayMs', () => { + expect(() => + defineBenchmarkConfig({ benchmarkSlug: 's', benchmarkName: 'n', participants, shapes: { s: { slug: 'ok', staggerDelayMs: -1 } } }), + ).toThrow('staggerDelayMs'); + }); }); describe('defineTask', () => { diff --git a/packages/benchsdk-runner/src/__tests__/cli.test.ts b/packages/benchsdk-runner/src/__tests__/cli.test.ts index 85faabab..922a9fcf 100644 --- a/packages/benchsdk-runner/src/__tests__/cli.test.ts +++ b/packages/benchsdk-runner/src/__tests__/cli.test.ts @@ -5,13 +5,19 @@ import { NoAvailableParticipantsError } from '../no-available-participants.js'; const fixture = (name: string) => `src/__tests__/fixtures/${name}`; describe('runBenchmarkFile', () => { + it('rejects retired imperative commands (there is no `create`)', async () => { + await expect(runBenchmarkFile(['create', 'benchmark', 'sandbox'])).rejects.toThrow(/Usage:/); + await expect(runBenchmarkFile(['create', 'run'])).rejects.toThrow(/Usage:/); + }); + it('rejects when the command is not `run`', async () => { - await expect(runBenchmarkFile([])).rejects.toThrow(/Usage: bench run/); - await expect(runBenchmarkFile(['nope', fixture('good.bench.ts')])).rejects.toThrow(/Usage: bench run/); + await expect(runBenchmarkFile([])).rejects.toThrow(/Usage:/); + await expect(runBenchmarkFile(['nope', fixture('good.bench.ts')])).rejects.toThrow(/Usage:/); }); - it('rejects when no file is given', async () => { - await expect(runBenchmarkFile(['run'])).rejects.toThrow(/Usage: bench run/); + it('rejects when no file is given, or a flag stands where the file should', async () => { + await expect(runBenchmarkFile(['run'])).rejects.toThrow(/Usage:/); + await expect(runBenchmarkFile(['run', '--shape', 'burst'])).rejects.toThrow(/Usage:/); }); it('rejects a module that does not export a config', async () => { diff --git a/packages/benchsdk-runner/src/__tests__/runner.test.ts b/packages/benchsdk-runner/src/__tests__/runner.test.ts index d6c0db22..7edffce3 100644 --- a/packages/benchsdk-runner/src/__tests__/runner.test.ts +++ b/packages/benchsdk-runner/src/__tests__/runner.test.ts @@ -49,15 +49,23 @@ describe('parseCliArgs', () => { expect(parseCliArgs(['--group-by=participant'])).toEqual({ groupBy: 'participant' }); }); - it('parses --slug and --name', () => { - expect(parseCliArgs(['--slug', 'sandbox-burst-local'])).toEqual({ slug: 'sandbox-burst-local' }); - expect(parseCliArgs(['--slug=sandbox-tti-local'])).toEqual({ slug: 'sandbox-tti-local' }); + it('parses --benchmark (and its --slug alias) and --name', () => { + expect(parseCliArgs(['--benchmark', 'sandbox-burst-local'])).toEqual({ benchmark: 'sandbox-burst-local' }); + expect(parseCliArgs(['--benchmark=sandbox-tti-local'])).toEqual({ benchmark: 'sandbox-tti-local' }); + expect(parseCliArgs(['--slug', 'sandbox-burst-local'])).toEqual({ benchmark: 'sandbox-burst-local' }); expect(parseCliArgs(['--name', 'Sandbox burst TTI'])).toEqual({ name: 'Sandbox burst TTI' }); expect(() => parseCliArgs(['--name', ' '])).toThrow('--name'); }); - it('throws on a non-slug --slug', () => { - expect(() => parseCliArgs(['--slug', 'Sandbox TTI'])).toThrow('--slug'); + it('parses --shape and --run-key', () => { + expect(parseCliArgs(['--shape', 'burst'])).toEqual({ shape: 'burst' }); + expect(parseCliArgs(['--run-key=ci-123'])).toEqual({ runKey: 'ci-123' }); + expect(() => parseCliArgs(['--shape', ''])).toThrow('--shape'); + expect(() => parseCliArgs(['--run-key', ''])).toThrow('--run-key'); + }); + + it('throws on a non-slug --benchmark', () => { + expect(() => parseCliArgs(['--benchmark', 'Sandbox TTI'])).toThrow('--benchmark'); expect(() => parseCliArgs(['--slug', ''])).toThrow('--slug'); }); @@ -164,14 +172,19 @@ describe('runBenchmark', () => { process.env.E2B_API_KEY = 'x'; process.env.MODAL_TOKEN = 'y'; process.env.BENCHMARKS_PLATFORM_API_KEY = 'test-key'; - calls = { upsertBenchmark: [], createRun: [], planWorkers: [], runWorker: [], taskData: [] }; + calls = { upsertBenchmark: [], createRun: [], planWorkers: [], upsertParticipant: [], getRun: [], runWorker: [], taskData: [] }; fakeClient = { upsertBenchmark: vi.fn(async (...a: any[]) => { calls.upsertBenchmark.push(a); return {}; }), createRun: vi.fn(async (...a: any[]) => { calls.createRun.push(a); return { run: { id: 'run-1' }, participants: [] }; }), planWorkers: vi.fn(async (...a: any[]) => { calls.planWorkers.push(a); return []; }), + upsertParticipant: vi.fn(async (...a: any[]) => { calls.upsertParticipant.push(a); return {}; }), + getRun: vi.fn(async (slug: string, runId: string) => { + calls.getRun.push([slug, runId]); + return { id: runId, totalTasks: 3, participantSized: runId === 'run-open' }; + }), runWorker: vi.fn(async (opts: any) => { calls.runWorker.push(opts); - const total = calls.createRun[0]?.[1]?.totalTasks ?? 1; + const total = calls.createRun[0]?.[1]?.totalTasks ?? calls.upsertParticipant[0]?.[3]?.totalTasks ?? 1; // The platform hands out globally-indexed task ranges; `taskRangeStart` // lets a test exercise a worker whose range doesn't start at 0. const start = taskRangeStart; @@ -294,6 +307,84 @@ describe('runBenchmark', () => { expect(calls.createRun[0][0]).toBe('sandbox-burst-local'); }); + it('shares a run by --run-key: get-or-creates it keyed, then registers only its own participant', async () => { + const config: BenchmarkConfig = { + benchmarkSlug: 'sandbox-tti-local', + benchmarkName: 'Sandbox TTI', + iterations: 2, + participants: [participants[0]], + }; + + const outcome = await runBenchmark(config, defineTask(async () => ({})), ['--run-key', 'ci-123', '--provider', 'e2b']); + + // The benchmark is still materialized from the file identity. + expect(calls.upsertBenchmark[0][0]).toBe('sandbox-tti-local'); + // One keyed get-or-create, carrying the key and no size/participant list. + expect(calls.createRun).toHaveLength(1); + expect(calls.createRun[0][1]).toMatchObject({ runKey: 'ci-123' }); + expect(calls.createRun[0][1].totalTasks).toBeUndefined(); + expect(calls.createRun[0][1].participants).toBeUndefined(); + // Registers only the provider it runs, sized to its own iteration count. + expect(calls.upsertParticipant[0].slice(0, 3)).toEqual(['sandbox-tti-local', 'run-1', 'e2b']); + expect(calls.upsertParticipant[0][3]).toMatchObject({ totalTasks: 2 }); + expect(calls.runWorker[0]).toMatchObject({ runId: 'run-1' }); + expect(outcome.runId).toBe('run-1'); + expect(outcome.participants[0].records).toHaveLength(2); + }); + + it('does not rename a benchmark it was merely retargeted at', async () => { + const config: BenchmarkConfig = { + benchmarkSlug: 'sandbox-tti-local', + benchmarkName: 'Sandbox TTI', + iterations: 1, + participants: [participants[0]], + }; + + await runBenchmark(config, defineTask(async () => ({})), ['--benchmark', 'sandbox-burst-local']); + expect(calls.upsertBenchmark).toEqual([]); + + await runBenchmark(config, defineTask(async () => ({})), [ + '--benchmark', + 'sandbox-burst-local', + '--name', + 'Sandbox burst TTI', + ]); + expect(calls.upsertBenchmark[0]).toEqual(['sandbox-burst-local', { name: 'Sandbox burst TTI' }]); + }); + + it('selects a declared shape by --shape, reporting under its slug and name', async () => { + const config: BenchmarkConfig = { + benchmarkSlug: 'sandbox-tti-local', + benchmarkName: 'Sandbox TTI', + iterations: 1, + participants: [participants[0]], + shapes: { + staggered: { slug: 'sandbox-staggered-local', name: 'Sandbox staggered TTI', staggerDelayMs: 200 }, + }, + }; + + const outcome = await runBenchmark(config, defineTask(async () => ({})), ['--shape', 'staggered']); + + expect(calls.upsertBenchmark[0][0]).toBe('sandbox-staggered-local'); + expect(calls.upsertBenchmark[0][1]).toMatchObject({ name: 'Sandbox staggered TTI' }); + expect(calls.createRun[0][0]).toBe('sandbox-staggered-local'); + // The shape's stable knob applies; scale knobs stay defaulted/overridable. + expect(outcome.config.staggerDelayMs).toBe(200); + }); + + it('rejects an unknown --shape, listing the declared ones', async () => { + const config: BenchmarkConfig = { + benchmarkSlug: 'sandbox-tti-local', + benchmarkName: 'Sandbox TTI', + participants: [participants[0]], + shapes: { burst: { slug: 'sandbox-burst-local' } }, + }; + + await expect( + runBenchmark(config, defineTask(async () => ({})), ['--shape', 'nope']), + ).rejects.toThrow('Known shapes: burst'); + }); + it('throws NoAvailableParticipantsError, listing the skips, when no participant has its env vars set', async () => { delete process.env.E2B_API_KEY; delete process.env.MODAL_TOKEN; diff --git a/packages/benchsdk-runner/src/bench-config.ts b/packages/benchsdk-runner/src/bench-config.ts index b27558ec..c1952ab2 100644 --- a/packages/benchsdk-runner/src/bench-config.ts +++ b/packages/benchsdk-runner/src/bench-config.ts @@ -19,6 +19,10 @@ * burst { iterations: N, concurrency: N } * staggered { iterations: N, concurrency: N, staggerDelayMs: 200 } * + * A benchmark can name these variants up front via `shapes`, so one file backs + * several platform benchmarks (`bench run --shape burst`) without + * restating each one's slug/name in scripts and CI. + * * A task is comprised of steps, declared via `ctx.step` inside a task function * — it supports closures, conditionals and try/finally, so values (a created * sandbox, say) flow naturally between steps. A task that declares no steps is @@ -37,6 +41,25 @@ import type { /** How tasks are ordered across participants. */ export type GroupBy = 'participant' | 'round'; +/** + * A named variant of a benchmark, selected with `--shape `. A shape + * carries only the parts that make it a distinct *benchmark* — its platform + * identity plus any stable distinguishing knob (e.g. staggered's delay). The + * scale knobs that vary per environment (`--iterations`, `--concurrency`) stay + * on the invocation, so a shape never sets a value only to have the CLI + * override it. + */ +export interface BenchmarkShape { + /** Platform slug this shape reports under (e.g. 'sandbox-burst-local'). */ + slug: string; + /** Display name shown on the platform; defaults to the slug. */ + name?: string; + /** Benchmark kind; defaults to the config's `benchmarkKind`. */ + kind?: string; + /** Default stagger delay (ms) for this shape; overridable with `--stagger-delay-ms`. */ + staggerDelayMs?: number; +} + /** * What a task returns: whatever it measured itself. This replaces the * assumption that the framework owns all timing. A plain data payload is @@ -132,6 +155,7 @@ export interface ResolvedRunConfig { */ export interface BenchmarkRunOutcome { runId: string; + /** Link to this run on the platform dashboard. */ dashboardUrl: string; participants: ParticipantRecords[]; config: ResolvedRunConfig; @@ -146,14 +170,22 @@ export interface BenchmarkRunOutcome { export interface BenchmarkConfig { /** * Stable platform slug for this benchmark (e.g. 'sandbox-tti-local'). - * Overridable per run with `--slug`, so one entrypoint can report under - * several benchmarks. + * Selectable per run with `--shape` (or overridable with `--benchmark`), so + * one entrypoint can report under several benchmarks. */ benchmarkSlug: string; /** Human-readable name shown on the platform. Overridable with `--name`. */ benchmarkName: string; /** Optional platform benchmark kind (e.g. 'sandbox'). */ benchmarkKind?: string; + /** + * Named variants of this benchmark, selected with `--shape `. Each + * shape swaps in its own platform identity (and optional stable knob) while + * reusing the same task and participants, so one bench file can back several + * platform benchmarks without duplicating the slug/name triple across + * package scripts and CI. + */ + shapes?: Record; /** * Total tasks to run per participant. Default: 1. Mutually exclusive with * `phases` — when `phases` is set, total iterations = sum of phase iterations. @@ -234,6 +266,19 @@ export function defineBenchmarkConfig= 0 (got ${shape.staggerDelayMs})`); + } + } + } return config; } diff --git a/packages/benchsdk-runner/src/cli.ts b/packages/benchsdk-runner/src/cli.ts index c76b4587..71089261 100644 --- a/packages/benchsdk-runner/src/cli.ts +++ b/packages/benchsdk-runner/src/cli.ts @@ -1,23 +1,31 @@ /** - * `bench run [--flags]` — the author-facing entrypoint. Imports a - * benchmark module, reads its `config` and `task` exports, and drives the run - * via the internal `runBenchmark`. CLI flags override the config's knobs, and - * `config.onComplete` (if any) fires once the run finishes. + * The author-facing entrypoint. `bench` is verbs-only — the benchmark and its + * runs are implicit, never nouns you type: + * + * bench run [--flags] execute a benchmark + * + * `run` imports a benchmark module, reads its `config` and `task` exports and + * drives `runBenchmark`; CLI flags override the config's knobs and + * `config.onComplete` (if any) fires once the run finishes. The benchmark is + * declared in the file (`--shape` picks a named variant) and materialized on + * run; a run is opened as a side effect, shared across sibling processes when + * they pass the same `--run-key`. There are no imperative `create` commands. * * The executable wrapper lives in `bin.ts`; this module has no side effects so * it can be unit-tested by calling `runBenchmarkFile` directly. */ import { resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; -import { runBenchmark } from './runner.js'; +import { parseCliArgs, runBenchmark } from './runner.js'; import { NoAvailableParticipantsError } from './no-available-participants.js'; import type { BaseParticipant } from '@benchsdk/client'; import type { BenchmarkConfig, BenchmarkTask } from './bench-config.js'; const USAGE = - 'Usage: bench run [--iterations N] [--concurrency N] ' + - '[--stagger-delay-ms N] [--group-by participant|round] [--provider a,b] ' + - "[--slug my-benchmark] [--name 'My benchmark']"; + 'Usage:\n' + + ' bench run [--shape name] [--provider a,b] [--run-key key]\n' + + ' [--benchmark slug] [--name "My benchmark"]\n' + + ' [--iterations N] [--concurrency N] [--stagger-delay-ms N] [--group-by participant|round]'; /** A benchmark module is expected to export `config` and `task`. */ interface BenchmarkModule { @@ -33,19 +41,17 @@ function isBenchmarkConfig(value: unknown): value is BenchmarkConfig { } /** - * Loads a benchmark file and runs it. Throws on bad usage / invalid exports and - * lets `NoAvailableParticipantsError` propagate so the caller can map it to a - * clean exit. Does not call `process.exit`. + * Dispatches one CLI invocation. Throws on bad usage / invalid exports and lets + * `NoAvailableParticipantsError` propagate so the caller can map it to a clean + * exit. Does not call `process.exit`. */ export async function runBenchmarkFile(argv: string[]): Promise { - const [command, file, ...rest] = argv; - if (command !== 'run' || !file) { - throw new Error(USAGE); - } + const [command, ...rest] = argv; - const moduleUrl = pathToFileURL(resolve(process.cwd(), file)).href; - const mod = (await import(moduleUrl)) as BenchmarkModule; + const [file, ...flags] = rest; + if (command !== 'run' || !file || file.startsWith('-')) throw new Error(USAGE); + const mod = (await import(pathToFileURL(resolve(process.cwd(), file)).href)) as BenchmarkModule; const config = mod.config; const task = mod.task ?? mod.default; @@ -56,7 +62,7 @@ export async function runBenchmarkFile(argv: string[]): Promise { throw new Error(`${file} must export a \`task\` created with defineTask.`); } - await runBenchmark(config as BenchmarkConfig, task as BenchmarkTask, rest); + await runBenchmark(config as BenchmarkConfig, task as BenchmarkTask, flags); } /** Executable entry: runs the file and maps outcomes to process exit codes. */ diff --git a/packages/benchsdk-runner/src/runner.ts b/packages/benchsdk-runner/src/runner.ts index b07477ad..950188cf 100644 --- a/packages/benchsdk-runner/src/runner.ts +++ b/packages/benchsdk-runner/src/runner.ts @@ -24,7 +24,6 @@ import { NoAvailableParticipantsError } from './no-available-participants.js'; import type { BaseParticipant, BenchmarkClient, - BenchmarkRun, JsonObject, RunWorkerContext, TaskResultRecord, @@ -34,6 +33,7 @@ import { TaskError } from './bench-config.js'; import type { BenchmarkConfig, BenchmarkRunOutcome, + BenchmarkShape, BenchmarkTask, GroupBy, ParticipantRecords, @@ -44,8 +44,16 @@ import type { import { LogBuffer } from './log-buffer.js'; export interface CliArgs { - slug?: string; + /** Which platform benchmark to report as (`--benchmark`, aka the benchmark slug). */ + benchmark?: string; name?: string; + /** Named variant from the bench file's `shapes` (`--shape`), swapping in its identity. */ + shape?: string; + /** + * Idempotency key (`--run-key`): sibling processes passing the same key share + * one run (get-or-created), instead of each opening its own. + */ + runKey?: string; iterations?: number; concurrency?: number; staggerDelayMs?: number; @@ -98,12 +106,14 @@ export function parseCliArgs(argv: string[]): CliArgs { const arg = argv[i]; const name = arg.includes('=') ? arg.slice(0, arg.indexOf('=')) : arg; switch (name) { - case '--slug': { + // `--slug` is the pre-`--benchmark` spelling, kept working for existing scripts. + case '--slug': + case '--benchmark': { const { value, nextIndex } = readValue(arg, i); if (!/^[a-z0-9][a-z0-9-]*$/.test(value)) { - throw new Error(`--slug expects a lowercase slug (got "${value}")`); + throw new Error(`${name} expects a lowercase benchmark slug (got "${value}")`); } - args.slug = value; + args.benchmark = value; i = nextIndex; break; } @@ -114,6 +124,20 @@ export function parseCliArgs(argv: string[]): CliArgs { i = nextIndex; break; } + case '--shape': { + const { value, nextIndex } = readValue(arg, i); + if (value.trim() === '') throw new Error('--shape expects a value'); + args.shape = value; + i = nextIndex; + break; + } + case '--run-key': { + const { value, nextIndex } = readValue(arg, i); + if (value.trim() === '') throw new Error('--run-key expects a value'); + args.runKey = value; + i = nextIndex; + break; + } case '--iterations': { const { value, nextIndex } = readValue(arg, i); args.iterations = intFlag(value, '--iterations'); @@ -234,40 +258,99 @@ function resolvePlatform(): { baseUrl: string; apiKey: string } { } /** - * Runs `config`'s `task` against `participants`. Selects participants by - * `--provider` (if given), env-gates them, then drives them per the resolved - * `groupBy`. `--slug`/`--name` retarget the whole run at a different platform - * benchmark, so one entrypoint can report under several slugs. + * Resolves `--shape ` against the config's declared `shapes`. Throws with + * the known names if the shape is unknown, so a typo fails loudly instead of + * silently running the base benchmark. */ -export async function runBenchmark( +function resolveShape( + config: BenchmarkConfig, + shapeName: string | undefined, +): BenchmarkShape | undefined { + if (!shapeName) return undefined; + const shape = config.shapes?.[shapeName]; + if (!shape) { + const known = Object.keys(config.shapes ?? {}); + throw new Error( + known.length > 0 + ? `Unknown --shape "${shapeName}". Known shapes: ${known.join(', ')}.` + : `Unknown --shape "${shapeName}": this benchmark declares no shapes.`, + ); + } + return shape; +} + +/** + * Swaps a shape's identity (and its stable knob) into the config. Only the + * parts that make it a distinct benchmark move here; scale knobs stay on the + * CLI, so `mergeConfig` still lets `--concurrency`/`--iterations` win. + */ +function applyShape( + config: BenchmarkConfig, + shape: BenchmarkShape | undefined, +): BenchmarkConfig { + if (!shape) return config; + const kind = shape.kind ?? config.benchmarkKind; + return { + ...config, + benchmarkSlug: shape.slug, + benchmarkName: shape.name ?? shape.slug, + ...(kind ? { benchmarkKind: kind } : {}), + ...(shape.staggerDelayMs !== undefined ? { staggerDelayMs: shape.staggerDelayMs } : {}), + }; +} + +/** Applies the `--benchmark`/`--name` overrides, so one entrypoint can report under several benchmarks. */ +function applyIdentityOverrides( fileConfig: BenchmarkConfig, - task: BenchmarkTask, - argv: string[] = [], -): Promise { - const args = parseCliArgs(argv); - const config = { + args: CliArgs, +): BenchmarkConfig { + return { ...fileConfig, - ...(args.slug ? { benchmarkSlug: args.slug } : {}), + ...(args.benchmark ? { benchmarkSlug: args.benchmark } : {}), ...(args.name ? { benchmarkName: args.name } : {}), }; - const resolved = mergeConfig(config, args); - const schedule = buildSchedule(config, resolved.iterations, task); - const totalTasks = schedule.length; +} - const selected = selectParticipants(config.participants, resolved.providers); - const { available, skipped } = filterParticipantsByEnv(selected); +function dashboardUrlFor(baseUrl: string, organizationSlug: string, benchmarkSlug: string, runId: string): string { + return `${baseUrl.replace(/\/api\/v1\/?$/, '')}/${organizationSlug}/benchmarks/${benchmarkSlug}/runs/${runId}`; +} +/** The participants a run covers: `--provider` selection, minus any whose env vars are unset. */ +function resolveParticipants(config: BenchmarkConfig, resolved: ResolvedRunConfig): T[] { + const { available, skipped } = filterParticipantsByEnv(selectParticipants(config.participants, resolved.providers)); for (const s of skipped) { console.log(`Skipping ${s.name}: missing ${s.missing.join(', ')}`); } + if (available.length === 0) throw new NoAvailableParticipantsError(skipped); + return available; +} - if (available.length === 0) { - throw new NoAvailableParticipantsError(skipped); - } +/** + * Runs `config`'s `task` against its participants. Selects participants by + * `--provider` (if given), env-gates them, then drives them per the resolved + * `groupBy`. `--shape` swaps in a declared variant's identity; `--benchmark`/ + * `--name` retarget the run at a different platform benchmark, so one entrypoint + * can report under several slugs. With `--run-key`, sibling processes (e.g. one + * CI job per provider) get-or-create one shared run and each registers only its + * own participants. + */ +export async function runBenchmark( + fileConfig: BenchmarkConfig, + task: BenchmarkTask, + argv: string[] = [], +): Promise { + const args = parseCliArgs(argv); + const shaped = applyShape(fileConfig, resolveShape(fileConfig, args.shape)); + const config = applyIdentityOverrides(shaped, args); + const resolved = mergeConfig(config, args); + const available = resolveParticipants(config, resolved); const { baseUrl, apiKey } = resolvePlatform(); const client = createBenchmarkClient({ baseUrl, apiKey }); + const schedule = buildSchedule(config, resolved.iterations, task); + const totalTasks = schedule.length; + const concurrencyLabel = resolved.groupBy === 'round' ? 'n/a (round mode)' : String(resolved.concurrency); console.log(`${config.benchmarkName} (self-contained)`); console.log(`Date: ${new Date().toISOString()}`); @@ -276,34 +359,65 @@ export async function runBenchmark( `staggerDelayMs=${resolved.staggerDelayMs}, groupBy=${resolved.groupBy}\n`, ); - await client.upsertBenchmark(config.benchmarkSlug, { - name: config.benchmarkName, - ...(config.benchmarkKind ? { kind: config.benchmarkKind } : {}), - }); - - const { run, organizationSlug } = await client.createRun(config.benchmarkSlug, { - name: `${config.benchmarkSlug} — ${totalTasks} iterations, concurrency ${resolved.concurrency}`, - totalTasks, - workerCount: 1, - participants: available.map((p) => p.name), - }); + // Declaratively materialize the benchmark from the file/shape identity, which + // is authoritative (its name lives in the file). A bare `--benchmark X` only + // *retargets* reporting at a benchmark this file doesn't name, so we don't + // upsert it — that would rename it to the file's own name. + const identityIsOurs = + args.shape !== undefined || + args.name !== undefined || + !args.benchmark || + args.benchmark === fileConfig.benchmarkSlug; + if (identityIsOurs) { + await client.upsertBenchmark(config.benchmarkSlug, { + name: config.benchmarkName, + ...(config.benchmarkKind ? { kind: config.benchmarkKind } : {}), + }); + } - const dashboardUrl = `${baseUrl.replace(/\/api\/v1\/?$/, '')}/${organizationSlug}/benchmarks/${config.benchmarkSlug}/runs/${run.id}`; - console.log(`Run created: ${run.id}`); - console.log(`View at: ${dashboardUrl}\n`); + let runId: string; + let dashboardUrl: string; + if (args.runKey) { + // Shared run: get-or-created by key, so sibling processes (one per provider) + // converge on one run. Opened participant-sized — register only the + // providers this process runs and let each sibling register its own, so the + // run lists exactly who's benchmarked and each brings its own task count. + const { run, organizationSlug } = await client.createRun(config.benchmarkSlug, { + name: config.benchmarkName, + runKey: args.runKey, + }); + runId = run.id; + dashboardUrl = dashboardUrlFor(baseUrl, organizationSlug, config.benchmarkSlug, run.id); + for (const participant of available) { + await client.upsertParticipant(config.benchmarkSlug, runId, participant.name, { totalTasks }); + } + console.log(`Shared run (key "${args.runKey}"): ${runId}`); + console.log(`View at: ${dashboardUrl}\n`); + } else { + const { run, organizationSlug } = await client.createRun(config.benchmarkSlug, { + name: `${config.benchmarkSlug} — ${totalTasks} iterations, concurrency ${resolved.concurrency}`, + totalTasks, + workerCount: 1, + participants: available.map((p) => p.name), + }); + runId = run.id; + dashboardUrl = dashboardUrlFor(baseUrl, organizationSlug, config.benchmarkSlug, run.id); + console.log(`Run created: ${runId}`); + console.log(`View at: ${dashboardUrl}\n`); + } const onResult = defaultOnResult; let participantRecords: ParticipantRecords[]; if (resolved.groupBy === 'round') { - participantRecords = await runGroupedByRound(config, schedule, available, resolved, client, run, baseUrl, apiKey, onResult); + participantRecords = await runGroupedByRound(config, schedule, available, resolved, client, runId, baseUrl, apiKey, onResult); } else { - participantRecords = await runGroupedByParticipant(config, schedule, available, resolved, client, run, onResult); + participantRecords = await runGroupedByParticipant(config, schedule, available, resolved, client, runId, onResult); } console.log(`All done. View at: ${dashboardUrl}`); const outcome: BenchmarkRunOutcome = { - runId: run.id, + runId, dashboardUrl, participants: participantRecords, config: resolved, @@ -325,7 +439,7 @@ async function runGroupedByParticipant( available: T[], resolved: ResolvedRunConfig, client: BenchmarkClient, - run: BenchmarkRun, + runId: string, onResult: OnResult, ): Promise { const participantRecords: ParticipantRecords[] = []; @@ -338,11 +452,11 @@ async function runGroupedByParticipant( // count can't inflate launch offsets: a task whose slot frees after its // scheduled launch time starts immediately instead of sleeping index*delay. let rampStartMs: number | undefined; - await client.planWorkers(config.benchmarkSlug, run.id, participant.name); + await client.planWorkers(config.benchmarkSlug, runId, participant.name); const result = await client.runWorker({ benchmarkSlug: config.benchmarkSlug, - runId: run.id, + runId: runId, participantSlug: participant.name, concurrency: resolved.concurrency, task: async (ctx: RunWorkerContext) => { @@ -372,7 +486,7 @@ async function runGroupedByParticipant( }); if (!result.assignment) { - console.error(` No pending worker to claim for run ${run.id} — it may already be fully claimed.`); + console.error(` No pending worker to claim for run ${runId} — it may already be fully claimed.`); participantRecords.push({ participant: participant.name, records: result.records ?? [] }); continue; } @@ -397,7 +511,7 @@ async function runGroupedByRound( available: T[], resolved: ResolvedRunConfig, client: BenchmarkClient, - run: BenchmarkRun, + runId: string, baseUrl: string, apiKey: string, onResult: OnResult, @@ -414,7 +528,7 @@ async function runGroupedByRound( // reads `targetConcurrency` as tasks-per-worker, so it must be the full // schedule length — otherwise only one task is planned and every record // past the first falls outside the worker's task range. - await client.planWorkers(config.benchmarkSlug, run.id, participant.name, { + await client.planWorkers(config.benchmarkSlug, runId, participant.name, { workerCount: 1, targetConcurrency: schedule.length, }); @@ -424,7 +538,7 @@ async function runGroupedByRound( baseUrl, apiKey, benchmarkSlug: config.benchmarkSlug, - runId: run.id, + runId: runId, participantSlug: participant.name, processKind: 'process', processKey: process.env.HOSTNAME ?? 'local', diff --git a/packages/benchsdk/src/types.ts b/packages/benchsdk/src/types.ts index 644400c5..df9b639c 100644 --- a/packages/benchsdk/src/types.ts +++ b/packages/benchsdk/src/types.ts @@ -28,7 +28,11 @@ export interface BenchmarkRun { benchmarkId: string; name?: string | null; status: BenchmarkRunStatus | string; + /** Idempotency key: runs created with the same key (per org + benchmark) are the same run. */ + runKey?: string | null; totalTasks: number; + /** The run declared no size: `totalTasks` is the sum of what its participants declare. */ + participantSized?: boolean; workerCount: number; config?: JsonObject; createdAt?: string; @@ -116,8 +120,14 @@ export interface UpdateBenchmarkInput { export interface CreateRunInput { name?: string; - totalTasks: number; - workerCount: number; + /** + * Idempotency key for get-or-create: sibling callers passing the same key + * (per org + benchmark) converge on one run instead of each opening its own. + */ + runKey?: string; + /** Omit to open a participant-sized run: each participant declares its own size when it registers. */ + totalTasks?: number; + workerCount?: number; participants?: string[]; config?: JsonObject; }