From 9d6ac9caef0127f74f4c5ea767f1096083edf79a Mon Sep 17 00:00:00 2001 From: garrison Date: Thu, 30 Jul 2026 20:29:07 +0000 Subject: [PATCH 1/8] feat(runner): bench create-run + --run-id so every provider reports into one run Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .changeset/benchsdk-runner-slug-flag.md | 2 + .github/workflows/sandbox-tti-benchmarks.yml | 62 ++++++++ .../benchsdk-runner/src/__tests__/cli.test.ts | 6 +- .../src/__tests__/runner.test.ts | 25 +++ packages/benchsdk-runner/src/bench-config.ts | 3 +- packages/benchsdk-runner/src/cli.ts | 22 ++- packages/benchsdk-runner/src/runner.ts | 146 ++++++++++++------ 7 files changed, 215 insertions(+), 51 deletions(-) diff --git a/.changeset/benchsdk-runner-slug-flag.md b/.changeset/benchsdk-runner-slug-flag.md index f100a1c7..de333d6e 100644 --- a/.changeset/benchsdk-runner-slug-flag.md +++ b/.changeset/benchsdk-runner-slug-flag.md @@ -3,3 +3,5 @@ --- Add `--slug` and `--name` CLI overrides for `benchmarkSlug`/`benchmarkName`, so one `*.bench.ts` can report under several platform benchmarks (e.g. the sandbox TTI entrypoint reporting sequential/staggered/burst runs of the same workload). + +Add `bench create-run `, which opens a platform run and prints its id, plus a `--run-id ` flag for `bench run` that joins that run instead of creating one. Sibling processes — one per provider, in parallel — then land in a single run (each still claiming its own worker), so their participants are directly comparable. diff --git a/.github/workflows/sandbox-tti-benchmarks.yml b/.github/workflows/sandbox-tti-benchmarks.yml index 19a5bb09..40de4288 100644 --- a/.github/workflows/sandbox-tti-benchmarks.yml +++ b/.github/workflows/sandbox-tti-benchmarks.yml @@ -49,8 +49,59 @@ permissions: pull-requests: write jobs: + create-runs: + name: Create platform runs + runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe + timeout-minutes: 10 + # One run per mode, holding every provider, so the platform can rank them + # against each other. Each provider job then claims its own worker inside + # the run it belongs to (see `--run-id` below). Best-effort: if this job + # can't create a run, the provider jobs fall back to opening their own. + continue-on-error: true + outputs: + sequential: ${{ steps.create.outputs.sequential }} + staggered: ${{ steps.create.outputs.staggered }} + burst: ${{ steps.create.outputs.burst }} + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: 'pnpm' + - name: Install dependencies + run: pnpm install --frozen-lockfile + - name: Create runs + id: create + run: | + . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUNLOOP_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' + + # `create-run` prints the run id on stdout and the dashboard URL on stderr. + CREATE=(npx tsx packages/benchsdk-runner/dist/bin.js create-run benchmarks/sandbox/tti.bench.ts) + COMMON=(--iterations ${{ github.event.inputs.iterations || (github.event_name == 'push' && '10') || '100' }}) + if [ -n "${{ github.event.inputs.provider }}" ]; then + COMMON+=(--provider "${{ github.event.inputs.provider }}") + fi + CONCURRENCY=${{ github.event.inputs.concurrency || (github.event_name == 'push' && '10') || '100' }} + MODE="${{ github.event.inputs.mode }}" + + if [ -z "$MODE" ] || [ "$MODE" = sequential ]; then + echo "sequential=$("${CREATE[@]}" "${COMMON[@]}")" >> "$GITHUB_OUTPUT" + fi + if [ -z "$MODE" ] || [ "$MODE" = staggered ]; then + echo "staggered=$("${CREATE[@]}" "${COMMON[@]}" --concurrency "$CONCURRENCY" --stagger-delay-ms 200 \ + --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)')" >> "$GITHUB_OUTPUT" + fi + if [ -z "$MODE" ] || [ "$MODE" = burst ]; then + echo "burst=$("${CREATE[@]}" "${COMMON[@]}" --concurrency "$CONCURRENCY" \ + --slug sandbox-burst-local --name 'Sandbox burst TTI (local)')" >> "$GITHUB_OUTPUT" + fi + bench: name: Bench ${{ matrix.provider }} + needs: create-runs + # Runs even if the run-creation job failed; see its `continue-on-error`. + if: always() runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe timeout-minutes: 60 env: @@ -118,6 +169,8 @@ jobs: # One entrypoint, three launch shapes: the knobs come from the CLI # and --slug/--name pick which platform benchmark each reports to. + # --run-id joins the shared run for that mode (this job still claims + # its own worker in it); without one, the run is created here. BENCH=(npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts) COMMON=(--provider ${{ matrix.provider }} \ --iterations ${{ github.event.inputs.iterations || (github.event_name == 'push' && '10') || '100' }}) @@ -129,6 +182,15 @@ jobs: --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)') BURST=("${BENCH[@]}" "${COMMON[@]}" --concurrency "$CONCURRENCY" \ --slug sandbox-burst-local --name 'Sandbox burst TTI (local)') + if [ -n "${{ needs.create-runs.outputs.sequential }}" ]; then + SEQUENTIAL+=(--run-id "${{ needs.create-runs.outputs.sequential }}") + fi + if [ -n "${{ needs.create-runs.outputs.staggered }}" ]; then + STAGGERED+=(--run-id "${{ needs.create-runs.outputs.staggered }}") + fi + if [ -n "${{ needs.create-runs.outputs.burst }}" ]; then + BURST+=(--run-id "${{ needs.create-runs.outputs.burst }}") + fi case "${{ github.event.inputs.mode }}" in sequential) "${SEQUENTIAL[@]}" ;; staggered) "${STAGGERED[@]}" ;; diff --git a/packages/benchsdk-runner/src/__tests__/cli.test.ts b/packages/benchsdk-runner/src/__tests__/cli.test.ts index 85faabab..867db759 100644 --- a/packages/benchsdk-runner/src/__tests__/cli.test.ts +++ b/packages/benchsdk-runner/src/__tests__/cli.test.ts @@ -6,12 +6,12 @@ const fixture = (name: string) => `src/__tests__/fixtures/${name}`; describe('runBenchmarkFile', () => { it('rejects when the command is not `run`', async () => { - await expect(runBenchmarkFile([])).rejects.toThrow(/Usage: bench run/); - await expect(runBenchmarkFile(['nope', fixture('good.bench.ts')])).rejects.toThrow(/Usage: bench run/); + await expect(runBenchmarkFile([])).rejects.toThrow(/Usage: bench /); + await expect(runBenchmarkFile(['nope', fixture('good.bench.ts')])).rejects.toThrow(/Usage: bench /); }); it('rejects when no file is given', async () => { - await expect(runBenchmarkFile(['run'])).rejects.toThrow(/Usage: bench run/); + await expect(runBenchmarkFile(['run'])).rejects.toThrow(/Usage: bench /); }); it('rejects a module that does not export a config', async () => { diff --git a/packages/benchsdk-runner/src/__tests__/runner.test.ts b/packages/benchsdk-runner/src/__tests__/runner.test.ts index d6c0db22..0ae7fea9 100644 --- a/packages/benchsdk-runner/src/__tests__/runner.test.ts +++ b/packages/benchsdk-runner/src/__tests__/runner.test.ts @@ -56,6 +56,12 @@ describe('parseCliArgs', () => { expect(() => parseCliArgs(['--name', ' '])).toThrow('--name'); }); + it('parses --run-id', () => { + expect(parseCliArgs(['--run-id', 'run-1'])).toEqual({ runId: 'run-1' }); + expect(parseCliArgs(['--run-id=run-1'])).toEqual({ runId: 'run-1' }); + expect(() => parseCliArgs(['--run-id', ''])).toThrow('--run-id'); + }); + it('throws on a non-slug --slug', () => { expect(() => parseCliArgs(['--slug', 'Sandbox TTI'])).toThrow('--slug'); expect(() => parseCliArgs(['--slug', ''])).toThrow('--slug'); @@ -294,6 +300,25 @@ describe('runBenchmark', () => { expect(calls.createRun[0][0]).toBe('sandbox-burst-local'); }); + it('joins the run named by --run-id instead of creating one', async () => { + const config: BenchmarkConfig = { + benchmarkSlug: 'sandbox-tti-local', + benchmarkName: 'Sandbox TTI', + iterations: 1, + participants: [participants[0]], + }; + + const outcome = await runBenchmark(config, defineTask(async () => ({})), ['--run-id', 'run-shared']); + + expect(calls.upsertBenchmark).toHaveLength(0); + expect(calls.createRun).toHaveLength(0); + // The participant still plans and claims its own worker, just inside the shared run. + expect(calls.planWorkers[0].slice(0, 3)).toEqual(['sandbox-tti-local', 'run-shared', 'e2b']); + expect(calls.runWorker[0]).toMatchObject({ runId: 'run-shared' }); + expect(outcome.runId).toBe('run-shared'); + expect(outcome.dashboardUrl).toBeUndefined(); + }); + it('throws NoAvailableParticipantsError, listing the skips, when no participant has its env vars set', async () => { delete process.env.E2B_API_KEY; delete process.env.MODAL_TOKEN; diff --git a/packages/benchsdk-runner/src/bench-config.ts b/packages/benchsdk-runner/src/bench-config.ts index b27558ec..74cc958e 100644 --- a/packages/benchsdk-runner/src/bench-config.ts +++ b/packages/benchsdk-runner/src/bench-config.ts @@ -132,7 +132,8 @@ export interface ResolvedRunConfig { */ export interface BenchmarkRunOutcome { runId: string; - dashboardUrl: string; + /** Absent when the process joined a run via `--run-id`: the org slug the URL needs comes from creating the run. */ + dashboardUrl?: string; participants: ParticipantRecords[]; config: ResolvedRunConfig; } diff --git a/packages/benchsdk-runner/src/cli.ts b/packages/benchsdk-runner/src/cli.ts index c76b4587..a29a2cc2 100644 --- a/packages/benchsdk-runner/src/cli.ts +++ b/packages/benchsdk-runner/src/cli.ts @@ -4,20 +4,25 @@ * via the internal `runBenchmark`. CLI flags override the config's knobs, and * `config.onComplete` (if any) fires once the run finishes. * + * `bench create-run [--flags]` opens a platform run for the same file + * without executing it and prints its id, so several `bench run --run-id ` + * processes — one per provider, in parallel — report into one comparable run + * while each still claims its own worker. + * * The executable wrapper lives in `bin.ts`; this module has no side effects so * it can be unit-tested by calling `runBenchmarkFile` directly. */ import { resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; -import { runBenchmark } from './runner.js'; +import { createRunOnly, runBenchmark } from './runner.js'; import { NoAvailableParticipantsError } from './no-available-participants.js'; import type { BaseParticipant } from '@benchsdk/client'; import type { BenchmarkConfig, BenchmarkTask } from './bench-config.js'; const USAGE = - 'Usage: bench run [--iterations N] [--concurrency N] ' + + 'Usage: bench [--iterations N] [--concurrency N] ' + '[--stagger-delay-ms N] [--group-by participant|round] [--provider a,b] ' + - "[--slug my-benchmark] [--name 'My benchmark']"; + "[--slug my-benchmark] [--name 'My benchmark'] [--run-id ] (run only)"; /** A benchmark module is expected to export `config` and `task`. */ interface BenchmarkModule { @@ -39,7 +44,7 @@ function isBenchmarkConfig(value: unknown): value is BenchmarkConfig { */ export async function runBenchmarkFile(argv: string[]): Promise { const [command, file, ...rest] = argv; - if (command !== 'run' || !file) { + if ((command !== 'run' && command !== 'create-run') || !file) { throw new Error(USAGE); } @@ -52,6 +57,15 @@ export async function runBenchmarkFile(argv: string[]): Promise { if (!isBenchmarkConfig(config)) { throw new Error(`${file} must export a \`config\` created with defineBenchmarkConfig (with participants).`); } + + if (command === 'create-run') { + const { runId, dashboardUrl } = await createRunOnly(config as BenchmarkConfig, rest); + console.error(`Run created: ${runId}\nView at: ${dashboardUrl}`); + // stdout carries the id alone so callers can capture it: RUN_ID=$(bench create-run ...) + console.log(runId); + return; + } + if (typeof task !== 'function') { throw new Error(`${file} must export a \`task\` created with defineTask.`); } diff --git a/packages/benchsdk-runner/src/runner.ts b/packages/benchsdk-runner/src/runner.ts index b07477ad..17c663eb 100644 --- a/packages/benchsdk-runner/src/runner.ts +++ b/packages/benchsdk-runner/src/runner.ts @@ -24,7 +24,6 @@ import { NoAvailableParticipantsError } from './no-available-participants.js'; import type { BaseParticipant, BenchmarkClient, - BenchmarkRun, JsonObject, RunWorkerContext, TaskResultRecord, @@ -46,6 +45,8 @@ import { LogBuffer } from './log-buffer.js'; export interface CliArgs { slug?: string; name?: string; + /** Join this existing platform run instead of creating one (`--run-id`). */ + runId?: string; iterations?: number; concurrency?: number; staggerDelayMs?: number; @@ -114,6 +115,13 @@ export function parseCliArgs(argv: string[]): CliArgs { i = nextIndex; break; } + case '--run-id': { + const { value, nextIndex } = readValue(arg, i); + if (value.trim() === '') throw new Error('--run-id expects a value'); + args.runId = value; + i = nextIndex; + break; + } case '--iterations': { const { value, nextIndex } = readValue(arg, i); args.iterations = intFlag(value, '--iterations'); @@ -233,11 +241,70 @@ function resolvePlatform(): { baseUrl: string; apiKey: string } { }; } +/** Applies the `--slug`/`--name` overrides, so one entrypoint can report under several benchmarks. */ +function applyIdentityOverrides( + fileConfig: BenchmarkConfig, + args: CliArgs, +): BenchmarkConfig { + return { + ...fileConfig, + ...(args.slug ? { benchmarkSlug: args.slug } : {}), + ...(args.name ? { benchmarkName: args.name } : {}), + }; +} + +function dashboardUrlFor(baseUrl: string, organizationSlug: string, benchmarkSlug: string, runId: string): string { + return `${baseUrl.replace(/\/api\/v1\/?$/, '')}/${organizationSlug}/benchmarks/${benchmarkSlug}/runs/${runId}`; +} + +/** The participants a run covers: `--provider` selection, minus any whose env vars are unset. */ +function resolveParticipants(config: BenchmarkConfig, resolved: ResolvedRunConfig): T[] { + const { available, skipped } = filterParticipantsByEnv(selectParticipants(config.participants, resolved.providers)); + for (const s of skipped) { + console.log(`Skipping ${s.name}: missing ${s.missing.join(', ')}`); + } + if (available.length === 0) throw new NoAvailableParticipantsError(skipped); + return available; +} + +/** + * Creates a platform run up front without executing anything, so several + * `runBenchmark` processes can report into it via `--run-id` — one run holding + * every participant is what makes them rankable against each other, and the + * processes stay independent (a provider per CI job). + */ +export async function createRunOnly( + fileConfig: BenchmarkConfig, + argv: string[] = [], +): Promise<{ runId: string; dashboardUrl: string }> { + const args = parseCliArgs(argv); + const config = applyIdentityOverrides(fileConfig, args); + const resolved = mergeConfig(config, args); + const available = resolveParticipants(config, resolved); + + const { baseUrl, apiKey } = resolvePlatform(); + const client = createBenchmarkClient({ baseUrl, apiKey }); + await client.upsertBenchmark(config.benchmarkSlug, { + name: config.benchmarkName, + ...(config.benchmarkKind ? { kind: config.benchmarkKind } : {}), + }); + const { run, organizationSlug } = await client.createRun(config.benchmarkSlug, { + name: `${config.benchmarkSlug} — ${resolved.iterations} iterations, concurrency ${resolved.concurrency}`, + totalTasks: resolved.iterations, + workerCount: 1, + participants: available.map((p) => p.name), + }); + + return { runId: run.id, dashboardUrl: dashboardUrlFor(baseUrl, organizationSlug, config.benchmarkSlug, run.id) }; +} + /** * Runs `config`'s `task` against `participants`. Selects participants by * `--provider` (if given), env-gates them, then drives them per the resolved * `groupBy`. `--slug`/`--name` retarget the whole run at a different platform - * benchmark, so one entrypoint can report under several slugs. + * benchmark, so one entrypoint can report under several slugs; `--run-id` + * joins a run created by `bench create-run` instead of opening a new one, so + * sibling processes (e.g. one CI job per provider) land in the same run. */ export async function runBenchmark( fileConfig: BenchmarkConfig, @@ -245,25 +312,12 @@ export async function runBenchmark( argv: string[] = [], ): Promise { const args = parseCliArgs(argv); - const config = { - ...fileConfig, - ...(args.slug ? { benchmarkSlug: args.slug } : {}), - ...(args.name ? { benchmarkName: args.name } : {}), - }; + const config = applyIdentityOverrides(fileConfig, args); const resolved = mergeConfig(config, args); const schedule = buildSchedule(config, resolved.iterations, task); const totalTasks = schedule.length; - const selected = selectParticipants(config.participants, resolved.providers); - const { available, skipped } = filterParticipantsByEnv(selected); - - for (const s of skipped) { - console.log(`Skipping ${s.name}: missing ${s.missing.join(', ')}`); - } - - if (available.length === 0) { - throw new NoAvailableParticipantsError(skipped); - } + const available = resolveParticipants(config, resolved); const { baseUrl, apiKey } = resolvePlatform(); const client = createBenchmarkClient({ baseUrl, apiKey }); @@ -276,34 +330,40 @@ export async function runBenchmark( `staggerDelayMs=${resolved.staggerDelayMs}, groupBy=${resolved.groupBy}\n`, ); - await client.upsertBenchmark(config.benchmarkSlug, { - name: config.benchmarkName, - ...(config.benchmarkKind ? { kind: config.benchmarkKind } : {}), - }); - - const { run, organizationSlug } = await client.createRun(config.benchmarkSlug, { - name: `${config.benchmarkSlug} — ${totalTasks} iterations, concurrency ${resolved.concurrency}`, - totalTasks, - workerCount: 1, - participants: available.map((p) => p.name), - }); - - const dashboardUrl = `${baseUrl.replace(/\/api\/v1\/?$/, '')}/${organizationSlug}/benchmarks/${config.benchmarkSlug}/runs/${run.id}`; - console.log(`Run created: ${run.id}`); - console.log(`View at: ${dashboardUrl}\n`); + let runId: string; + let dashboardUrl: string | undefined; + if (args.runId) { + runId = args.runId; + console.log(`Joining run: ${runId}\n`); + } else { + await client.upsertBenchmark(config.benchmarkSlug, { + name: config.benchmarkName, + ...(config.benchmarkKind ? { kind: config.benchmarkKind } : {}), + }); + const { run, organizationSlug } = await client.createRun(config.benchmarkSlug, { + name: `${config.benchmarkSlug} — ${totalTasks} iterations, concurrency ${resolved.concurrency}`, + totalTasks, + workerCount: 1, + participants: available.map((p) => p.name), + }); + runId = run.id; + dashboardUrl = dashboardUrlFor(baseUrl, organizationSlug, config.benchmarkSlug, run.id); + console.log(`Run created: ${runId}`); + console.log(`View at: ${dashboardUrl}\n`); + } const onResult = defaultOnResult; let participantRecords: ParticipantRecords[]; if (resolved.groupBy === 'round') { - participantRecords = await runGroupedByRound(config, schedule, available, resolved, client, run, baseUrl, apiKey, onResult); + participantRecords = await runGroupedByRound(config, schedule, available, resolved, client, runId, baseUrl, apiKey, onResult); } else { - participantRecords = await runGroupedByParticipant(config, schedule, available, resolved, client, run, onResult); + participantRecords = await runGroupedByParticipant(config, schedule, available, resolved, client, runId, onResult); } - console.log(`All done. View at: ${dashboardUrl}`); + console.log(dashboardUrl ? `All done. View at: ${dashboardUrl}` : `All done. Run: ${runId}`); const outcome: BenchmarkRunOutcome = { - runId: run.id, + runId, dashboardUrl, participants: participantRecords, config: resolved, @@ -325,7 +385,7 @@ async function runGroupedByParticipant( available: T[], resolved: ResolvedRunConfig, client: BenchmarkClient, - run: BenchmarkRun, + runId: string, onResult: OnResult, ): Promise { const participantRecords: ParticipantRecords[] = []; @@ -338,11 +398,11 @@ async function runGroupedByParticipant( // count can't inflate launch offsets: a task whose slot frees after its // scheduled launch time starts immediately instead of sleeping index*delay. let rampStartMs: number | undefined; - await client.planWorkers(config.benchmarkSlug, run.id, participant.name); + await client.planWorkers(config.benchmarkSlug, runId, participant.name); const result = await client.runWorker({ benchmarkSlug: config.benchmarkSlug, - runId: run.id, + runId: runId, participantSlug: participant.name, concurrency: resolved.concurrency, task: async (ctx: RunWorkerContext) => { @@ -372,7 +432,7 @@ async function runGroupedByParticipant( }); if (!result.assignment) { - console.error(` No pending worker to claim for run ${run.id} — it may already be fully claimed.`); + console.error(` No pending worker to claim for run ${runId} — it may already be fully claimed.`); participantRecords.push({ participant: participant.name, records: result.records ?? [] }); continue; } @@ -397,7 +457,7 @@ async function runGroupedByRound( available: T[], resolved: ResolvedRunConfig, client: BenchmarkClient, - run: BenchmarkRun, + runId: string, baseUrl: string, apiKey: string, onResult: OnResult, @@ -414,7 +474,7 @@ async function runGroupedByRound( // reads `targetConcurrency` as tasks-per-worker, so it must be the full // schedule length — otherwise only one task is planned and every record // past the first falls outside the worker's task range. - await client.planWorkers(config.benchmarkSlug, run.id, participant.name, { + await client.planWorkers(config.benchmarkSlug, runId, participant.name, { workerCount: 1, targetConcurrency: schedule.length, }); @@ -424,7 +484,7 @@ async function runGroupedByRound( baseUrl, apiKey, benchmarkSlug: config.benchmarkSlug, - runId: run.id, + runId: runId, participantSlug: participant.name, processKind: 'process', processKey: process.env.HOSTNAME ?? 'local', From 8285687422a5caf85978f88d2753c983caac76a5 Mon Sep 17 00:00:00 2001 From: garrison Date: Thu, 30 Jul 2026 20:33:40 +0000 Subject: [PATCH 2/8] fix(runner): create the shared run empty and let each joiner register itself Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/sandbox-tti-benchmarks.yml | 13 +++++++------ .../src/__tests__/runner.test.ts | 6 ++++-- packages/benchsdk-runner/src/runner.ts | 19 +++++++++++++------ 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/.github/workflows/sandbox-tti-benchmarks.yml b/.github/workflows/sandbox-tti-benchmarks.yml index 40de4288..9bcc2346 100644 --- a/.github/workflows/sandbox-tti-benchmarks.yml +++ b/.github/workflows/sandbox-tti-benchmarks.yml @@ -53,10 +53,10 @@ jobs: name: Create platform runs runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe timeout-minutes: 10 - # One run per mode, holding every provider, so the platform can rank them - # against each other. Each provider job then claims its own worker inside - # the run it belongs to (see `--run-id` below). Best-effort: if this job - # can't create a run, the provider jobs fall back to opening their own. + # One (empty) run per mode, which every provider job then joins via + # `--run-id` below, registering itself and claiming its own worker — so the + # platform can rank the providers against each other. Best-effort: if this + # job can't create a run, the provider jobs fall back to opening their own. continue-on-error: true outputs: sequential: ${{ steps.create.outputs.sequential }} @@ -74,9 +74,10 @@ jobs: - name: Create runs id: create run: | - . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUNLOOP_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' + . benchmarks/scripts/load-vault-secrets.sh '^(BENCHMARKS_PLATFORM_API_KEY)$' - # `create-run` prints the run id on stdout and the dashboard URL on stderr. + # `create-run` only talks to the platform — no provider credentials + # needed — and prints the run id alone on stdout (diagnostics go to stderr). CREATE=(npx tsx packages/benchsdk-runner/dist/bin.js create-run benchmarks/sandbox/tti.bench.ts) COMMON=(--iterations ${{ github.event.inputs.iterations || (github.event_name == 'push' && '10') || '100' }}) if [ -n "${{ github.event.inputs.provider }}" ]; then diff --git a/packages/benchsdk-runner/src/__tests__/runner.test.ts b/packages/benchsdk-runner/src/__tests__/runner.test.ts index 0ae7fea9..c35d807d 100644 --- a/packages/benchsdk-runner/src/__tests__/runner.test.ts +++ b/packages/benchsdk-runner/src/__tests__/runner.test.ts @@ -170,11 +170,12 @@ describe('runBenchmark', () => { process.env.E2B_API_KEY = 'x'; process.env.MODAL_TOKEN = 'y'; process.env.BENCHMARKS_PLATFORM_API_KEY = 'test-key'; - calls = { upsertBenchmark: [], createRun: [], planWorkers: [], runWorker: [], taskData: [] }; + calls = { upsertBenchmark: [], createRun: [], planWorkers: [], upsertParticipant: [], runWorker: [], taskData: [] }; fakeClient = { upsertBenchmark: vi.fn(async (...a: any[]) => { calls.upsertBenchmark.push(a); return {}; }), createRun: vi.fn(async (...a: any[]) => { calls.createRun.push(a); return { run: { id: 'run-1' }, participants: [] }; }), planWorkers: vi.fn(async (...a: any[]) => { calls.planWorkers.push(a); return []; }), + upsertParticipant: vi.fn(async (...a: any[]) => { calls.upsertParticipant.push(a); return {}; }), runWorker: vi.fn(async (opts: any) => { calls.runWorker.push(opts); const total = calls.createRun[0]?.[1]?.totalTasks ?? 1; @@ -312,7 +313,8 @@ describe('runBenchmark', () => { expect(calls.upsertBenchmark).toHaveLength(0); expect(calls.createRun).toHaveLength(0); - // The participant still plans and claims its own worker, just inside the shared run. + // The participant registers itself in the shared run, then plans and claims its own worker. + expect(calls.upsertParticipant[0].slice(0, 3)).toEqual(['sandbox-tti-local', 'run-shared', 'e2b']); expect(calls.planWorkers[0].slice(0, 3)).toEqual(['sandbox-tti-local', 'run-shared', 'e2b']); expect(calls.runWorker[0]).toMatchObject({ runId: 'run-shared' }); expect(outcome.runId).toBe('run-shared'); diff --git a/packages/benchsdk-runner/src/runner.ts b/packages/benchsdk-runner/src/runner.ts index 17c663eb..05f02ee5 100644 --- a/packages/benchsdk-runner/src/runner.ts +++ b/packages/benchsdk-runner/src/runner.ts @@ -268,10 +268,14 @@ function resolveParticipants(config: BenchmarkConfig< } /** - * Creates a platform run up front without executing anything, so several - * `runBenchmark` processes can report into it via `--run-id` — one run holding - * every participant is what makes them rankable against each other, and the - * processes stay independent (a provider per CI job). + * Creates an empty platform run up front without executing anything, so + * several `runBenchmark` processes can report into it via `--run-id` — one run + * holding every provider is what makes them rankable against each other, while + * the processes stay independent (a provider per CI job). + * + * Deliberately registers no participants: each joining process upserts its own, + * so the run lists exactly the providers that are actually being benchmarked + * rather than everything the bench file could reach. */ export async function createRunOnly( fileConfig: BenchmarkConfig, @@ -280,7 +284,6 @@ export async function createRunOnly( const args = parseCliArgs(argv); const config = applyIdentityOverrides(fileConfig, args); const resolved = mergeConfig(config, args); - const available = resolveParticipants(config, resolved); const { baseUrl, apiKey } = resolvePlatform(); const client = createBenchmarkClient({ baseUrl, apiKey }); @@ -292,7 +295,6 @@ export async function createRunOnly( name: `${config.benchmarkSlug} — ${resolved.iterations} iterations, concurrency ${resolved.concurrency}`, totalTasks: resolved.iterations, workerCount: 1, - participants: available.map((p) => p.name), }); return { runId: run.id, dashboardUrl: dashboardUrlFor(baseUrl, organizationSlug, config.benchmarkSlug, run.id) }; @@ -334,6 +336,11 @@ export async function runBenchmark( let dashboardUrl: string | undefined; if (args.runId) { runId = args.runId; + // The run was opened empty by `create-run`; register the participants this + // process is responsible for before planning their workers. + for (const participant of available) { + await client.upsertParticipant(config.benchmarkSlug, runId, participant.name, { totalTasks }); + } console.log(`Joining run: ${runId}\n`); } else { await client.upsertBenchmark(config.benchmarkSlug, { From 1d1658524e1782d932eae09ad42149eaa1ba10d5 Mon Sep 17 00:00:00 2001 From: garrison Date: Thu, 30 Jul 2026 20:33:47 +0000 Subject: [PATCH 3/8] chore(ci): drop the no-op --provider from create-run Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/sandbox-tti-benchmarks.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/sandbox-tti-benchmarks.yml b/.github/workflows/sandbox-tti-benchmarks.yml index 9bcc2346..056725cb 100644 --- a/.github/workflows/sandbox-tti-benchmarks.yml +++ b/.github/workflows/sandbox-tti-benchmarks.yml @@ -80,9 +80,6 @@ jobs: # needed — and prints the run id alone on stdout (diagnostics go to stderr). CREATE=(npx tsx packages/benchsdk-runner/dist/bin.js create-run benchmarks/sandbox/tti.bench.ts) COMMON=(--iterations ${{ github.event.inputs.iterations || (github.event_name == 'push' && '10') || '100' }}) - if [ -n "${{ github.event.inputs.provider }}" ]; then - COMMON+=(--provider "${{ github.event.inputs.provider }}") - fi CONCURRENCY=${{ github.event.inputs.concurrency || (github.event_name == 'push' && '10') || '100' }} MODE="${{ github.event.inputs.mode }}" From 44149204c8032b9f37fb46e710515739deb8f60b Mon Sep 17 00:00:00 2001 From: garrison Date: Thu, 30 Jul 2026 20:58:28 +0000 Subject: [PATCH 4/8] refactor(runner): verb-first CLI (bench create benchmark|run) and let the run own its size Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .changeset/benchsdk-runner-slug-flag.md | 6 +- .github/workflows/sandbox-tti-benchmarks.yml | 45 ++++---- .../benchsdk-runner/src/__tests__/cli.test.ts | 18 +++- .../src/__tests__/runner.test.ts | 34 ++++-- packages/benchsdk-runner/src/cli.ts | 101 ++++++++++++------ packages/benchsdk-runner/src/runner.ts | 95 ++++++++++------ 6 files changed, 204 insertions(+), 95 deletions(-) diff --git a/.changeset/benchsdk-runner-slug-flag.md b/.changeset/benchsdk-runner-slug-flag.md index de333d6e..1e12cd7e 100644 --- a/.changeset/benchsdk-runner-slug-flag.md +++ b/.changeset/benchsdk-runner-slug-flag.md @@ -2,6 +2,8 @@ "@benchsdk/runner": minor --- -Add `--slug` and `--name` CLI overrides for `benchmarkSlug`/`benchmarkName`, so one `*.bench.ts` can report under several platform benchmarks (e.g. the sandbox TTI entrypoint reporting sequential/staggered/burst runs of the same workload). +Verb-first CLI: `bench create benchmark [--name] [--kind]` and `bench create run [file] [--benchmark slug] [--iterations N]`, which prints a run id, alongside the existing `bench run `. -Add `bench create-run `, which opens a platform run and prints its id, plus a `--run-id ` flag for `bench run` that joins that run instead of creating one. Sibling processes — one per provider, in parallel — then land in a single run (each still claiming its own worker), so their participants are directly comparable. +`bench run` gains `--run-id ` to report into an already-open run instead of creating one — so sibling processes (one per provider, in parallel, each claiming its own worker) land in a single run whose participants are directly comparable. A joined run owns its own size, so `--iterations` is rejected alongside `--run-id`. + +`--slug` is now `--benchmark` (naming the resource, not the identifier); the old spelling still works. diff --git a/.github/workflows/sandbox-tti-benchmarks.yml b/.github/workflows/sandbox-tti-benchmarks.yml index 056725cb..31e8a939 100644 --- a/.github/workflows/sandbox-tti-benchmarks.yml +++ b/.github/workflows/sandbox-tti-benchmarks.yml @@ -76,23 +76,25 @@ jobs: run: | . benchmarks/scripts/load-vault-secrets.sh '^(BENCHMARKS_PLATFORM_API_KEY)$' - # `create-run` only talks to the platform — no provider credentials - # needed — and prints the run id alone on stdout (diagnostics go to stderr). - CREATE=(npx tsx packages/benchsdk-runner/dist/bin.js create-run benchmarks/sandbox/tti.bench.ts) - COMMON=(--iterations ${{ github.event.inputs.iterations || (github.event_name == 'push' && '10') || '100' }}) - CONCURRENCY=${{ github.event.inputs.concurrency || (github.event_name == 'push' && '10') || '100' }} + # `bench create` only talks to the platform — no provider credentials — + # and prints the run id alone on stdout (diagnostics go to stderr). + CLI=(npx tsx packages/benchsdk-runner/dist/bin.js) + ITERATIONS=${{ github.event.inputs.iterations || (github.event_name == 'push' && '10') || '100' }} MODE="${{ github.event.inputs.mode }}" + open_run() { + "${CLI[@]}" create benchmark "$1" --name "$2" --kind sandbox >/dev/null + "${CLI[@]}" create run --benchmark "$1" --iterations "$ITERATIONS" + } + if [ -z "$MODE" ] || [ "$MODE" = sequential ]; then - echo "sequential=$("${CREATE[@]}" "${COMMON[@]}")" >> "$GITHUB_OUTPUT" + echo "sequential=$(open_run sandbox-tti-local 'Sandbox TTI (local)')" >> "$GITHUB_OUTPUT" fi if [ -z "$MODE" ] || [ "$MODE" = staggered ]; then - echo "staggered=$("${CREATE[@]}" "${COMMON[@]}" --concurrency "$CONCURRENCY" --stagger-delay-ms 200 \ - --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)')" >> "$GITHUB_OUTPUT" + echo "staggered=$(open_run sandbox-staggered-local 'Sandbox staggered TTI (local)')" >> "$GITHUB_OUTPUT" fi if [ -z "$MODE" ] || [ "$MODE" = burst ]; then - echo "burst=$("${CREATE[@]}" "${COMMON[@]}" --concurrency "$CONCURRENCY" \ - --slug sandbox-burst-local --name 'Sandbox burst TTI (local)')" >> "$GITHUB_OUTPUT" + echo "burst=$(open_run sandbox-burst-local 'Sandbox burst TTI (local)')" >> "$GITHUB_OUTPUT" fi bench: @@ -165,29 +167,36 @@ jobs: run: | . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUNLOOP_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' - # One entrypoint, three launch shapes: the knobs come from the CLI - # and --slug/--name pick which platform benchmark each reports to. - # --run-id joins the shared run for that mode (this job still claims - # its own worker in it); without one, the run is created here. + # One entrypoint, three launch shapes: the knobs come from the CLI and + # --benchmark picks which platform benchmark each reports to. --run-id + # joins the shared run for that mode (this job still registers itself + # and claims its own worker in it); the run owns the iteration count, + # so --iterations is only passed when we open our own run. BENCH=(npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts) - COMMON=(--provider ${{ matrix.provider }} \ - --iterations ${{ github.event.inputs.iterations || (github.event_name == 'push' && '10') || '100' }}) + COMMON=(--provider ${{ matrix.provider }}) + ITERATIONS=${{ github.event.inputs.iterations || (github.event_name == 'push' && '10') || '100' }} CONCURRENCY=${{ github.event.inputs.concurrency || (github.event_name == 'push' && '10') || '100' }} # Sequential is one-at-a-time by definition, so it never gets # --concurrency (the config default pins it to 1). SEQUENTIAL=("${BENCH[@]}" "${COMMON[@]}") STAGGERED=("${BENCH[@]}" "${COMMON[@]}" --concurrency "$CONCURRENCY" --stagger-delay-ms 200 \ - --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)') + --benchmark sandbox-staggered-local --name 'Sandbox staggered TTI (local)') BURST=("${BENCH[@]}" "${COMMON[@]}" --concurrency "$CONCURRENCY" \ - --slug sandbox-burst-local --name 'Sandbox burst TTI (local)') + --benchmark sandbox-burst-local --name 'Sandbox burst TTI (local)') if [ -n "${{ needs.create-runs.outputs.sequential }}" ]; then SEQUENTIAL+=(--run-id "${{ needs.create-runs.outputs.sequential }}") + else + SEQUENTIAL+=(--iterations "$ITERATIONS") fi if [ -n "${{ needs.create-runs.outputs.staggered }}" ]; then STAGGERED+=(--run-id "${{ needs.create-runs.outputs.staggered }}") + else + STAGGERED+=(--iterations "$ITERATIONS") fi if [ -n "${{ needs.create-runs.outputs.burst }}" ]; then BURST+=(--run-id "${{ needs.create-runs.outputs.burst }}") + else + BURST+=(--iterations "$ITERATIONS") fi case "${{ github.event.inputs.mode }}" in sequential) "${SEQUENTIAL[@]}" ;; diff --git a/packages/benchsdk-runner/src/__tests__/cli.test.ts b/packages/benchsdk-runner/src/__tests__/cli.test.ts index 867db759..19b6a1e8 100644 --- a/packages/benchsdk-runner/src/__tests__/cli.test.ts +++ b/packages/benchsdk-runner/src/__tests__/cli.test.ts @@ -5,13 +5,25 @@ import { NoAvailableParticipantsError } from '../no-available-participants.js'; const fixture = (name: string) => `src/__tests__/fixtures/${name}`; describe('runBenchmarkFile', () => { + it('rejects an unknown verb, and an unknown noun after `create`', async () => { + await expect(runBenchmarkFile(['create', 'sandbox'])).rejects.toThrow(/Usage:/); + await expect(runBenchmarkFile(['create'])).rejects.toThrow(/Usage:/); + }); + + it('rejects `create run` with neither a file nor --benchmark', async () => { + await expect(runBenchmarkFile(['create', 'run'])).rejects.toThrow(/--benchmark/); + await expect(runBenchmarkFile(['create', 'run', '--benchmark', 'sandbox-tti-local'])).rejects.toThrow( + /--iterations/, + ); + }); + it('rejects when the command is not `run`', async () => { - await expect(runBenchmarkFile([])).rejects.toThrow(/Usage: bench /); - await expect(runBenchmarkFile(['nope', fixture('good.bench.ts')])).rejects.toThrow(/Usage: bench /); + await expect(runBenchmarkFile([])).rejects.toThrow(/Usage:/); + await expect(runBenchmarkFile(['nope', fixture('good.bench.ts')])).rejects.toThrow(/Usage:/); }); it('rejects when no file is given', async () => { - await expect(runBenchmarkFile(['run'])).rejects.toThrow(/Usage: bench /); + await expect(runBenchmarkFile(['run'])).rejects.toThrow(/Usage:/); }); it('rejects a module that does not export a config', async () => { diff --git a/packages/benchsdk-runner/src/__tests__/runner.test.ts b/packages/benchsdk-runner/src/__tests__/runner.test.ts index c35d807d..e235254c 100644 --- a/packages/benchsdk-runner/src/__tests__/runner.test.ts +++ b/packages/benchsdk-runner/src/__tests__/runner.test.ts @@ -49,10 +49,12 @@ describe('parseCliArgs', () => { expect(parseCliArgs(['--group-by=participant'])).toEqual({ groupBy: 'participant' }); }); - it('parses --slug and --name', () => { - expect(parseCliArgs(['--slug', 'sandbox-burst-local'])).toEqual({ slug: 'sandbox-burst-local' }); - expect(parseCliArgs(['--slug=sandbox-tti-local'])).toEqual({ slug: 'sandbox-tti-local' }); + it('parses --benchmark (and its --slug alias), --name and --kind', () => { + expect(parseCliArgs(['--benchmark', 'sandbox-burst-local'])).toEqual({ benchmark: 'sandbox-burst-local' }); + expect(parseCliArgs(['--benchmark=sandbox-tti-local'])).toEqual({ benchmark: 'sandbox-tti-local' }); + expect(parseCliArgs(['--slug', 'sandbox-burst-local'])).toEqual({ benchmark: 'sandbox-burst-local' }); expect(parseCliArgs(['--name', 'Sandbox burst TTI'])).toEqual({ name: 'Sandbox burst TTI' }); + expect(parseCliArgs(['--kind', 'sandbox'])).toEqual({ kind: 'sandbox' }); expect(() => parseCliArgs(['--name', ' '])).toThrow('--name'); }); @@ -62,8 +64,8 @@ describe('parseCliArgs', () => { expect(() => parseCliArgs(['--run-id', ''])).toThrow('--run-id'); }); - it('throws on a non-slug --slug', () => { - expect(() => parseCliArgs(['--slug', 'Sandbox TTI'])).toThrow('--slug'); + it('throws on a non-slug --benchmark', () => { + expect(() => parseCliArgs(['--benchmark', 'Sandbox TTI'])).toThrow('--benchmark'); expect(() => parseCliArgs(['--slug', ''])).toThrow('--slug'); }); @@ -170,15 +172,16 @@ describe('runBenchmark', () => { process.env.E2B_API_KEY = 'x'; process.env.MODAL_TOKEN = 'y'; process.env.BENCHMARKS_PLATFORM_API_KEY = 'test-key'; - calls = { upsertBenchmark: [], createRun: [], planWorkers: [], upsertParticipant: [], runWorker: [], taskData: [] }; + calls = { upsertBenchmark: [], createRun: [], planWorkers: [], upsertParticipant: [], getRun: [], runWorker: [], taskData: [] }; fakeClient = { upsertBenchmark: vi.fn(async (...a: any[]) => { calls.upsertBenchmark.push(a); return {}; }), createRun: vi.fn(async (...a: any[]) => { calls.createRun.push(a); return { run: { id: 'run-1' }, participants: [] }; }), planWorkers: vi.fn(async (...a: any[]) => { calls.planWorkers.push(a); return []; }), upsertParticipant: vi.fn(async (...a: any[]) => { calls.upsertParticipant.push(a); return {}; }), + getRun: vi.fn(async (slug: string, runId: string) => { calls.getRun.push([slug, runId]); return { id: runId, totalTasks: 3 }; }), runWorker: vi.fn(async (opts: any) => { calls.runWorker.push(opts); - const total = calls.createRun[0]?.[1]?.totalTasks ?? 1; + const total = calls.createRun[0]?.[1]?.totalTasks ?? calls.upsertParticipant[0]?.[3]?.totalTasks ?? 1; // The platform hands out globally-indexed task ranges; `taskRangeStart` // lets a test exercise a worker whose range doesn't start at 0. const start = taskRangeStart; @@ -321,6 +324,23 @@ describe('runBenchmark', () => { expect(outcome.dashboardUrl).toBeUndefined(); }); + it('takes its iteration count from the joined run, and rejects --iterations alongside --run-id', async () => { + const config: BenchmarkConfig = { + benchmarkSlug: 'sandbox-tti-local', + benchmarkName: 'Sandbox TTI', + iterations: 1, + participants: [participants[0]], + }; + + const outcome = await runBenchmark(config, defineTask(async () => ({})), ['--run-id', 'run-shared']); + expect(outcome.config.iterations).toBe(3); + expect(outcome.participants[0].records).toHaveLength(3); + + await expect( + runBenchmark(config, defineTask(async () => ({})), ['--run-id', 'run-shared', '--iterations', '5']), + ).rejects.toThrow('--iterations cannot be combined with --run-id'); + }); + it('throws NoAvailableParticipantsError, listing the skips, when no participant has its env vars set', async () => { delete process.env.E2B_API_KEY; delete process.env.MODAL_TOKEN; diff --git a/packages/benchsdk-runner/src/cli.ts b/packages/benchsdk-runner/src/cli.ts index a29a2cc2..f59ce643 100644 --- a/packages/benchsdk-runner/src/cli.ts +++ b/packages/benchsdk-runner/src/cli.ts @@ -1,28 +1,36 @@ /** - * `bench run [--flags]` — the author-facing entrypoint. Imports a - * benchmark module, reads its `config` and `task` exports, and drives the run - * via the internal `runBenchmark`. CLI flags override the config's knobs, and - * `config.onComplete` (if any) fires once the run finishes. + * The author-facing entrypoint. Verb first, then the noun it acts on: * - * `bench create-run [--flags]` opens a platform run for the same file - * without executing it and prints its id, so several `bench run --run-id ` - * processes — one per provider, in parallel — report into one comparable run - * while each still claims its own worker. + * bench run [--flags] execute a benchmark + * bench create benchmark declare a benchmark on the platform + * bench create run --benchmark open a run, printing its id + * + * `run` imports a benchmark module, reads its `config` and `task` exports and + * drives `runBenchmark`; CLI flags override the config's knobs and + * `config.onComplete` (if any) fires once the run finishes. With `--run-id` it + * reports into a run opened by `create run` instead of opening its own, so one + * process per provider — in parallel, each claiming its own worker — still ends + * up in a single run the platform can rank them in. + * + * `create` only talks to the platform: no bench file to load (it's just a + * source of defaults) and no provider credentials. * * The executable wrapper lives in `bin.ts`; this module has no side effects so * it can be unit-tested by calling `runBenchmarkFile` directly. */ import { resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; -import { createRunOnly, runBenchmark } from './runner.js'; +import { createBenchmark, createRun, mergeConfig, parseCliArgs, runBenchmark } from './runner.js'; import { NoAvailableParticipantsError } from './no-available-participants.js'; import type { BaseParticipant } from '@benchsdk/client'; import type { BenchmarkConfig, BenchmarkTask } from './bench-config.js'; const USAGE = - 'Usage: bench [--iterations N] [--concurrency N] ' + - '[--stagger-delay-ms N] [--group-by participant|round] [--provider a,b] ' + - "[--slug my-benchmark] [--name 'My benchmark'] [--run-id ] (run only)"; + 'Usage:\n' + + ' bench run [--benchmark slug] [--provider a,b] [--run-id id]\n' + + ' [--iterations N] [--concurrency N] [--stagger-delay-ms N] [--group-by participant|round]\n' + + " bench create benchmark [--name 'My benchmark'] [--kind sandbox]\n" + + ' bench create run [file.bench.ts] [--benchmark slug] [--iterations N]'; /** A benchmark module is expected to export `config` and `task`. */ interface BenchmarkModule { @@ -37,40 +45,71 @@ function isBenchmarkConfig(value: unknown): value is BenchmarkConfig { return typeof candidate.benchmarkSlug === 'string' && Array.isArray(candidate.participants); } +async function loadConfig(file: string): Promise { + const mod = (await import(pathToFileURL(resolve(process.cwd(), file)).href)) as BenchmarkModule; + if (!isBenchmarkConfig(mod.config)) { + throw new Error(`${file} must export a \`config\` created with defineBenchmarkConfig (with participants).`); + } + return mod.config; +} + +/** `bench create ...`. Prints created run ids on stdout, alone, so callers can capture them. */ +async function create(argv: string[]): Promise { + const [noun, ...rest] = argv; + + if (noun === 'benchmark') { + const [slug] = rest; + if (!slug || slug.startsWith('-')) throw new Error(USAGE); + const args = parseCliArgs(rest); + await createBenchmark(slug, { name: args.name, kind: args.kind }); + console.error(`Benchmark ready: ${slug}`); + return; + } + + if (noun === 'run') { + // The file, when given, only supplies defaults for the flags below. + const file = rest[0] && !rest[0].startsWith('-') ? rest[0] : undefined; + const args = parseCliArgs(file ? rest.slice(1) : rest); + const fileConfig = file ? await loadConfig(file) : undefined; + const benchmarkSlug = args.benchmark ?? fileConfig?.benchmarkSlug; + if (!benchmarkSlug) throw new Error('bench create run needs --benchmark (or a file to read it from).'); + const iterations = fileConfig ? mergeConfig(fileConfig, args).iterations : args.iterations; + if (!iterations) throw new Error('bench create run needs --iterations N (or a file to read it from).'); + + const { runId, dashboardUrl } = await createRun({ benchmarkSlug, iterations }); + console.error(`Run created: ${runId}\nView at: ${dashboardUrl}`); + console.log(runId); + return; + } + + throw new Error(USAGE); +} + /** - * Loads a benchmark file and runs it. Throws on bad usage / invalid exports and - * lets `NoAvailableParticipantsError` propagate so the caller can map it to a - * clean exit. Does not call `process.exit`. + * Dispatches one CLI invocation. Throws on bad usage / invalid exports and lets + * `NoAvailableParticipantsError` propagate so the caller can map it to a clean + * exit. Does not call `process.exit`. */ export async function runBenchmarkFile(argv: string[]): Promise { - const [command, file, ...rest] = argv; - if ((command !== 'run' && command !== 'create-run') || !file) { - throw new Error(USAGE); - } + const [command, ...rest] = argv; + + if (command === 'create') return create(rest); - const moduleUrl = pathToFileURL(resolve(process.cwd(), file)).href; - const mod = (await import(moduleUrl)) as BenchmarkModule; + const [file, ...flags] = rest; + if (command !== 'run' || !file) throw new Error(USAGE); + const mod = (await import(pathToFileURL(resolve(process.cwd(), file)).href)) as BenchmarkModule; const config = mod.config; const task = mod.task ?? mod.default; if (!isBenchmarkConfig(config)) { throw new Error(`${file} must export a \`config\` created with defineBenchmarkConfig (with participants).`); } - - if (command === 'create-run') { - const { runId, dashboardUrl } = await createRunOnly(config as BenchmarkConfig, rest); - console.error(`Run created: ${runId}\nView at: ${dashboardUrl}`); - // stdout carries the id alone so callers can capture it: RUN_ID=$(bench create-run ...) - console.log(runId); - return; - } - if (typeof task !== 'function') { throw new Error(`${file} must export a \`task\` created with defineTask.`); } - await runBenchmark(config as BenchmarkConfig, task as BenchmarkTask, rest); + await runBenchmark(config as BenchmarkConfig, task as BenchmarkTask, flags); } /** Executable entry: runs the file and maps outcomes to process exit codes. */ diff --git a/packages/benchsdk-runner/src/runner.ts b/packages/benchsdk-runner/src/runner.ts index 05f02ee5..2982c5d9 100644 --- a/packages/benchsdk-runner/src/runner.ts +++ b/packages/benchsdk-runner/src/runner.ts @@ -43,8 +43,10 @@ import type { import { LogBuffer } from './log-buffer.js'; export interface CliArgs { - slug?: string; + /** Which platform benchmark to report as (`--benchmark`, aka the benchmark slug). */ + benchmark?: string; name?: string; + kind?: string; /** Join this existing platform run instead of creating one (`--run-id`). */ runId?: string; iterations?: number; @@ -99,12 +101,14 @@ export function parseCliArgs(argv: string[]): CliArgs { const arg = argv[i]; const name = arg.includes('=') ? arg.slice(0, arg.indexOf('=')) : arg; switch (name) { - case '--slug': { + // `--slug` is the pre-`--benchmark` spelling, kept working for existing scripts. + case '--slug': + case '--benchmark': { const { value, nextIndex } = readValue(arg, i); if (!/^[a-z0-9][a-z0-9-]*$/.test(value)) { - throw new Error(`--slug expects a lowercase slug (got "${value}")`); + throw new Error(`${name} expects a lowercase benchmark slug (got "${value}")`); } - args.slug = value; + args.benchmark = value; i = nextIndex; break; } @@ -115,6 +119,13 @@ export function parseCliArgs(argv: string[]): CliArgs { i = nextIndex; break; } + case '--kind': { + const { value, nextIndex } = readValue(arg, i); + if (value.trim() === '') throw new Error('--kind expects a value'); + args.kind = value; + i = nextIndex; + break; + } case '--run-id': { const { value, nextIndex } = readValue(arg, i); if (value.trim() === '') throw new Error('--run-id expects a value'); @@ -241,14 +252,14 @@ function resolvePlatform(): { baseUrl: string; apiKey: string } { }; } -/** Applies the `--slug`/`--name` overrides, so one entrypoint can report under several benchmarks. */ +/** Applies the `--benchmark`/`--name` overrides, so one entrypoint can report under several benchmarks. */ function applyIdentityOverrides( fileConfig: BenchmarkConfig, args: CliArgs, ): BenchmarkConfig { return { ...fileConfig, - ...(args.slug ? { benchmarkSlug: args.slug } : {}), + ...(args.benchmark ? { benchmarkSlug: args.benchmark } : {}), ...(args.name ? { benchmarkName: args.name } : {}), }; } @@ -267,37 +278,38 @@ function resolveParticipants(config: BenchmarkConfig< return available; } +/** `bench create benchmark `: the only object with a user-chosen identity, so it's created explicitly. */ +export async function createBenchmark(slug: string, options: { name?: string; kind?: string } = {}): Promise { + const { baseUrl, apiKey } = resolvePlatform(); + const client = createBenchmarkClient({ baseUrl, apiKey }); + await client.upsertBenchmark(slug, { + name: options.name ?? slug, + ...(options.kind ? { kind: options.kind } : {}), + }); +} + /** - * Creates an empty platform run up front without executing anything, so - * several `runBenchmark` processes can report into it via `--run-id` — one run - * holding every provider is what makes them rankable against each other, while - * the processes stay independent (a provider per CI job). + * `bench create run --benchmark `: opens an empty run and returns its id, + * so several `bench run --run-id` processes report into one comparable run + * while staying independent (a provider per CI job). A pure platform call — no + * bench file, no provider credentials. * - * Deliberately registers no participants: each joining process upserts its own, - * so the run lists exactly the providers that are actually being benchmarked - * rather than everything the bench file could reach. + * Registers no participants: each joining process upserts its own, so the run + * lists exactly the providers being benchmarked rather than everything the + * bench file could reach. */ -export async function createRunOnly( - fileConfig: BenchmarkConfig, - argv: string[] = [], -): Promise<{ runId: string; dashboardUrl: string }> { - const args = parseCliArgs(argv); - const config = applyIdentityOverrides(fileConfig, args); - const resolved = mergeConfig(config, args); - +export async function createRun(options: { + benchmarkSlug: string; + iterations: number; +}): Promise<{ runId: string; dashboardUrl: string }> { const { baseUrl, apiKey } = resolvePlatform(); const client = createBenchmarkClient({ baseUrl, apiKey }); - await client.upsertBenchmark(config.benchmarkSlug, { - name: config.benchmarkName, - ...(config.benchmarkKind ? { kind: config.benchmarkKind } : {}), - }); - const { run, organizationSlug } = await client.createRun(config.benchmarkSlug, { - name: `${config.benchmarkSlug} — ${resolved.iterations} iterations, concurrency ${resolved.concurrency}`, - totalTasks: resolved.iterations, + const { run, organizationSlug } = await client.createRun(options.benchmarkSlug, { + name: `${options.benchmarkSlug} — ${options.iterations} iterations`, + totalTasks: options.iterations, workerCount: 1, }); - - return { runId: run.id, dashboardUrl: dashboardUrlFor(baseUrl, organizationSlug, config.benchmarkSlug, run.id) }; + return { runId: run.id, dashboardUrl: dashboardUrlFor(baseUrl, organizationSlug, options.benchmarkSlug, run.id) }; } /** @@ -314,16 +326,31 @@ export async function runBenchmark( argv: string[] = [], ): Promise { const args = parseCliArgs(argv); + if (args.runId && args.iterations !== undefined) { + throw new Error('--iterations cannot be combined with --run-id: the run already owns its size.'); + } const config = applyIdentityOverrides(fileConfig, args); - const resolved = mergeConfig(config, args); - const schedule = buildSchedule(config, resolved.iterations, task); - const totalTasks = schedule.length; - + let resolved = mergeConfig(config, args); const available = resolveParticipants(config, resolved); const { baseUrl, apiKey } = resolvePlatform(); const client = createBenchmarkClient({ baseUrl, apiKey }); + // A joined run is the single source of truth for how many tasks there are, so + // sibling processes can't disagree about the size of the run they share. + if (args.runId) { + const run = await client.getRun(config.benchmarkSlug, args.runId); + if (config.phases?.length && run.totalTasks !== resolved.iterations) { + throw new Error( + `Run ${args.runId} has ${run.totalTasks} tasks but this benchmark's phases total ${resolved.iterations}.`, + ); + } + resolved = { ...resolved, iterations: run.totalTasks }; + } + + const schedule = buildSchedule(config, resolved.iterations, task); + const totalTasks = schedule.length; + const concurrencyLabel = resolved.groupBy === 'round' ? 'n/a (round mode)' : String(resolved.concurrency); console.log(`${config.benchmarkName} (self-contained)`); console.log(`Date: ${new Date().toISOString()}`); From 4680cc49a16f40393a39b893436dae36071f63ae Mon Sep 17 00:00:00 2001 From: garrison Date: Thu, 30 Jul 2026 20:58:48 +0000 Subject: [PATCH 5/8] docs(bench): show the shared-run invocation in the TTI header Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- benchmarks/sandbox/tti.bench.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/benchmarks/sandbox/tti.bench.ts b/benchmarks/sandbox/tti.bench.ts index bbb7a80c..2d2f5cc9 100644 --- a/benchmarks/sandbox/tti.bench.ts +++ b/benchmarks/sandbox/tti.bench.ts @@ -9,11 +9,17 @@ * burst --concurrency N: all slots open at once, so this launches that * many real sandboxes simultaneously — raise N deliberately * staggered --concurrency N --stagger-delay-ms D: task i starts at i * D - * Each shape reports under its own platform benchmark via `--slug`/`--name`. + * Each shape reports under its own platform benchmark via `--benchmark`. * * bench run benchmarks/sandbox/tti.bench.ts --iterations 5 --provider e2b,modal - * bench run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 10 --concurrency 10 - * bench run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 10 --concurrency 10 --stagger-delay-ms 200 + * bench run benchmarks/sandbox/tti.bench.ts --benchmark sandbox-burst-local --iterations 10 --concurrency 10 + * bench run benchmarks/sandbox/tti.bench.ts --benchmark sandbox-staggered-local --iterations 10 --concurrency 10 --stagger-delay-ms 200 + * + * To rank providers against each other, open one run and have every provider + * join it (each still claims its own worker): + * + * ID=$(bench create run --benchmark sandbox-burst-local --iterations 100) + * bench run benchmarks/sandbox/tti.bench.ts --benchmark sandbox-burst-local --provider e2b --run-id "$ID" */ import '../src/env.js'; import path from 'node:path'; From e0309ba1da78a6eab82208172664167a75064654 Mon Sep 17 00:00:00 2001 From: garrison Date: Thu, 30 Jul 2026 21:13:45 +0000 Subject: [PATCH 6/8] feat(runner): open runs without a size; joiners bring their own iteration count Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .changeset/benchsdk-runner-slug-flag.md | 4 +-- .changeset/participant-sized-runs.md | 5 +++ .github/workflows/sandbox-tti-benchmarks.yml | 20 +++++------ .../benchsdk-runner/src/__tests__/cli.test.ts | 3 -- .../src/__tests__/runner.test.ts | 28 +++++++++++++-- packages/benchsdk-runner/src/cli.ts | 6 ++-- packages/benchsdk-runner/src/runner.ts | 36 +++++++++++-------- packages/benchsdk/src/types.ts | 7 ++-- 8 files changed, 71 insertions(+), 38 deletions(-) create mode 100644 .changeset/participant-sized-runs.md diff --git a/.changeset/benchsdk-runner-slug-flag.md b/.changeset/benchsdk-runner-slug-flag.md index 1e12cd7e..161f967f 100644 --- a/.changeset/benchsdk-runner-slug-flag.md +++ b/.changeset/benchsdk-runner-slug-flag.md @@ -2,8 +2,8 @@ "@benchsdk/runner": minor --- -Verb-first CLI: `bench create benchmark [--name] [--kind]` and `bench create run [file] [--benchmark slug] [--iterations N]`, which prints a run id, alongside the existing `bench run `. +Verb-first CLI: `bench create benchmark [--name] [--kind]` and `bench create run [--benchmark slug] [--iterations N] [file]`, which prints a run id (no size needed — the run takes its total from the providers that join it), alongside the existing `bench run `. -`bench run` gains `--run-id ` to report into an already-open run instead of creating one — so sibling processes (one per provider, in parallel, each claiming its own worker) land in a single run whose participants are directly comparable. A joined run owns its own size, so `--iterations` is rejected alongside `--run-id`. +`bench run` gains `--run-id ` to report into an already-open run instead of creating one — so sibling processes (one per provider, in parallel, each claiming its own worker) land in a single run whose participants are directly comparable. A run created *with* a size owns it, so an `--iterations` that disagrees with it is rejected. `--slug` is now `--benchmark` (naming the resource, not the identifier); the old spelling still works. diff --git a/.changeset/participant-sized-runs.md b/.changeset/participant-sized-runs.md new file mode 100644 index 00000000..6d8b8121 --- /dev/null +++ b/.changeset/participant-sized-runs.md @@ -0,0 +1,5 @@ +--- +"@benchsdk/client": minor +--- + +`createRun` no longer requires `totalTasks`: omit it to open a participant-sized run, whose total is the sum of what its participants declare when they register. `BenchmarkRun.participantSized` reports which kind a run is. diff --git a/.github/workflows/sandbox-tti-benchmarks.yml b/.github/workflows/sandbox-tti-benchmarks.yml index 31e8a939..fe4d910f 100644 --- a/.github/workflows/sandbox-tti-benchmarks.yml +++ b/.github/workflows/sandbox-tti-benchmarks.yml @@ -79,12 +79,13 @@ jobs: # `bench create` only talks to the platform — no provider credentials — # and prints the run id alone on stdout (diagnostics go to stderr). CLI=(npx tsx packages/benchsdk-runner/dist/bin.js) - ITERATIONS=${{ github.event.inputs.iterations || (github.event_name == 'push' && '10') || '100' }} MODE="${{ github.event.inputs.mode }}" + # No size here: the run takes its total from the providers that + # register into it, so this job needs to know nothing about the matrix. open_run() { "${CLI[@]}" create benchmark "$1" --name "$2" --kind sandbox >/dev/null - "${CLI[@]}" create run --benchmark "$1" --iterations "$ITERATIONS" + "${CLI[@]}" create run --benchmark "$1" } if [ -z "$MODE" ] || [ "$MODE" = sequential ]; then @@ -170,11 +171,12 @@ jobs: # One entrypoint, three launch shapes: the knobs come from the CLI and # --benchmark picks which platform benchmark each reports to. --run-id # joins the shared run for that mode (this job still registers itself - # and claims its own worker in it); the run owns the iteration count, - # so --iterations is only passed when we open our own run. + # and claims its own worker in it, bringing its own iteration count). BENCH=(npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts) - COMMON=(--provider ${{ matrix.provider }}) - ITERATIONS=${{ github.event.inputs.iterations || (github.event_name == 'push' && '10') || '100' }} + COMMON=( + --provider ${{ matrix.provider }} + --iterations ${{ github.event.inputs.iterations || (github.event_name == 'push' && '10') || '100' }} + ) CONCURRENCY=${{ github.event.inputs.concurrency || (github.event_name == 'push' && '10') || '100' }} # Sequential is one-at-a-time by definition, so it never gets # --concurrency (the config default pins it to 1). @@ -185,18 +187,12 @@ jobs: --benchmark sandbox-burst-local --name 'Sandbox burst TTI (local)') if [ -n "${{ needs.create-runs.outputs.sequential }}" ]; then SEQUENTIAL+=(--run-id "${{ needs.create-runs.outputs.sequential }}") - else - SEQUENTIAL+=(--iterations "$ITERATIONS") fi if [ -n "${{ needs.create-runs.outputs.staggered }}" ]; then STAGGERED+=(--run-id "${{ needs.create-runs.outputs.staggered }}") - else - STAGGERED+=(--iterations "$ITERATIONS") fi if [ -n "${{ needs.create-runs.outputs.burst }}" ]; then BURST+=(--run-id "${{ needs.create-runs.outputs.burst }}") - else - BURST+=(--iterations "$ITERATIONS") fi case "${{ github.event.inputs.mode }}" in sequential) "${SEQUENTIAL[@]}" ;; diff --git a/packages/benchsdk-runner/src/__tests__/cli.test.ts b/packages/benchsdk-runner/src/__tests__/cli.test.ts index 19b6a1e8..52ba6b06 100644 --- a/packages/benchsdk-runner/src/__tests__/cli.test.ts +++ b/packages/benchsdk-runner/src/__tests__/cli.test.ts @@ -12,9 +12,6 @@ describe('runBenchmarkFile', () => { it('rejects `create run` with neither a file nor --benchmark', async () => { await expect(runBenchmarkFile(['create', 'run'])).rejects.toThrow(/--benchmark/); - await expect(runBenchmarkFile(['create', 'run', '--benchmark', 'sandbox-tti-local'])).rejects.toThrow( - /--iterations/, - ); }); it('rejects when the command is not `run`', async () => { diff --git a/packages/benchsdk-runner/src/__tests__/runner.test.ts b/packages/benchsdk-runner/src/__tests__/runner.test.ts index e235254c..8550f120 100644 --- a/packages/benchsdk-runner/src/__tests__/runner.test.ts +++ b/packages/benchsdk-runner/src/__tests__/runner.test.ts @@ -178,7 +178,10 @@ describe('runBenchmark', () => { createRun: vi.fn(async (...a: any[]) => { calls.createRun.push(a); return { run: { id: 'run-1' }, participants: [] }; }), planWorkers: vi.fn(async (...a: any[]) => { calls.planWorkers.push(a); return []; }), upsertParticipant: vi.fn(async (...a: any[]) => { calls.upsertParticipant.push(a); return {}; }), - getRun: vi.fn(async (slug: string, runId: string) => { calls.getRun.push([slug, runId]); return { id: runId, totalTasks: 3 }; }), + getRun: vi.fn(async (slug: string, runId: string) => { + calls.getRun.push([slug, runId]); + return { id: runId, totalTasks: 3, participantSized: runId === 'run-open' }; + }), runWorker: vi.fn(async (opts: any) => { calls.runWorker.push(opts); const total = calls.createRun[0]?.[1]?.totalTasks ?? calls.upsertParticipant[0]?.[3]?.totalTasks ?? 1; @@ -324,7 +327,7 @@ describe('runBenchmark', () => { expect(outcome.dashboardUrl).toBeUndefined(); }); - it('takes its iteration count from the joined run, and rejects --iterations alongside --run-id', async () => { + it('takes its iteration count from a sized run, and rejects an --iterations that disagrees', async () => { const config: BenchmarkConfig = { benchmarkSlug: 'sandbox-tti-local', benchmarkName: 'Sandbox TTI', @@ -338,7 +341,26 @@ describe('runBenchmark', () => { await expect( runBenchmark(config, defineTask(async () => ({})), ['--run-id', 'run-shared', '--iterations', '5']), - ).rejects.toThrow('--iterations cannot be combined with --run-id'); + ).rejects.toThrow('disagrees with run run-shared'); + }); + + it('brings its own iteration count to a participant-sized run', async () => { + const config: BenchmarkConfig = { + benchmarkSlug: 'sandbox-tti-local', + benchmarkName: 'Sandbox TTI', + iterations: 1, + participants: [participants[0]], + }; + + const outcome = await runBenchmark(config, defineTask(async () => ({})), [ + '--run-id', + 'run-open', + '--iterations', + '4', + ]); + + expect(outcome.config.iterations).toBe(4); + expect(calls.upsertParticipant[0][3]).toMatchObject({ totalTasks: 4 }); }); it('throws NoAvailableParticipantsError, listing the skips, when no participant has its env vars set', async () => { diff --git a/packages/benchsdk-runner/src/cli.ts b/packages/benchsdk-runner/src/cli.ts index f59ce643..b5151c89 100644 --- a/packages/benchsdk-runner/src/cli.ts +++ b/packages/benchsdk-runner/src/cli.ts @@ -73,8 +73,10 @@ async function create(argv: string[]): Promise { const fileConfig = file ? await loadConfig(file) : undefined; const benchmarkSlug = args.benchmark ?? fileConfig?.benchmarkSlug; if (!benchmarkSlug) throw new Error('bench create run needs --benchmark (or a file to read it from).'); - const iterations = fileConfig ? mergeConfig(fileConfig, args).iterations : args.iterations; - if (!iterations) throw new Error('bench create run needs --iterations N (or a file to read it from).'); + // No size at all is fine: the run then takes its total from the + // participants that register, which is what lets one run be opened before + // anyone knows which providers will join or how much each will do. + const iterations = args.iterations ?? (fileConfig ? mergeConfig(fileConfig, args).iterations : undefined); const { runId, dashboardUrl } = await createRun({ benchmarkSlug, iterations }); console.error(`Run created: ${runId}\nView at: ${dashboardUrl}`); diff --git a/packages/benchsdk-runner/src/runner.ts b/packages/benchsdk-runner/src/runner.ts index 2982c5d9..2eb3d54b 100644 --- a/packages/benchsdk-runner/src/runner.ts +++ b/packages/benchsdk-runner/src/runner.ts @@ -300,14 +300,17 @@ export async function createBenchmark(slug: string, options: { name?: string; ki */ export async function createRun(options: { benchmarkSlug: string; - iterations: number; + iterations?: number; }): Promise<{ runId: string; dashboardUrl: string }> { const { baseUrl, apiKey } = resolvePlatform(); const client = createBenchmarkClient({ baseUrl, apiKey }); + // Without `iterations` the run is participant-sized: it takes its total from + // the participants that register, so opening a run needs nothing but a name. const { run, organizationSlug } = await client.createRun(options.benchmarkSlug, { - name: `${options.benchmarkSlug} — ${options.iterations} iterations`, - totalTasks: options.iterations, - workerCount: 1, + name: options.iterations + ? `${options.benchmarkSlug} — ${options.iterations} iterations` + : options.benchmarkSlug, + ...(options.iterations ? { totalTasks: options.iterations, workerCount: 1 } : {}), }); return { runId: run.id, dashboardUrl: dashboardUrlFor(baseUrl, organizationSlug, options.benchmarkSlug, run.id) }; } @@ -326,9 +329,6 @@ export async function runBenchmark( argv: string[] = [], ): Promise { const args = parseCliArgs(argv); - if (args.runId && args.iterations !== undefined) { - throw new Error('--iterations cannot be combined with --run-id: the run already owns its size.'); - } const config = applyIdentityOverrides(fileConfig, args); let resolved = mergeConfig(config, args); const available = resolveParticipants(config, resolved); @@ -336,16 +336,24 @@ export async function runBenchmark( const { baseUrl, apiKey } = resolvePlatform(); const client = createBenchmarkClient({ baseUrl, apiKey }); - // A joined run is the single source of truth for how many tasks there are, so - // sibling processes can't disagree about the size of the run they share. + // A run that declared a size is the single source of truth for it, so sibling + // processes can't disagree about the run they share. A participant-sized run + // declared none — each joiner brings its own iteration count. if (args.runId) { const run = await client.getRun(config.benchmarkSlug, args.runId); - if (config.phases?.length && run.totalTasks !== resolved.iterations) { - throw new Error( - `Run ${args.runId} has ${run.totalTasks} tasks but this benchmark's phases total ${resolved.iterations}.`, - ); + if (!run.participantSized) { + if (args.iterations !== undefined && args.iterations !== run.totalTasks) { + throw new Error( + `--iterations ${args.iterations} disagrees with run ${args.runId}, which was created with ${run.totalTasks}.`, + ); + } + if (config.phases?.length && run.totalTasks !== resolved.iterations) { + throw new Error( + `Run ${args.runId} has ${run.totalTasks} tasks but this benchmark's phases total ${resolved.iterations}.`, + ); + } + resolved = { ...resolved, iterations: run.totalTasks }; } - resolved = { ...resolved, iterations: run.totalTasks }; } const schedule = buildSchedule(config, resolved.iterations, task); diff --git a/packages/benchsdk/src/types.ts b/packages/benchsdk/src/types.ts index 644400c5..81dac11f 100644 --- a/packages/benchsdk/src/types.ts +++ b/packages/benchsdk/src/types.ts @@ -29,6 +29,8 @@ export interface BenchmarkRun { name?: string | null; status: BenchmarkRunStatus | string; totalTasks: number; + /** The run declared no size: `totalTasks` is the sum of what its participants declare. */ + participantSized?: boolean; workerCount: number; config?: JsonObject; createdAt?: string; @@ -116,8 +118,9 @@ export interface UpdateBenchmarkInput { export interface CreateRunInput { name?: string; - totalTasks: number; - workerCount: number; + /** Omit to open a participant-sized run: each participant declares its own size when it registers. */ + totalTasks?: number; + workerCount?: number; participants?: string[]; config?: JsonObject; } From b055aea0fdb553c514816dbcd0c3a283c38cfd8f Mon Sep 17 00:00:00 2001 From: garrison Date: Thu, 30 Jul 2026 21:41:24 +0000 Subject: [PATCH 7/8] fix(runner): don't rename a benchmark that --benchmark merely retargets Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .changeset/benchsdk-runner-slug-flag.md | 2 ++ .../src/__tests__/runner.test.ts | 20 +++++++++++++++++++ packages/benchsdk-runner/src/runner.ts | 13 ++++++++---- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/.changeset/benchsdk-runner-slug-flag.md b/.changeset/benchsdk-runner-slug-flag.md index 161f967f..1ea474c6 100644 --- a/.changeset/benchsdk-runner-slug-flag.md +++ b/.changeset/benchsdk-runner-slug-flag.md @@ -7,3 +7,5 @@ Verb-first CLI: `bench create benchmark [--name] [--kind]` and `bench cre `bench run` gains `--run-id ` to report into an already-open run instead of creating one — so sibling processes (one per provider, in parallel, each claiming its own worker) land in a single run whose participants are directly comparable. A run created *with* a size owns it, so an `--iterations` that disagrees with it is rejected. `--slug` is now `--benchmark` (naming the resource, not the identifier); the old spelling still works. + +`bench run --benchmark ` no longer upserts (and so can no longer rename) a benchmark it was merely retargeted at — pass `--name` to claim its identity, or create it with `bench create benchmark`. diff --git a/packages/benchsdk-runner/src/__tests__/runner.test.ts b/packages/benchsdk-runner/src/__tests__/runner.test.ts index 8550f120..a69e6458 100644 --- a/packages/benchsdk-runner/src/__tests__/runner.test.ts +++ b/packages/benchsdk-runner/src/__tests__/runner.test.ts @@ -344,6 +344,26 @@ describe('runBenchmark', () => { ).rejects.toThrow('disagrees with run run-shared'); }); + it('does not rename a benchmark it was merely retargeted at', async () => { + const config: BenchmarkConfig = { + benchmarkSlug: 'sandbox-tti-local', + benchmarkName: 'Sandbox TTI', + iterations: 1, + participants: [participants[0]], + }; + + await runBenchmark(config, defineTask(async () => ({})), ['--benchmark', 'sandbox-burst-local']); + expect(calls.upsertBenchmark).toEqual([]); + + await runBenchmark(config, defineTask(async () => ({})), [ + '--benchmark', + 'sandbox-burst-local', + '--name', + 'Sandbox burst TTI', + ]); + expect(calls.upsertBenchmark[0]).toEqual(['sandbox-burst-local', { name: 'Sandbox burst TTI' }]); + }); + it('brings its own iteration count to a participant-sized run', async () => { const config: BenchmarkConfig = { benchmarkSlug: 'sandbox-tti-local', diff --git a/packages/benchsdk-runner/src/runner.ts b/packages/benchsdk-runner/src/runner.ts index 2eb3d54b..a737e3b2 100644 --- a/packages/benchsdk-runner/src/runner.ts +++ b/packages/benchsdk-runner/src/runner.ts @@ -378,10 +378,15 @@ export async function runBenchmark( } console.log(`Joining run: ${runId}\n`); } else { - await client.upsertBenchmark(config.benchmarkSlug, { - name: config.benchmarkName, - ...(config.benchmarkKind ? { kind: config.benchmarkKind } : {}), - }); + // Retargeted at a benchmark this file doesn't name, so its identity isn't + // ours to write: upserting would rename it to the file's own name. It has + // to exist already (`bench create benchmark`), and run creation says so. + if (!args.benchmark || args.benchmark === fileConfig.benchmarkSlug || args.name) { + await client.upsertBenchmark(config.benchmarkSlug, { + name: config.benchmarkName, + ...(config.benchmarkKind ? { kind: config.benchmarkKind } : {}), + }); + } const { run, organizationSlug } = await client.createRun(config.benchmarkSlug, { name: `${config.benchmarkSlug} — ${totalTasks} iterations, concurrency ${resolved.concurrency}`, totalTasks, From 9b153cee635644edba13ee6a41b89b14d4bd03d3 Mon Sep 17 00:00:00 2001 From: garrison Date: Fri, 31 Jul 2026 20:36:44 +0000 Subject: [PATCH 8/8] feat(runner): verbs-only bench run with --shape and --run-key Retire the imperative create commands: the benchmark is declared in the .bench.ts file (via optional named shapes) and materialized on run, and a run is opened as a side effect. --shape selects a named variant (swapping platform identity + stable knobs); --run-key get-or-creates a shared run so sibling provider jobs converge on one comparable run. Collapse the per-shape slug/name/knob triple out of package scripts and the sandbox TTI workflow (drop the create-runs job; providers pass a shared run key including GITHUB_RUN_ATTEMPT). Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .changeset/benchsdk-client-run-key.md | 5 + .changeset/benchsdk-runner-shapes-run-key.md | 11 ++ .changeset/benchsdk-runner-slug-flag.md | 11 -- .github/workflows/sandbox-tti-benchmarks.yml | 80 ++------ benchmarks/sandbox/tti.bench.ts | 33 ++-- package.json | 42 ++-- .../src/__tests__/bench-config.test.ts | 25 +++ .../benchsdk-runner/src/__tests__/cli.test.ts | 13 +- .../src/__tests__/runner.test.ts | 88 +++++---- packages/benchsdk-runner/src/bench-config.ts | 52 ++++- packages/benchsdk-runner/src/cli.ts | 71 ++----- packages/benchsdk-runner/src/runner.ts | 185 +++++++++--------- packages/benchsdk/src/types.ts | 7 + 13 files changed, 305 insertions(+), 318 deletions(-) create mode 100644 .changeset/benchsdk-client-run-key.md create mode 100644 .changeset/benchsdk-runner-shapes-run-key.md delete mode 100644 .changeset/benchsdk-runner-slug-flag.md diff --git a/.changeset/benchsdk-client-run-key.md b/.changeset/benchsdk-client-run-key.md new file mode 100644 index 00000000..90795488 --- /dev/null +++ b/.changeset/benchsdk-client-run-key.md @@ -0,0 +1,5 @@ +--- +"@benchsdk/client": minor +--- + +`createRun` accepts an optional `runKey`: callers passing the same key (per org + benchmark) get-or-create one shared run instead of each opening its own. `BenchmarkRun.runKey` reports the key a run was created with. diff --git a/.changeset/benchsdk-runner-shapes-run-key.md b/.changeset/benchsdk-runner-shapes-run-key.md new file mode 100644 index 00000000..bf198f3c --- /dev/null +++ b/.changeset/benchsdk-runner-shapes-run-key.md @@ -0,0 +1,11 @@ +--- +"@benchsdk/runner": minor +--- + +Verbs-only CLI: `bench run ` is the one mutating command. The benchmark is declared in the file and materialized (upserted) as a side effect of running it, and a run is opened as a side effect too — there are no imperative `bench create benchmark` / `bench create run` commands. + +`bench run` gains `--shape `: a bench file can declare named `shapes`, each swapping in its own platform identity (`slug`/`name`, optional `kind`) and a stable knob (`staggerDelayMs`) while reusing the same task and participants. This collapses the per-shape slug/name/knob triple that was duplicated across package scripts and CI. + +`bench run` gains `--run-key `: sibling processes passing the same key (per org + benchmark) get-or-create one shared run instead of each opening its own, so provider jobs running in parallel land in a single, directly-comparable run. Each process registers only the participants it runs. The key binding is permanent, so callers that need a fresh run (e.g. a CI re-run) vary the key (e.g. include `GITHUB_RUN_ATTEMPT`). + +`--slug` remains a working alias for `--benchmark`. diff --git a/.changeset/benchsdk-runner-slug-flag.md b/.changeset/benchsdk-runner-slug-flag.md deleted file mode 100644 index 1ea474c6..00000000 --- a/.changeset/benchsdk-runner-slug-flag.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -"@benchsdk/runner": minor ---- - -Verb-first CLI: `bench create benchmark [--name] [--kind]` and `bench create run [--benchmark slug] [--iterations N] [file]`, which prints a run id (no size needed — the run takes its total from the providers that join it), alongside the existing `bench run `. - -`bench run` gains `--run-id ` to report into an already-open run instead of creating one — so sibling processes (one per provider, in parallel, each claiming its own worker) land in a single run whose participants are directly comparable. A run created *with* a size owns it, so an `--iterations` that disagrees with it is rejected. - -`--slug` is now `--benchmark` (naming the resource, not the identifier); the old spelling still works. - -`bench run --benchmark ` no longer upserts (and so can no longer rename) a benchmark it was merely retargeted at — pass `--name` to claim its identity, or create it with `bench create benchmark`. diff --git a/.github/workflows/sandbox-tti-benchmarks.yml b/.github/workflows/sandbox-tti-benchmarks.yml index fe4d910f..0db92f3d 100644 --- a/.github/workflows/sandbox-tti-benchmarks.yml +++ b/.github/workflows/sandbox-tti-benchmarks.yml @@ -49,60 +49,8 @@ permissions: pull-requests: write jobs: - create-runs: - name: Create platform runs - runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe - timeout-minutes: 10 - # One (empty) run per mode, which every provider job then joins via - # `--run-id` below, registering itself and claiming its own worker — so the - # platform can rank the providers against each other. Best-effort: if this - # job can't create a run, the provider jobs fall back to opening their own. - continue-on-error: true - outputs: - sequential: ${{ steps.create.outputs.sequential }} - staggered: ${{ steps.create.outputs.staggered }} - burst: ${{ steps.create.outputs.burst }} - steps: - - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: 'pnpm' - - name: Install dependencies - run: pnpm install --frozen-lockfile - - name: Create runs - id: create - run: | - . benchmarks/scripts/load-vault-secrets.sh '^(BENCHMARKS_PLATFORM_API_KEY)$' - - # `bench create` only talks to the platform — no provider credentials — - # and prints the run id alone on stdout (diagnostics go to stderr). - CLI=(npx tsx packages/benchsdk-runner/dist/bin.js) - MODE="${{ github.event.inputs.mode }}" - - # No size here: the run takes its total from the providers that - # register into it, so this job needs to know nothing about the matrix. - open_run() { - "${CLI[@]}" create benchmark "$1" --name "$2" --kind sandbox >/dev/null - "${CLI[@]}" create run --benchmark "$1" - } - - if [ -z "$MODE" ] || [ "$MODE" = sequential ]; then - echo "sequential=$(open_run sandbox-tti-local 'Sandbox TTI (local)')" >> "$GITHUB_OUTPUT" - fi - if [ -z "$MODE" ] || [ "$MODE" = staggered ]; then - echo "staggered=$(open_run sandbox-staggered-local 'Sandbox staggered TTI (local)')" >> "$GITHUB_OUTPUT" - fi - if [ -z "$MODE" ] || [ "$MODE" = burst ]; then - echo "burst=$(open_run sandbox-burst-local 'Sandbox burst TTI (local)')" >> "$GITHUB_OUTPUT" - fi - bench: name: Bench ${{ matrix.provider }} - needs: create-runs - # Runs even if the run-creation job failed; see its `continue-on-error`. - if: always() runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe timeout-minutes: 60 env: @@ -168,32 +116,26 @@ jobs: run: | . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUNLOOP_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' - # One entrypoint, three launch shapes: the knobs come from the CLI and - # --benchmark picks which platform benchmark each reports to. --run-id - # joins the shared run for that mode (this job still registers itself - # and claims its own worker in it, bringing its own iteration count). + # One entrypoint, three launch shapes selected with --shape; the scale + # knobs come from the CLI. Every provider passes the same --run-key, so + # each shape's providers get-or-create one shared run and rank against + # each other (each still registers itself and claims its own worker). + # The shapes are distinct benchmark slugs, so one key value yields one + # run per shape; the run attempt is in the key so a workflow re-run + # opens fresh runs instead of rejoining the completed ones. BENCH=(npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts) + RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" COMMON=( --provider ${{ matrix.provider }} --iterations ${{ github.event.inputs.iterations || (github.event_name == 'push' && '10') || '100' }} + --run-key "$RUN_KEY" ) CONCURRENCY=${{ github.event.inputs.concurrency || (github.event_name == 'push' && '10') || '100' }} # Sequential is one-at-a-time by definition, so it never gets # --concurrency (the config default pins it to 1). SEQUENTIAL=("${BENCH[@]}" "${COMMON[@]}") - STAGGERED=("${BENCH[@]}" "${COMMON[@]}" --concurrency "$CONCURRENCY" --stagger-delay-ms 200 \ - --benchmark sandbox-staggered-local --name 'Sandbox staggered TTI (local)') - BURST=("${BENCH[@]}" "${COMMON[@]}" --concurrency "$CONCURRENCY" \ - --benchmark sandbox-burst-local --name 'Sandbox burst TTI (local)') - if [ -n "${{ needs.create-runs.outputs.sequential }}" ]; then - SEQUENTIAL+=(--run-id "${{ needs.create-runs.outputs.sequential }}") - fi - if [ -n "${{ needs.create-runs.outputs.staggered }}" ]; then - STAGGERED+=(--run-id "${{ needs.create-runs.outputs.staggered }}") - fi - if [ -n "${{ needs.create-runs.outputs.burst }}" ]; then - BURST+=(--run-id "${{ needs.create-runs.outputs.burst }}") - fi + STAGGERED=("${BENCH[@]}" "${COMMON[@]}" --shape staggered --concurrency "$CONCURRENCY") + BURST=("${BENCH[@]}" "${COMMON[@]}" --shape burst --concurrency "$CONCURRENCY") case "${{ github.event.inputs.mode }}" in sequential) "${SEQUENTIAL[@]}" ;; staggered) "${STAGGERED[@]}" ;; diff --git a/benchmarks/sandbox/tti.bench.ts b/benchmarks/sandbox/tti.bench.ts index 2d2f5cc9..a1b9aa7f 100644 --- a/benchmarks/sandbox/tti.bench.ts +++ b/benchmarks/sandbox/tti.bench.ts @@ -3,23 +3,23 @@ * first command (`node -v`) succeeding, excluding destroy. Declarative — * exports `config` + `task`; `bench run` owns the entrypoint. * - * One workload, three launch shapes — the only thing that differs is the - * framework's own knobs, so they're CLI flags rather than separate files: - * sequential one at a time (the config default: concurrency 1) - * burst --concurrency N: all slots open at once, so this launches that - * many real sandboxes simultaneously — raise N deliberately - * staggered --concurrency N --stagger-delay-ms D: task i starts at i * D - * Each shape reports under its own platform benchmark via `--benchmark`. + * One workload, three launch shapes. Each shape is its own platform benchmark + * but the same task, so they're declared once in `shapes` and picked with + * `--shape`; the scale knobs (`--iterations`/`--concurrency`) stay on the CLI: + * sequential the base config (concurrency 1) — no `--shape` + * burst --concurrency N: all slots open at once, launching that many + * real sandboxes simultaneously — raise N deliberately + * staggered its 200ms delay is baked into the shape; task i starts at i * D * * bench run benchmarks/sandbox/tti.bench.ts --iterations 5 --provider e2b,modal - * bench run benchmarks/sandbox/tti.bench.ts --benchmark sandbox-burst-local --iterations 10 --concurrency 10 - * bench run benchmarks/sandbox/tti.bench.ts --benchmark sandbox-staggered-local --iterations 10 --concurrency 10 --stagger-delay-ms 200 + * bench run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 10 --concurrency 10 + * bench run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 10 --concurrency 10 * - * To rank providers against each other, open one run and have every provider - * join it (each still claims its own worker): + * To rank providers against each other, run each provider with the same + * `--run-key`: they get-or-create one shared run and each claims its own worker. * - * ID=$(bench create run --benchmark sandbox-burst-local --iterations 100) - * bench run benchmarks/sandbox/tti.bench.ts --benchmark sandbox-burst-local --provider e2b --run-id "$ID" + * bench run benchmarks/sandbox/tti.bench.ts --shape burst --provider e2b --run-key "$GITHUB_RUN_ID" + * bench run benchmarks/sandbox/tti.bench.ts --shape burst --provider modal --run-key "$GITHUB_RUN_ID" */ import '../src/env.js'; import path from 'node:path'; @@ -58,6 +58,13 @@ export const config = defineBenchmarkConfig({ benchmarkKind: 'sandbox', iterations: 2, concurrency: 1, + // The launch shapes: same task, distinct platform identities. `--shape burst` + // just swaps the slug/name — the caller brings `--concurrency`. Staggered + // also carries its defining 200ms delay so no caller has to remember it. + shapes: { + burst: { slug: 'sandbox-burst-local', name: 'Sandbox burst TTI (local)' }, + staggered: { slug: 'sandbox-staggered-local', name: 'Sandbox staggered TTI (local)', staggerDelayMs: 200 }, + }, participants: providers, onComplete: (outcome) => { const { resultsDir, mode } = legacyShape(outcome.config); diff --git a/package.json b/package.json index f0a5d010..e3d0b3f3 100644 --- a/package.json +++ b/package.json @@ -5,29 +5,29 @@ "type": "module", "scripts": { "typecheck": "tsc --noEmit", - "bench": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3", + "bench": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3", "bench:sequential": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts", - "bench:staggered": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200", - "bench:burst": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3", + "bench:staggered": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3", + "bench:burst": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3", "bench:sandbox:dax": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/dax.bench.ts", - "bench:blaxel": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider blaxel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider blaxel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider blaxel", - "bench:codesandbox": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider codesandbox && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider codesandbox && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider codesandbox", - "bench:daytona": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider daytona && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider daytona && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider daytona", - "bench:e2b": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider e2b && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider e2b && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider e2b", - "bench:hopx": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider hopx && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider hopx && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider hopx", - "bench:isorun": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider isorun && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider isorun && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider isorun", - "bench:modal": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider modal && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider modal && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider modal", - "bench:namespace": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider namespace && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider namespace && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider namespace", - "bench:northflank": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider northflank && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider northflank && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider northflank", - "bench:railway": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider railway && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider railway && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider railway", - "bench:render": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider render && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider render && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider render", - "bench:runloop": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider runloop && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider runloop && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider runloop", - "bench:vercel": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider vercel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider vercel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider vercel", - "bench:just-bash": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider just-bash && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider just-bash && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider just-bash", - "bench:sprites": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider sprites && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider sprites && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider sprites", - "bench:sandbox0": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider sandbox0 && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider sandbox0 && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider sandbox0", - "bench:opencomputer": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider opencomputer && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider opencomputer && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider opencomputer", - "bench:tensorlake": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider tensorlake && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider tensorlake && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider tensorlake", + "bench:blaxel": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider blaxel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider blaxel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider blaxel", + "bench:codesandbox": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider codesandbox && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider codesandbox && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider codesandbox", + "bench:daytona": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider daytona && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider daytona && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider daytona", + "bench:e2b": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider e2b && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider e2b && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider e2b", + "bench:hopx": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider hopx && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider hopx && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider hopx", + "bench:isorun": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider isorun && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider isorun && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider isorun", + "bench:modal": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider modal && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider modal && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider modal", + "bench:namespace": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider namespace && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider namespace && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider namespace", + "bench:northflank": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider northflank && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider northflank && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider northflank", + "bench:railway": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider railway && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider railway && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider railway", + "bench:render": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider render && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider render && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider render", + "bench:runloop": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider runloop && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider runloop && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider runloop", + "bench:vercel": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider vercel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider vercel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider vercel", + "bench:just-bash": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider just-bash && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider just-bash && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider just-bash", + "bench:sprites": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider sprites && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider sprites && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider sprites", + "bench:sandbox0": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider sandbox0 && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider sandbox0 && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider sandbox0", + "bench:opencomputer": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider opencomputer && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider opencomputer && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider opencomputer", + "bench:tensorlake": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider tensorlake && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider tensorlake && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider tensorlake", "bench:browser": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/browser.bench.ts", "bench:browser:browserbase": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/browser.bench.ts --provider browserbase", "bench:browser:hyperbrowser": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/browser.bench.ts --provider hyperbrowser", diff --git a/packages/benchsdk-runner/src/__tests__/bench-config.test.ts b/packages/benchsdk-runner/src/__tests__/bench-config.test.ts index d390074b..da7df4c4 100644 --- a/packages/benchsdk-runner/src/__tests__/bench-config.test.ts +++ b/packages/benchsdk-runner/src/__tests__/bench-config.test.ts @@ -85,6 +85,31 @@ describe('defineBenchmarkConfig', () => { defineBenchmarkConfig({ benchmarkSlug: 's', benchmarkName: 'n', participants, phases: [{ name: 'cold', iterations: 1 }, { name: 'cold', iterations: 1 }] }), ).toThrow('duplicate phase name'); }); + + it('accepts a valid shapes map', () => { + const config = defineBenchmarkConfig({ + benchmarkSlug: 's', + benchmarkName: 'n', + participants, + shapes: { + burst: { slug: 'sandbox-burst-local', name: 'Burst' }, + staggered: { slug: 'sandbox-staggered-local', staggerDelayMs: 200 }, + }, + }); + expect(config.shapes?.burst.slug).toBe('sandbox-burst-local'); + }); + + it('rejects a shape without a lowercase slug', () => { + expect(() => + defineBenchmarkConfig({ benchmarkSlug: 's', benchmarkName: 'n', participants, shapes: { burst: { slug: 'Burst' } } }), + ).toThrow('burst'); + }); + + it('rejects a shape with a negative staggerDelayMs', () => { + expect(() => + defineBenchmarkConfig({ benchmarkSlug: 's', benchmarkName: 'n', participants, shapes: { s: { slug: 'ok', staggerDelayMs: -1 } } }), + ).toThrow('staggerDelayMs'); + }); }); describe('defineTask', () => { diff --git a/packages/benchsdk-runner/src/__tests__/cli.test.ts b/packages/benchsdk-runner/src/__tests__/cli.test.ts index 52ba6b06..922a9fcf 100644 --- a/packages/benchsdk-runner/src/__tests__/cli.test.ts +++ b/packages/benchsdk-runner/src/__tests__/cli.test.ts @@ -5,13 +5,9 @@ import { NoAvailableParticipantsError } from '../no-available-participants.js'; const fixture = (name: string) => `src/__tests__/fixtures/${name}`; describe('runBenchmarkFile', () => { - it('rejects an unknown verb, and an unknown noun after `create`', async () => { - await expect(runBenchmarkFile(['create', 'sandbox'])).rejects.toThrow(/Usage:/); - await expect(runBenchmarkFile(['create'])).rejects.toThrow(/Usage:/); - }); - - it('rejects `create run` with neither a file nor --benchmark', async () => { - await expect(runBenchmarkFile(['create', 'run'])).rejects.toThrow(/--benchmark/); + it('rejects retired imperative commands (there is no `create`)', async () => { + await expect(runBenchmarkFile(['create', 'benchmark', 'sandbox'])).rejects.toThrow(/Usage:/); + await expect(runBenchmarkFile(['create', 'run'])).rejects.toThrow(/Usage:/); }); it('rejects when the command is not `run`', async () => { @@ -19,8 +15,9 @@ describe('runBenchmarkFile', () => { await expect(runBenchmarkFile(['nope', fixture('good.bench.ts')])).rejects.toThrow(/Usage:/); }); - it('rejects when no file is given', async () => { + it('rejects when no file is given, or a flag stands where the file should', async () => { await expect(runBenchmarkFile(['run'])).rejects.toThrow(/Usage:/); + await expect(runBenchmarkFile(['run', '--shape', 'burst'])).rejects.toThrow(/Usage:/); }); it('rejects a module that does not export a config', async () => { diff --git a/packages/benchsdk-runner/src/__tests__/runner.test.ts b/packages/benchsdk-runner/src/__tests__/runner.test.ts index a69e6458..7edffce3 100644 --- a/packages/benchsdk-runner/src/__tests__/runner.test.ts +++ b/packages/benchsdk-runner/src/__tests__/runner.test.ts @@ -49,19 +49,19 @@ describe('parseCliArgs', () => { expect(parseCliArgs(['--group-by=participant'])).toEqual({ groupBy: 'participant' }); }); - it('parses --benchmark (and its --slug alias), --name and --kind', () => { + it('parses --benchmark (and its --slug alias) and --name', () => { expect(parseCliArgs(['--benchmark', 'sandbox-burst-local'])).toEqual({ benchmark: 'sandbox-burst-local' }); expect(parseCliArgs(['--benchmark=sandbox-tti-local'])).toEqual({ benchmark: 'sandbox-tti-local' }); expect(parseCliArgs(['--slug', 'sandbox-burst-local'])).toEqual({ benchmark: 'sandbox-burst-local' }); expect(parseCliArgs(['--name', 'Sandbox burst TTI'])).toEqual({ name: 'Sandbox burst TTI' }); - expect(parseCliArgs(['--kind', 'sandbox'])).toEqual({ kind: 'sandbox' }); expect(() => parseCliArgs(['--name', ' '])).toThrow('--name'); }); - it('parses --run-id', () => { - expect(parseCliArgs(['--run-id', 'run-1'])).toEqual({ runId: 'run-1' }); - expect(parseCliArgs(['--run-id=run-1'])).toEqual({ runId: 'run-1' }); - expect(() => parseCliArgs(['--run-id', ''])).toThrow('--run-id'); + it('parses --shape and --run-key', () => { + expect(parseCliArgs(['--shape', 'burst'])).toEqual({ shape: 'burst' }); + expect(parseCliArgs(['--run-key=ci-123'])).toEqual({ runKey: 'ci-123' }); + expect(() => parseCliArgs(['--shape', ''])).toThrow('--shape'); + expect(() => parseCliArgs(['--run-key', ''])).toThrow('--run-key'); }); it('throws on a non-slug --benchmark', () => { @@ -307,41 +307,29 @@ describe('runBenchmark', () => { expect(calls.createRun[0][0]).toBe('sandbox-burst-local'); }); - it('joins the run named by --run-id instead of creating one', async () => { + it('shares a run by --run-key: get-or-creates it keyed, then registers only its own participant', async () => { const config: BenchmarkConfig = { benchmarkSlug: 'sandbox-tti-local', benchmarkName: 'Sandbox TTI', - iterations: 1, - participants: [participants[0]], - }; - - const outcome = await runBenchmark(config, defineTask(async () => ({})), ['--run-id', 'run-shared']); - - expect(calls.upsertBenchmark).toHaveLength(0); - expect(calls.createRun).toHaveLength(0); - // The participant registers itself in the shared run, then plans and claims its own worker. - expect(calls.upsertParticipant[0].slice(0, 3)).toEqual(['sandbox-tti-local', 'run-shared', 'e2b']); - expect(calls.planWorkers[0].slice(0, 3)).toEqual(['sandbox-tti-local', 'run-shared', 'e2b']); - expect(calls.runWorker[0]).toMatchObject({ runId: 'run-shared' }); - expect(outcome.runId).toBe('run-shared'); - expect(outcome.dashboardUrl).toBeUndefined(); - }); - - it('takes its iteration count from a sized run, and rejects an --iterations that disagrees', async () => { - const config: BenchmarkConfig = { - benchmarkSlug: 'sandbox-tti-local', - benchmarkName: 'Sandbox TTI', - iterations: 1, + iterations: 2, participants: [participants[0]], }; - const outcome = await runBenchmark(config, defineTask(async () => ({})), ['--run-id', 'run-shared']); - expect(outcome.config.iterations).toBe(3); - expect(outcome.participants[0].records).toHaveLength(3); + const outcome = await runBenchmark(config, defineTask(async () => ({})), ['--run-key', 'ci-123', '--provider', 'e2b']); - await expect( - runBenchmark(config, defineTask(async () => ({})), ['--run-id', 'run-shared', '--iterations', '5']), - ).rejects.toThrow('disagrees with run run-shared'); + // The benchmark is still materialized from the file identity. + expect(calls.upsertBenchmark[0][0]).toBe('sandbox-tti-local'); + // One keyed get-or-create, carrying the key and no size/participant list. + expect(calls.createRun).toHaveLength(1); + expect(calls.createRun[0][1]).toMatchObject({ runKey: 'ci-123' }); + expect(calls.createRun[0][1].totalTasks).toBeUndefined(); + expect(calls.createRun[0][1].participants).toBeUndefined(); + // Registers only the provider it runs, sized to its own iteration count. + expect(calls.upsertParticipant[0].slice(0, 3)).toEqual(['sandbox-tti-local', 'run-1', 'e2b']); + expect(calls.upsertParticipant[0][3]).toMatchObject({ totalTasks: 2 }); + expect(calls.runWorker[0]).toMatchObject({ runId: 'run-1' }); + expect(outcome.runId).toBe('run-1'); + expect(outcome.participants[0].records).toHaveLength(2); }); it('does not rename a benchmark it was merely retargeted at', async () => { @@ -364,23 +352,37 @@ describe('runBenchmark', () => { expect(calls.upsertBenchmark[0]).toEqual(['sandbox-burst-local', { name: 'Sandbox burst TTI' }]); }); - it('brings its own iteration count to a participant-sized run', async () => { + it('selects a declared shape by --shape, reporting under its slug and name', async () => { const config: BenchmarkConfig = { benchmarkSlug: 'sandbox-tti-local', benchmarkName: 'Sandbox TTI', iterations: 1, participants: [participants[0]], + shapes: { + staggered: { slug: 'sandbox-staggered-local', name: 'Sandbox staggered TTI', staggerDelayMs: 200 }, + }, }; - const outcome = await runBenchmark(config, defineTask(async () => ({})), [ - '--run-id', - 'run-open', - '--iterations', - '4', - ]); + const outcome = await runBenchmark(config, defineTask(async () => ({})), ['--shape', 'staggered']); + + expect(calls.upsertBenchmark[0][0]).toBe('sandbox-staggered-local'); + expect(calls.upsertBenchmark[0][1]).toMatchObject({ name: 'Sandbox staggered TTI' }); + expect(calls.createRun[0][0]).toBe('sandbox-staggered-local'); + // The shape's stable knob applies; scale knobs stay defaulted/overridable. + expect(outcome.config.staggerDelayMs).toBe(200); + }); - expect(outcome.config.iterations).toBe(4); - expect(calls.upsertParticipant[0][3]).toMatchObject({ totalTasks: 4 }); + it('rejects an unknown --shape, listing the declared ones', async () => { + const config: BenchmarkConfig = { + benchmarkSlug: 'sandbox-tti-local', + benchmarkName: 'Sandbox TTI', + participants: [participants[0]], + shapes: { burst: { slug: 'sandbox-burst-local' } }, + }; + + await expect( + runBenchmark(config, defineTask(async () => ({})), ['--shape', 'nope']), + ).rejects.toThrow('Known shapes: burst'); }); it('throws NoAvailableParticipantsError, listing the skips, when no participant has its env vars set', async () => { diff --git a/packages/benchsdk-runner/src/bench-config.ts b/packages/benchsdk-runner/src/bench-config.ts index 74cc958e..c1952ab2 100644 --- a/packages/benchsdk-runner/src/bench-config.ts +++ b/packages/benchsdk-runner/src/bench-config.ts @@ -19,6 +19,10 @@ * burst { iterations: N, concurrency: N } * staggered { iterations: N, concurrency: N, staggerDelayMs: 200 } * + * A benchmark can name these variants up front via `shapes`, so one file backs + * several platform benchmarks (`bench run --shape burst`) without + * restating each one's slug/name in scripts and CI. + * * A task is comprised of steps, declared via `ctx.step` inside a task function * — it supports closures, conditionals and try/finally, so values (a created * sandbox, say) flow naturally between steps. A task that declares no steps is @@ -37,6 +41,25 @@ import type { /** How tasks are ordered across participants. */ export type GroupBy = 'participant' | 'round'; +/** + * A named variant of a benchmark, selected with `--shape `. A shape + * carries only the parts that make it a distinct *benchmark* — its platform + * identity plus any stable distinguishing knob (e.g. staggered's delay). The + * scale knobs that vary per environment (`--iterations`, `--concurrency`) stay + * on the invocation, so a shape never sets a value only to have the CLI + * override it. + */ +export interface BenchmarkShape { + /** Platform slug this shape reports under (e.g. 'sandbox-burst-local'). */ + slug: string; + /** Display name shown on the platform; defaults to the slug. */ + name?: string; + /** Benchmark kind; defaults to the config's `benchmarkKind`. */ + kind?: string; + /** Default stagger delay (ms) for this shape; overridable with `--stagger-delay-ms`. */ + staggerDelayMs?: number; +} + /** * What a task returns: whatever it measured itself. This replaces the * assumption that the framework owns all timing. A plain data payload is @@ -132,8 +155,8 @@ export interface ResolvedRunConfig { */ export interface BenchmarkRunOutcome { runId: string; - /** Absent when the process joined a run via `--run-id`: the org slug the URL needs comes from creating the run. */ - dashboardUrl?: string; + /** Link to this run on the platform dashboard. */ + dashboardUrl: string; participants: ParticipantRecords[]; config: ResolvedRunConfig; } @@ -147,14 +170,22 @@ export interface BenchmarkRunOutcome { export interface BenchmarkConfig { /** * Stable platform slug for this benchmark (e.g. 'sandbox-tti-local'). - * Overridable per run with `--slug`, so one entrypoint can report under - * several benchmarks. + * Selectable per run with `--shape` (or overridable with `--benchmark`), so + * one entrypoint can report under several benchmarks. */ benchmarkSlug: string; /** Human-readable name shown on the platform. Overridable with `--name`. */ benchmarkName: string; /** Optional platform benchmark kind (e.g. 'sandbox'). */ benchmarkKind?: string; + /** + * Named variants of this benchmark, selected with `--shape `. Each + * shape swaps in its own platform identity (and optional stable knob) while + * reusing the same task and participants, so one bench file can back several + * platform benchmarks without duplicating the slug/name triple across + * package scripts and CI. + */ + shapes?: Record; /** * Total tasks to run per participant. Default: 1. Mutually exclusive with * `phases` — when `phases` is set, total iterations = sum of phase iterations. @@ -235,6 +266,19 @@ export function defineBenchmarkConfig= 0 (got ${shape.staggerDelayMs})`); + } + } + } return config; } diff --git a/packages/benchsdk-runner/src/cli.ts b/packages/benchsdk-runner/src/cli.ts index b5151c89..71089261 100644 --- a/packages/benchsdk-runner/src/cli.ts +++ b/packages/benchsdk-runner/src/cli.ts @@ -1,36 +1,31 @@ /** - * The author-facing entrypoint. Verb first, then the noun it acts on: + * The author-facing entrypoint. `bench` is verbs-only — the benchmark and its + * runs are implicit, never nouns you type: * * bench run [--flags] execute a benchmark - * bench create benchmark declare a benchmark on the platform - * bench create run --benchmark open a run, printing its id * * `run` imports a benchmark module, reads its `config` and `task` exports and * drives `runBenchmark`; CLI flags override the config's knobs and - * `config.onComplete` (if any) fires once the run finishes. With `--run-id` it - * reports into a run opened by `create run` instead of opening its own, so one - * process per provider — in parallel, each claiming its own worker — still ends - * up in a single run the platform can rank them in. - * - * `create` only talks to the platform: no bench file to load (it's just a - * source of defaults) and no provider credentials. + * `config.onComplete` (if any) fires once the run finishes. The benchmark is + * declared in the file (`--shape` picks a named variant) and materialized on + * run; a run is opened as a side effect, shared across sibling processes when + * they pass the same `--run-key`. There are no imperative `create` commands. * * The executable wrapper lives in `bin.ts`; this module has no side effects so * it can be unit-tested by calling `runBenchmarkFile` directly. */ import { resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; -import { createBenchmark, createRun, mergeConfig, parseCliArgs, runBenchmark } from './runner.js'; +import { parseCliArgs, runBenchmark } from './runner.js'; import { NoAvailableParticipantsError } from './no-available-participants.js'; import type { BaseParticipant } from '@benchsdk/client'; import type { BenchmarkConfig, BenchmarkTask } from './bench-config.js'; const USAGE = 'Usage:\n' + - ' bench run [--benchmark slug] [--provider a,b] [--run-id id]\n' + - ' [--iterations N] [--concurrency N] [--stagger-delay-ms N] [--group-by participant|round]\n' + - " bench create benchmark [--name 'My benchmark'] [--kind sandbox]\n" + - ' bench create run [file.bench.ts] [--benchmark slug] [--iterations N]'; + ' bench run [--shape name] [--provider a,b] [--run-key key]\n' + + ' [--benchmark slug] [--name "My benchmark"]\n' + + ' [--iterations N] [--concurrency N] [--stagger-delay-ms N] [--group-by participant|round]'; /** A benchmark module is expected to export `config` and `task`. */ interface BenchmarkModule { @@ -45,48 +40,6 @@ function isBenchmarkConfig(value: unknown): value is BenchmarkConfig { return typeof candidate.benchmarkSlug === 'string' && Array.isArray(candidate.participants); } -async function loadConfig(file: string): Promise { - const mod = (await import(pathToFileURL(resolve(process.cwd(), file)).href)) as BenchmarkModule; - if (!isBenchmarkConfig(mod.config)) { - throw new Error(`${file} must export a \`config\` created with defineBenchmarkConfig (with participants).`); - } - return mod.config; -} - -/** `bench create ...`. Prints created run ids on stdout, alone, so callers can capture them. */ -async function create(argv: string[]): Promise { - const [noun, ...rest] = argv; - - if (noun === 'benchmark') { - const [slug] = rest; - if (!slug || slug.startsWith('-')) throw new Error(USAGE); - const args = parseCliArgs(rest); - await createBenchmark(slug, { name: args.name, kind: args.kind }); - console.error(`Benchmark ready: ${slug}`); - return; - } - - if (noun === 'run') { - // The file, when given, only supplies defaults for the flags below. - const file = rest[0] && !rest[0].startsWith('-') ? rest[0] : undefined; - const args = parseCliArgs(file ? rest.slice(1) : rest); - const fileConfig = file ? await loadConfig(file) : undefined; - const benchmarkSlug = args.benchmark ?? fileConfig?.benchmarkSlug; - if (!benchmarkSlug) throw new Error('bench create run needs --benchmark (or a file to read it from).'); - // No size at all is fine: the run then takes its total from the - // participants that register, which is what lets one run be opened before - // anyone knows which providers will join or how much each will do. - const iterations = args.iterations ?? (fileConfig ? mergeConfig(fileConfig, args).iterations : undefined); - - const { runId, dashboardUrl } = await createRun({ benchmarkSlug, iterations }); - console.error(`Run created: ${runId}\nView at: ${dashboardUrl}`); - console.log(runId); - return; - } - - throw new Error(USAGE); -} - /** * Dispatches one CLI invocation. Throws on bad usage / invalid exports and lets * `NoAvailableParticipantsError` propagate so the caller can map it to a clean @@ -95,10 +48,8 @@ async function create(argv: string[]): Promise { export async function runBenchmarkFile(argv: string[]): Promise { const [command, ...rest] = argv; - if (command === 'create') return create(rest); - const [file, ...flags] = rest; - if (command !== 'run' || !file) throw new Error(USAGE); + if (command !== 'run' || !file || file.startsWith('-')) throw new Error(USAGE); const mod = (await import(pathToFileURL(resolve(process.cwd(), file)).href)) as BenchmarkModule; const config = mod.config; diff --git a/packages/benchsdk-runner/src/runner.ts b/packages/benchsdk-runner/src/runner.ts index a737e3b2..950188cf 100644 --- a/packages/benchsdk-runner/src/runner.ts +++ b/packages/benchsdk-runner/src/runner.ts @@ -33,6 +33,7 @@ import { TaskError } from './bench-config.js'; import type { BenchmarkConfig, BenchmarkRunOutcome, + BenchmarkShape, BenchmarkTask, GroupBy, ParticipantRecords, @@ -46,9 +47,13 @@ export interface CliArgs { /** Which platform benchmark to report as (`--benchmark`, aka the benchmark slug). */ benchmark?: string; name?: string; - kind?: string; - /** Join this existing platform run instead of creating one (`--run-id`). */ - runId?: string; + /** Named variant from the bench file's `shapes` (`--shape`), swapping in its identity. */ + shape?: string; + /** + * Idempotency key (`--run-key`): sibling processes passing the same key share + * one run (get-or-created), instead of each opening its own. + */ + runKey?: string; iterations?: number; concurrency?: number; staggerDelayMs?: number; @@ -119,17 +124,17 @@ export function parseCliArgs(argv: string[]): CliArgs { i = nextIndex; break; } - case '--kind': { + case '--shape': { const { value, nextIndex } = readValue(arg, i); - if (value.trim() === '') throw new Error('--kind expects a value'); - args.kind = value; + if (value.trim() === '') throw new Error('--shape expects a value'); + args.shape = value; i = nextIndex; break; } - case '--run-id': { + case '--run-key': { const { value, nextIndex } = readValue(arg, i); - if (value.trim() === '') throw new Error('--run-id expects a value'); - args.runId = value; + if (value.trim() === '') throw new Error('--run-key expects a value'); + args.runKey = value; i = nextIndex; break; } @@ -252,6 +257,48 @@ function resolvePlatform(): { baseUrl: string; apiKey: string } { }; } +/** + * Resolves `--shape ` against the config's declared `shapes`. Throws with + * the known names if the shape is unknown, so a typo fails loudly instead of + * silently running the base benchmark. + */ +function resolveShape( + config: BenchmarkConfig, + shapeName: string | undefined, +): BenchmarkShape | undefined { + if (!shapeName) return undefined; + const shape = config.shapes?.[shapeName]; + if (!shape) { + const known = Object.keys(config.shapes ?? {}); + throw new Error( + known.length > 0 + ? `Unknown --shape "${shapeName}". Known shapes: ${known.join(', ')}.` + : `Unknown --shape "${shapeName}": this benchmark declares no shapes.`, + ); + } + return shape; +} + +/** + * Swaps a shape's identity (and its stable knob) into the config. Only the + * parts that make it a distinct benchmark move here; scale knobs stay on the + * CLI, so `mergeConfig` still lets `--concurrency`/`--iterations` win. + */ +function applyShape( + config: BenchmarkConfig, + shape: BenchmarkShape | undefined, +): BenchmarkConfig { + if (!shape) return config; + const kind = shape.kind ?? config.benchmarkKind; + return { + ...config, + benchmarkSlug: shape.slug, + benchmarkName: shape.name ?? shape.slug, + ...(kind ? { benchmarkKind: kind } : {}), + ...(shape.staggerDelayMs !== undefined ? { staggerDelayMs: shape.staggerDelayMs } : {}), + }; +} + /** Applies the `--benchmark`/`--name` overrides, so one entrypoint can report under several benchmarks. */ function applyIdentityOverrides( fileConfig: BenchmarkConfig, @@ -278,50 +325,14 @@ function resolveParticipants(config: BenchmarkConfig< return available; } -/** `bench create benchmark `: the only object with a user-chosen identity, so it's created explicitly. */ -export async function createBenchmark(slug: string, options: { name?: string; kind?: string } = {}): Promise { - const { baseUrl, apiKey } = resolvePlatform(); - const client = createBenchmarkClient({ baseUrl, apiKey }); - await client.upsertBenchmark(slug, { - name: options.name ?? slug, - ...(options.kind ? { kind: options.kind } : {}), - }); -} - /** - * `bench create run --benchmark `: opens an empty run and returns its id, - * so several `bench run --run-id` processes report into one comparable run - * while staying independent (a provider per CI job). A pure platform call — no - * bench file, no provider credentials. - * - * Registers no participants: each joining process upserts its own, so the run - * lists exactly the providers being benchmarked rather than everything the - * bench file could reach. - */ -export async function createRun(options: { - benchmarkSlug: string; - iterations?: number; -}): Promise<{ runId: string; dashboardUrl: string }> { - const { baseUrl, apiKey } = resolvePlatform(); - const client = createBenchmarkClient({ baseUrl, apiKey }); - // Without `iterations` the run is participant-sized: it takes its total from - // the participants that register, so opening a run needs nothing but a name. - const { run, organizationSlug } = await client.createRun(options.benchmarkSlug, { - name: options.iterations - ? `${options.benchmarkSlug} — ${options.iterations} iterations` - : options.benchmarkSlug, - ...(options.iterations ? { totalTasks: options.iterations, workerCount: 1 } : {}), - }); - return { runId: run.id, dashboardUrl: dashboardUrlFor(baseUrl, organizationSlug, options.benchmarkSlug, run.id) }; -} - -/** - * Runs `config`'s `task` against `participants`. Selects participants by + * Runs `config`'s `task` against its participants. Selects participants by * `--provider` (if given), env-gates them, then drives them per the resolved - * `groupBy`. `--slug`/`--name` retarget the whole run at a different platform - * benchmark, so one entrypoint can report under several slugs; `--run-id` - * joins a run created by `bench create-run` instead of opening a new one, so - * sibling processes (e.g. one CI job per provider) land in the same run. + * `groupBy`. `--shape` swaps in a declared variant's identity; `--benchmark`/ + * `--name` retarget the run at a different platform benchmark, so one entrypoint + * can report under several slugs. With `--run-key`, sibling processes (e.g. one + * CI job per provider) get-or-create one shared run and each registers only its + * own participants. */ export async function runBenchmark( fileConfig: BenchmarkConfig, @@ -329,33 +340,14 @@ export async function runBenchmark( argv: string[] = [], ): Promise { const args = parseCliArgs(argv); - const config = applyIdentityOverrides(fileConfig, args); - let resolved = mergeConfig(config, args); + const shaped = applyShape(fileConfig, resolveShape(fileConfig, args.shape)); + const config = applyIdentityOverrides(shaped, args); + const resolved = mergeConfig(config, args); const available = resolveParticipants(config, resolved); const { baseUrl, apiKey } = resolvePlatform(); const client = createBenchmarkClient({ baseUrl, apiKey }); - // A run that declared a size is the single source of truth for it, so sibling - // processes can't disagree about the run they share. A participant-sized run - // declared none — each joiner brings its own iteration count. - if (args.runId) { - const run = await client.getRun(config.benchmarkSlug, args.runId); - if (!run.participantSized) { - if (args.iterations !== undefined && args.iterations !== run.totalTasks) { - throw new Error( - `--iterations ${args.iterations} disagrees with run ${args.runId}, which was created with ${run.totalTasks}.`, - ); - } - if (config.phases?.length && run.totalTasks !== resolved.iterations) { - throw new Error( - `Run ${args.runId} has ${run.totalTasks} tasks but this benchmark's phases total ${resolved.iterations}.`, - ); - } - resolved = { ...resolved, iterations: run.totalTasks }; - } - } - const schedule = buildSchedule(config, resolved.iterations, task); const totalTasks = schedule.length; @@ -367,26 +359,41 @@ export async function runBenchmark( `staggerDelayMs=${resolved.staggerDelayMs}, groupBy=${resolved.groupBy}\n`, ); + // Declaratively materialize the benchmark from the file/shape identity, which + // is authoritative (its name lives in the file). A bare `--benchmark X` only + // *retargets* reporting at a benchmark this file doesn't name, so we don't + // upsert it — that would rename it to the file's own name. + const identityIsOurs = + args.shape !== undefined || + args.name !== undefined || + !args.benchmark || + args.benchmark === fileConfig.benchmarkSlug; + if (identityIsOurs) { + await client.upsertBenchmark(config.benchmarkSlug, { + name: config.benchmarkName, + ...(config.benchmarkKind ? { kind: config.benchmarkKind } : {}), + }); + } + let runId: string; - let dashboardUrl: string | undefined; - if (args.runId) { - runId = args.runId; - // The run was opened empty by `create-run`; register the participants this - // process is responsible for before planning their workers. + let dashboardUrl: string; + if (args.runKey) { + // Shared run: get-or-created by key, so sibling processes (one per provider) + // converge on one run. Opened participant-sized — register only the + // providers this process runs and let each sibling register its own, so the + // run lists exactly who's benchmarked and each brings its own task count. + const { run, organizationSlug } = await client.createRun(config.benchmarkSlug, { + name: config.benchmarkName, + runKey: args.runKey, + }); + runId = run.id; + dashboardUrl = dashboardUrlFor(baseUrl, organizationSlug, config.benchmarkSlug, run.id); for (const participant of available) { await client.upsertParticipant(config.benchmarkSlug, runId, participant.name, { totalTasks }); } - console.log(`Joining run: ${runId}\n`); + console.log(`Shared run (key "${args.runKey}"): ${runId}`); + console.log(`View at: ${dashboardUrl}\n`); } else { - // Retargeted at a benchmark this file doesn't name, so its identity isn't - // ours to write: upserting would rename it to the file's own name. It has - // to exist already (`bench create benchmark`), and run creation says so. - if (!args.benchmark || args.benchmark === fileConfig.benchmarkSlug || args.name) { - await client.upsertBenchmark(config.benchmarkSlug, { - name: config.benchmarkName, - ...(config.benchmarkKind ? { kind: config.benchmarkKind } : {}), - }); - } const { run, organizationSlug } = await client.createRun(config.benchmarkSlug, { name: `${config.benchmarkSlug} — ${totalTasks} iterations, concurrency ${resolved.concurrency}`, totalTasks, @@ -408,7 +415,7 @@ export async function runBenchmark( participantRecords = await runGroupedByParticipant(config, schedule, available, resolved, client, runId, onResult); } - console.log(dashboardUrl ? `All done. View at: ${dashboardUrl}` : `All done. Run: ${runId}`); + console.log(`All done. View at: ${dashboardUrl}`); const outcome: BenchmarkRunOutcome = { runId, dashboardUrl, diff --git a/packages/benchsdk/src/types.ts b/packages/benchsdk/src/types.ts index 81dac11f..df9b639c 100644 --- a/packages/benchsdk/src/types.ts +++ b/packages/benchsdk/src/types.ts @@ -28,6 +28,8 @@ export interface BenchmarkRun { benchmarkId: string; name?: string | null; status: BenchmarkRunStatus | string; + /** Idempotency key: runs created with the same key (per org + benchmark) are the same run. */ + runKey?: string | null; totalTasks: number; /** The run declared no size: `totalTasks` is the sum of what its participants declare. */ participantSized?: boolean; @@ -118,6 +120,11 @@ export interface UpdateBenchmarkInput { export interface CreateRunInput { name?: string; + /** + * Idempotency key for get-or-create: sibling callers passing the same key + * (per org + benchmark) converge on one run instead of each opening its own. + */ + runKey?: string; /** Omit to open a participant-sized run: each participant declares its own size when it registers. */ totalTasks?: number; workerCount?: number;