diff --git a/benchmarks/ai-gateway/ai-gateway.bench.ts b/benchmarks/ai-gateway/ai-gateway.bench.ts index ac2160b3..b504abe7 100644 --- a/benchmarks/ai-gateway/ai-gateway.bench.ts +++ b/benchmarks/ai-gateway/ai-gateway.bench.ts @@ -1,11 +1,12 @@ /** - * AI Gateway benchmark, built on @benchsdk/runner's runBenchmark with - * `groupBy: 'round'`. Fairness is the whole point (see AI_GATEWAYS.md): every - * gateway's Nth iteration must run at roughly the same point in time as every - * other gateway's Nth iteration, so no gateway is favored by running during a - * different network condition. `groupBy: 'round'` provides exactly that — one - * task per gateway per round, taking turns — replacing the bespoke round-robin - * loop this file used to hand-roll. + * AI Gateway benchmark. Declarative — exports `config` + `task`; `bench run` + * owns the entrypoint. Configured with `groupBy: 'round'`. Fairness is the + * whole point (see AI_GATEWAYS.md): every gateway's Nth iteration must run at + * roughly the same point in time as every other gateway's Nth iteration, so no + * gateway is favored by running during a different network condition. + * `groupBy: 'round'` provides exactly that — one task per gateway per round, + * taking turns — replacing the bespoke round-robin loop this file used to + * hand-roll. * * Declared as two phases (cold, warm); the framework owns the phase boundary * and exposes it via `ctx.phase`, so the task branches on phase identity, not diff --git a/benchmarks/browser/browser-task.ts b/benchmarks/browser/browser-task.ts deleted file mode 100644 index 442db779..00000000 --- a/benchmarks/browser/browser-task.ts +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Shared browser-lifecycle workload for the browser benchmark. One iteration = - * create a session, connect over CDP, navigate to example.com, then release - * (always, even on failure). Orchestration (how many, how parallel) is owned by - * @benchsdk/runner's runBenchmark — this file only describes what one iteration - * does. - */ -import { chromium } from 'playwright-core'; -import type { BenchmarkTask, TaskContext, TaskResult } from '@benchsdk/runner'; -import { TaskError } from '@benchsdk/runner'; -import { withTimeout } from '../src/util/timeout.js'; -import type { BrowserProviderConfig } from './types.js'; - -/** - * Build a browser task that caches one provider instance per participant name - * (the legacy benchmark created the provider once per provider). - */ -export function makeBrowserTask(): BenchmarkTask { - const providerCache = new Map(); - - return async function browserTask(ctx: TaskContext): Promise { - const { participant, step } = ctx; - const timeout = participant.timeout ?? 120_000; - const sessionCreateOptions = participant.sessionCreateOptions ?? {}; - - let provider = providerCache.get(participant.name); - if (!provider) { - provider = participant.createBrowserProvider(); - providerCache.set(participant.name, provider); - } - - const timings = { createMs: 0, connectMs: 0, navigateMs: 0, releaseMs: 0, totalMs: 0 }; - const totalStart = performance.now(); - let session: { sessionId: string; connectUrl: string } | undefined; - let browser: any; - - try { - const createStart = performance.now(); - session = (await step('create', () => - withTimeout(provider.session.create(sessionCreateOptions), timeout, 'Session creation timed out'), - )) as { sessionId: string; connectUrl: string }; - timings.createMs = performance.now() - createStart; - - try { - const connectStart = performance.now(); - const page = await step('connect', async () => { - browser = await withTimeout( - chromium.connectOverCDP(session!.connectUrl), - 30_000, - 'CDP connection timed out', - ); - const [context] = browser.contexts(); - if (!context) throw new Error('No default browser context found'); - const [p] = context.pages(); - if (!p) throw new Error('No default page found'); - return p; - }); - timings.connectMs = performance.now() - connectStart; - - const navStart = performance.now(); - await step('navigate', () => - withTimeout( - page.goto('https://www.example.com', { waitUntil: 'load' }), - 30_000, - 'Navigation timed out', - ), - ); - timings.navigateMs = performance.now() - navStart; - } finally { - if (browser) await browser.close().catch(() => {}); - const releaseStart = performance.now(); - await step( - 'release', - () => withTimeout(provider.session.destroy(session!.sessionId), 15_000, 'Session destroy timed out'), - { reportConcurrency: false }, - ); - timings.releaseMs = performance.now() - releaseStart; - } - - timings.totalMs = performance.now() - totalStart; - return { data: { ...timings } }; - } catch (err) { - timings.totalMs = performance.now() - totalStart; - const message = err instanceof Error ? err.message : String(err); - throw new TaskError(message, { code: 'BROWSER_ERROR', data: { ...timings, errorMessage: message } }); - } - }; -} diff --git a/benchmarks/browser/browser-throughput-task.ts b/benchmarks/browser/browser-throughput-task.ts deleted file mode 100644 index c6d99cf7..00000000 --- a/benchmarks/browser/browser-throughput-task.ts +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Shared throughput workload for the browser-throughput benchmark. One task = - * one session running the fixed 10-action loop (navigate, waitForSelector, - * screenshot, textContent, click, goBack, ...). Reuses the legacy - * `runThroughputIteration` verbatim so per-action timings/results are identical; - * this file only adapts that into the CLI's task/TaskResult shape. - * - * Runs under `groupBy: 'round'` so every provider runs its Nth session against - * the same article URL (via navUrlForIteration(taskIndex)) before anyone starts - * their (N+1)th — matching the legacy interleaved throughput benchmark. - */ -import type { BenchmarkTask, TaskContext, TaskResult } from '@benchsdk/runner'; -import { TaskError } from '@benchsdk/runner'; -import type { JsonValue } from '@benchsdk/client'; -import { navUrlForIteration, runThroughputIteration } from './throughput-benchmark.js'; -import type { ThroughputProviderConfig } from './throughput-types.js'; - -/** Build a throughput task caching one provider instance per participant name. */ -export function makeThroughputTask(): BenchmarkTask { - const providerCache = new Map(); - - return async function throughputTask(ctx: TaskContext): Promise { - const { participant, taskIndex } = ctx; - const timeout = participant.timeout ?? 120_000; - const sessionCreateOptions = participant.sessionCreateOptions ?? {}; - - let provider = providerCache.get(participant.name); - if (!provider) { - provider = participant.createBrowserProvider(); - providerCache.set(participant.name, provider); - } - - const navigateUrl = navUrlForIteration(taskIndex); - // runThroughputIteration wraps the session lifecycle in real `ctx.step` - // calls (create/connect/actions/release) and reports throughput via - // `ctx.measure` inside the `actions` step — no fabricated step records. - const result = await runThroughputIteration(provider, timeout, sessionCreateOptions, navigateUrl, ctx); - - const data = { - createMs: result.createMs, - connectMs: result.connectMs, - releaseMs: result.releaseMs, - totalMs: result.totalMs, - taskMs: result.taskMs, - actionsCompleted: result.actionsCompleted, - actionsPerSecond: result.actionsPerSecond, - actions: result.actions as unknown as JsonValue, - ...(result.error ? { errorMessage: result.error } : {}), - }; - - // A session-level failure (create/connect/etc.) is a task error. Partial - // action completion is NOT an error — it's a success record with - // actionsCompleted < ACTIONS_PER_SESSION, exactly as the legacy runner did. - if (result.error) { - throw new TaskError(result.error, { code: 'THROUGHPUT_ERROR', data }); - } - - // Steps (create/connect/actions/release) and the throughput metric on - // `actions` are recorded by the framework via the `ctx.step`/`ctx.measure` - // calls inside runThroughputIteration; the task just returns task-level data. - return { data }; - }; -} diff --git a/benchmarks/browser/browser-throughput.bench.ts b/benchmarks/browser/browser-throughput.bench.ts index 40c52259..ed105bab 100644 --- a/benchmarks/browser/browser-throughput.bench.ts +++ b/benchmarks/browser/browser-throughput.bench.ts @@ -4,16 +4,25 @@ * Nth session against the same article before anyone starts their (N+1)th. * Declarative — exports `config` + `task`; `bench run` owns the entrypoint. * + * The session lifecycle (create / connect / actions / release) runs as real + * framework steps via `ctx.step`, so the platform records each phase's true + * timing and status. `actionsPerSecond`/`actionsCompleted` are non-latency + * throughput metrics, so they're reported with `ctx.measure` inside `actions`. + * * bench run benchmarks/browser/browser-throughput.bench.ts * bench run benchmarks/browser/browser-throughput.bench.ts --iterations 5 --provider browserbase */ import '../src/env.js'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { defineBenchmarkConfig, defineTask } from '@benchsdk/runner'; +import { chromium, type Browser, type Page } from 'playwright-core'; +import { defineBenchmarkConfig, defineTask, TaskError } from '@benchsdk/runner'; +import type { JsonValue } from '@benchsdk/client'; +import { withTimeout } from '../src/util/timeout.js'; import { throughputProviders } from './throughput-providers.js'; -import { makeThroughputTask } from './browser-throughput-task.js'; import { writeThroughputLegacyResults } from './browser-throughput-legacy-results.js'; +import { ACTIONS_PER_LOOP, LOOPS_PER_SESSION } from './throughput-types.js'; +import type { ActionResult, ThroughputProviderConfig } from './throughput-types.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -34,4 +43,278 @@ export const config = defineBenchmarkConfig({ }), }); -export const task = defineTask(makeThroughputTask()); +const RANDOM_URL = 'https://en.wikipedia.org/wiki/Special:Random'; +const FIRST_HEADING = '#firstHeading'; +const ACTION_TIMEOUT_MS = 30_000; + +// Match article-body links across classic MediaWiki HTML (relative "/wiki/Foo"), +// Parsoid read-HTML (protocol-relative "//en.wikipedia.org/wiki/Foo"), and the +// newer absolute form ("https://en.wikipedia.org/wiki/Foo"). +// We can't use :not([href*=":"]) because that also rejects https: URLs. Instead +// we select all /wiki/ links in the content area and filter in JS by checking +// that the path segment after /wiki/ has no namespace colon (Help:, File:, etc.). +const ARTICLE_LINK_SELECTOR = '#mw-content-text a[href*="/wiki/"]'; + +// Providers must be benchmarked against the same pages to be comparable, but in +// CI each provider runs as a separate job/process — so `Special:Random` makes +// each one draw different articles. The workflow resolves one concrete article +// URL per iteration up front and injects the list via THROUGHPUT_URLS (a JSON +// array). Iteration i then navigates to urls[i] for *every* provider: the same +// page across providers, but a different page each iteration so we don't measure +// a warmed cache. Absent the env var (local runs), navigation stays random. +const NAV_URLS: string[] = parseNavUrls(); + +function parseNavUrls(): string[] { + const raw = process.env.THROUGHPUT_URLS?.trim(); + if (!raw) return []; + // Accept either a JSON array or a plain newline/whitespace-delimited list, so + // the workflow can emit the URLs with pure bash and not depend on jq/python + // being present on the runner image. + if (raw.startsWith('[')) { + try { + const parsed = JSON.parse(raw); + if (Array.isArray(parsed)) return parsed.filter((u): u is string => typeof u === 'string' && u.length > 0); + } catch { + // Malformed JSON falls through to whitespace splitting below. + } + } + return raw.split(/\s+/).filter(u => u.length > 0); +} + +/** The article URL iteration `i` should navigate to (same across providers). */ +function navUrlForIteration(i: number): string { + if (NAV_URLS.length > 0) return NAV_URLS[i % NAV_URLS.length]; + return RANDOM_URL; +} + +/** Returns true if an href points to a real article (not a namespace page). */ +function isArticleLink(href: string | null): boolean { + if (!href) return false; + const match = href.match(/\/wiki\/([^#]*)/); + if (!match) return false; + return !match[1].includes(':'); +} + +async function timeAction( + fn: () => Promise, +): Promise<{ durationMs: number; success: boolean; error?: string; value?: T }> { + const start = performance.now(); + try { + const value = await withTimeout(fn(), ACTION_TIMEOUT_MS, 'Action timed out'); + return { durationMs: performance.now() - start, success: true, value }; + } catch (err) { + const error = err instanceof Error ? err.message : String(err); + return { durationMs: performance.now() - start, success: false, error }; + } +} + +async function runActionLoop(page: Page, results: ActionResult[], navigateUrl: string): Promise { + for (let loop = 0; loop < LOOPS_PER_SESSION; loop++) { + const baseIdx = loop * ACTIONS_PER_LOOP; + + // 1. Navigate to the article for this iteration (shared across providers) + { + const r = await timeAction(() => + page.goto(navigateUrl, { waitUntil: 'load' }) as Promise, + ); + results.push({ index: baseIdx + 1, type: 'navigate', durationMs: r.durationMs, success: r.success, error: r.error }); + } + + // 2. Wait for #firstHeading + { + const r = await timeAction(() => page.waitForSelector(FIRST_HEADING)); + results.push({ index: baseIdx + 2, type: 'waitForSelector', durationMs: r.durationMs, success: r.success, error: r.error }); + } + + // 3. Screenshot + { + const r = await timeAction(() => page.screenshot()); + results.push({ index: baseIdx + 3, type: 'screenshot', durationMs: r.durationMs, success: r.success, error: r.error }); + } + + // 4. Read text content of #firstHeading + { + const r = await timeAction(() => page.textContent(FIRST_HEADING)); + results.push({ index: baseIdx + 4, type: 'textContent', durationMs: r.durationMs, success: r.success, error: r.error }); + } + + // 5. Click first article link (filter out meta pages like Help:, File:, etc.) + let clickSucceeded = false; + { + const r = await timeAction(async () => { + // Wait for any /wiki/ link to appear, then filter in JS for article + // links (no namespace colon after /wiki/). This handles relative, + // protocol-relative, and absolute https:// URL formats. + await page.waitForSelector(ARTICLE_LINK_SELECTOR, { timeout: 10_000 }); + const links = await page.$$(ARTICLE_LINK_SELECTOR); + for (const link of links) { + const href = await link.getAttribute('href'); + if (isArticleLink(href)) { + await link.click(); + return; + } + } + throw new Error('No article body link found on page'); + }); + clickSucceeded = r.success; + results.push({ index: baseIdx + 5, type: 'click', durationMs: r.durationMs, success: r.success, error: r.error }); + } + + // If the click failed (e.g. stub article with no body links), skip the + // remaining actions that depend on having navigated to a new page. + // Without this, goBack navigates to a blank page and the final + // waitForSelector times out for 30s, inflating task time. + if (!clickSucceeded) { + for (const idx of [6, 7, 8, 9, 10]) { + results.push({ index: baseIdx + idx, type: idx <= 8 ? (idx === 6 || idx === 10 ? 'waitForSelector' : idx === 7 ? 'screenshot' : 'textContent') : 'goBack', durationMs: 0, success: false, error: 'skipped: click failed' }); + } + continue; + } + + // 6. Wait for #firstHeading on the new page + { + const r = await timeAction(() => page.waitForSelector(FIRST_HEADING)); + results.push({ index: baseIdx + 6, type: 'waitForSelector', durationMs: r.durationMs, success: r.success, error: r.error }); + } + + // 7. Screenshot the new page + { + const r = await timeAction(() => page.screenshot()); + results.push({ index: baseIdx + 7, type: 'screenshot', durationMs: r.durationMs, success: r.success, error: r.error }); + } + + // 8. Read text content of #firstHeading on the new page + { + const r = await timeAction(() => page.textContent(FIRST_HEADING)); + results.push({ index: baseIdx + 8, type: 'textContent', durationMs: r.durationMs, success: r.success, error: r.error }); + } + + // 9. Go back. Use `waitUntil: 'commit'` because back-forward cache restores + // fire `pageshow` instead of `load`, and Playwright's default + // `waitUntil: 'load'` hangs for the full timeout on a bfcache restore. + // Resolving on commit returns as soon as the navigation lands; the next + // waitForSelector confirms #firstHeading is present. + { + const r = await timeAction(() => page.goBack({ waitUntil: 'commit' }) as Promise); + results.push({ index: baseIdx + 9, type: 'goBack', durationMs: r.durationMs, success: r.success, error: r.error }); + } + + // 10. Wait for #firstHeading on the previous page + { + const r = await timeAction(() => page.waitForSelector(FIRST_HEADING)); + results.push({ index: baseIdx + 10, type: 'waitForSelector', durationMs: r.durationMs, success: r.success, error: r.error }); + } + } +} + +/** One provider instance per participant, reused across that provider's sessions. */ +const providerCache = new Map(); + +export const task = defineTask(async (ctx) => { + const { participant, taskIndex, step, measure } = ctx; + const timeout = participant.timeout ?? 120_000; + const sessionCreateOptions = participant.sessionCreateOptions ?? {}; + + let provider = providerCache.get(participant.name); + if (!provider) { + provider = participant.createBrowserProvider(); + providerCache.set(participant.name, provider); + } + + const navigateUrl = navUrlForIteration(taskIndex); + const totalStart = performance.now(); + const actions: ActionResult[] = []; + let createMs = 0; + let connectMs = 0; + let releaseMs = 0; + let taskMs = 0; + let actionsCompleted = 0; + let actionsPerSecond = 0; + + let session: { sessionId: string; connectUrl: string } | undefined; + let browser: Browser | undefined; + let iterationError: string | undefined; + + try { + const createStart = performance.now(); + session = await step('create', () => + withTimeout( + provider.session.create(sessionCreateOptions), + timeout, + 'Session creation timed out', + ), + ) as { sessionId: string; connectUrl: string }; + createMs = performance.now() - createStart; + + const connectStart = performance.now(); + const page = await step('connect', async () => { + browser = await withTimeout( + chromium.connectOverCDP(session!.connectUrl), + 30_000, + 'CDP connection timed out', + ); + const [context] = browser.contexts(); + if (!context) throw new Error('No default browser context found'); + const [existingPage] = context.pages(); + return existingPage ?? (await context.newPage()); + }); + connectMs = performance.now() - connectStart; + + // Individual action failures are recorded but do not abort the session. + // Throughput is a rate the step latency can't express, so report it as step + // data via `measure`. + await step('actions', async () => { + await runActionLoop(page, actions, navigateUrl); + taskMs = actions.reduce((sum, a) => sum + a.durationMs, 0); + actionsCompleted = actions.filter((a) => a.success).length; + actionsPerSecond = taskMs > 0 ? actionsCompleted / (taskMs / 1000) : 0; + measure({ actionsPerSecond, actionsCompleted }); + }); + } catch (err) { + iterationError = err instanceof Error ? err.message : String(err); + } finally { + if (browser) { + await browser.close().catch(() => { }); + } + if (session) { + const releaseStart = performance.now(); + try { + await step( + 'release', + () => + withTimeout( + provider.session.destroy(session!.sessionId), + 15_000, + 'Session destroy timed out', + ), + { reportConcurrency: false }, + ); + } catch { + // Swallow release errors — they're recorded via releaseMs but should + // not mask the more important action timings. + } + releaseMs = performance.now() - releaseStart; + } + } + + const data = { + createMs, + connectMs, + releaseMs, + totalMs: performance.now() - totalStart, + taskMs, + actionsCompleted, + actionsPerSecond, + actions: actions as unknown as JsonValue, + ...(iterationError ? { errorMessage: iterationError } : {}), + }; + + // A session-level failure (create/connect/etc.) is a task error. Partial + // action completion is NOT an error — it's a success record with + // actionsCompleted < ACTIONS_PER_SESSION. + if (iterationError) { + throw new TaskError(iterationError, { code: 'THROUGHPUT_ERROR', data }); + } + + return { data }; +}); diff --git a/benchmarks/browser/browser.bench.ts b/benchmarks/browser/browser.bench.ts index eb093600..f4e49f85 100644 --- a/benchmarks/browser/browser.bench.ts +++ b/benchmarks/browser/browser.bench.ts @@ -9,9 +9,11 @@ import '../src/env.js'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { defineBenchmarkConfig, defineTask } from '@benchsdk/runner'; +import { chromium } from 'playwright-core'; +import { defineBenchmarkConfig, defineTask, TaskError } from '@benchsdk/runner'; +import { withTimeout } from '../src/util/timeout.js'; import { browserProviders } from './providers.js'; -import { makeBrowserTask } from './browser-task.js'; +import type { BrowserProviderConfig } from './types.js'; import { writeBrowserLegacyResults } from './legacy-results.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -32,4 +34,73 @@ export const config = defineBenchmarkConfig({ }), }); -export const task = defineTask(makeBrowserTask()); +/** One provider instance per participant, reused across that provider's iterations. */ +const providerCache = new Map(); + +export const task = defineTask(async (ctx) => { + const { participant, step } = ctx; + const timeout = participant.timeout ?? 120_000; + const sessionCreateOptions = participant.sessionCreateOptions ?? {}; + + let provider = providerCache.get(participant.name); + if (!provider) { + provider = participant.createBrowserProvider(); + providerCache.set(participant.name, provider); + } + + const timings = { createMs: 0, connectMs: 0, navigateMs: 0, releaseMs: 0, totalMs: 0 }; + const totalStart = performance.now(); + let session: { sessionId: string; connectUrl: string } | undefined; + let browser: any; + + try { + const createStart = performance.now(); + session = (await step('create', () => + withTimeout(provider.session.create(sessionCreateOptions), timeout, 'Session creation timed out'), + )) as { sessionId: string; connectUrl: string }; + timings.createMs = performance.now() - createStart; + + try { + const connectStart = performance.now(); + const page = await step('connect', async () => { + browser = await withTimeout( + chromium.connectOverCDP(session!.connectUrl), + 30_000, + 'CDP connection timed out', + ); + const [context] = browser.contexts(); + if (!context) throw new Error('No default browser context found'); + const [p] = context.pages(); + if (!p) throw new Error('No default page found'); + return p; + }); + timings.connectMs = performance.now() - connectStart; + + const navStart = performance.now(); + await step('navigate', () => + withTimeout( + page.goto('https://www.example.com', { waitUntil: 'load' }), + 30_000, + 'Navigation timed out', + ), + ); + timings.navigateMs = performance.now() - navStart; + } finally { + if (browser) await browser.close().catch(() => {}); + const releaseStart = performance.now(); + await step( + 'release', + () => withTimeout(provider.session.destroy(session!.sessionId), 15_000, 'Session destroy timed out'), + { reportConcurrency: false }, + ); + timings.releaseMs = performance.now() - releaseStart; + } + + timings.totalMs = performance.now() - totalStart; + return { data: { ...timings } }; + } catch (err) { + timings.totalMs = performance.now() - totalStart; + const message = err instanceof Error ? err.message : String(err); + throw new TaskError(message, { code: 'BROWSER_ERROR', data: { ...timings, errorMessage: message } }); + } +}); diff --git a/benchmarks/browser/throughput-benchmark.ts b/benchmarks/browser/throughput-benchmark.ts index d5f54afd..92e44baf 100644 --- a/benchmarks/browser/throughput-benchmark.ts +++ b/benchmarks/browser/throughput-benchmark.ts @@ -1,72 +1,18 @@ -import { chromium, type Browser, type Page } from 'playwright-core'; -import type { TaskContext } from '@benchsdk/runner'; -import { withTimeout } from '../src/util/timeout.js'; +/** + * Throughput result summarization + legacy JSON writer. The workload itself + * lives in browser-throughput.bench.ts; this module only turns collected + * iterations into the `results/browser-throughput/` JSON shape. + */ import { ACTION_TYPES, - ACTIONS_PER_LOOP, ACTIONS_PER_SESSION, - LOOPS_PER_SESSION, - type ActionResult, type ActionType, type ThroughputBenchmarkResult, - type ThroughputProviderConfig, type ThroughputStats, type ThroughputStatsTriple, type ThroughputTimingResult, } from './throughput-types.js'; -const RANDOM_URL = 'https://en.wikipedia.org/wiki/Special:Random'; -const FIRST_HEADING = '#firstHeading'; - -// Providers must be benchmarked against the same pages to be comparable, but in -// CI each provider runs as a separate job/process — so `Special:Random` makes -// each one draw different articles. The workflow resolves one concrete article -// URL per iteration up front and injects the list via THROUGHPUT_URLS (a JSON -// array). Iteration i then navigates to urls[i] for *every* provider: the same -// page across providers, but a different page each iteration so we don't measure -// a warmed cache. Absent the env var (local runs), navigation stays random. -const NAV_URLS: string[] = parseNavUrls(); - -function parseNavUrls(): string[] { - const raw = process.env.THROUGHPUT_URLS?.trim(); - if (!raw) return []; - // Accept either a JSON array or a plain newline/whitespace-delimited list, so - // the workflow can emit the URLs with pure bash and not depend on jq/python - // being present on the runner image. - if (raw.startsWith('[')) { - try { - const parsed = JSON.parse(raw); - if (Array.isArray(parsed)) return parsed.filter((u): u is string => typeof u === 'string' && u.length > 0); - } catch { - // Malformed JSON falls through to whitespace splitting below. - } - } - return raw.split(/\s+/).filter(u => u.length > 0); -} - -/** The article URL iteration `i` should navigate to (same across providers). */ -export function navUrlForIteration(i: number): string { - if (NAV_URLS.length > 0) return NAV_URLS[i % NAV_URLS.length]; - return RANDOM_URL; -} -// Match article-body links across classic MediaWiki HTML (relative "/wiki/Foo"), -// Parsoid read-HTML (protocol-relative "//en.wikipedia.org/wiki/Foo"), and the -// newer absolute form ("https://en.wikipedia.org/wiki/Foo"). -// We can't use :not([href*=":"]) because that also rejects https: URLs. Instead -// we select all /wiki/ links in the content area and filter in JS by checking -// that the path segment after /wiki/ has no namespace colon (Help:, File:, etc.). -const ARTICLE_LINK_SELECTOR = '#mw-content-text a[href*="/wiki/"]'; - -/** Returns true if an href points to a real article (not a namespace page). */ -function isArticleLink(href: string | null): boolean { - if (!href) return false; - const match = href.match(/\/wiki\/([^#]*)/); - if (!match) return false; - return !match[1].includes(':'); -} - -const ACTION_TIMEOUT_MS = 30_000; - function round(n: number): number { return Math.round(n * 100) / 100; } @@ -99,224 +45,6 @@ function computeStats(values: number[]): ThroughputStatsTriple { }; } -async function timeAction( - fn: () => Promise, -): Promise<{ durationMs: number; success: boolean; error?: string; value?: T }> { - const start = performance.now(); - try { - const value = await withTimeout(fn(), ACTION_TIMEOUT_MS, 'Action timed out'); - return { durationMs: performance.now() - start, success: true, value }; - } catch (err) { - const error = err instanceof Error ? err.message : String(err); - return { durationMs: performance.now() - start, success: false, error }; - } -} - -async function runActionLoop(page: Page, results: ActionResult[], navigateUrl: string): Promise { - for (let loop = 0; loop < LOOPS_PER_SESSION; loop++) { - const baseIdx = loop * ACTIONS_PER_LOOP; - - // 1. Navigate to the article for this iteration (shared across providers) - { - const r = await timeAction(() => - page.goto(navigateUrl, { waitUntil: 'load' }) as Promise, - ); - results.push({ index: baseIdx + 1, type: 'navigate', durationMs: r.durationMs, success: r.success, error: r.error }); - } - - // 2. Wait for #firstHeading - { - const r = await timeAction(() => page.waitForSelector(FIRST_HEADING)); - results.push({ index: baseIdx + 2, type: 'waitForSelector', durationMs: r.durationMs, success: r.success, error: r.error }); - } - - // 3. Screenshot - { - const r = await timeAction(() => page.screenshot()); - results.push({ index: baseIdx + 3, type: 'screenshot', durationMs: r.durationMs, success: r.success, error: r.error }); - } - - // 4. Read text content of #firstHeading - { - const r = await timeAction(() => page.textContent(FIRST_HEADING)); - results.push({ index: baseIdx + 4, type: 'textContent', durationMs: r.durationMs, success: r.success, error: r.error }); - } - - // 5. Click first article link (filter out meta pages like Help:, File:, etc.) - let clickSucceeded = false; - { - const r = await timeAction(async () => { - // Wait for any /wiki/ link to appear, then filter in JS for article - // links (no namespace colon after /wiki/). This handles relative, - // protocol-relative, and absolute https:// URL formats. - await page.waitForSelector(ARTICLE_LINK_SELECTOR, { timeout: 10_000 }); - const links = await page.$$(ARTICLE_LINK_SELECTOR); - for (const link of links) { - const href = await link.getAttribute('href'); - if (isArticleLink(href)) { - await link.click(); - return; - } - } - throw new Error('No article body link found on page'); - }); - clickSucceeded = r.success; - results.push({ index: baseIdx + 5, type: 'click', durationMs: r.durationMs, success: r.success, error: r.error }); - } - - // If the click failed (e.g. stub article with no body links), skip the - // remaining actions that depend on having navigated to a new page. - // Without this, goBack navigates to a blank page and the final - // waitForSelector times out for 30s, inflating task time. - if (!clickSucceeded) { - for (const idx of [6, 7, 8, 9, 10]) { - results.push({ index: baseIdx + idx, type: idx <= 8 ? (idx === 6 || idx === 10 ? 'waitForSelector' : idx === 7 ? 'screenshot' : 'textContent') : 'goBack', durationMs: 0, success: false, error: 'skipped: click failed' }); - } - continue; - } - - // 6. Wait for #firstHeading on the new page - { - const r = await timeAction(() => page.waitForSelector(FIRST_HEADING)); - results.push({ index: baseIdx + 6, type: 'waitForSelector', durationMs: r.durationMs, success: r.success, error: r.error }); - } - - // 7. Screenshot the new page - { - const r = await timeAction(() => page.screenshot()); - results.push({ index: baseIdx + 7, type: 'screenshot', durationMs: r.durationMs, success: r.success, error: r.error }); - } - - // 8. Read text content of #firstHeading on the new page - { - const r = await timeAction(() => page.textContent(FIRST_HEADING)); - results.push({ index: baseIdx + 8, type: 'textContent', durationMs: r.durationMs, success: r.success, error: r.error }); - } - - // 9. Go back. Use `waitUntil: 'commit'` because back-forward cache restores - // fire `pageshow` instead of `load`, and Playwright's default - // `waitUntil: 'load'` hangs for the full timeout on a bfcache restore. - // Resolving on commit returns as soon as the navigation lands; the next - // waitForSelector confirms #firstHeading is present. - { - const r = await timeAction(() => page.goBack({ waitUntil: 'commit' }) as Promise); - results.push({ index: baseIdx + 9, type: 'goBack', durationMs: r.durationMs, success: r.success, error: r.error }); - } - - // 10. Wait for #firstHeading on the previous page - { - const r = await timeAction(() => page.waitForSelector(FIRST_HEADING)); - results.push({ index: baseIdx + 10, type: 'waitForSelector', durationMs: r.durationMs, success: r.success, error: r.error }); - } - } -} - -// The session lifecycle (create / connect / actions / release) runs as real -// framework steps via `ctx.step`, so the platform records each phase's true -// timing and status rather than us fabricating step records after the fact. -// `actionsPerSecond`/`actionsCompleted` are non-latency throughput metrics, so -// they're reported with `ctx.measure` inside the `actions` step. -type StepCtx = Pick, 'step' | 'measure'>; - -export async function runThroughputIteration( - provider: any, - timeout: number, - sessionCreateOptions: Record, - navigateUrl: string, - { step, measure }: StepCtx, -): Promise { - const totalStart = performance.now(); - const actions: ActionResult[] = []; - let createMs = 0; - let connectMs = 0; - let releaseMs = 0; - let taskMs = 0; - let actionsCompleted = 0; - let actionsPerSecond = 0; - - let session: { sessionId: string; connectUrl: string } | undefined; - let browser: Browser | undefined; - let iterationError: string | undefined; - - try { - // 1. Create session - const createStart = performance.now(); - session = await step('create', () => - withTimeout( - provider.session.create(sessionCreateOptions), - timeout, - 'Session creation timed out', - ), - ) as { sessionId: string; connectUrl: string }; - createMs = performance.now() - createStart; - - // 2. Connect over CDP - const connectStart = performance.now(); - const page = await step('connect', async () => { - browser = await withTimeout( - chromium.connectOverCDP(session!.connectUrl), - 30_000, - 'CDP connection timed out', - ); - const [context] = browser.contexts(); - if (!context) throw new Error('No default browser context found'); - const [existingPage] = context.pages(); - return existingPage ?? (await context.newPage()); - }); - connectMs = performance.now() - connectStart; - - // 3. Run the 10-action loop. Individual action failures are recorded but - // do not abort the session. Throughput is a rate the step latency can't - // express, so report it as step data via `measure`. - await step('actions', async () => { - await runActionLoop(page, actions, navigateUrl); - taskMs = actions.reduce((sum, a) => sum + a.durationMs, 0); - actionsCompleted = actions.filter((a) => a.success).length; - actionsPerSecond = taskMs > 0 ? actionsCompleted / (taskMs / 1000) : 0; - measure({ actionsPerSecond, actionsCompleted }); - }); - } catch (err) { - iterationError = err instanceof Error ? err.message : String(err); - } finally { - if (browser) { - await browser.close().catch(() => { }); - } - if (session) { - const releaseStart = performance.now(); - try { - await step( - 'release', - () => - withTimeout( - provider.session.destroy(session!.sessionId), - 15_000, - 'Session destroy timed out', - ), - { reportConcurrency: false }, - ); - } catch { - // Swallow release errors — they're recorded via releaseMs but should - // not mask the more important action timings. - } - releaseMs = performance.now() - releaseStart; - } - } - - const totalMs = performance.now() - totalStart; - - return { - createMs, - connectMs, - actions, - releaseMs, - totalMs, - taskMs, - actionsCompleted, - actionsPerSecond, - ...(iterationError ? { error: iterationError } : {}), - }; -} - export function summarizeIterations(iterations: ThroughputTimingResult[]): ThroughputStats { const createValues = iterations.map(i => i.createMs).filter(v => v > 0); const taskValues = iterations.map(i => i.taskMs).filter(v => v > 0); diff --git a/benchmarks/sandbox/dax.bench.ts b/benchmarks/sandbox/dax.bench.ts index 6f57e46a..9fcdace5 100644 --- a/benchmarks/sandbox/dax.bench.ts +++ b/benchmarks/sandbox/dax.bench.ts @@ -1,11 +1,11 @@ /** * Dax benchmark: runs the OpenCode build (scripts/dax-benchmark.sh) inside a * freshly created sandbox once per iteration and measures its build phases - * (prepare / bun-download / bun-unpack / clone / install / typecheck). Config - * lives here; platform orchestration is owned by @benchsdk/runner's runBenchmark. - * The phase-parsing + resource-sizing logic is reused from ./dax.ts so the - * legacy `results/sandbox-dax/` JSON shape is preserved verbatim via the - * legacy-results adapter (TEMPORARY local-JSON bridge — see legacy-results). + * (prepare / bun-download / bun-unpack / clone / install / typecheck). + * Declarative — exports `config` + `task`; `bench run` owns the entrypoint and + * platform orchestration. The legacy `results/sandbox-dax/` JSON shape is + * preserved verbatim via the legacy-results adapter (TEMPORARY local-JSON + * bridge — see legacy-results); its summarizer/writer lives in ./dax.ts. * * `groupBy: 'round'` is used so the task owns its own measured `latencyMs` * (the build's totalMs) and can attach pre-measured phase steps; the local @@ -16,16 +16,17 @@ * bench run benchmarks/sandbox/dax.bench.ts --provider e2b,modal */ import '../src/env.js'; +import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { defineBenchmarkConfig, defineTask, TaskError } from '@benchsdk/runner'; -import type { TaskContext, TaskResult } from '@benchsdk/runner'; import type { JsonObject, TaskStepRecord } from '@benchsdk/client'; +import { VMTier } from '@codesandbox/sdk'; import { withTimeout } from '../src/util/timeout.js'; import { formatError } from '../src/util/error.js'; import { providers } from './providers.js'; import type { ProviderConfig } from './types.js'; -import { getSandboxOptionsWithResources, runDaxIteration } from './dax.js'; +import { BENCH_SCRIPT_PATH } from './dax.js'; import type { DaxTimingResult } from './dax.js'; import { writeDaxLegacyResults } from './dax-legacy-results.js'; @@ -34,6 +35,164 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url)); const timeout = 600_000; const destroyTimeoutMs = 15_000; +export const config = defineBenchmarkConfig({ + benchmarkSlug: 'sandbox-dax-local', + benchmarkName: 'Dax sandbox benchmark (local)', + benchmarkKind: 'sandbox', + iterations: 3, + concurrency: 1, + groupBy: 'round', + defaultProviders: ['e2b', 'modal', 'tensorlake'], + participants: providers, + onComplete: (outcome) => + writeDaxLegacyResults(outcome.participants, { + resultsDir: path.resolve(__dirname, '../../results/sandbox-dax'), + }), +}); + +// Standardized resource sizing for fair comparison across providers. +// Target: 8 vCPU, 16 GiB RAM. +// Each provider uses different parameter names and units, so we map per-provider. +// Providers not listed here don't support CPU/memory configuration at sandbox creation time. +// Note: E2B sets CPU/memory at template build time, not at sandbox creation. +// Note: lightning is sized via LIGHTNING_INSTANCE_TYPE=cpu-8 (8 vCPU / 16 GiB), applied on +// the provider factory in providers.ts — the SDK ignores an instanceType passed to create(). +const DAX_RESOURCE_OPTIONS: Record> = { + modal: { cpu: 4, cpuLimit: 4, memoryMiB: 16384 }, // Modal: 1 core = 2 vCPUs, so 4 cores = 8 vCPUs + tenki: { cpuCores: 8, memoryMb: 16384, diskSizeGb: 20 }, // default disk cannot hold the OpenCode install + tensorlake: { cpus: 8, memoryMb: 16384 }, + isorun: { vcpus: 8, memMiB: 16384 }, + runloop: { launch_parameters: { resource_size_request: 'CUSTOM_SIZE', custom_cpu_cores: 8, custom_gb_memory: 16 } }, + upstash: { size: 'large' }, // large = 8 cores, 16 GB + vercel: { resources: { vcpus: 8 } }, // no memory control + blaxel: { memory: 16384 }, // CPU derived: cores = memory_MB / 2048 = 8 + beam: { cpu: 8, memory: 16384 }, // cpu = cores, memory = MiB + codesandbox: { vmTier: VMTier.Small }, // Small = 8 CPU, 16 GiB + daytona: { resources: { cpu: 8, memory: 16 } }, // memory in GiB; requires image-based creation (see providers.ts) + northflank: { deploymentPlan: process.env.NORTHFLANK_DEPLOYMENT_PLAN || 'nf-compute-50', ephemeralStorageSize: 5120 }, // 5 GiB ephemeral storage + declaw: { templateId: 'node-large' }, // node-large template: 8 vCPU / 16 GiB RAM / 8 GiB disk + superserve: { templateId: 'node22-8cpu-16gb' }, // 8 vCPU / 16 GiB template built in the pre-step + createos: { shape: 's-8vcpu-16gb', ephemeralDiskMb: 61440 }, // 8 vCPU, 16 GiB RAM, 60 GiB disk + opencomputer: { cpuCount: 4, memoryMB: 16384, timeout: 600_000 }, + sandbox0: { memory: 16384 }, // Sandbox0 exposes only `memory` +}; + +function getSandboxOptionsWithResources(providerName: string, baseOptions?: Record): Record { + const resourceOpts = DAX_RESOURCE_OPTIONS[providerName]; + if (!resourceOpts) return baseOptions ?? {}; + return { ...baseOptions, ...resourceOpts }; +} + +/** Runs the build script inside `sandbox` and parses its structured output. */ +async function runDaxBuild(sandbox: any, providerName: string, buildTimeout: number): Promise { + // Load the benchmark script from the local filesystem rather than fetching + // it over HTTP inside the sandbox. This eliminates a curl dependency + // (several providers don't ship curl in their sandboxes). + const benchScript = fs.readFileSync(BENCH_SCRIPT_PATH, 'utf8'); + + // Write the benchmark script to /tmp inside the sandbox via a single-quoted + // heredoc (so $ and backticks in the script are not expanded) and execute it + // directly with bash. A random marker avoids collisions with anything + // appearing on its own line inside the script. Running the benchmark script + // directly (without a Node.js wrapper) lets providers that ship a different + // Node.js version pre-installed (e.g. Vercel) reuse their own binary. + const marker = '__DAX_BENCH_HEREDOC_' + Math.random().toString(36).slice(2) + '__'; + const shellCmd = + `cat > /tmp/dax-benchmark.sh <<'${marker}'\n` + + benchScript + + `\n${marker}\n` + + `BENCH_PROVIDER=${providerName} BENCH_REGION=unknown bash /tmp/dax-benchmark.sh`; + + const totalStart = Date.now(); + const result = await withTimeout( + sandbox.runCommand(shellCmd, { timeout: buildTimeout }), + buildTimeout, + 'Dax benchmark timed out', + ) as { exitCode: number; stdout?: string; stderr?: string }; + const totalMs = Date.now() - totalStart; + + const stdout = result.stdout || ''; + const stderr = result.stderr || ''; + const exitCode = result.exitCode; + + // Parse structured output lines emitted by the benchmark script. + const phases: Record = {}; + const meta: Record = {}; + const disk: Record = {}; + let benchError: string | null = null; + let doneCommit: string | null = null; + + for (const line of stdout.split('\n')) { + if (line.startsWith('BENCH_PHASE\t')) { + const parts = line.split('\t'); + if (parts.length >= 3) phases[parts[1]] = parseInt(parts[2], 10); + } else if (line.startsWith('BENCH_META\t')) { + const parts = line.split('\t'); + if (parts.length >= 3) meta[parts[1]] = parts[2]; + } else if (line.startsWith('BENCH_DISK\t')) { + const parts = line.split('\t'); + if (parts.length >= 3) disk[parts[1]] = parseInt(parts[2], 10); + } else if (line.startsWith('BENCH_DONE\t')) { + const parts = line.split('\t'); + if (parts.length >= 2) doneCommit = parts[1]; + } + } + + for (const line of stderr.split('\n')) { + if (line.startsWith('BENCH_ERROR\t')) { + const parts = line.split('\t'); + benchError = parts.slice(1).join(': '); + } + } + + if (exitCode !== 0 && !benchError) { + // Include last few lines of stderr for diagnostics + const tail = stderr.trim().split('\n').slice(-3).join(' | '); + benchError = 'Script exited with code ' + exitCode + (tail ? ': ' + tail : ''); + } + + // Count completed phases + const phaseKeys = ['prepare', 'cache_clear', 'bun_download', 'bun_unpack', 'clone', 'install', 'typecheck']; + const rawPhasesCompleted = phaseKeys.filter(k => phases[k] !== undefined).length; + // The script's phase() function emits BENCH_PHASE even for the failing phase (it prints timing before checking exit code). + // When there's an error, the last phase that emitted a BENCH_PHASE line is the one that failed, so don't count it. + const phasesCompleted = benchError ? Math.max(0, rawPhasesCompleted - 1) : rawPhasesCompleted; + // Determine which phase failed so we can exclude its timing from the result. + // The failed phase is the last one that emitted BENCH_PHASE (index rawPhasesCompleted - 1). + const failedPhaseKey = benchError && rawPhasesCompleted > 0 ? phaseKeys[rawPhasesCompleted - 1] : null; + + // If no phases completed, the script didn't actually run (e.g. heredoc failure) + if (phasesCompleted === 0 && !benchError) { + const tail = stderr.trim().split('\n').slice(-2).join(' | '); + benchError = 'No benchmark phases completed' + (tail ? ': ' + tail : ''); + } + + return { + totalMs, + phasesCompleted, + phasesTotal: phaseKeys.length, + prepareMs: failedPhaseKey === 'prepare' ? undefined : phases.prepare, + cacheClearMs: failedPhaseKey === 'cache_clear' ? undefined : phases.cache_clear, + bunDownloadMs: failedPhaseKey === 'bun_download' ? undefined : phases.bun_download, + bunUnpackMs: failedPhaseKey === 'bun_unpack' ? undefined : phases.bun_unpack, + cloneMs: failedPhaseKey === 'clone' ? undefined : phases.clone, + installMs: failedPhaseKey === 'install' ? undefined : phases.install, + typecheckMs: failedPhaseKey === 'typecheck' ? undefined : phases.typecheck, + diskAfterClone: disk.after_clone, + diskAfterInstall: disk.after_install, + diskAfterTypecheck: disk.after_typecheck, + commit: doneCommit || meta.commit, + bunVersion: meta.bun_version, + nodeVersion: meta.node_version, + architecture: meta.architecture, + kernel: meta.kernel, + logicalCpus: meta.logical_cpus, + cpuModel: meta.cpu_model, + memoryKib: meta.memory_kib, + ...(benchError ? { error: benchError } : {}), + }; +} + /** Emit each measured build phase as a pre-measured platform step. */ function daxPhaseSteps(t: DaxTimingResult): TaskStepRecord[] { const phases: [string, number | undefined][] = [ @@ -50,7 +209,7 @@ function daxPhaseSteps(t: DaxTimingResult): TaskStepRecord[] { .map(([name, latencyMs]) => ({ name, status: 'success', latencyMs })); } -async function daxTask(ctx: TaskContext): Promise { +export const task = defineTask(async (ctx) => { const p = ctx.participant; const compute = p.createCompute(); const opts = getSandboxOptionsWithResources(p.name, p.sandboxOptions); @@ -65,7 +224,7 @@ async function daxTask(ctx: TaskContext): Promise { let timing: DaxTimingResult; try { - timing = await ctx.step('build', () => runDaxIteration(sandbox, p.name, p.timeout ?? timeout)); + timing = await ctx.step('build', () => runDaxBuild(sandbox, p.name, p.timeout ?? timeout)); } finally { await ctx .step('destroy', () => withTimeout(sandbox.destroy(), p.destroyTimeoutMs ?? destroyTimeoutMs, 'Destroy timeout'), { @@ -81,21 +240,4 @@ async function daxTask(ctx: TaskContext): Promise { throw new TaskError(timing.error, { code: 'dax_failed', data, steps }); } return { data, steps, latencyMs: timing.totalMs }; -} - -export const config = defineBenchmarkConfig({ - benchmarkSlug: 'sandbox-dax-local', - benchmarkName: 'Dax sandbox benchmark (local)', - benchmarkKind: 'sandbox', - iterations: 3, - concurrency: 1, - groupBy: 'round', - defaultProviders: ['e2b', 'modal', 'tensorlake'], - participants: providers, - onComplete: (outcome) => - writeDaxLegacyResults(outcome.participants, { - resultsDir: path.resolve(__dirname, '../../results/sandbox-dax'), - }), }); - -export const task = defineTask(daxTask); diff --git a/benchmarks/sandbox/dax.ts b/benchmarks/sandbox/dax.ts index 2fa55bda..333304bd 100644 --- a/benchmarks/sandbox/dax.ts +++ b/benchmarks/sandbox/dax.ts @@ -1,49 +1,16 @@ +/** + * Dax result types, summarization, and the legacy `results/sandbox-dax/` JSON + * writer. The workload (build execution + phase parsing) lives in + * dax.bench.ts. + */ import fs from 'fs'; import os from 'os'; import path from 'path'; -import type { ProviderConfig, Stats } from './types.js'; +import type { Stats } from './types.js'; import { computeStats } from '../src/util/stats.js'; -import { withTimeout } from '../src/util/timeout.js'; -import { VMTier } from '@codesandbox/sdk'; -// The benchmark script is now loaded from the local filesystem (scripts/dax-benchmark.sh) -// rather than fetched over HTTP from upstream. This avoids a curl dependency inside the -// sandbox (some providers don't ship curl). The previous upstream URL was: -// https://raw.githubusercontent.com/anomalyco/opencode/provider-benchmark/script/provider-benchmark.sh -const BENCH_SCRIPT_PATH = path.resolve(import.meta.dirname, '../scripts/dax-benchmark.sh'); - -// Standardized resource sizing for fair comparison across providers. -// Target: 8 vCPU, 16 GiB RAM. -// Each provider uses different parameter names and units, so we map per-provider. -// Providers not listed here don't support CPU/memory configuration at sandbox creation time. -// Note: E2B sets CPU/memory at template build time, not at sandbox creation. -// Note: lightning is sized via LIGHTNING_INSTANCE_TYPE=cpu-8 (8 vCPU / 16 GiB), applied on -// the provider factory in providers.ts — the SDK ignores an instanceType passed to create(). -const DAX_RESOURCE_OPTIONS: Record> = { - modal: { cpu: 4, cpuLimit: 4, memoryMiB: 16384 }, // Modal: 1 core = 2 vCPUs, so 4 cores = 8 vCPUs - tenki: { cpuCores: 8, memoryMb: 16384, diskSizeGb: 20 }, // default disk cannot hold the OpenCode install - tensorlake: { cpus: 8, memoryMb: 16384 }, - isorun: { vcpus: 8, memMiB: 16384 }, - runloop: { launch_parameters: { resource_size_request: 'CUSTOM_SIZE', custom_cpu_cores: 8, custom_gb_memory: 16 } }, - upstash: { size: 'large' }, // large = 8 cores, 16 GB - vercel: { resources: { vcpus: 8 } }, // no memory control - blaxel: { memory: 16384 }, // CPU derived: cores = memory_MB / 2048 = 8 - beam: { cpu: 8, memory: 16384 }, // cpu = cores, memory = MiB - codesandbox: { vmTier: VMTier.Small }, // Small = 8 CPU, 16 GiB - daytona: { resources: { cpu: 8, memory: 16 } }, // memory in GiB; requires image-based creation (see providers.ts) - northflank: { deploymentPlan: process.env.NORTHFLANK_DEPLOYMENT_PLAN || 'nf-compute-50', ephemeralStorageSize: 5120 }, // 5 GiB ephemeral storage - declaw: { templateId: 'node-large' }, // node-large template: 8 vCPU / 16 GiB RAM / 8 GiB disk - superserve: { templateId: 'node22-8cpu-16gb' }, // 8 vCPU / 16 GiB template built in the pre-step - createos: { shape: 's-8vcpu-16gb', ephemeralDiskMb: 61440 }, // 8 vCPU, 16 GiB RAM, 60 GiB disk - opencomputer: { cpuCount: 4, memoryMB: 16384, timeout: 600_000 }, - sandbox0: { memory: 16384 }, // Sandbox0 exposes only `memory` -}; - -export function getSandboxOptionsWithResources(providerName: string, baseOptions?: Record): Record { - const resourceOpts = DAX_RESOURCE_OPTIONS[providerName]; - if (!resourceOpts) return baseOptions ?? {}; - return { ...baseOptions, ...resourceOpts }; -} +/** The build script the dax task uploads into each sandbox. */ +export const BENCH_SCRIPT_PATH = path.resolve(import.meta.dirname, '../scripts/dax-benchmark.sh'); export interface DaxTimingResult { totalMs: number; @@ -88,115 +55,6 @@ export interface DaxBenchmarkResult { skipReason?: string; } -export async function runDaxIteration(sandbox: any, providerName: string, timeout: number): Promise { - // Load the benchmark script from the local filesystem rather than fetching - // it over HTTP inside the sandbox. This eliminates a curl dependency - // (several providers don't ship curl in their sandboxes). - const benchScript = fs.readFileSync(BENCH_SCRIPT_PATH, 'utf8'); - - // Write the benchmark script to /tmp inside the sandbox via a single-quoted - // heredoc (so $ and backticks in the script are not expanded) and execute it - // directly with bash. A random marker avoids collisions with anything - // appearing on its own line inside the script. Running the benchmark script - // directly (without a Node.js wrapper) lets providers that ship a different - // Node.js version pre-installed (e.g. Vercel) reuse their own binary. - const marker = '__DAX_BENCH_HEREDOC_' + Math.random().toString(36).slice(2) + '__'; - const shellCmd = - `cat > /tmp/dax-benchmark.sh <<'${marker}'\n` + - benchScript + - `\n${marker}\n` + - `BENCH_PROVIDER=${providerName} BENCH_REGION=unknown bash /tmp/dax-benchmark.sh`; - - const totalStart = Date.now(); - const result = await withTimeout( - sandbox.runCommand(shellCmd, { timeout }), - timeout, - 'Dax benchmark timed out', - ) as { exitCode: number; stdout?: string; stderr?: string }; - const totalMs = Date.now() - totalStart; - - const stdout = result.stdout || ''; - const stderr = result.stderr || ''; - const exitCode = result.exitCode; - - // Parse structured output lines emitted by the benchmark script. - const phases: Record = {}; - const meta: Record = {}; - const disk: Record = {}; - let benchError: string | null = null; - let doneCommit: string | null = null; - - for (const line of stdout.split('\n')) { - if (line.startsWith('BENCH_PHASE\t')) { - const parts = line.split('\t'); - if (parts.length >= 3) phases[parts[1]] = parseInt(parts[2], 10); - } else if (line.startsWith('BENCH_META\t')) { - const parts = line.split('\t'); - if (parts.length >= 3) meta[parts[1]] = parts[2]; - } else if (line.startsWith('BENCH_DISK\t')) { - const parts = line.split('\t'); - if (parts.length >= 3) disk[parts[1]] = parseInt(parts[2], 10); - } else if (line.startsWith('BENCH_DONE\t')) { - const parts = line.split('\t'); - if (parts.length >= 2) doneCommit = parts[1]; - } - } - - for (const line of stderr.split('\n')) { - if (line.startsWith('BENCH_ERROR\t')) { - const parts = line.split('\t'); - benchError = parts.slice(1).join(': '); - } - } - - if (exitCode !== 0 && !benchError) { - // Include last few lines of stderr for diagnostics - const tail = stderr.trim().split('\n').slice(-3).join(' | '); - benchError = 'Script exited with code ' + exitCode + (tail ? ': ' + tail : ''); - } - - // Count completed phases - const phaseKeys = ['prepare', 'cache_clear', 'bun_download', 'bun_unpack', 'clone', 'install', 'typecheck']; - const rawPhasesCompleted = phaseKeys.filter(k => phases[k] !== undefined).length; - // The script's phase() function emits BENCH_PHASE even for the failing phase (it prints timing before checking exit code). - // When there's an error, the last phase that emitted a BENCH_PHASE line is the one that failed, so don't count it. - const phasesCompleted = benchError ? Math.max(0, rawPhasesCompleted - 1) : rawPhasesCompleted; - // Determine which phase failed so we can exclude its timing from the result. - // The failed phase is the last one that emitted BENCH_PHASE (index rawPhasesCompleted - 1). - const failedPhaseKey = benchError && rawPhasesCompleted > 0 ? phaseKeys[rawPhasesCompleted - 1] : null; - - // If no phases completed, the script didn't actually run (e.g. heredoc failure) - if (phasesCompleted === 0 && !benchError) { - const tail = stderr.trim().split('\n').slice(-2).join(' | '); - benchError = 'No benchmark phases completed' + (tail ? ': ' + tail : ''); - } - - return { - totalMs, - phasesCompleted, - phasesTotal: phaseKeys.length, - prepareMs: failedPhaseKey === 'prepare' ? undefined : phases.prepare, - cacheClearMs: failedPhaseKey === 'cache_clear' ? undefined : phases.cache_clear, - bunDownloadMs: failedPhaseKey === 'bun_download' ? undefined : phases.bun_download, - bunUnpackMs: failedPhaseKey === 'bun_unpack' ? undefined : phases.bun_unpack, - cloneMs: failedPhaseKey === 'clone' ? undefined : phases.clone, - installMs: failedPhaseKey === 'install' ? undefined : phases.install, - typecheckMs: failedPhaseKey === 'typecheck' ? undefined : phases.typecheck, - diskAfterClone: disk.after_clone, - diskAfterInstall: disk.after_install, - diskAfterTypecheck: disk.after_typecheck, - commit: doneCommit || meta.commit, - bunVersion: meta.bun_version, - nodeVersion: meta.node_version, - architecture: meta.architecture, - kernel: meta.kernel, - logicalCpus: meta.logical_cpus, - cpuModel: meta.cpu_model, - memoryKib: meta.memory_kib, - ...(benchError ? { error: benchError } : {}), - }; -} - export function summarize(results: DaxTimingResult[]): DaxBenchmarkResult['summary'] { const empty = { median: 0, p95: 0, p99: 0 }; const pick = (key: keyof DaxTimingResult) => { diff --git a/benchmarks/storage/snapshot-fork-task.ts b/benchmarks/storage/snapshot-fork-task.ts deleted file mode 100644 index 9b93e187..00000000 --- a/benchmarks/storage/snapshot-fork-task.ts +++ /dev/null @@ -1,149 +0,0 @@ -/** - * Shared snapshot/fork workload for the snapshot-fork benchmark. One iteration: - * seed dataset -> snapshot -> fork(from snapshot) -> fork(from live) -> - * read-back-from-fork (verify) -> teardown. - * Every created resource is torn down in a `finally` so a mid-iteration failure - * does not leak real storage. Orchestration is owned by @benchsdk/runner's - * runBenchmark — this file only describes what one iteration does. - */ -import crypto from 'node:crypto'; -import type { Storage } from '@storagesdk/core'; -import type { BenchmarkTask, TaskContext, TaskResult } from '@benchsdk/runner'; -import { TaskError } from '@benchsdk/runner'; -import { withTimeout } from '../src/util/timeout.js'; -import { formatError } from '../src/util/error.js'; -import type { StorageProviderConfig } from './types.js'; -import type { DatasetPreset, DatasetSpec } from './snapshot-fork-types.js'; - -function randomId(): string { - return Math.random().toString(36).substring(2, 15); -} - -/** Best-effort cleanup that never throws — logs and swallows. */ -async function safeCleanup(label: string, fn: () => Promise): Promise { - try { - await withTimeout(Promise.resolve(fn()), 30_000, `${label} timed out`); - } catch (err) { - console.warn(` [cleanup] ${label} failed: ${formatError(err)}`); - } -} - -/** - * Build a snapshot/fork task bound to a dataset preset. Caches one `Storage` - * per participant name and a single random payload buffer (the legacy benchmark - * created payload + storage once per provider). - */ -export function makeSnapshotForkTask( - _dataset: DatasetPreset, - spec: DatasetSpec, -): BenchmarkTask { - const storageCache = new Map(); - const payload = crypto.randomBytes(spec.objectSizeBytes); - const datasetBytes = spec.objectCount * spec.objectSizeBytes; - - return async function snapshotForkTask(ctx: TaskContext): Promise { - const { participant, step } = ctx; - const timeout = participant.timeout ?? 60_000; - - let storage = storageCache.get(participant.name); - if (!storage) { - storage = participant.createStorage(); - storageCache.set(participant.name, storage); - } - - const runId = `${Date.now()}-${randomId()}`; - const prefix = `snapfork/${runId}/`; - const keys = Array.from({ length: spec.objectCount }, (_, i) => `${prefix}obj-${i}`); - const snapshotName = `snap-${runId}`; - const forkFromSnapName = `fork-snap-${runId}`; - const forkFromLiveName = `fork-live-${runId}`; - - let snapshotId: string | undefined; - let forkFromSnapCreated = false; - let forkFromLiveCreated = false; - - try { - const seedStart = performance.now(); - await step('seed', () => - withTimeout(Promise.all(keys.map((k) => storage!.upload(k, payload))), timeout, 'Seed upload timed out'), - ); - const seedMs = performance.now() - seedStart; - - const snapStart = performance.now(); - const snapshot = await step<{ id: string }>('snapshot-create', () => - withTimeout(storage!.snapshots.create({ name: snapshotName }), timeout, 'Snapshot create timed out'), - ); - const snapshotCreateMs = performance.now() - snapStart; - snapshotId = snapshot.id; - - const forkSnapStart = performance.now(); - await step('fork-from-snapshot', () => - withTimeout( - storage!.forks.create({ name: forkFromSnapName, fromSnapshot: snapshot.id }), - timeout, - 'Fork-from-snapshot timed out', - ), - ); - const forkFromSnapshotMs = performance.now() - forkSnapStart; - forkFromSnapCreated = true; - - const forkLiveStart = performance.now(); - await step('fork-from-live', () => - withTimeout(storage!.forks.create({ name: forkFromLiveName }), timeout, 'Fork-from-live timed out'), - ); - const forkFromLiveMs = performance.now() - forkLiveStart; - forkFromLiveCreated = true; - - const readStart = performance.now(); - const bytes = await step<{ length: number }>('fork-first-read', () => - withTimeout( - storage!.forks.get(forkFromSnapName).download(keys[0], { as: 'bytes' }), - timeout, - 'Fork read timed out', - ), - ); - const forkFirstReadMs = performance.now() - readStart; - const verified = bytes.length === payload.length; - - return { - data: { - seedMs, - snapshotCreateMs, - forkFromSnapshotMs, - forkFromLiveMs, - forkFirstReadMs, - verified, - datasetBytes, - objectCount: spec.objectCount, - }, - }; - } catch (err) { - const message = formatError(err); - throw new TaskError(message, { - code: 'SNAPSHOT_FORK_ERROR', - data: { - seedMs: 0, - snapshotCreateMs: 0, - forkFromSnapshotMs: 0, - forkFromLiveMs: 0, - forkFirstReadMs: 0, - verified: false, - datasetBytes, - objectCount: spec.objectCount, - errorMessage: message, - }, - }); - } finally { - if (forkFromSnapCreated) { - await safeCleanup(`fork delete ${forkFromSnapName}`, () => storage!.forks.delete(forkFromSnapName)); - } - if (forkFromLiveCreated) { - await safeCleanup(`fork delete ${forkFromLiveName}`, () => storage!.forks.delete(forkFromLiveName)); - } - if (snapshotId) { - await safeCleanup(`snapshot delete ${snapshotId}`, () => storage!.snapshots.delete(snapshotId!)); - } - await safeCleanup(`object delete ${prefix}*`, () => Promise.all(keys.map((k) => storage!.delete(k)))); - } - }; -} diff --git a/benchmarks/storage/snapshot-fork.bench.ts b/benchmarks/storage/snapshot-fork.bench.ts index 6548a524..73124bb9 100644 --- a/benchmarks/storage/snapshot-fork.bench.ts +++ b/benchmarks/storage/snapshot-fork.bench.ts @@ -1,8 +1,8 @@ /** * Storage snapshot/fork benchmark: per-iteration seed -> snapshot -> fork -> * verify, per provider (concurrency 1 = sequential; each iteration creates real - * snapshots/forks). Config lives here; orchestration is owned by @benchsdk/runner's - * runBenchmark. + * snapshots/forks). Declarative — exports `config` + `task`; `bench run` owns + * the entrypoint and orchestration. * * bench run benchmarks/storage/snapshot-fork.bench.ts * bench run benchmarks/storage/snapshot-fork.bench.ts --dataset wide --iterations 5 --provider tigris @@ -10,9 +10,12 @@ import '../src/env.js'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { defineBenchmarkConfig, defineTask } from '@benchsdk/runner'; +import crypto from 'node:crypto'; +import { defineBenchmarkConfig, defineTask, TaskError } from '@benchsdk/runner'; +import type { Storage } from '@storagesdk/core'; +import { withTimeout } from '../src/util/timeout.js'; +import { formatError } from '../src/util/error.js'; import { storageProviders } from './providers.js'; -import { makeSnapshotForkTask } from './snapshot-fork-task.js'; import { writeSnapshotForkLegacyResults } from './snapshot-fork-legacy-results.js'; import { DATASET_PRESETS } from './snapshot-fork-types.js'; import type { DatasetPreset } from './snapshot-fork-types.js'; @@ -60,4 +63,132 @@ export const config = defineBenchmarkConfig({ }), }); -export const task = defineTask(makeSnapshotForkTask(dataset, spec)); +function randomId(): string { + return Math.random().toString(36).substring(2, 15); +} + +/** Best-effort cleanup that never throws — logs and swallows. */ +async function safeCleanup(label: string, fn: () => Promise): Promise { + try { + await withTimeout(Promise.resolve(fn()), 30_000, `${label} timed out`); + } catch (err) { + console.warn(` [cleanup] ${label} failed: ${formatError(err)}`); + } +} + +/** One `Storage` per participant, and a single random payload buffer for the run. */ +const storageCache = new Map(); +const payload = crypto.randomBytes(spec.objectSizeBytes); +const datasetBytes = spec.objectCount * spec.objectSizeBytes; + +/** + * One iteration: seed dataset -> snapshot -> fork(from snapshot) -> fork(from + * live) -> read-back-from-fork (verify) -> teardown. Every created resource is + * torn down in the `finally` so a mid-iteration failure does not leak real + * storage. + */ +export const task = defineTask(async (ctx) => { + const { participant, step } = ctx; + const timeout = participant.timeout ?? 60_000; + + let storage = storageCache.get(participant.name); + if (!storage) { + storage = participant.createStorage(); + storageCache.set(participant.name, storage); + } + + const runId = `${Date.now()}-${randomId()}`; + const prefix = `snapfork/${runId}/`; + const keys = Array.from({ length: spec.objectCount }, (_, i) => `${prefix}obj-${i}`); + const snapshotName = `snap-${runId}`; + const forkFromSnapName = `fork-snap-${runId}`; + const forkFromLiveName = `fork-live-${runId}`; + + let snapshotId: string | undefined; + let forkFromSnapCreated = false; + let forkFromLiveCreated = false; + + try { + const seedStart = performance.now(); + await step('seed', () => + withTimeout(Promise.all(keys.map((k) => storage!.upload(k, payload))), timeout, 'Seed upload timed out'), + ); + const seedMs = performance.now() - seedStart; + + const snapStart = performance.now(); + const snapshot = await step<{ id: string }>('snapshot-create', () => + withTimeout(storage!.snapshots.create({ name: snapshotName }), timeout, 'Snapshot create timed out'), + ); + const snapshotCreateMs = performance.now() - snapStart; + snapshotId = snapshot.id; + + const forkSnapStart = performance.now(); + await step('fork-from-snapshot', () => + withTimeout( + storage!.forks.create({ name: forkFromSnapName, fromSnapshot: snapshot.id }), + timeout, + 'Fork-from-snapshot timed out', + ), + ); + const forkFromSnapshotMs = performance.now() - forkSnapStart; + forkFromSnapCreated = true; + + const forkLiveStart = performance.now(); + await step('fork-from-live', () => + withTimeout(storage!.forks.create({ name: forkFromLiveName }), timeout, 'Fork-from-live timed out'), + ); + const forkFromLiveMs = performance.now() - forkLiveStart; + forkFromLiveCreated = true; + + const readStart = performance.now(); + const bytes = await step<{ length: number }>('fork-first-read', () => + withTimeout( + storage!.forks.get(forkFromSnapName).download(keys[0], { as: 'bytes' }), + timeout, + 'Fork read timed out', + ), + ); + const forkFirstReadMs = performance.now() - readStart; + const verified = bytes.length === payload.length; + + return { + data: { + seedMs, + snapshotCreateMs, + forkFromSnapshotMs, + forkFromLiveMs, + forkFirstReadMs, + verified, + datasetBytes, + objectCount: spec.objectCount, + }, + }; + } catch (err) { + const message = formatError(err); + throw new TaskError(message, { + code: 'SNAPSHOT_FORK_ERROR', + data: { + seedMs: 0, + snapshotCreateMs: 0, + forkFromSnapshotMs: 0, + forkFromLiveMs: 0, + forkFirstReadMs: 0, + verified: false, + datasetBytes, + objectCount: spec.objectCount, + errorMessage: message, + }, + }); + } finally { + if (forkFromSnapCreated) { + await safeCleanup(`fork delete ${forkFromSnapName}`, () => storage!.forks.delete(forkFromSnapName)); + } + if (forkFromLiveCreated) { + await safeCleanup(`fork delete ${forkFromLiveName}`, () => storage!.forks.delete(forkFromLiveName)); + } + if (snapshotId) { + await safeCleanup(`snapshot delete ${snapshotId}`, () => storage!.snapshots.delete(snapshotId!)); + } + await safeCleanup(`object delete ${prefix}*`, () => Promise.all(keys.map((k) => storage!.delete(k)))); + } +}); diff --git a/benchmarks/storage/storage-task.ts b/benchmarks/storage/storage-task.ts deleted file mode 100644 index e6f4966c..00000000 --- a/benchmarks/storage/storage-task.ts +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Shared storage upload/download workload for the storage benchmark. One - * iteration = upload a buffer, download it, compute throughput, then delete - * (best-effort cleanup). Orchestration (how many, how parallel) is owned by - * @benchsdk/runner's runBenchmark — this file only describes what one iteration - * does. - */ -import type { BenchmarkTask, TaskContext, TaskResult } from '@benchsdk/runner'; -import { TaskError } from '@benchsdk/runner'; -import type { Storage } from '@storagesdk/core'; -import { withTimeout } from '../src/util/timeout.js'; -import { formatError } from '../src/util/error.js'; -import type { StorageProviderConfig } from './types.js'; - -function randomId(): string { - return Math.random().toString(36).substring(2, 15); -} - -/** - * Build a storage task bound to a shared test buffer + file size. The factory - * caches one `Storage` instance per participant name inside its closure so the - * adapter (and its credentials lookup) isn't recreated on every iteration — - * the legacy benchmark created storage once per provider; this preserves that - * behavior while keeping the per-iteration task signature the CLI expects. - */ -export function makeStorageTask( - testData: Buffer, - fileSizeBytes: number, -): BenchmarkTask { - const storageCache = new Map(); - - return async function storageTask(ctx: TaskContext): Promise { - const { participant, step, measure } = ctx; - const timeout = participant.timeout ?? 30_000; - - let storage = storageCache.get(participant.name); - if (!storage) { - storage = participant.createStorage(); - storageCache.set(participant.name, storage); - } - - const key = `benchmark-${Date.now()}-${randomId()}`; - - try { - // Upload timing - const uploadStart = performance.now(); - await step('upload', () => - withTimeout(storage!.upload(key, testData), timeout, 'Upload timed out'), - ); - const uploadMs = performance.now() - uploadStart; - - // Download timing — request raw bytes so we measure a full object fetch. - // Throughput (Mbps) is a rate, not a duration, so it can't be inferred - // from the step's latency; measure it inside the `download` step so it - // lands on that step's data (platform step_data_json). - let downloadMs = 0; - let throughputMbps = 0; - await step('download', async () => { - const downloadStart = performance.now(); - await withTimeout(storage!.download(key, { as: 'bytes' }), timeout, 'Download timed out'); - downloadMs = performance.now() - downloadStart; - throughputMbps = (fileSizeBytes * 8) / (downloadMs / 1000) / 1_000_000; - measure({ throughputMbps }); - }); - - // Cleanup: best-effort delete; failures are warned but don't fail the task. - await step( - 'delete', - () => withTimeout(storage!.delete(key), 10_000, 'Delete timed out'), - { reportConcurrency: false }, - ).catch((err) => console.warn(` [cleanup] delete failed: ${formatError(err)}`)); - - return { data: { uploadMs, downloadMs, throughputMbps, fileSizeBytes } }; - } catch (err) { - // Attempt cleanup even on failure (best-effort, mirrors legacy catch). - try { - await withTimeout(storage!.delete(key), 10_000, 'Delete timed out'); - } catch { - // Ignore cleanup errors on the failure path. - } - const message = formatError(err); - throw new TaskError(message, { - code: 'STORAGE_ERROR', - data: { uploadMs: 0, downloadMs: 0, throughputMbps: 0, fileSizeBytes }, - }); - } - }; -} diff --git a/benchmarks/storage/storage.bench.ts b/benchmarks/storage/storage.bench.ts index e1734fbd..18fb082f 100644 --- a/benchmarks/storage/storage.bench.ts +++ b/benchmarks/storage/storage.bench.ts @@ -11,12 +11,14 @@ import '../src/env.js'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import crypto from 'node:crypto'; -import { defineBenchmarkConfig, defineTask } from '@benchsdk/runner'; +import { defineBenchmarkConfig, defineTask, TaskError } from '@benchsdk/runner'; +import type { Storage } from '@storagesdk/core'; +import { withTimeout } from '../src/util/timeout.js'; +import { formatError } from '../src/util/error.js'; import { storageProviders } from './providers.js'; -import { makeStorageTask } from './storage-task.js'; import { writeStorageLegacyResults } from './legacy-results.js'; import { FILE_SIZE_BYTES } from './types.js'; -import type { StorageFileSize } from './types.js'; +import type { StorageFileSize, StorageProviderConfig } from './types.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -53,4 +55,69 @@ export const config = defineBenchmarkConfig({ }), }); -export const task = defineTask(makeStorageTask(testData, fileSizeBytes)); +function randomId(): string { + return Math.random().toString(36).substring(2, 15); +} + +/** + * One `Storage` instance per participant, so the adapter (and its credentials + * lookup) isn't recreated on every iteration. + */ +const storageCache = new Map(); + +export const task = defineTask(async (ctx) => { + const { participant, step, measure } = ctx; + const timeout = participant.timeout ?? 30_000; + + let storage = storageCache.get(participant.name); + if (!storage) { + storage = participant.createStorage(); + storageCache.set(participant.name, storage); + } + + const key = `benchmark-${Date.now()}-${randomId()}`; + + try { + // Upload timing + const uploadStart = performance.now(); + await step('upload', () => + withTimeout(storage!.upload(key, testData), timeout, 'Upload timed out'), + ); + const uploadMs = performance.now() - uploadStart; + + // Download timing — request raw bytes so we measure a full object fetch. + // Throughput (Mbps) is a rate, not a duration, so it can't be inferred + // from the step's latency; measure it inside the `download` step so it + // lands on that step's data (platform step_data_json). + let downloadMs = 0; + let throughputMbps = 0; + await step('download', async () => { + const downloadStart = performance.now(); + await withTimeout(storage!.download(key, { as: 'bytes' }), timeout, 'Download timed out'); + downloadMs = performance.now() - downloadStart; + throughputMbps = (fileSizeBytes * 8) / (downloadMs / 1000) / 1_000_000; + measure({ throughputMbps }); + }); + + // Cleanup: best-effort delete; failures are warned but don't fail the task. + await step( + 'delete', + () => withTimeout(storage!.delete(key), 10_000, 'Delete timed out'), + { reportConcurrency: false }, + ).catch((err) => console.warn(` [cleanup] delete failed: ${formatError(err)}`)); + + return { data: { uploadMs, downloadMs, throughputMbps, fileSizeBytes } }; + } catch (err) { + // Attempt cleanup even on failure. + try { + await withTimeout(storage!.delete(key), 10_000, 'Delete timed out'); + } catch { + // Ignore cleanup errors on the failure path. + } + const message = formatError(err); + throw new TaskError(message, { + code: 'STORAGE_ERROR', + data: { uploadMs: 0, downloadMs: 0, throughputMbps: 0, fileSizeBytes }, + }); + } +});