From e7b20151282451b6ed32622cd028ecce015d7312 Mon Sep 17 00:00:00 2001 From: Karolis Date: Tue, 21 Jul 2026 11:50:31 +0400 Subject: [PATCH] feat: run lightning at cpu-8 (8 vCPU / 16 GiB) in dax benchmark Lightning was in the dax matrix but had no resource sizing, so it ran on the SDK default cpu-1 (1 vCPU) while other providers standardized to 8 vCPU / 16 GiB, making results incomparable. Lightning's @computesdk/lightning wrapper sets instanceType from the provider factory config and overrides any instanceType passed to sandbox.create(), so it can't be sized through DAX_RESOURCE_OPTIONS like most providers. Instead, read LIGHTNING_INSTANCE_TYPE in createCompute (default cpu-1, the SDK default) and set it to cpu-8 in the dax workflow, mirroring how northflank reads its plan from env with a fallback. This scopes the 8 vCPU sizing to the dax benchmark and leaves other modes (e.g. TTI) on the default. Verified against lightning.ai: sandbox reports 8 logical CPUs, all 7 dax phases complete. --- .github/workflows/sandbox-dax-benchmarks.yml | 1 + env.example | 3 +++ src/sandbox/dax.ts | 2 ++ src/sandbox/providers.ts | 10 +++++++++- 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sandbox-dax-benchmarks.yml b/.github/workflows/sandbox-dax-benchmarks.yml index 24025cf1..0ea6bffc 100644 --- a/.github/workflows/sandbox-dax-benchmarks.yml +++ b/.github/workflows/sandbox-dax-benchmarks.yml @@ -115,6 +115,7 @@ jobs: HOPX_API_KEY: ${{ secrets.HOPX_API_KEY }} ISORUN_API_KEY: ${{ secrets.ISORUN_API_KEY }} LIGHTNING_API_KEY: ${{ secrets.LIGHTNING_API_KEY }} + LIGHTNING_INSTANCE_TYPE: cpu-8 # 8 vCPU / 16 GiB, matching the standardized dax profile MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} NORTHFLANK_TOKEN: ${{ secrets.NORTHFLANK_TOKEN }} diff --git a/env.example b/env.example index ac10806d..fc408aa6 100644 --- a/env.example +++ b/env.example @@ -106,6 +106,9 @@ CREATEOS_SANDBOX_API_KEY=your_createos_sandbox_api_key ######### LIGHTNING ######## LIGHTNING_API_KEY=your_lightning_api_key +# Instance type for new sandboxes (cpu-1, cpu-2, ... cpu-16). Defaults to cpu-1. +# The dax benchmark uses cpu-8 (8 vCPU / 16 GiB) for a fair cross-provider comparison. +LIGHTNING_INSTANCE_TYPE=cpu-1 ######### AI GATEWAYS ######## OPENROUTER_API_KEY=your_openrouter_api_key diff --git a/src/sandbox/dax.ts b/src/sandbox/dax.ts index 98d1d0da..d0376465 100644 --- a/src/sandbox/dax.ts +++ b/src/sandbox/dax.ts @@ -17,6 +17,8 @@ const BENCH_SCRIPT_PATH = path.resolve(import.meta.dirname, '../../scripts/dax-b // Each provider uses different parameter names and units, so we map per-provider. // Providers not listed here don't support CPU/memory configuration at sandbox creation time. // Note: E2B sets CPU/memory at template build time, not at sandbox creation. +// Note: lightning is sized via LIGHTNING_INSTANCE_TYPE=cpu-8 (8 vCPU / 16 GiB), applied on +// the provider factory in providers.ts — the SDK ignores an instanceType passed to create(). const DAX_RESOURCE_OPTIONS: Record> = { modal: { cpu: 4, cpuLimit: 4, memoryMiB: 16384 }, // Modal: 1 core = 2 vCPUs, so 4 cores = 8 vCPUs tensorlake: { cpus: 8, memoryMb: 16384 }, diff --git a/src/sandbox/providers.ts b/src/sandbox/providers.ts index d833c0c7..f2f5c3bc 100644 --- a/src/sandbox/providers.ts +++ b/src/sandbox/providers.ts @@ -115,7 +115,15 @@ export const providers: ProviderConfig[] = [ { name: 'lightning', requiredEnvVars: ['LIGHTNING_API_KEY'], - createCompute: () => lightning({ apiKey: process.env.LIGHTNING_API_KEY! }), + // Lightning sizes sandboxes via `instanceType` on the provider factory (the SDK + // ignores an instanceType passed to sandbox.create()), so it can't be sized through + // DAX_RESOURCE_OPTIONS like most providers. The dax workflow sets + // LIGHTNING_INSTANCE_TYPE=cpu-8 for the standardized 8 vCPU / 16 GiB profile; + // it defaults to cpu-1 (the SDK default) everywhere else. + createCompute: () => lightning({ + apiKey: process.env.LIGHTNING_API_KEY!, + instanceType: process.env.LIGHTNING_INSTANCE_TYPE || 'cpu-1', + }), }, { name: 'modal',