Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .github/workflows/sandbox-dax-benchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ jobs:
matrix:
provider:
- archil
- arker
- beam
- blaxel
- cloud-run
Expand Down Expand Up @@ -103,8 +104,9 @@ jobs:
- name: Run dax benchmark
env:
LIGHTNING_INSTANCE_TYPE: cpu-8 # 8 vCPU / 16 GiB, matching the standardized dax profile
ARKER_PLATFORMS: graviton4 # dax only; elsewhere Arker picks the platform automatically
run: |
. benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUNLOOP_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)'
. benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|ARKER_API_KEY|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUNLOOP_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)'

npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/dax.bench.ts \
--provider ${{ matrix.provider }} \
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/sandbox-tti-benchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ jobs:
matrix:
provider:
- archil
- arker
- beam
- blaxel
- cloud-run
Expand Down Expand Up @@ -114,7 +115,7 @@ jobs:
- name: Run benchmark
if: env.SHOULD_RUN == 'true'
run: |
. benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUNLOOP_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)'
. benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|ARKER_API_KEY|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUNLOOP_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)'

# One entrypoint, three launch shapes: the knobs come from the CLI
# and --slug/--name pick which platform benchmark each reports to.
Expand Down
3 changes: 3 additions & 0 deletions benchmarks/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,9 @@ OPENCOMPUTER_API_URL=https://app.opencomputer.dev
TILION_API_KEY=your_tilion_api_key
TILION_BASE_URL=https://api.tilion.dev

######### ARKER ########
ARKER_API_KEY=your_arker_api_key

######### AI GATEWAYS ########
OPENROUTER_API_KEY=your_openrouter_api_key
VERCEL_AI_GATEWAY_API_KEY=your_vercel_ai_gateway_api_key
Expand Down
1 change: 1 addition & 0 deletions benchmarks/sandbox/dax.bench.ts
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ export const config = defineBenchmarkConfig({
// Note: lightning is sized via LIGHTNING_INSTANCE_TYPE=cpu-8 (8 vCPU / 16 GiB), applied on
// the provider factory in providers.ts — the SDK ignores an instanceType passed to create().
const DAX_RESOURCE_OPTIONS: Record<string, Record<string, any>> = {
arker: { templateId: 'ubuntu-full-8' }, // 8 vCPU / 16 GiB golden
modal: { cpu: 4, cpuLimit: 4, memoryMiB: 16384 }, // Modal: 1 core = 2 vCPUs, so 4 cores = 8 vCPUs
tenki: { cpuCores: 8, memoryMb: 16384, diskSizeGb: 20 }, // default disk cannot hold the OpenCode install
tensorlake: { cpus: 8, memoryMb: 16384 },
Expand Down
9 changes: 9 additions & 0 deletions benchmarks/sandbox/providers.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { archil } from '@computesdk/archil';
import { arker } from '@computesdk/arker';
import { beam } from '@computesdk/beam';
import { blaxel } from '@computesdk/blaxel';
import { codesandbox } from '@computesdk/codesandbox';
Expand Down Expand Up @@ -43,6 +44,14 @@ export const providers: ProviderConfig[] = [
createCompute: () => archil({ apiKey: process.env.ARCHIL_API_KEY!, region: process.env.ARCHIL_REGION! }),
sandboxOptions: { diskId: process.env.ARCHIL_DISK_ID! }
},
{
// Arker sizes via `platforms` on the factory, not sandbox.create(), so like
// lightning it is set from the dax workflow env (ARKER_PLATFORMS=graviton4).
name: 'arker',
requiredEnvVars: ['ARKER_API_KEY'],
createCompute: () => arker({ apiKey: process.env.ARKER_API_KEY! }),
sandboxOptions: { templateId: 'ubuntu-small' },
},
Comment thread
aatishparson-arker marked this conversation as resolved.
{
// Activated in daily + PR benchmarks once BEAM_TOKEN / BEAM_WORKSPACE_ID secrets landed.
name: 'beam',
Expand Down
3 changes: 3 additions & 0 deletions benchmarks/scripts/provider-vars.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@
"ARCHIL_REGION",
"ARCHIL_DISK_ID"
],
"arker": [
"ARKER_API_KEY"
],
"beam": [
"BEAM_TOKEN",
"BEAM_WORKSPACE_ID"
Expand Down
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@
"@codesandbox/sdk": "^2.4.2",
"@computesdk/anchorbrowser": "^0.2.6",
"@computesdk/archil": "^0.4.7",
"@computesdk/arker": "^0.1.2",
"@computesdk/beam": "^0.3.1",
"@computesdk/blaxel": "^1.6.18",
"@computesdk/browserbase": "^0.3.10",
Expand Down
62 changes: 62 additions & 0 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading