feat: add Run Cloud to the Dax sandbox benchmark

.github/workflows/sandbox-dax-benchmarks.yml

@@ -6,6 +6,7 @@ on:
     paths:
       - 'benchmarks/sandbox/dax.ts'
       - 'benchmarks/sandbox/providers.ts'
+      - 'benchmarks/sandbox/run-cloud.ts'
       - 'benchmarks/src/run.ts'
       - 'benchmarks/src/merge-results.ts'
       - 'benchmarks/scripts/dax-benchmark.sh'
@@ -63,6 +64,7 @@ jobs:
           - namespace
           - northflank
           - opencomputer
+          - run-cloud
           - runloop
           - superserve
           - tenki
@@ -130,6 +132,8 @@ jobs:
           NORTHFLANK_PROJECT_ID: ${{ secrets.NORTHFLANK_PROJECT_ID }}
           OPENCOMPUTER_API_KEY: ${{ secrets.OPENCOMPUTER_API_KEY }}
           OPENCOMPUTER_API_URL: ${{ secrets.OPENCOMPUTER_API_URL }}
+          RUN_CLOUD_API_KEY: ${{ secrets.RUN_CLOUD_API_KEY }}
+          RUN_CLOUD_API_URL: https://api.run.cloud
           RUNLOOP_API_KEY: ${{ secrets.RUNLOOP_API_KEY }}
           SUPERSERVE_API_KEY: ${{ secrets.SUPERSERVE_API_KEY }}
           TENKI_API_KEY: ${{ secrets.TENKI_API_KEY }}

benchmarks/.env.example

@@ -114,6 +114,11 @@ LIGHTNING_INSTANCE_TYPE=cpu-1
 OPENCOMPUTER_API_KEY=your_opencomputer_api_key
 OPENCOMPUTER_API_URL=https://app.opencomputer.dev
 
+######### RUN CLOUD ########
+RUN_CLOUD_API_KEY=your_run_cloud_api_key
+# Defaults to https://api.run.cloud
+RUN_CLOUD_API_URL=https://api.run.cloud
+
 ######### TILION ########
 TILION_API_KEY=your_tilion_api_key
 TILION_BASE_URL=https://api.tilion.dev

benchmarks/sandbox/dax.ts

@@ -35,6 +35,7 @@ const DAX_RESOURCE_OPTIONS: Record<string, Record<string, any>> = {
   superserve:   { templateId: 'node22-8cpu-16gb' },           // 8 vCPU / 16 GiB template built in the pre-step
   createos:     { shape: 's-8vcpu-16gb', ephemeralDiskMb: 61440 }, // 8 vCPU, 16 GiB RAM, 60 GiB disk
   opencomputer: { cpuCount: 4, memoryMB: 16384, timeout: 600_000 },
+  'run-cloud':  { cpu: 8, memory: 16384, disk: 40 },       // cpu = cores, memory = MiB, disk = GiB
 };
 
 function getSandboxOptionsWithResources(providerName: string, baseOptions?: Record<string, any>): Record<string, any> {

benchmarks/sandbox/providers.ts

@@ -26,6 +26,7 @@ import { tenki } from '@computesdk/tenki';
 import { tensorlake } from '@computesdk/tensorlake'
 import { upstash } from '@computesdk/upstash';
 import { vercel } from '@computesdk/vercel';
+import { runCloud } from './run-cloud.js';
 import type { ProviderConfig } from './types.js';
 
 /**
@@ -175,6 +176,15 @@ export const providers: ProviderConfig[] = [
     requiredEnvVars: ['RUNLOOP_API_KEY'],
     createCompute: () => runloop({ apiKey: process.env.RUNLOOP_API_KEY! }),
   },
+  {
+    name: 'run-cloud',
+    requiredEnvVars: ['RUN_CLOUD_API_KEY'],
+    createCompute: () => runCloud({
+      apiKey: process.env.RUN_CLOUD_API_KEY!,
+      apiUrl: process.env.RUN_CLOUD_API_URL,
+    }),
+    sandboxOptions: { disk: 40 },
+  },
   {
     name: 'sprites',
     requiredEnvVars: ['SPRITES_TOKEN'],

benchmarks/sandbox/run-cloud.ts

@@ -0,0 +1,171 @@
+import {
+  Client,
+  type CreateSandboxOptions,
+  type ExecResult,
+} from '@run-cloud/sdk';
+import { randomUUID } from 'node:crypto';
+
+interface RunCommandOptions {
+  timeout?: number;
+}
+
+interface RunCloudComputeOptions {
+  apiKey: string;
+  apiUrl?: string;
+}
+
+export function runCloud(options: RunCloudComputeOptions) {
+  const client = new Client(options);
+
+  return {
+    sandbox: {
+      async create(createOptions: CreateSandboxOptions = {}) {
+        const sandbox = await client.sandboxes.create(createOptions);
+
+        return {
+          id: sandbox.id,
+          runCommand(
+            command: string,
+            commandOptions: RunCommandOptions = {},
+          ): Promise<ExecResult> {
+            // The public API proxy terminates long synchronous requests. Detach
+            // benchmark workloads and poll them through short exec requests.
+            if ((commandOptions.timeout ?? 0) > 45_000) {
+              return runLongCommand(
+                client,
+                sandbox.id,
+                command,
+                commandOptions.timeout!,
+              );
+            }
+
+            return client.sandboxes.exec(sandbox.id, command, {
+              timeoutSeconds: toTimeoutSeconds(commandOptions.timeout),
+            });
+          },
+          destroy(): Promise<void> {
+            return client.sandboxes.destroy(sandbox.id);
+          },
+        };
+      },
+    },
+  };
+}
+
+async function runLongCommand(
+  client: Client,
+  sandboxId: string,
+  command: string,
+  timeoutMs: number,
+): Promise<ExecResult> {
+  const runId = randomUUID().replaceAll('-', '');
+  const prefix = `/tmp/run-cloud-benchmark-${runId}`;
+  const scriptPath = `${prefix}.sh`;
+  const stdoutPath = `${prefix}.stdout`;
+  const stderrPath = `${prefix}.stderr`;
+  const statusPath = `${prefix}.status`;
+  const unitName = `run-cloud-benchmark-${runId}`;
+  const encodedCommand = Buffer.from(command).toString('base64');
+
+  const launch = [
+    `printf '%s' '${encodedCommand}' | base64 -d > '${scriptPath}'`,
+    `chmod 700 '${scriptPath}'`,
+    `systemd-run --unit='${unitName}' --collect --quiet --property=OOMPolicy=continue /bin/sh -c 'bash "$1" >"$2" 2>"$3"; printf "%s" "$?" >"$4"' _ '${scriptPath}' '${stdoutPath}' '${stderrPath}' '${statusPath}'`,
+  ].join('\n');
+
+  const launched = await client.sandboxes.exec(sandboxId, launch, {
+    timeoutSeconds: 30,
+  });
+  if (launched.exitCode !== 0) return launched;
+
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    let status: ExecResult;
+    try {
+      status = await client.sandboxes.exec(
+        sandboxId,
+        `if [ -f '${statusPath}' ]; then cat '${statusPath}'; else printf pending; fi`,
+        { timeoutSeconds: 30 },
+      );
+    } catch (error) {
+      if (!isTransientApiError(error)) throw error;
+      await new Promise((resolve) => setTimeout(resolve, 500));
+      continue;
+    }
+
+    if (status.stdout.trim() !== 'pending') {
+      const exitCode = Number.parseInt(status.stdout.trim(), 10);
+      const stdout = await execWithTransientRetry(
+        client,
+        sandboxId,
+        ['cat', stdoutPath],
+        deadline,
+      );
+      const stderr = await execWithTransientRetry(
+        client,
+        sandboxId,
+        ['cat', stderrPath],
+        deadline,
+      );
+      await cleanupCommandFiles(client, sandboxId, prefix);
+
+      return {
+        exit_code: Number.isFinite(exitCode) ? exitCode : 1,
+        exitCode: Number.isFinite(exitCode) ? exitCode : 1,
+        stdout: stdout.stdout,
+        stderr: stderr.stdout,
+      };
+    }
+
+    await new Promise((resolve) => setTimeout(resolve, 500));
+  }
+
+  await client.sandboxes.exec(
+    sandboxId,
+    `systemctl stop '${unitName}' 2>/dev/null || true`,
+    { timeoutSeconds: 30 },
+  ).catch(() => {});
+  await cleanupCommandFiles(client, sandboxId, prefix);
+  throw new Error(`Run Cloud command timed out after ${timeoutMs}ms`);
+}
+
+async function execWithTransientRetry(
+  client: Client,
+  sandboxId: string,
+  command: string[],
+  deadline: number,
+): Promise<ExecResult> {
+  while (Date.now() < deadline) {
+    try {
+      return await client.sandboxes.exec(sandboxId, command, {
+        timeoutSeconds: 30,
+      });
+    } catch (error) {
+      if (!isTransientApiError(error)) throw error;
+      await new Promise((resolve) => setTimeout(resolve, 500));
+    }
+  }
+
+  throw new Error('Run Cloud API remained unavailable while collecting benchmark output');
+}
+
+function isTransientApiError(error: unknown): boolean {
+  return error instanceof Error
+    && /run\.cloud API (?:429|5\d\d)\b/.test(error.message);
+}
+
+async function cleanupCommandFiles(
+  client: Client,
+  sandboxId: string,
+  prefix: string,
+): Promise<void> {
+  await client.sandboxes.exec(
+    sandboxId,
+    `rm -f '${prefix}.sh' '${prefix}.stdout' '${prefix}.stderr' '${prefix}.status'`,
+    { timeoutSeconds: 30 },
+  ).catch(() => {});
+}
+
+function toTimeoutSeconds(timeoutMs?: number): number | undefined {
+  return timeoutMs === undefined ? undefined : Math.ceil(timeoutMs / 1_000);
+}

package.json

@@ -21,6 +21,7 @@
     "bench:northflank": "tsx benchmarks/src/run.ts --provider northflank",
     "bench:railway": "tsx benchmarks/src/run.ts --provider railway",
     "bench:render": "tsx benchmarks/src/run.ts --provider render",
+    "bench:run-cloud": "tsx benchmarks/src/run.ts --provider run-cloud",
     "bench:runloop": "tsx benchmarks/src/run.ts --provider runloop",
     "bench:vercel": "tsx benchmarks/src/run.ts --provider vercel",
     "bench:just-bash": "tsx benchmarks/src/run.ts --provider just-bash",
@@ -132,6 +133,7 @@
     "@computesdk/vercel": "^1.7.31",
     "@superserve/sdk": "^0.8.1",
     "@google-cloud/storage": "^7.21.0",
+    "@run-cloud/sdk": "^0.5.1",
     "@storagesdk/adapters": "^0.7.1",
     "@storagesdk/core": "^0.4.2",
     "@tigrisdata/storage": "^3.16.0",