init untested

Author: maxffarrell

.env.example

@@ -1 +1,5 @@
-VERCEL_OIDC_TOKEN="create with vercel cli"
+# Enable bun.secrets for secure credential storage (1=enabled, 0=disabled)
+# When enabled, secrets are stored in the OS credential manager instead of .env
+USE_BUN_SECRETS=1
+
+VERCEL_OIDC_TOKEN="create with vercel cli"

AGENTS.md

@@ -22,9 +22,51 @@ bun run generate-report.ts results/result-2024-12-07-14-30-45.json
 
 # Run TypeScript type checking
 bun tsc --noEmit
+
+# Manage secrets (bun.secrets integration)
+bun run secrets
+
+# Test secrets integration
+bun run test:secrets
+```
+
+## Environment Variables & Secrets Management
+
+This project supports both traditional environment variables and bun.secrets for secure credential storage. See [docs/secrets.md](./docs/secrets.md) for complete bun.secrets documentation.
+
+### bun.secrets Integration
+
+Enable secure credential storage using your operating system's native credential manager:
+
+```bash
+# Enable bun.secrets (recommended for development)
+export USE_BUN_SECRETS=1
+
+# Initialize secrets from existing environment variables
+bun run secrets init
+
+# Manage secrets interactively
+bun run secrets
 ```
 
-## Environment Variables
+**Benefits:**
+- ✅ Encrypted storage using OS credential manager
+- ✅ No plain text API keys in .env files
+- ✅ Survives application restarts
+- ✅ User-level access control
+
+**Quick Setup:**
+```bash
+# 1. Enable bun.secrets
+export USE_BUN_SECRETS=1
+
+# 2. Set your API keys
+bun run secrets set
+# Select "ANTHROPIC_API_KEY" and enter your key
+
+# 3. Verify setup
+bun run secrets status
+```
 
 ### MODEL Configuration
 
@@ -106,11 +148,27 @@ MCP_SERVER_URL=
 
 ### Required API Keys
 
-- `ANTHROPIC_API_KEY`: Required when using `anthropic/*` models
+The following API keys are required for different model providers. These can be stored as environment variables or using bun.secrets:
+
+- `ANTHROPIC_API_KEY`: Required when using `anthropic/*` models (get at https://console.anthropic.com/)
 - `OPENAI_API_KEY`: Required when using `openai/*` models (get at https://platform.openai.com/api-keys)
 - `OPENROUTER_API_KEY`: Required when using `openrouter/*` models (get at https://openrouter.ai/keys)
+- `VERCEL_OIDC_TOKEN`: Optional for Vercel integration (create with `vercel link`)
 - No API key required for `lmstudio/*` models (runs locally)
 
+**With bun.secrets:**
+```bash
+export USE_BUN_SECRETS=1
+bun run secrets set  # Select which keys to configure
+```
+
+**With environment variables:**
+```bash
+export ANTHROPIC_API_KEY="your-key-here"
+export OPENAI_API_KEY="your-key-here"
+export OPENROUTER_API_KEY="your-key-here"
+```
+
 ### Provider Routing
 
 The benchmark tool automatically routes to the correct provider based on the `MODEL` prefix:

README.md

@@ -10,7 +10,44 @@ To install dependencies:
 bun install
 ```
 
-## Setup
+## bun.secrets Integration
+
+This project includes secure credential storage using bun.secrets. Store API keys in your OS credential manager instead of plain text .env files.
+
+### Quick Start
+
+```bash
+# 1. Enable bun.secrets
+export USE_BUN_SECRETS=1
+
+# 2. Initialize from existing .env (optional)
+bun run secrets init
+
+# 3. Set your API keys
+bun run secrets set
+
+# 4. Run the benchmark
+bun run index.ts
+```
+
+### Commands
+
+- `bun run secrets` - Interactive secrets management
+- `bun run secrets status` - Show current status
+- `bun run secrets set` - Set a secret value
+- `bun run secrets init` - Initialize from environment variables
+- `bun run test:secrets` - Test the integration
+
+### Supported Secrets
+
+- `VERCEL_OIDC_TOKEN` - Vercel authentication
+- `ANTHROPIC_API_KEY` - Anthropic Claude API
+- `OPENAI_API_KEY` - OpenAI API
+- `OPENROUTER_API_KEY` - OpenRouter API
+
+See [docs/secrets.md](./docs/secrets.md) for complete documentation.
+
+---
 
 Configure your API keys in `.env`:
 

docs/secrets.md

@@ -0,0 +1,190 @@
+# bun.secrets Integration
+
+This project includes a modular `bun.secrets` integration for secure credential storage and management. This allows you to store API keys and other sensitive data in your operating system's native credential manager instead of plain text `.env` files.
+
+## Quick Start
+
+1. **Enable bun.secrets:**
+   ```bash
+   export USE_BUN_SECRETS=1
+   ```
+
+2. **Initialize your secrets from existing environment variables:**
+   ```bash
+   bun run secrets init
+   ```
+
+3. **Manage secrets interactively:**
+   ```bash
+   bun run secrets
+   ```
+
+## Available Commands
+
+### `bun run secrets`
+Interactive CLI for managing secrets:
+- 📊 Show status
+- 📋 List all secrets
+- 🔐 Set a secret
+- 🔍 Get a secret
+- 🗑️ Delete a secret
+- 🚀 Initialize from environment
+
+### `bun run secrets <action>`
+Direct command execution:
+```bash
+bun run secrets status   # Show current status
+bun run secrets list     # List all configured secrets
+bun run secrets set      # Set a secret value
+bun run secrets get      # Retrieve a secret value
+bun run secrets delete   # Delete a secret
+bun run secrets init     # Initialize from environment variables
+```
+
+### `bun run test:secrets`
+Test the secrets integration and verify functionality.
+
+## Environment Variables
+
+### USE_BUN_SECRETS
+- `0` or `false`: Use traditional environment variables (default)
+- `1` or `true`: Use bun.secrets for credential storage
+
+When enabled, the application will:
+1. First try to retrieve secrets from bun.secrets
+2. Fall back to environment variables if not found
+3. Store new secrets using bun.secrets
+
+### Supported Secrets
+
+The system supports these secrets by default:
+
+| Secret Key | Description | Environment Variable |
+|------------|-------------|---------------------|
+| `VERCEL_OIDC_TOKEN` | Vercel OIDC authentication token | `VERCEL_OIDC_TOKEN` |
+| `ANTHROPIC_API_KEY` | Anthropic Claude API key | `ANTHROPIC_API_KEY` |
+| `OPENAI_API_KEY` | OpenAI API key | `OPENAI_API_KEY` |
+| `OPENROUTER_API_KEY` | OpenRouter API key | `OPENROUTER_API_KEY` |
+
+## Platform-Specific Storage
+
+- **macOS**: Keychain Services
+- **Linux**: libsecret (GNOME Keyring, KWallet, etc.)
+- **Windows**: Windows Credential Manager
+
+## Usage Examples
+
+### Setting Secrets
+
+```bash
+# Enable bun.secrets
+export USE_BUN_SECRETS=1
+
+# Set API key interactively
+bun run secrets set
+# Select "ANTHROPIC_API_KEY" from the menu
+# Enter your key when prompted
+```
+
+### Migrating from .env Files
+
+```bash
+# With your .env file in place and USE_BUN_SECRETS=1
+bun run secrets init
+# This will copy all environment variables into bun.secrets
+```
+
+### Programmatic Usage
+
+```typescript
+import { getSecret, setSecret, isBunSecretsEnabled } from "./lib/secrets.ts";
+
+// Check if bun.secrets is enabled
+if (isBunSecretsEnabled()) {
+  // Get a secret
+  const apiKey = await getSecret("ANTHROPIC_API_KEY");
+  
+  // Set a secret
+  await setSecret("CUSTOM_SECRET", "super-secret-value");
+}
+```
+
+## Security Considerations
+
+- ✅ Credentials are encrypted by the operating system
+- ✅ Access is restricted to the user who stored them
+- ✅ No plain text storage of sensitive data
+- ✅ Memory is zeroed after use
+- ✅ Survives application restarts
+
+## Migration from Environment Variables
+
+1. **Enable bun.secrets:**
+   ```bash
+   export USE_BUN_SECRETS=1
+   ```
+
+2. **Initialize from current environment:**
+   ```bash
+   bun run secrets init
+   ```
+
+3. **Verify migration:**
+   ```bash
+   bun run secrets list
+   ```
+
+4. **Optional:** Remove sensitive values from .env files
+
+## Custom Secret Definitions
+
+The system is modular and can be extended with custom secrets:
+
+```typescript
+import { DEFAULT_SECRETS, type SecretDefinition } from "./lib/secrets.ts";
+
+const customSecrets: SecretDefinition = {
+  ...DEFAULT_SECRETS,
+  MY_CUSTOM_API_KEY: {
+    service: "my-app",
+    name: "custom-api-key",
+    envVar: "MY_CUSTOM_API_KEY",
+  },
+};
+
+// Use custom secrets
+await setSecret("MY_CUSTOM_API_KEY", "my-secret-value", customSecrets);
+```
+
+## Troubleshooting
+
+### "bun.secrets is not enabled"
+Set `USE_BUN_SECRETS=1` in your environment.
+
+### "Failed to retrieve secret"
+- Check that the secret exists: `bun run secrets list`
+- Verify the secret name in the configuration
+- Check OS-specific permissions for credential storage
+
+### Linux: "Secret service daemon not running"
+Install and start a secret service:
+```bash
+# Ubuntu/Debian
+sudo apt-get install libsecret-1-0
+# GNOME Keyring should start automatically
+
+# Or for systems without GUI
+sudo apt-get install gnome-keyring
+eval $(gnome-keyring-daemon --start)
+```
+
+### Windows: Credential Manager access denied
+Ensure you're running with appropriate permissions to access Windows Credential Manager.
+
+## Best Practices
+
+1. **Enable bun.secrets in development** for better security
+2. **Use environment variables in CI/CD** (bun.secrets is not for production secrets)
+3. **Initialize secrets** when switching between bun.secrets and environment variables
+4. **Regularly check status** with `bun run secrets status`
+5. **Remove old .env values** after successful migration to avoid confusion

index.ts

@@ -31,6 +31,8 @@ import {
   type ModelPricing,
   type ModelPricingLookup,
 } from "./lib/pricing.ts";
+import { createModel, validateModelSecrets } from "./lib/gateway.ts";
+import { isBunSecretsEnabled } from "./lib/secrets.ts";
 import type { LanguageModel } from "ai";
 import {
   intro,
@@ -161,6 +163,11 @@ async function validateAndConfirmPricing(
 async function selectOptions(): Promise<SelectOptionsResult> {
   intro("🚀 Svelte AI Bench");
 
+  // Show secrets status
+  const secretsEnabled = isBunSecretsEnabled();
+  const secretsStatus = secretsEnabled ? "🔐 bun.secrets enabled" : "🌐 Environment variables";
+  note(secretsStatus, "Authentication Method");
+
   const available_models = await gateway.getAvailableModels();
 
   const models = await multiselect({
@@ -196,6 +203,31 @@ async function selectOptions(): Promise<SelectOptionsResult> {
 
   const selectedModels = models.filter((model) => model !== "custom");
 
+  // Validate that we have the required secrets for the selected models
+  const validation = await validateModelSecrets(selectedModels);
+  if (!validation.valid) {
+    const errorMessage = [
+      "Missing required API keys:",
+      ...validation.missing.map(missing => `  • ${missing}`),
+      "",
+      secretsEnabled 
+        ? "Run 'bun run secrets' to configure your secrets."
+        : "Set the required environment variables or enable bun.secrets.",
+    ].join("\n");
+    
+    note(errorMessage, "❌ Authentication Error");
+    
+    const proceed = await confirm({
+      message: "Continue anyway? Some models may fail.",
+      initialValue: false,
+    });
+
+    if (isCancel(proceed) || !proceed) {
+      cancel("Operation cancelled.");
+      process.exit(0);
+    }
+  }
+
   // Validate pricing for selected models
   const pricing = await validateAndConfirmPricing(selectedModels);
 
@@ -608,8 +640,8 @@ async function main() {
       );
     }
 
-    // Get the model from gateway
-    const model = gateway.languageModel(modelId);
+    // Get the model with secret integration
+    const model = await createModel(modelId);
 
     // Run all tests
     const testResults: SingleTestResult[] = [];