From 0805a6269df6598487df18290130fcc64dac5b98 Mon Sep 17 00:00:00 2001 From: David Turnbull Date: Fri, 3 Oct 2025 14:30:03 +1000 Subject: [PATCH 01/20] feat: providers package --- PRD.md | 426 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 426 insertions(+) create mode 100644 PRD.md diff --git a/PRD.md b/PRD.md new file mode 100644 index 000000000..85196283a --- /dev/null +++ b/PRD.md @@ -0,0 +1,426 @@ +# PRD: Unified Provider Architecture + +## Problem Statement + +There is significant drift between `packages/cli` and `packages/compiler` in their LLM provider implementations: + +### Current State + +**packages/cli supports:** +- OpenAI, Anthropic, Google, OpenRouter, Ollama, Mistral + +**packages/compiler supports:** +- Groq, Google, OpenRouter, Ollama, Mistral, Lingo.dev + +**packages/spec config schema defines:** +- `["openai", "anthropic", "google", "ollama", "openrouter", "mistral"]` + +### Drift Issues + +1. **Incomplete provider coverage** - CLI missing Groq; Compiler missing OpenAI/Anthropic +2. **Inconsistent implementations** - Different patterns for API key resolution, error handling, client creation +3. **No shared constants** - Provider IDs, metadata, env var names duplicated +4. **Config schema mismatch** - Spec doesn't list Groq as valid provider +5. **Maintenance burden** - Bug fixes must be applied twice; new providers require dual implementation + +## Goals + +1. **Eliminate drift** - Make it architecturally impossible for packages to diverge +2. **Support all 7 providers** - Groq, OpenAI, Anthropic, Google, OpenRouter, Ollama, Mistral +3. **Single source of truth** - One implementation of provider logic shared by both packages +4. **Consistent UX** - Same error messages, behavior, key resolution across all tooling +5. **Future-proof** - Easy to add new providers; other packages can consume provider logic + +## Architecture Decisions + +### Package Structure + +Create two new shared packages: + +``` +packages/config → .lingodotdevrc reading (user credentials) +packages/providers → Provider registry, API key resolution, client factories +packages/spec → i18n.json schema (CLI-only, existing) +packages/cli → Uses spec, config, providers +packages/compiler → Uses config, providers (NOT spec) +``` + +### Dependency Graph + +``` +packages/config (standalone, no deps) + ↓ +packages/providers (depends on config) + ↓ +packages/cli (depends on spec, config, providers) +packages/compiler (depends on config, providers) +``` + +**Key principle:** Compiler never depends on CLI-specific concerns (i18n.json schema in spec) + +### Design Decisions + +| Decision | Choice | Rationale | +|----------|--------|-----------| +| **Provider consolidation** | All 7 providers in both packages | Complete feature parity | +| **Abstraction level** | Maximal - full client factories | Eliminates all duplication, enforces consistency | +| **Error handling** | Structured error classes | Maximum optionality - consumers format as needed | +| **RC file reading** | Providers owns it (Phase 1), extracted to config (Phase 2) | Self-contained initially, shared later | +| **API design** | Layered (4 layers) | Flexibility - escape hatches for custom scenarios | + +## Implementation Plan + +### Phase 1: Create `packages/providers` (with duplicated rc reading) + +**Goal:** Establish provider package with complete functionality + +**Tasks:** +1. Create package scaffold with dependencies +2. Implement 4-layer API (see API Design section) +3. Port provider logic from CLI + compiler +4. Add all 7 providers with full support +5. Write comprehensive tests +6. Document API + +### Phase 2: Migrate CLI to use `packages/providers` + +**Goal:** Validate provider package works in production CLI + +**Tasks:** +1. Update CLI dependencies +2. Replace `cli/localizer/explicit.ts` with provider imports +3. Update `cli/utils/settings.ts` to use provider constants +4. Remove duplicated provider code +5. Ensure tests pass +6. Verify CLI functionality unchanged + +### Phase 3: Migrate compiler to use `packages/providers` + +**Goal:** Validate provider package works in production compiler + +**Tasks:** +1. Update compiler dependencies +2. Replace `compiler/lib/lcp/api/index.ts` provider logic with imports +3. Remove `compiler/utils/llm-api-key.ts` +4. Remove `compiler/lib/lcp/api/provider-details.ts` +5. Ensure tests pass +6. Verify compiler functionality unchanged + +### Phase 4: Extract rc reading to `packages/config` + +**Goal:** Eliminate remaining duplication in config reading + +**Tasks:** +1. Create `packages/config` package +2. Move `.lingodotdevrc` schema + reading logic from providers +3. Update providers to depend on config +4. Update CLI to use config for auth settings +5. Remove duplicated rc reading from CLI + +### Phase 5: Update `packages/spec` provider enum + +**Goal:** Ensure config schema matches implementation + +**Tasks:** +1. Add "groq" to provider ID enum in spec +2. Update config version (1.9 → 2.0) +3. Update JSON schema generation +4. Document migration path + +## API Design: `packages/providers` + +### Layer 1: Constants & Metadata + +```typescript +/** + * All supported provider IDs + */ +export const SUPPORTED_PROVIDERS = [ + 'groq', + 'openai', + 'anthropic', + 'google', + 'openrouter', + 'ollama', + 'mistral', +] as const; + +export type ProviderId = typeof SUPPORTED_PROVIDERS[number]; + +/** + * Metadata for each provider + */ +export interface ProviderMetadata { + name: string; // Display name (e.g., "Groq", "OpenAI") + apiKeyEnvVar?: string; // Environment variable name (e.g., "GROQ_API_KEY") + apiKeyConfigKey?: string; // RC file key path (e.g., "llm.groqApiKey") + getKeyLink: string; // Link to get API key + docsLink: string; // Link to API docs +} + +export const PROVIDER_METADATA: Record; +``` + +### Layer 2: Utilities + +```typescript +/** + * Get API key for provider from environment or rc file + * Returns undefined if not found + */ +export function getProviderApiKey(providerId: ProviderId): string | undefined; + +/** + * Resolve API key with custom sources + * @throws ProviderKeyMissingError if not found and required=true + */ +export function resolveProviderApiKey( + providerId: ProviderId, + options?: { + sources?: KeySources; + required?: boolean; + } +): string | undefined; + +/** + * Key sources for resolution + */ +export interface KeySources { + env?: Record; + rc?: RcData; +} + +/** + * Structured error: API key not found + */ +export class ProviderKeyMissingError extends Error { + constructor( + public providerId: ProviderId, + public envVar?: string, + public configKey?: string + ); +} + +/** + * Structured error: Authentication failed + */ +export class ProviderAuthFailedError extends Error { + constructor( + public providerId: ProviderId, + public originalError: Error + ); +} + +/** + * Structured error: Unsupported provider + */ +export class UnsupportedProviderError extends Error { + constructor(public providerId: string); +} +``` + +### Layer 3: Factory + +```typescript +/** + * Create AI SDK LanguageModel client for provider + * @throws ProviderKeyMissingError if key required but not found + * @throws UnsupportedProviderError if provider not supported + */ +export function createProviderClient( + providerId: ProviderId, + modelId: string, + options?: ClientOptions +): LanguageModel; + +export interface ClientOptions { + apiKey?: string; // Override API key (skip resolution) + baseUrl?: string; // Custom base URL for provider API + skipAuth?: boolean; // Skip authentication check (for Ollama) +} +``` + +### Layer 4: High-level + +```typescript +/** + * Get provider client + metadata in one call + * Convenience wrapper around createProviderClient + PROVIDER_METADATA + */ +export function getProvider( + providerId: ProviderId, + modelId: string, + options?: ClientOptions +): { + client: LanguageModel; + metadata: ProviderMetadata; +}; +``` + +## API Design: `packages/config` + +```typescript +/** + * Read and parse .lingodotdevrc file from home directory + */ +export function getRcConfig(): RcConfig; + +/** + * RC file data structure + */ +export interface RcConfig { + auth?: { + apiKey?: string; + apiUrl?: string; + webUrl?: string; + }; + llm?: { + groqApiKey?: string; + openaiApiKey?: string; + anthropicApiKey?: string; + googleApiKey?: string; + openrouterApiKey?: string; + mistralApiKey?: string; + }; +} + +/** + * Zod schema for validation + */ +export const rcConfigSchema: ZodType; +``` + +## Migration Examples + +### Before (CLI) + +```typescript +// packages/cli/src/cli/localizer/explicit.ts +switch (provider.id) { + case "openai": + return createAiSdkLocalizer({ + factory: (params) => createOpenAI(params).languageModel(provider.model), + id: provider.id, + prompt: provider.prompt, + apiKeyName: "OPENAI_API_KEY", + baseUrl: provider.baseUrl, + }); + // ... 5 more cases +} +``` + +### After (CLI) + +```typescript +// packages/cli/src/cli/localizer/explicit.ts +import { createProviderClient } from "@lingo.dev/providers"; + +const client = createProviderClient(provider.id, provider.model, { + baseUrl: provider.baseUrl, +}); +``` + +### Before (Compiler) + +```typescript +// packages/compiler/src/lib/lcp/api/index.ts (lines 291-392) +switch (providerId) { + case "groq": { + if (isRunningInCIOrDocker()) { + const groqFromEnv = getGroqKeyFromEnv(); + if (!groqFromEnv) { + this._failMissingLLMKeyCi(providerId); + } + } + const groqKey = getGroqKey(); + if (!groqKey) { + throw new Error("⚠️ GROQ API key not found..."); + } + return createGroq({ apiKey: groqKey })(modelId); + } + // ... 4 more cases +} +``` + +### After (Compiler) + +```typescript +// packages/compiler/src/lib/lcp/api/index.ts +import { createProviderClient, ProviderKeyMissingError } from "@lingo.dev/providers"; + +try { + return createProviderClient(providerId, modelId); +} catch (error) { + if (error instanceof ProviderKeyMissingError) { + // Custom error formatting for compiler context + this._failMissingLLMKeyCi(error.providerId); + } + throw error; +} +``` + +## Success Criteria + +### Functionality +- [ ] All 7 providers work in CLI +- [ ] All 7 providers work in compiler +- [ ] API key resolution works (env + rc file) +- [ ] Error handling preserves existing behavior +- [ ] No breaking changes for end users + +### Code Quality +- [ ] Zero duplication of provider logic +- [ ] All provider constants shared +- [ ] Comprehensive test coverage (>90%) +- [ ] Full API documentation +- [ ] Type safety maintained + +### Architecture +- [ ] Compiler doesn't depend on CLI concerns +- [ ] Clean dependency graph (no cycles) +- [ ] Layered API provides escape hatches +- [ ] Easy to add new providers (single location) + +### Validation +- [ ] CLI tests pass +- [ ] Compiler tests pass +- [ ] Integration tests pass +- [ ] Manual smoke testing complete + +## Risks & Mitigations + +| Risk | Impact | Mitigation | +|------|--------|------------| +| Breaking changes in migration | High | Phased rollout, comprehensive testing, feature flags | +| API key resolution differs subtly | Medium | Unit test all edge cases, validate against existing behavior | +| Dependency conflicts with AI SDK packages | Medium | Lock versions, test compatibility | +| RC file format assumptions differ | Low | Extract and validate schema early (Phase 4) | + +## Future Enhancements + +Once core architecture is in place: + +1. **Rate limiting** - Add rate limit handling to provider clients +2. **Retries** - Built-in retry logic with exponential backoff +3. **Observability** - Structured logging, metrics, tracing +4. **Caching** - Cache provider clients, metadata +5. **Validation** - Runtime validation of API keys, connectivity checks +6. **Provider plugins** - Allow custom provider implementations +7. **Cost tracking** - Track token usage per provider + +## Timeline Estimate + +- **Phase 1** (packages/providers): 2-3 days +- **Phase 2** (CLI migration): 1 day +- **Phase 3** (compiler migration): 1 day +- **Phase 4** (packages/config): 1 day +- **Phase 5** (spec update): 0.5 days +- **Testing & validation**: 1 day + +**Total: ~1 week** + +## Open Questions + +None - all architectural decisions have been made. + +Ready to begin implementation with Phase 1. From d63e60269d6546a566820ccd8bac7f318cfe5a86 Mon Sep 17 00:00:00 2001 From: David Turnbull Date: Fri, 3 Oct 2025 14:44:43 +1000 Subject: [PATCH 02/20] feat: phase 1 --- packages/providers/package.json | 49 +++++++++++++++++++ packages/providers/src/constants.ts | 11 +++++ packages/providers/src/errors.ts | 33 +++++++++++++ packages/providers/src/factory.spec.ts | 16 ++++++ packages/providers/src/factory.ts | 59 ++++++++++++++++++++++ packages/providers/src/highlevel.ts | 14 ++++++ packages/providers/src/index.ts | 20 ++++++++ packages/providers/src/keys.spec.ts | 24 +++++++++ packages/providers/src/keys.ts | 68 ++++++++++++++++++++++++++ packages/providers/src/metadata.ts | 61 +++++++++++++++++++++++ packages/providers/src/rc.ts | 32 ++++++++++++ packages/providers/tsconfig.json | 21 ++++++++ packages/providers/tsup.config.ts | 15 ++++++ pnpm-lock.yaml | 49 +++++++++++++++++++ 14 files changed, 472 insertions(+) create mode 100644 packages/providers/package.json create mode 100644 packages/providers/src/constants.ts create mode 100644 packages/providers/src/errors.ts create mode 100644 packages/providers/src/factory.spec.ts create mode 100644 packages/providers/src/factory.ts create mode 100644 packages/providers/src/highlevel.ts create mode 100644 packages/providers/src/index.ts create mode 100644 packages/providers/src/keys.spec.ts create mode 100644 packages/providers/src/keys.ts create mode 100644 packages/providers/src/metadata.ts create mode 100644 packages/providers/src/rc.ts create mode 100644 packages/providers/tsconfig.json create mode 100644 packages/providers/tsup.config.ts diff --git a/packages/providers/package.json b/packages/providers/package.json new file mode 100644 index 000000000..ec064665f --- /dev/null +++ b/packages/providers/package.json @@ -0,0 +1,49 @@ +{ + "name": "@lingo.dev/providers", + "version": "0.1.0", + "description": "Unified provider registry, API key resolution, and client factories for Lingo.dev", + "private": false, + "publishConfig": { + "access": "public" + }, + "type": "module", + "sideEffects": false, + "main": "build/index.cjs", + "module": "build/index.mjs", + "types": "build/index.d.ts", + "files": [ + "build" + ], + "scripts": { + "dev": "tsup --watch", + "build": "pnpm typecheck && tsup", + "typecheck": "tsc --noEmit", + "test": "vitest run", + "test:watch": "vitest" + }, + "keywords": [], + "author": "", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/anthropic": "^1.2.11", + "@ai-sdk/google": "^1.2.19", + "@ai-sdk/groq": "^1.2.3", + "@ai-sdk/mistral": "^1.2.8", + "@ai-sdk/openai": "^1.3.22", + "@openrouter/ai-sdk-provider": "^0.7.1", + "ai": "^4.3.15", + "dotenv": "^16.4.7", + "ini": "^5.0.0", + "ollama-ai-provider": "^1.2.0" + }, + "devDependencies": { + "@types/node": "^22.13.5", + "@types/ini": "^4.1.1", + "tsup": "^8.3.5", + "typescript": "^5.8.3", + "vitest": "^3.1.2" + }, + "engines": { + "node": ">=18" + } +} diff --git a/packages/providers/src/constants.ts b/packages/providers/src/constants.ts new file mode 100644 index 000000000..707fdf686 --- /dev/null +++ b/packages/providers/src/constants.ts @@ -0,0 +1,11 @@ +export const SUPPORTED_PROVIDERS = [ + "groq", + "openai", + "anthropic", + "google", + "openrouter", + "ollama", + "mistral", +] as const; + +export type ProviderId = (typeof SUPPORTED_PROVIDERS)[number]; diff --git a/packages/providers/src/errors.ts b/packages/providers/src/errors.ts new file mode 100644 index 000000000..8b061793f --- /dev/null +++ b/packages/providers/src/errors.ts @@ -0,0 +1,33 @@ +import { ProviderId } from "./constants"; + +export class ProviderKeyMissingError extends Error { + constructor( + public providerId: ProviderId, + public envVar?: string, + public configKey?: string, + ) { + super( + `API key for provider "${providerId}" not found` + + (envVar ? ` (env: ${envVar})` : "") + + (configKey ? ` (rc: ${configKey})` : ""), + ); + this.name = "ProviderKeyMissingError"; + } +} + +export class ProviderAuthFailedError extends Error { + constructor( + public providerId: ProviderId, + public originalError: Error, + ) { + super(`Authentication failed for provider "${providerId}": ${originalError.message}`); + this.name = "ProviderAuthFailedError"; + } +} + +export class UnsupportedProviderError extends Error { + constructor(public providerId: string) { + super(`Unsupported provider: ${providerId}`); + this.name = "UnsupportedProviderError"; + } +} diff --git a/packages/providers/src/factory.spec.ts b/packages/providers/src/factory.spec.ts new file mode 100644 index 000000000..e891e77a3 --- /dev/null +++ b/packages/providers/src/factory.spec.ts @@ -0,0 +1,16 @@ +import { describe, it, expect } from "vitest"; +import { createProviderClient } from "./factory"; + +describe("createProviderClient", () => { + it("creates ollama client without api key (skipAuth)", () => { + const client = createProviderClient("ollama", "llama3"); + expect(client).toBeTruthy(); + }); + + it("creates openrouter client with explicit apiKey override", () => { + const client = createProviderClient("openrouter", "gpt-4o-mini", { + apiKey: "dummy", + }); + expect(client).toBeTruthy(); + }); +}); diff --git a/packages/providers/src/factory.ts b/packages/providers/src/factory.ts new file mode 100644 index 000000000..59a2abcf4 --- /dev/null +++ b/packages/providers/src/factory.ts @@ -0,0 +1,59 @@ +import { LanguageModel } from "ai"; +import { ProviderId } from "./constants"; +import { resolveProviderApiKey } from "./keys"; +import { UnsupportedProviderError } from "./errors"; +import { createOpenAI } from "@ai-sdk/openai"; +import { createAnthropic } from "@ai-sdk/anthropic"; +import { createGoogleGenerativeAI } from "@ai-sdk/google"; +import { createOpenRouter } from "@openrouter/ai-sdk-provider"; +import { createOllama } from "ollama-ai-provider"; +import { createMistral } from "@ai-sdk/mistral"; +import { createGroq } from "@ai-sdk/groq"; + +export interface ClientOptions { + apiKey?: string; + baseUrl?: string; + skipAuth?: boolean; +} + +export function createProviderClient( + providerId: ProviderId, + modelId: string, + options?: ClientOptions, +): LanguageModel { + const skipAuth = options?.skipAuth === true || providerId === "ollama"; + const apiKey = options?.apiKey ?? resolveProviderApiKey(providerId, { required: !skipAuth }); + + switch (providerId) { + case "openai": { + const client = createOpenAI({ apiKey, baseURL: options?.baseUrl }); + return client(modelId); + } + case "anthropic": { + const client = createAnthropic({ apiKey, baseURL: options?.baseUrl }); + return client(modelId); + } + case "google": { + const client = createGoogleGenerativeAI({ apiKey }); + return client(modelId); + } + case "openrouter": { + const client = createOpenRouter({ apiKey, baseURL: options?.baseUrl }); + return client(modelId); + } + case "ollama": { + const client = createOllama(); + return client(modelId); + } + case "mistral": { + const client = createMistral({ apiKey, baseURL: options?.baseUrl }); + return client(modelId); + } + case "groq": { + const client = createGroq({ apiKey }); + return client(modelId); + } + default: + throw new UnsupportedProviderError(providerId); + } +} diff --git a/packages/providers/src/highlevel.ts b/packages/providers/src/highlevel.ts new file mode 100644 index 000000000..f5f9e77c0 --- /dev/null +++ b/packages/providers/src/highlevel.ts @@ -0,0 +1,14 @@ +import { ProviderId } from "./constants"; +import { ProviderMetadata, PROVIDER_METADATA } from "./metadata"; +import { createProviderClient, ClientOptions } from "./factory"; +import { LanguageModel } from "ai"; + +export function getProvider( + providerId: ProviderId, + modelId: string, + options?: ClientOptions, +): { client: LanguageModel; metadata: ProviderMetadata } { + const client = createProviderClient(providerId, modelId, options); + const metadata = PROVIDER_METADATA[providerId]; + return { client, metadata }; +} diff --git a/packages/providers/src/index.ts b/packages/providers/src/index.ts new file mode 100644 index 000000000..b25a8b465 --- /dev/null +++ b/packages/providers/src/index.ts @@ -0,0 +1,20 @@ +export { SUPPORTED_PROVIDERS, type ProviderId } from "./constants"; +export { + type ProviderMetadata, + PROVIDER_METADATA, +} from "./metadata"; +export { + getProviderApiKey, + resolveProviderApiKey, + type KeySources, +} from "./keys"; +export { + ProviderKeyMissingError, + ProviderAuthFailedError, + UnsupportedProviderError, +} from "./errors"; +export { + createProviderClient, + type ClientOptions, +} from "./factory"; +export { getProvider } from "./highlevel"; diff --git a/packages/providers/src/keys.spec.ts b/packages/providers/src/keys.spec.ts new file mode 100644 index 000000000..07e072ee2 --- /dev/null +++ b/packages/providers/src/keys.spec.ts @@ -0,0 +1,24 @@ +import { describe, it, expect } from "vitest"; +import { resolveProviderApiKey } from "./keys"; +import { ProviderKeyMissingError } from "./errors"; + +describe("resolveProviderApiKey", () => { + it("prefers env over rc when both provided", () => { + const key = resolveProviderApiKey("openai", { + sources: { + env: { OPENAI_API_KEY: "env-key" }, + rc: { llm: { openaiApiKey: "rc-key" } }, + }, + }); + expect(key).toBe("env-key"); + }); + + it("throws when required and key missing", () => { + expect(() => + resolveProviderApiKey("mistral", { + sources: { env: {}, rc: {} }, + required: true, + }), + ).toThrow(ProviderKeyMissingError); + }); +}); diff --git a/packages/providers/src/keys.ts b/packages/providers/src/keys.ts new file mode 100644 index 000000000..0e64efcb3 --- /dev/null +++ b/packages/providers/src/keys.ts @@ -0,0 +1,68 @@ +import path from "path"; +import fs from "fs"; +import dotenv from "dotenv"; +import { ProviderId } from "./constants"; +import { PROVIDER_METADATA } from "./metadata"; +import { readRc, RcData } from "./rc"; +import { ProviderKeyMissingError } from "./errors"; + +let dotenvLoaded = false; +function loadDotEnvOnce() { + if (dotenvLoaded) return; + const candidates = [ + path.resolve(process.cwd(), ".env"), + path.resolve(process.cwd(), ".env.local"), + path.resolve(process.cwd(), ".env.development"), + ]; + for (const file of candidates) { + if (fs.existsSync(file)) { + dotenv.config({ path: file }); + } + } + dotenvLoaded = true; +} + +function getByPath(obj: any, keyPath?: string): any { + if (!obj || !keyPath) return undefined; + return keyPath.split(".").reduce((acc, key) => (acc ? acc[key] : undefined), obj); +} + +export interface KeySources { + env?: Record; + rc?: RcData; +} + +export function getProviderApiKey(providerId: ProviderId): string | undefined { + return resolveProviderApiKey(providerId, { required: false }); +} + +export function resolveProviderApiKey( + providerId: ProviderId, + options?: { sources?: KeySources; required?: boolean }, +): string | undefined { + const meta = PROVIDER_METADATA[providerId]; + if (!meta) return undefined; + + const sources = options?.sources ?? {}; + + let envVal: string | undefined; + if (sources.env) { + envVal = meta.apiKeyEnvVar ? sources.env[meta.apiKeyEnvVar] : undefined; + } else { + loadDotEnvOnce(); + envVal = meta.apiKeyEnvVar ? process.env[meta.apiKeyEnvVar] : undefined; + } + + const rc = sources.rc ?? readRc(); + const rcVal = getByPath(rc, meta.apiKeyConfigKey); + + const key = envVal || rcVal; + if (!key && options?.required) { + throw new ProviderKeyMissingError( + providerId, + meta.apiKeyEnvVar, + meta.apiKeyConfigKey, + ); + } + return key; +} diff --git a/packages/providers/src/metadata.ts b/packages/providers/src/metadata.ts new file mode 100644 index 000000000..b9d570643 --- /dev/null +++ b/packages/providers/src/metadata.ts @@ -0,0 +1,61 @@ +import { ProviderId } from "./constants"; + +export interface ProviderMetadata { + name: string; + apiKeyEnvVar?: string; + apiKeyConfigKey?: string; + getKeyLink: string; + docsLink: string; +} + +export const PROVIDER_METADATA: Record = { + groq: { + name: "Groq", + apiKeyEnvVar: "GROQ_API_KEY", + apiKeyConfigKey: "llm.groqApiKey", + getKeyLink: "https://groq.com", + docsLink: "https://console.groq.com/docs/errors", + }, + openai: { + name: "OpenAI", + apiKeyEnvVar: "OPENAI_API_KEY", + apiKeyConfigKey: "llm.openaiApiKey", + getKeyLink: "https://platform.openai.com", + docsLink: "https://platform.openai.com/docs", + }, + anthropic: { + name: "Anthropic", + apiKeyEnvVar: "ANTHROPIC_API_KEY", + apiKeyConfigKey: "llm.anthropicApiKey", + getKeyLink: "https://console.anthropic.com", + docsLink: "https://docs.anthropic.com", + }, + google: { + name: "Google", + apiKeyEnvVar: "GOOGLE_API_KEY", + apiKeyConfigKey: "llm.googleApiKey", + getKeyLink: "https://ai.google.dev/", + docsLink: "https://ai.google.dev/gemini-api/docs/troubleshooting", + }, + openrouter: { + name: "OpenRouter", + apiKeyEnvVar: "OPENROUTER_API_KEY", + apiKeyConfigKey: "llm.openrouterApiKey", + getKeyLink: "https://openrouter.ai", + docsLink: "https://openrouter.ai/docs", + }, + ollama: { + name: "Ollama", + apiKeyEnvVar: undefined, + apiKeyConfigKey: undefined, + getKeyLink: "https://ollama.com/download", + docsLink: "https://github.com/ollama/ollama/tree/main/docs", + }, + mistral: { + name: "Mistral", + apiKeyEnvVar: "MISTRAL_API_KEY", + apiKeyConfigKey: "llm.mistralApiKey", + getKeyLink: "https://console.mistral.ai", + docsLink: "https://docs.mistral.ai", + }, +}; diff --git a/packages/providers/src/rc.ts b/packages/providers/src/rc.ts new file mode 100644 index 000000000..f076426ae --- /dev/null +++ b/packages/providers/src/rc.ts @@ -0,0 +1,32 @@ +import os from "os"; +import path from "path"; +import fs from "fs"; +import Ini from "ini"; + +export interface RcData { + auth?: { + apiKey?: string; + apiUrl?: string; + webUrl?: string; + }; + llm?: { + groqApiKey?: string; + openaiApiKey?: string; + anthropicApiKey?: string; + googleApiKey?: string; + openrouterApiKey?: string; + mistralApiKey?: string; + }; + [key: string]: any; +} + +export function readRc(): RcData { + const settingsFile = ".lingodotdevrc"; + const homedir = os.homedir(); + const settingsFilePath = path.join(homedir, settingsFile); + const content = fs.existsSync(settingsFilePath) + ? fs.readFileSync(settingsFilePath, "utf-8") + : ""; + const data = Ini.parse(content); + return data as RcData; +} diff --git a/packages/providers/tsconfig.json b/packages/providers/tsconfig.json new file mode 100644 index 000000000..d7b0f86bc --- /dev/null +++ b/packages/providers/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "rootDir": "src", + "outDir": "build", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "strict": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "allowSyntheticDefaultImports": true, + "skipLibCheck": true, + "moduleResolution": "Bundler", + "module": "ESNext", + "target": "ESNext", + "allowUnreachableCode": true, + "types": ["vitest/globals"] + }, + "include": ["src/**/*.ts"], + "exclude": ["src/**/*.spec.ts"] +} diff --git a/packages/providers/tsup.config.ts b/packages/providers/tsup.config.ts new file mode 100644 index 000000000..2d13ece73 --- /dev/null +++ b/packages/providers/tsup.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + clean: true, + target: "esnext", + entry: ["src/index.ts"], + outDir: "build", + format: ["cjs", "esm"], + dts: true, + cjsInterop: true, + splitting: true, + outExtension: (ctx) => ({ + js: ctx.format === "cjs" ? ".cjs" : ".mjs", + }), +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8358a3c52..44e2d421e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -815,6 +815,55 @@ importers: specifier: ^3.2.4 version: 3.2.4(@types/debug@4.1.12)(@types/node@22.17.2)(@vitest/ui@3.2.4)(jiti@2.5.1)(jsdom@25.0.1)(lightningcss@1.30.1)(terser@5.36.0)(tsx@4.20.3)(yaml@2.7.0) + packages/providers: + dependencies: + '@ai-sdk/anthropic': + specifier: ^1.2.11 + version: 1.2.11(zod@3.25.76) + '@ai-sdk/google': + specifier: ^1.2.19 + version: 1.2.19(zod@3.25.76) + '@ai-sdk/groq': + specifier: ^1.2.3 + version: 1.2.9(zod@3.25.76) + '@ai-sdk/mistral': + specifier: ^1.2.8 + version: 1.2.8(zod@3.25.76) + '@ai-sdk/openai': + specifier: ^1.3.22 + version: 1.3.22(zod@3.25.76) + '@openrouter/ai-sdk-provider': + specifier: ^0.7.1 + version: 0.7.1(ai@4.3.15(react@19.1.0)(zod@3.25.76))(zod@3.25.76) + ai: + specifier: ^4.3.15 + version: 4.3.15(react@19.1.0)(zod@3.25.76) + dotenv: + specifier: ^16.4.7 + version: 16.4.7 + ini: + specifier: ^5.0.0 + version: 5.0.0 + ollama-ai-provider: + specifier: ^1.2.0 + version: 1.2.0(zod@3.25.76) + devDependencies: + '@types/ini': + specifier: ^4.1.1 + version: 4.1.1 + '@types/node': + specifier: ^22.13.5 + version: 22.17.2 + tsup: + specifier: ^8.3.5 + version: 8.3.5(@swc/core@1.13.5)(jiti@2.5.1)(postcss@8.5.4)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.7.0) + typescript: + specifier: ^5.8.3 + version: 5.8.3 + vitest: + specifier: ^3.1.2 + version: 3.2.4(@types/debug@4.1.12)(@types/node@22.17.2)(@vitest/ui@3.2.4)(jiti@2.5.1)(jsdom@25.0.1)(lightningcss@1.30.1)(terser@5.36.0)(tsx@4.20.3)(yaml@2.7.0) + packages/react: dependencies: js-cookie: From 173da9c3184d28eeafab10dca91f5c10816e6937 Mon Sep 17 00:00:00 2001 From: David Turnbull Date: Fri, 3 Oct 2025 14:49:04 +1000 Subject: [PATCH 03/20] chore: rename --- packages/providers/src/{highlevel.ts => get-provider.ts} | 0 packages/providers/src/index.ts | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename packages/providers/src/{highlevel.ts => get-provider.ts} (100%) diff --git a/packages/providers/src/highlevel.ts b/packages/providers/src/get-provider.ts similarity index 100% rename from packages/providers/src/highlevel.ts rename to packages/providers/src/get-provider.ts diff --git a/packages/providers/src/index.ts b/packages/providers/src/index.ts index b25a8b465..d466e15b4 100644 --- a/packages/providers/src/index.ts +++ b/packages/providers/src/index.ts @@ -17,4 +17,4 @@ export { createProviderClient, type ClientOptions, } from "./factory"; -export { getProvider } from "./highlevel"; +export { getProvider } from "./get-provider"; From 5ad36fc7f8f9b30a3dc24a2757b5ba461954aaf8 Mon Sep 17 00:00:00 2001 From: David Turnbull Date: Fri, 3 Oct 2025 15:02:26 +1000 Subject: [PATCH 04/20] feat: phase 2 --- packages/cli/package.json | 1 + packages/cli/src/cli/localizer/explicit.ts | 168 +++++++-------------- packages/cli/src/cli/processor/index.ts | 92 ++++------- packages/cli/src/cli/utils/settings.ts | 50 ++---- 4 files changed, 98 insertions(+), 213 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index de68d1bc0..096d90e17 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -122,6 +122,7 @@ "author": "", "license": "Apache-2.0", "dependencies": { + "@lingo.dev/providers": "workspace:*", "@ai-sdk/anthropic": "^1.2.11", "@ai-sdk/google": "^1.2.19", "@ai-sdk/mistral": "^1.2.8", diff --git a/packages/cli/src/cli/localizer/explicit.ts b/packages/cli/src/cli/localizer/explicit.ts index c055f56e3..6dcf2a62a 100644 --- a/packages/cli/src/cli/localizer/explicit.ts +++ b/packages/cli/src/cli/localizer/explicit.ts @@ -1,8 +1,3 @@ -import { createAnthropic } from "@ai-sdk/anthropic"; -import { createGoogleGenerativeAI } from "@ai-sdk/google"; -import { createOpenAI } from "@ai-sdk/openai"; -import { createOpenRouter } from "@openrouter/ai-sdk-provider"; -import { createMistral } from "@ai-sdk/mistral"; import { I18nConfig } from "@lingo.dev/_spec"; import chalk from "chalk"; import dedent from "dedent"; @@ -10,135 +5,84 @@ import { ILocalizer, LocalizerData } from "./_types"; import { LanguageModel, Message, generateText } from "ai"; import { colors } from "../constants"; import { jsonrepair } from "jsonrepair"; -import { createOllama } from "ollama-ai-provider"; +import { + createProviderClient, + ProviderKeyMissingError, + PROVIDER_METADATA, + SUPPORTED_PROVIDERS, + type ProviderId, +} from "@lingo.dev/providers"; export default function createExplicitLocalizer( provider: NonNullable, ): ILocalizer { - const settings = provider.settings || {}; + const supported = new Set(SUPPORTED_PROVIDERS as readonly string[]); - switch (provider.id) { - default: + if (!supported.has(provider.id as any)) { + throw new Error( + dedent` + You're trying to use unsupported provider: ${chalk.dim(provider.id)}. + + To fix this issue: + 1. Switch to one of the supported providers, or + 2. Remove the ${chalk.italic("provider")} node from your i18n.json configuration to switch to ${chalk.hex( + colors.green, + )("Lingo.dev")} + + ${chalk.hex(colors.blue)("Docs: https://lingo.dev/go/docs")} + `, + ); + } + + const skipAuth = provider.id === "ollama"; + try { + const model = createProviderClient(provider.id as ProviderId, provider.model, { + baseUrl: provider.baseUrl, + skipAuth, + }); + return createLocalizerFromModel({ + model, + id: provider.id, + prompt: provider.prompt, + skipAuth, + }); + } catch (error: unknown) { + if (error instanceof ProviderKeyMissingError) { + const meta = PROVIDER_METADATA[error.providerId]; + const envVar = meta?.apiKeyEnvVar; throw new Error( dedent` - You're trying to use unsupported provider: ${chalk.dim(provider.id)}. + You're trying to use raw ${chalk.dim(provider.id)} API for translation. ${ + envVar + ? `However, ${chalk.dim(envVar)} environment variable is not set.` + : "However, that provider is unavailable." + } To fix this issue: - 1. Switch to one of the supported providers, or - 2. Remove the ${chalk.italic( - "provider", - )} node from your i18n.json configuration to switch to ${chalk.hex( + 1. ${ + envVar + ? `Set ${chalk.dim(envVar)} in your environment variables` + : "Set the environment variable for your provider (if required)" + }, or + 2. Remove the ${chalk.italic("provider")} node from your i18n.json configuration to switch to ${chalk.hex( colors.green, )("Lingo.dev")} ${chalk.hex(colors.blue)("Docs: https://lingo.dev/go/docs")} `, ); - case "openai": - return createAiSdkLocalizer({ - factory: (params) => createOpenAI(params).languageModel(provider.model), - id: provider.id, - prompt: provider.prompt, - apiKeyName: "OPENAI_API_KEY", - baseUrl: provider.baseUrl, - settings, - }); - case "anthropic": - return createAiSdkLocalizer({ - factory: (params) => - createAnthropic(params).languageModel(provider.model), - id: provider.id, - prompt: provider.prompt, - apiKeyName: "ANTHROPIC_API_KEY", - baseUrl: provider.baseUrl, - settings, - }); - case "google": - return createAiSdkLocalizer({ - factory: (params) => - createGoogleGenerativeAI(params).languageModel(provider.model), - id: provider.id, - prompt: provider.prompt, - apiKeyName: "GOOGLE_API_KEY", - baseUrl: provider.baseUrl, - settings, - }); - case "openrouter": - return createAiSdkLocalizer({ - factory: (params) => - createOpenRouter(params).languageModel(provider.model), - id: provider.id, - prompt: provider.prompt, - apiKeyName: "OPENROUTER_API_KEY", - baseUrl: provider.baseUrl, - settings, - }); - case "ollama": - return createAiSdkLocalizer({ - factory: (_params) => createOllama().languageModel(provider.model), - id: provider.id, - prompt: provider.prompt, - skipAuth: true, - settings, - }); - case "mistral": - return createAiSdkLocalizer({ - factory: (params) => - createMistral(params).languageModel(provider.model), - id: provider.id, - prompt: provider.prompt, - apiKeyName: "MISTRAL_API_KEY", - baseUrl: provider.baseUrl, - settings, - }); + } + throw error as Error; } } -function createAiSdkLocalizer(params: { - factory: (params: { apiKey?: string; baseUrl?: string }) => LanguageModel; +function createLocalizerFromModel(params: { + model: LanguageModel; id: NonNullable["id"]; prompt: string; - apiKeyName?: string; - baseUrl?: string; skipAuth?: boolean; - settings?: { temperature?: number }; }): ILocalizer { - const skipAuth = params.skipAuth === true; - - const apiKey = process.env[params?.apiKeyName ?? ""]; - if ((!skipAuth && !apiKey) || !params.apiKeyName) { - throw new Error( - dedent` - You're trying to use raw ${chalk.dim(params.id)} API for translation. ${ - params.apiKeyName - ? `However, ${chalk.dim( - params.apiKeyName, - )} environment variable is not set.` - : "However, that provider is unavailable." - } - - To fix this issue: - 1. ${ - params.apiKeyName - ? `Set ${chalk.dim( - params.apiKeyName, - )} in your environment variables` - : "Set the environment variable for your provider (if required)" - }, or - 2. Remove the ${chalk.italic( - "provider", - )} node from your i18n.json configuration to switch to ${chalk.hex( - colors.green, - )("Lingo.dev")} - - ${chalk.hex(colors.blue)("Docs: https://lingo.dev/go/docs")} - `, - ); - } - - const model = params.factory( - skipAuth ? {} : { apiKey, baseUrl: params.baseUrl }, - ); + const { model } = params; return { id: params.id, diff --git a/packages/cli/src/cli/processor/index.ts b/packages/cli/src/cli/processor/index.ts index 3f898d04a..696a359be 100644 --- a/packages/cli/src/cli/processor/index.ts +++ b/packages/cli/src/cli/processor/index.ts @@ -4,13 +4,14 @@ import dedent from "dedent"; import { LocalizerFn } from "./_base"; import { createLingoLocalizer } from "./lingo"; import { createBasicTranslator } from "./basic"; -import { createOpenAI } from "@ai-sdk/openai"; import { colors } from "../constants"; -import { createAnthropic } from "@ai-sdk/anthropic"; -import { createGoogleGenerativeAI } from "@ai-sdk/google"; -import { createOpenRouter } from "@openrouter/ai-sdk-provider"; -import { createMistral } from "@ai-sdk/mistral"; -import { createOllama } from "ollama-ai-provider"; +import { + createProviderClient, + ProviderKeyMissingError, + PROVIDER_METADATA, + SUPPORTED_PROVIDERS, + type ProviderId, +} from "@lingo.dev/providers"; export default function createProcessor( provider: I18nConfig["provider"], @@ -68,66 +69,25 @@ function getPureModelProvider(provider: I18nConfig["provider"]) { ${chalk.hex(colors.blue)("Docs: https://lingo.dev/go/docs")} `; - switch (provider?.id) { - case "openai": { - if (!process.env.OPENAI_API_KEY) { - throw new Error( - createMissingKeyErrorMessage("OpenAI", "OPENAI_API_KEY"), - ); - } - return createOpenAI({ - apiKey: process.env.OPENAI_API_KEY, - baseURL: provider.baseUrl, - })(provider.model); - } - case "anthropic": { - if (!process.env.ANTHROPIC_API_KEY) { - throw new Error( - createMissingKeyErrorMessage("Anthropic", "ANTHROPIC_API_KEY"), - ); - } - return createAnthropic({ - apiKey: process.env.ANTHROPIC_API_KEY, - })(provider.model); - } - case "google": { - if (!process.env.GOOGLE_API_KEY) { - throw new Error( - createMissingKeyErrorMessage("Google", "GOOGLE_API_KEY"), - ); - } - return createGoogleGenerativeAI({ - apiKey: process.env.GOOGLE_API_KEY, - })(provider.model); - } - case "openrouter": { - if (!process.env.OPENROUTER_API_KEY) { - throw new Error( - createMissingKeyErrorMessage("OpenRouter", "OPENROUTER_API_KEY"), - ); - } - return createOpenRouter({ - apiKey: process.env.OPENROUTER_API_KEY, - baseURL: provider.baseUrl, - })(provider.model); - } - case "ollama": { - // No API key check needed for Ollama - return createOllama()(provider.model); - } - case "mistral": { - if (!process.env.MISTRAL_API_KEY) { - throw new Error( - createMissingKeyErrorMessage("Mistral", "MISTRAL_API_KEY"), - ); - } - return createMistral({ - apiKey: process.env.MISTRAL_API_KEY, - baseURL: provider.baseUrl, - })(provider.model); - } - default: { - throw new Error(createUnsupportedProviderErrorMessage(provider?.id)); + const supported = new Set(SUPPORTED_PROVIDERS as readonly string[]); + + if (!supported.has(provider?.id as any)) { + throw new Error(createUnsupportedProviderErrorMessage(provider?.id)); + } + + const skipAuth = provider?.id === "ollama"; + try { + return createProviderClient(provider!.id as ProviderId, provider!.model, { + baseUrl: provider!.baseUrl, + skipAuth, + }); + } catch (error: unknown) { + if (error instanceof ProviderKeyMissingError) { + const meta = PROVIDER_METADATA[error.providerId]; + throw new Error( + createMissingKeyErrorMessage(meta?.name ?? error.providerId, meta?.apiKeyEnvVar), + ); } + throw error as Error; } } diff --git a/packages/cli/src/cli/utils/settings.ts b/packages/cli/src/cli/utils/settings.ts index 8d3db2928..9883f2d17 100644 --- a/packages/cli/src/cli/utils/settings.ts +++ b/packages/cli/src/cli/utils/settings.ts @@ -4,6 +4,7 @@ import _ from "lodash"; import Z from "zod"; import fs from "fs"; import Ini from "ini"; +import { PROVIDER_METADATA } from "@lingo.dev/providers"; export type CliSettings = Z.infer; @@ -177,41 +178,20 @@ function _envVarsInfo() { `ℹ️ Using LINGODOTDEV_API_KEY env var instead of credentials from user config`, ); } - if (env.OPENAI_API_KEY && systemFile.llm?.openaiApiKey) { - console.info( - "\x1b[36m%s\x1b[0m", - `ℹ️ Using OPENAI_API_KEY env var instead of key from user config.`, - ); - } - if (env.ANTHROPIC_API_KEY && systemFile.llm?.anthropicApiKey) { - console.info( - "\x1b[36m%s\x1b[0m", - `ℹ️ Using ANTHROPIC_API_KEY env var instead of key from user config`, - ); - } - if (env.GROQ_API_KEY && systemFile.llm?.groqApiKey) { - console.info( - "\x1b[36m%s\x1b[0m", - `ℹ️ Using GROQ_API_KEY env var instead of key from user config`, - ); - } - if (env.GOOGLE_API_KEY && systemFile.llm?.googleApiKey) { - console.info( - "\x1b[36m%s\x1b[0m", - `ℹ️ Using GOOGLE_API_KEY env var instead of key from user config`, - ); - } - if (env.OPENROUTER_API_KEY && systemFile.llm?.openrouterApiKey) { - console.info( - "\x1b[36m%s\x1b[0m", - `ℹ️ Using OPENROUTER_API_KEY env var instead of key from user config`, - ); - } - if (env.MISTRAL_API_KEY && systemFile.llm?.mistralApiKey) { - console.info( - "\x1b[36m%s\x1b[0m", - `ℹ️ Using MISTRAL_API_KEY env var instead of key from user config`, - ); + // Provider-specific env vs rc info using shared metadata + for (const meta of Object.values(PROVIDER_METADATA)) { + const envVar = meta.apiKeyEnvVar; + const cfgKey = meta.apiKeyConfigKey; + if (!envVar || !cfgKey) continue; + const cfgSuffix = cfgKey.startsWith("llm.") ? cfgKey.slice(4) : undefined; + const envVal = (env as any)[envVar]; + const rcVal = cfgSuffix ? (systemFile.llm as any)?.[cfgSuffix] : undefined; + if (envVal && rcVal) { + console.info( + "\x1b[36m%s\x1b[0m", + `ℹ️ Using ${envVar} env var instead of key from user config`, + ); + } } if (env.LINGODOTDEV_API_URL) { console.info( From 6c3645ca26da0f3a9ce68f93b2c880753f819c33 Mon Sep 17 00:00:00 2001 From: David Turnbull Date: Fri, 3 Oct 2025 15:13:19 +1000 Subject: [PATCH 05/20] feat: phase 3 --- packages/compiler/package.json | 1 + packages/compiler/src/index.ts | 83 ++++---- packages/compiler/src/lib/lcp/api/index.ts | 196 ++++++------------ .../src/lib/lcp/api/provider-details.spec.ts | 15 -- .../src/lib/lcp/api/provider-details.ts | 55 ----- packages/compiler/src/utils/llm-api-key.ts | 84 -------- .../compiler/src/utils/llm-api-keys.spec.ts | 54 ----- pnpm-lock.yaml | 3 + 8 files changed, 107 insertions(+), 384 deletions(-) delete mode 100644 packages/compiler/src/lib/lcp/api/provider-details.spec.ts delete mode 100644 packages/compiler/src/lib/lcp/api/provider-details.ts delete mode 100644 packages/compiler/src/utils/llm-api-key.ts delete mode 100644 packages/compiler/src/utils/llm-api-keys.spec.ts diff --git a/packages/compiler/package.json b/packages/compiler/package.json index 946650444..e4bc9dc5e 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -38,6 +38,7 @@ "vitest": "^2.1.4" }, "dependencies": { + "@lingo.dev/providers": "workspace:*", "@ai-sdk/google": "^1.2.19", "@ai-sdk/groq": "^1.2.3", "@ai-sdk/mistral": "^1.2.8", diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 94fcb6056..a5d80819b 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -7,45 +7,26 @@ import { defaultParams } from "./_base"; import { LCP_DICTIONARY_FILE_NAME } from "./_const"; import { LCPCache } from "./lib/lcp/cache"; import { getInvalidLocales } from "./utils/locales"; -import { - getGroqKeyFromEnv, - getGroqKeyFromRc, - getGoogleKeyFromEnv, - getGoogleKeyFromRc, - getMistralKeyFromEnv, - getMistralKeyFromRc, - getLingoDotDevKeyFromEnv, - getLingoDotDevKeyFromRc, -} from "./utils/llm-api-key"; import { isRunningInCIOrDocker } from "./utils/env"; -import { providerDetails } from "./lib/lcp/api/provider-details"; import { loadDictionary, transformComponent } from "./_loader-utils"; import trackEvent from "./utils/observability"; +import { PROVIDER_METADATA, resolveProviderApiKey } from "@lingo.dev/providers"; +import { getRc } from "./utils/rc"; -const keyCheckers: Record< - string, - { - checkEnv: () => string | undefined; - checkRc: () => string | undefined; +function getProviderDetails(providerId: string) { + const meta = (PROVIDER_METADATA as any)[providerId]; + if (meta) return meta; + if (providerId === "lingo.dev") { + return { + name: "Lingo.dev", + apiKeyEnvVar: "LINGODOTDEV_API_KEY", + apiKeyConfigKey: "auth.apiKey", + getKeyLink: "https://lingo.dev", + docsLink: "https://lingo.dev/docs", + }; } -> = { - groq: { - checkEnv: getGroqKeyFromEnv, - checkRc: getGroqKeyFromRc, - }, - google: { - checkEnv: getGoogleKeyFromEnv, - checkRc: getGoogleKeyFromRc, - }, - mistral: { - checkEnv: getMistralKeyFromEnv, - checkRc: getMistralKeyFromRc, - }, - "lingo.dev": { - checkEnv: getLingoDotDevKeyFromEnv, - checkRc: getLingoDotDevKeyFromRc, - }, -}; + return undefined; +} const alreadySentBuildEvent = { value: false }; @@ -347,9 +328,7 @@ function getConfiguredProviders(models: Record): string[] { .filter(Boolean) // Remove empty strings .uniq() // Get unique providers .filter( - (providerId) => - providerDetails.hasOwnProperty(providerId) && - keyCheckers.hasOwnProperty(providerId), + (providerId) => !!(PROVIDER_METADATA as any)[providerId], ) // Only check for known and implemented providers .value(); } @@ -371,19 +350,37 @@ function validateLLMKeyDetails(configuredProviders: string[]): void { { foundInEnv: boolean; foundInRc: boolean; - details: (typeof providerDetails)[string]; + details: ReturnType; } > = {}; const missingProviders: string[] = []; const foundProviders: string[] = []; for (const providerId of configuredProviders) { - const details = providerDetails[providerId]; - const checkers = keyCheckers[providerId]; - if (!details || !checkers) continue; // Should not happen due to filter above + const details = getProviderDetails(providerId); + if (!details) continue; - const foundInEnv = !!checkers.checkEnv(); - const foundInRc = !!checkers.checkRc(); + const foundInEnv = (() => { + if (providerId === "lingo.dev") { + return !!process.env["LINGODOTDEV_API_KEY"]; + } + const envVar = details.apiKeyEnvVar; + if (!envVar) return false; + // Isolate env-only resolution + return !!resolveProviderApiKey(providerId as any, { + sources: { env: { [envVar]: process.env[envVar] }, rc: {} as any }, + }); + })(); + + const foundInRc = (() => { + const rc = getRc(); + if (providerId === "lingo.dev") { + return typeof _.get(rc, "auth.apiKey") === "string"; + } + return !!resolveProviderApiKey(providerId as any, { + sources: { env: {}, rc: rc as any }, + }); + })(); keyStatuses[providerId] = { foundInEnv, foundInRc, details }; diff --git a/packages/compiler/src/lib/lcp/api/index.ts b/packages/compiler/src/lib/lcp/api/index.ts index 0c7cf5541..b06a873e7 100644 --- a/packages/compiler/src/lib/lcp/api/index.ts +++ b/packages/compiler/src/lib/lcp/api/index.ts @@ -1,8 +1,3 @@ -import { createGroq } from "@ai-sdk/groq"; -import { createGoogleGenerativeAI } from "@ai-sdk/google"; -import { createOpenRouter } from "@openrouter/ai-sdk-provider"; -import { createOllama } from "ollama-ai-provider"; -import { createMistral } from "@ai-sdk/mistral"; import { generateText } from "ai"; import { LingoDotDevEngine } from "@lingo.dev/_sdk"; import { DictionarySchema } from "../schema"; @@ -11,22 +6,18 @@ import { getLocaleModel } from "../../../utils/locales"; import getSystemPrompt from "./prompt"; import { obj2xml, xml2obj } from "./xml2obj"; import shots from "./shots"; -import { - getGroqKey, - getGroqKeyFromEnv, - getGoogleKey, - getGoogleKeyFromEnv, - getOpenRouterKey, - getOpenRouterKeyFromEnv, - getMistralKey, - getMistralKeyFromEnv, - getLingoDotDevKeyFromEnv, - getLingoDotDevKey, -} from "../../../utils/llm-api-key"; import dedent from "dedent"; import { isRunningInCIOrDocker } from "../../../utils/env"; import { LanguageModel } from "ai"; -import { providerDetails } from "./provider-details"; +import { + createProviderClient, + ProviderKeyMissingError, + PROVIDER_METADATA, + type ProviderId, +} from "@lingo.dev/providers"; +import * as dotenv from "dotenv"; +import path from "path"; +import { getRc } from "../../../utils/rc"; export class LCPAPI { static async translate( @@ -139,14 +130,31 @@ export class LCPAPI { } private static _createLingoDotDevEngine() { - // Specific check for CI/CD or Docker missing GROQ key + const getEnvWithDotenv = (name: string): string | undefined => { + if (process.env[name]) return process.env[name]; + const result = dotenv.config({ + path: [ + path.resolve(process.cwd(), ".env"), + path.resolve(process.cwd(), ".env.local"), + path.resolve(process.cwd(), ".env.development"), + ], + }); + return result?.parsed?.[name]; + }; + if (isRunningInCIOrDocker()) { - const apiKeyFromEnv = getLingoDotDevKeyFromEnv(); + const apiKeyFromEnv = getEnvWithDotenv("LINGODOTDEV_API_KEY"); if (!apiKeyFromEnv) { this._failMissingLLMKeyCi("lingo.dev"); } } - const apiKey = getLingoDotDevKey(); + const apiKey = + getEnvWithDotenv("LINGODOTDEV_API_KEY") || + ((): string | undefined => { + const rc = getRc(); + const val = _.get(rc, "auth.apiKey"); + return typeof val === "string" ? val : undefined; + })(); if (!apiKey) { throw new Error( "⚠️ Lingo.dev API key not found. Please set LINGODOTDEV_API_KEY environment variable or configure it user-wide.", @@ -223,7 +231,24 @@ export class LCPAPI { } try { - const aiModel = this._createAiModel(provider, model, targetLocale); + const aiModel = ((): LanguageModel => { + try { + return createProviderClient(provider as ProviderId, model); + } catch (error: unknown) { + if (error instanceof ProviderKeyMissingError) { + if (isRunningInCIOrDocker()) { + this._failMissingLLMKeyCi(error.providerId); + } else { + this._failLLMFailureLocal( + provider, + targetLocale, + error.message, + ); + } + } + throw error as Error; + } + })(); console.log( `ℹ️ Using raw LLM API ("${provider}":"${model}") to translate from "${sourceLocale}" to "${targetLocale}"`, @@ -279,116 +304,21 @@ export class LCPAPI { } } - /** - * Instantiates an AI model based on provider and model ID. - * Includes CI/CD API key checks. - * @param providerId The ID of the AI provider (e.g., "groq", "google"). - * @param modelId The ID of the specific model (e.g., "llama3-8b-8192", "gemini-2.0-flash"). - * @param targetLocale The target locale being translated to (for logging/error messages). - * @returns An instantiated AI LanguageModel. - * @throws Error if the provider is not supported or API key is missing in CI/CD. - */ - private static _createAiModel( - providerId: string, - modelId: string, - targetLocale: string, - ): LanguageModel { - switch (providerId) { - case "groq": { - // Specific check for CI/CD or Docker missing GROQ key - if (isRunningInCIOrDocker()) { - const groqFromEnv = getGroqKeyFromEnv(); - if (!groqFromEnv) { - this._failMissingLLMKeyCi(providerId); - } - } - const groqKey = getGroqKey(); - if (!groqKey) { - throw new Error( - "⚠️ GROQ API key not found. Please set GROQ_API_KEY environment variable or configure it user-wide.", - ); - } - console.log( - `Creating Groq client for ${targetLocale} using model ${modelId}`, - ); - return createGroq({ apiKey: groqKey })(modelId); - } - - case "google": { - // Specific check for CI/CD or Docker missing Google key - if (isRunningInCIOrDocker()) { - const googleFromEnv = getGoogleKeyFromEnv(); - if (!googleFromEnv) { - this._failMissingLLMKeyCi(providerId); - } - } - const googleKey = getGoogleKey(); - if (!googleKey) { - throw new Error( - "⚠️ Google API key not found. Please set GOOGLE_API_KEY environment variable or configure it user-wide.", - ); - } - console.log( - `Creating Google Generative AI client for ${targetLocale} using model ${modelId}`, - ); - return createGoogleGenerativeAI({ apiKey: googleKey })(modelId); - } - case "openrouter": { - // Specific check for CI/CD or Docker missing OpenRouter key - if (isRunningInCIOrDocker()) { - const openRouterFromEnv = getOpenRouterKeyFromEnv(); - if (!openRouterFromEnv) { - this._failMissingLLMKeyCi(providerId); - } - } - const openRouterKey = getOpenRouterKey(); - if (!openRouterKey) { - throw new Error( - "⚠️ OpenRouter API key not found. Please set OPENROUTER_API_KEY environment variable or configure it user-wide.", - ); - } - console.log( - `Creating OpenRouter client for ${targetLocale} using model ${modelId}`, - ); - return createOpenRouter({ - apiKey: openRouterKey, - })(modelId); - } - - case "ollama": { - // No API key check needed for Ollama - console.log( - `Creating Ollama client for ${targetLocale} using model ${modelId} at default Ollama address`, - ); - return createOllama()(modelId); - } - - case "mistral": { - // Specific check for CI/CD or Docker missing Mistral key - if (isRunningInCIOrDocker()) { - const mistralFromEnv = getMistralKeyFromEnv(); - if (!mistralFromEnv) { - this._failMissingLLMKeyCi(providerId); - } - } - const mistralKey = getMistralKey(); - if (!mistralKey) { - throw new Error( - "⚠️ Mistral API key not found. Please set MISTRAL_API_KEY environment variable or configure it user-wide.", - ); - } - console.log( - `Creating Mistral client for ${targetLocale} using model ${modelId}`, - ); - return createMistral({ apiKey: mistralKey })(modelId); - } - - default: { - throw new Error( - `⚠️ Provider "${providerId}" for locale "${targetLocale}" is not supported. Only "groq", "google", "openrouter", "ollama", and "mistral" providers are supported at the moment.`, - ); - } + // details lookup compatible with providers metadata + lingo.dev special-case + private static _getProviderDetails(providerId: string) { + if ((PROVIDER_METADATA as any)[providerId]) { + return (PROVIDER_METADATA as any)[providerId]; + } + if (providerId === "lingo.dev") { + return { + name: "Lingo.dev", + apiKeyEnvVar: "LINGODOTDEV_API_KEY", + apiKeyConfigKey: "auth.apiKey", + getKeyLink: "https://lingo.dev", + docsLink: "https://lingo.dev/docs", + }; } + return undefined; } /** @@ -398,7 +328,7 @@ export class LCPAPI { * @param providerId The ID of the LLM provider whose key is missing. */ private static _failMissingLLMKeyCi(providerId: string): void { - let details = providerDetails[providerId]; + let details = this._getProviderDetails(providerId); if (!details) { // Fallback for unsupported provider in failure message logic console.error( @@ -443,7 +373,7 @@ export class LCPAPI { targetLocale: string, errorMessage: string, ): void { - const details = providerDetails[providerId]; + const details = this._getProviderDetails(providerId); if (!details) { // Fallback console.error( diff --git a/packages/compiler/src/lib/lcp/api/provider-details.spec.ts b/packages/compiler/src/lib/lcp/api/provider-details.spec.ts deleted file mode 100644 index 79f3a79d8..000000000 --- a/packages/compiler/src/lib/lcp/api/provider-details.spec.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { describe, it, expect } from "vitest"; -import { providerDetails } from "./provider-details"; - -describe("provider-details", () => { - it("should provide data for all supported providers", () => { - expect(Object.keys(providerDetails)).toEqual([ - "groq", - "google", - "openrouter", - "ollama", - "mistral", - "lingo.dev", - ]); - }); -}); diff --git a/packages/compiler/src/lib/lcp/api/provider-details.ts b/packages/compiler/src/lib/lcp/api/provider-details.ts deleted file mode 100644 index b4468720d..000000000 --- a/packages/compiler/src/lib/lcp/api/provider-details.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { openrouter } from "@openrouter/ai-sdk-provider"; - -export const providerDetails: Record< - string, - { - name: string; // Display name (e.g., "Groq", "Google") - apiKeyEnvVar?: string; // Environment variable name (e.g., "GROQ_API_KEY") - apiKeyConfigKey?: string; // Config key if applicable (e.g., "llm.groqApiKey") - getKeyLink: string; // Link to get API key - docsLink: string; // Link to API docs for troubleshooting - } -> = { - groq: { - name: "Groq", - apiKeyEnvVar: "GROQ_API_KEY", - apiKeyConfigKey: "llm.groqApiKey", - getKeyLink: "https://groq.com", - docsLink: "https://console.groq.com/docs/errors", - }, - google: { - name: "Google", - apiKeyEnvVar: "GOOGLE_API_KEY", - apiKeyConfigKey: "llm.googleApiKey", - getKeyLink: "https://ai.google.dev/", - docsLink: "https://ai.google.dev/gemini-api/docs/troubleshooting", - }, - openrouter: { - name: "OpenRouter", - apiKeyEnvVar: "OPENROUTER_API_KEY", - apiKeyConfigKey: "llm.openrouterApiKey", - getKeyLink: "https://openrouter.ai", - docsLink: "https://openrouter.ai/docs", - }, - ollama: { - name: "Ollama", - apiKeyEnvVar: undefined, // Ollama doesn't require an API key - apiKeyConfigKey: undefined, // Ollama doesn't require an API key - getKeyLink: "https://ollama.com/download", - docsLink: "https://github.com/ollama/ollama/tree/main/docs", - }, - mistral: { - name: "Mistral", - apiKeyEnvVar: "MISTRAL_API_KEY", - apiKeyConfigKey: "llm.mistralApiKey", - getKeyLink: "https://console.mistral.ai", - docsLink: "https://docs.mistral.ai", - }, - "lingo.dev": { - name: "Lingo.dev", - apiKeyEnvVar: "LINGODOTDEV_API_KEY", - apiKeyConfigKey: "auth.apiKey", - getKeyLink: "https://lingo.dev", - docsLink: "https://lingo.dev/docs", - }, -}; diff --git a/packages/compiler/src/utils/llm-api-key.ts b/packages/compiler/src/utils/llm-api-key.ts deleted file mode 100644 index bf1b243ad..000000000 --- a/packages/compiler/src/utils/llm-api-key.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { getRc } from "./rc"; -import _ from "lodash"; -import * as dotenv from "dotenv"; -import path from "path"; - -// Generic function to retrieve key from process.env, with .env file as fallback -export function getKeyFromEnv(envVarName: string): string | undefined { - if (process.env[envVarName]) { - return process.env[envVarName]; - } - const result = dotenv.config({ - path: [ - path.resolve(process.cwd(), ".env"), - path.resolve(process.cwd(), ".env.local"), - path.resolve(process.cwd(), ".env.development"), - ], - }); - return result?.parsed?.[envVarName]; -} - -// Generic function to retrieve key from .lingodotdevrc file -function getKeyFromRc(rcPath: string): string | undefined { - const rc = getRc(); - const result = _.get(rc, rcPath); - return typeof result === "string" ? result : undefined; -} - -export function getGroqKey() { - return getGroqKeyFromEnv() || getGroqKeyFromRc(); -} - -export function getGroqKeyFromRc() { - return getKeyFromRc("llm.groqApiKey"); -} - -export function getGroqKeyFromEnv() { - return getKeyFromEnv("GROQ_API_KEY"); -} - -export function getLingoDotDevKeyFromEnv() { - return getKeyFromEnv("LINGODOTDEV_API_KEY"); -} - -export function getLingoDotDevKeyFromRc() { - return getKeyFromRc("auth.apiKey"); -} - -export function getLingoDotDevKey() { - return getLingoDotDevKeyFromEnv() || getLingoDotDevKeyFromRc(); -} - -export function getGoogleKey() { - return getGoogleKeyFromEnv() || getGoogleKeyFromRc(); -} - -export function getGoogleKeyFromRc() { - return getKeyFromRc("llm.googleApiKey"); -} - -export function getGoogleKeyFromEnv() { - return getKeyFromEnv("GOOGLE_API_KEY"); -} - -export function getOpenRouterKey() { - return getOpenRouterKeyFromEnv() || getOpenRouterKeyFromRc(); -} -export function getOpenRouterKeyFromRc() { - return getKeyFromRc("llm.openrouterApiKey"); -} -export function getOpenRouterKeyFromEnv() { - return getKeyFromEnv("OPENROUTER_API_KEY"); -} - -export function getMistralKey() { - return getMistralKeyFromEnv() || getMistralKeyFromRc(); -} - -export function getMistralKeyFromRc() { - return getKeyFromRc("llm.mistralApiKey"); -} - -export function getMistralKeyFromEnv() { - return getKeyFromEnv("MISTRAL_API_KEY"); -} diff --git a/packages/compiler/src/utils/llm-api-keys.spec.ts b/packages/compiler/src/utils/llm-api-keys.spec.ts deleted file mode 100644 index c6c6b5ca8..000000000 --- a/packages/compiler/src/utils/llm-api-keys.spec.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import * as dotenv from "dotenv"; -import * as path from "path"; -import { getKeyFromEnv } from "./llm-api-key"; - -const ORIGINAL_ENV = { ...process.env }; - -vi.mock("dotenv"); - -describe("LLM API keys", () => { - describe("getKeyFromEnv", () => { - beforeEach(() => { - vi.resetModules(); - process.env = { ...ORIGINAL_ENV }; - }); - - afterEach(() => { - process.env = { ...ORIGINAL_ENV }; - vi.restoreAllMocks(); - }); - - it("returns API key from process.env if set", () => { - process.env.FOOBAR_API_KEY = "env-key"; - expect(getKeyFromEnv("FOOBAR_API_KEY")).toBe("env-key"); - }); - - it("returns API key from .env file if not in process.env", () => { - delete process.env.FOOBAR_API_KEY; - const fakeEnv = { FOOBAR_API_KEY: "file-key" }; - const configMock = vi - .mocked(dotenv.config) - .mockImplementation((opts: any) => { - if (opts && opts.processEnv) { - Object.assign(opts.processEnv, fakeEnv); - } - return { parsed: fakeEnv }; - }); - expect(getKeyFromEnv("FOOBAR_API_KEY")).toBe("file-key"); - expect(configMock).toHaveBeenCalledWith({ - path: [ - path.resolve(process.cwd(), ".env"), - path.resolve(process.cwd(), ".env.local"), - path.resolve(process.cwd(), ".env.development"), - ], - }); - }); - - it("returns undefined if no GROQ_API_KEY in env or .env file", () => { - delete process.env.GROQ_API_KEY; - vi.mocked(dotenv.config).mockResolvedValue({ parsed: {} }); - expect(getKeyFromEnv("FOOBAR_API_KEY")).toBeUndefined(); - }); - }); -}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 44e2d421e..34f5ebbb9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -723,6 +723,9 @@ importers: '@lingo.dev/_spec': specifier: workspace:* version: link:../spec + '@lingo.dev/providers': + specifier: workspace:* + version: link:../providers '@openrouter/ai-sdk-provider': specifier: ^0.7.1 version: 0.7.1(ai@4.3.15(react@19.1.0)(zod@3.25.76))(zod@3.25.76) From 57b9c86b445aa2f1c69b0646f1f4bd625fcb1fa4 Mon Sep 17 00:00:00 2001 From: David Turnbull Date: Fri, 3 Oct 2025 15:19:41 +1000 Subject: [PATCH 06/20] feat: phase 4 --- packages/cli/package.json | 2 +- packages/cli/src/cli/utils/settings.ts | 31 ++++++++++---- packages/config/package.json | 41 +++++++++++++++++++ .../src/rc.ts => config/src/index.ts} | 25 +++++++++-- packages/config/tsconfig.json | 21 ++++++++++ packages/config/tsup.config.ts | 15 +++++++ packages/providers/package.json | 2 +- packages/providers/src/keys.ts | 6 +-- 8 files changed, 127 insertions(+), 16 deletions(-) create mode 100644 packages/config/package.json rename packages/{providers/src/rc.ts => config/src/index.ts} (50%) create mode 100644 packages/config/tsconfig.json create mode 100644 packages/config/tsup.config.ts diff --git a/packages/cli/package.json b/packages/cli/package.json index 096d90e17..148cf07ff 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -122,6 +122,7 @@ "author": "", "license": "Apache-2.0", "dependencies": { + "@lingo.dev/config": "workspace:*", "@lingo.dev/providers": "workspace:*", "@ai-sdk/anthropic": "^1.2.11", "@ai-sdk/google": "^1.2.19", @@ -169,7 +170,6 @@ "glob": "<11.0.0", "gradient-string": "^3.0.0", "gray-matter": "^4.0.3", - "ini": "^5.0.0", "ink": "^4.2.0", "ink-progress-bar": "^3.0.0", "ink-spinner": "^5.0.0", diff --git a/packages/cli/src/cli/utils/settings.ts b/packages/cli/src/cli/utils/settings.ts index 9883f2d17..0c7ac8c1f 100644 --- a/packages/cli/src/cli/utils/settings.ts +++ b/packages/cli/src/cli/utils/settings.ts @@ -3,7 +3,7 @@ import path from "path"; import _ from "lodash"; import Z from "zod"; import fs from "fs"; -import Ini from "ini"; +import { getRcConfig } from "@lingo.dev/config"; import { PROVIDER_METADATA } from "@lingo.dev/providers"; export type CliSettings = Z.infer; @@ -113,12 +113,7 @@ function _loadEnv() { } function _loadSystemFile() { - const settingsFilePath = _getSettingsFilePath(); - const content = fs.existsSync(settingsFilePath) - ? fs.readFileSync(settingsFilePath, "utf-8") - : ""; - const data = Ini.parse(content); - + const data = getRcConfig(); return Z.object({ auth: Z.object({ apiKey: Z.string().optional(), @@ -140,7 +135,27 @@ function _loadSystemFile() { function _saveSystemFile(settings: CliSettings) { const settingsFilePath = _getSettingsFilePath(); - const content = Ini.stringify(settings); + const content = [ + `[auth]`, + `apiKey=${settings.auth.apiKey}`, + `apiUrl=${settings.auth.apiUrl}`, + `webUrl=${settings.auth.webUrl}`, + ``, + `[llm]`, + settings.llm.openaiApiKey ? `openaiApiKey=${settings.llm.openaiApiKey}` : ``, + settings.llm.anthropicApiKey + ? `anthropicApiKey=${settings.llm.anthropicApiKey}` + : ``, + settings.llm.groqApiKey ? `groqApiKey=${settings.llm.groqApiKey}` : ``, + settings.llm.googleApiKey ? `googleApiKey=${settings.llm.googleApiKey}` : ``, + settings.llm.openrouterApiKey + ? `openrouterApiKey=${settings.llm.openrouterApiKey}` + : ``, + settings.llm.mistralApiKey ? `mistralApiKey=${settings.llm.mistralApiKey}` : ``, + ``, + ] + .filter(Boolean) + .join("\n"); fs.writeFileSync(settingsFilePath, content); } diff --git a/packages/config/package.json b/packages/config/package.json new file mode 100644 index 000000000..7b226c154 --- /dev/null +++ b/packages/config/package.json @@ -0,0 +1,41 @@ +{ + "name": "@lingo.dev/config", + "version": "0.1.0", + "description": "Lingo.dev user configuration (.lingodotdevrc) reader and schema", + "private": false, + "publishConfig": { + "access": "public" + }, + "type": "module", + "sideEffects": false, + "main": "build/index.cjs", + "module": "build/index.mjs", + "types": "build/index.d.ts", + "files": [ + "build" + ], + "scripts": { + "dev": "tsup --watch", + "build": "pnpm typecheck && tsup", + "typecheck": "tsc --noEmit", + "test": "vitest run", + "test:watch": "vitest" + }, + "keywords": [], + "author": "", + "license": "Apache-2.0", + "dependencies": { + "ini": "^5.0.0", + "zod": "^3.25.76" + }, + "devDependencies": { + "@types/ini": "^4.1.1", + "@types/node": "^22.13.5", + "tsup": "^8.3.5", + "typescript": "^5.8.3", + "vitest": "^3.1.2" + }, + "engines": { + "node": ">=18" + } +} diff --git a/packages/providers/src/rc.ts b/packages/config/src/index.ts similarity index 50% rename from packages/providers/src/rc.ts rename to packages/config/src/index.ts index f076426ae..2c0b645b7 100644 --- a/packages/providers/src/rc.ts +++ b/packages/config/src/index.ts @@ -2,8 +2,9 @@ import os from "os"; import path from "path"; import fs from "fs"; import Ini from "ini"; +import Z from "zod"; -export interface RcData { +export interface RcConfig { auth?: { apiKey?: string; apiUrl?: string; @@ -20,7 +21,25 @@ export interface RcData { [key: string]: any; } -export function readRc(): RcData { +export const rcConfigSchema = Z.object({ + auth: Z.object({ + apiKey: Z.string().optional(), + apiUrl: Z.string().optional(), + webUrl: Z.string().optional(), + }).optional(), + llm: Z.object({ + groqApiKey: Z.string().optional(), + openaiApiKey: Z.string().optional(), + anthropicApiKey: Z.string().optional(), + googleApiKey: Z.string().optional(), + openrouterApiKey: Z.string().optional(), + mistralApiKey: Z.string().optional(), + }).optional(), +}) + .passthrough() + .transform((v) => v as RcConfig); + +export function getRcConfig(): RcConfig { const settingsFile = ".lingodotdevrc"; const homedir = os.homedir(); const settingsFilePath = path.join(homedir, settingsFile); @@ -28,5 +47,5 @@ export function readRc(): RcData { ? fs.readFileSync(settingsFilePath, "utf-8") : ""; const data = Ini.parse(content); - return data as RcData; + return rcConfigSchema.parse(data); } diff --git a/packages/config/tsconfig.json b/packages/config/tsconfig.json new file mode 100644 index 000000000..d7b0f86bc --- /dev/null +++ b/packages/config/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "rootDir": "src", + "outDir": "build", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "strict": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "allowSyntheticDefaultImports": true, + "skipLibCheck": true, + "moduleResolution": "Bundler", + "module": "ESNext", + "target": "ESNext", + "allowUnreachableCode": true, + "types": ["vitest/globals"] + }, + "include": ["src/**/*.ts"], + "exclude": ["src/**/*.spec.ts"] +} diff --git a/packages/config/tsup.config.ts b/packages/config/tsup.config.ts new file mode 100644 index 000000000..2d13ece73 --- /dev/null +++ b/packages/config/tsup.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + clean: true, + target: "esnext", + entry: ["src/index.ts"], + outDir: "build", + format: ["cjs", "esm"], + dts: true, + cjsInterop: true, + splitting: true, + outExtension: (ctx) => ({ + js: ctx.format === "cjs" ? ".cjs" : ".mjs", + }), +}); diff --git a/packages/providers/package.json b/packages/providers/package.json index ec064665f..5e98b5f75 100644 --- a/packages/providers/package.json +++ b/packages/providers/package.json @@ -25,6 +25,7 @@ "author": "", "license": "Apache-2.0", "dependencies": { + "@lingo.dev/config": "workspace:*", "@ai-sdk/anthropic": "^1.2.11", "@ai-sdk/google": "^1.2.19", "@ai-sdk/groq": "^1.2.3", @@ -33,7 +34,6 @@ "@openrouter/ai-sdk-provider": "^0.7.1", "ai": "^4.3.15", "dotenv": "^16.4.7", - "ini": "^5.0.0", "ollama-ai-provider": "^1.2.0" }, "devDependencies": { diff --git a/packages/providers/src/keys.ts b/packages/providers/src/keys.ts index 0e64efcb3..486846bac 100644 --- a/packages/providers/src/keys.ts +++ b/packages/providers/src/keys.ts @@ -3,7 +3,7 @@ import fs from "fs"; import dotenv from "dotenv"; import { ProviderId } from "./constants"; import { PROVIDER_METADATA } from "./metadata"; -import { readRc, RcData } from "./rc"; +import { getRcConfig, type RcConfig } from "@lingo.dev/config"; import { ProviderKeyMissingError } from "./errors"; let dotenvLoaded = false; @@ -29,7 +29,7 @@ function getByPath(obj: any, keyPath?: string): any { export interface KeySources { env?: Record; - rc?: RcData; + rc?: RcConfig; } export function getProviderApiKey(providerId: ProviderId): string | undefined { @@ -53,7 +53,7 @@ export function resolveProviderApiKey( envVal = meta.apiKeyEnvVar ? process.env[meta.apiKeyEnvVar] : undefined; } - const rc = sources.rc ?? readRc(); + const rc = sources.rc ?? getRcConfig(); const rcVal = getByPath(rc, meta.apiKeyConfigKey); const key = envVal || rcVal; From ac1374d4f94a5d23dd1bf1e1e31cc9cb7a2e4921 Mon Sep 17 00:00:00 2001 From: David Turnbull Date: Fri, 3 Oct 2025 15:29:34 +1000 Subject: [PATCH 07/20] feat: phase 5 --- packages/spec/package.json | 1 + packages/spec/src/config.ts | 37 +++++++++++++++++++++++++++--------- packages/spec/tsup.config.ts | 1 + pnpm-lock.yaml | 3 +++ 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/packages/spec/package.json b/packages/spec/package.json index f1138ffac..06e1fe27a 100644 --- a/packages/spec/package.json +++ b/packages/spec/package.json @@ -25,6 +25,7 @@ "author": "", "license": "Apache-2.0", "dependencies": { + "@lingo.dev/providers": "workspace:*", "zod": "^3.25.76", "zod-to-json-schema": "^3.24.5" }, diff --git a/packages/spec/src/config.ts b/packages/spec/src/config.ts index 34ab0aaa8..f1af7a187 100644 --- a/packages/spec/src/config.ts +++ b/packages/spec/src/config.ts @@ -1,4 +1,5 @@ import Z from "zod"; +import { SUPPORTED_PROVIDERS } from "@lingo.dev/providers"; import { localeCodeSchema } from "./locales"; import { bucketTypeSchema } from "./formats"; @@ -289,15 +290,16 @@ export const configV1_4Definition = extendConfigDefinition( // v1.4 -> v1.5 // Changes: add "provider" field to the config +const providerIdEnumValues = + SUPPORTED_PROVIDERS as unknown as [ + (typeof SUPPORTED_PROVIDERS)[number], + ...((typeof SUPPORTED_PROVIDERS)[number])[], + ]; + const providerSchema = Z.object({ - id: Z.enum([ - "openai", - "anthropic", - "google", - "ollama", - "openrouter", - "mistral", - ]).describe("Identifier of the translation provider service."), + id: Z.enum(providerIdEnumValues).describe( + "Identifier of the translation provider service.", + ), model: Z.string().describe("Model name to use for translations."), prompt: Z.string().describe( "Prompt template used when requesting translations.", @@ -486,7 +488,24 @@ export const configV1_10Definition = extendConfigDefinition( ); // exports -export const LATEST_CONFIG_DEFINITION = configV1_10Definition; +// v1.10 -> v2.0 +// Changes: Use SUPPORTED_PROVIDERS from providers package for provider enum +export const configV2_0Definition = extendConfigDefinition( + configV1_10Definition, + { + createSchema: (baseSchema) => baseSchema, + createDefaultValue: (baseDefaultValue) => ({ + ...baseDefaultValue, + version: 2.0, + }), + createUpgrader: (oldConfig) => ({ + ...oldConfig, + version: 2.0, + }), + }, +); + +export const LATEST_CONFIG_DEFINITION = configV2_0Definition; export type I18nConfig = Z.infer<(typeof LATEST_CONFIG_DEFINITION)["schema"]>; diff --git a/packages/spec/tsup.config.ts b/packages/spec/tsup.config.ts index 297ea8cb4..5a1a959c4 100644 --- a/packages/spec/tsup.config.ts +++ b/packages/spec/tsup.config.ts @@ -10,6 +10,7 @@ export default defineConfig({ dts: true, cjsInterop: true, splitting: true, + external: ["@lingo.dev/providers"], outExtension: (ctx) => ({ js: ctx.format === "cjs" ? ".cjs" : ".mjs", }), diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 34f5ebbb9..be333eba5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -949,6 +949,9 @@ importers: packages/spec: dependencies: + '@lingo.dev/providers': + specifier: workspace:* + version: link:../providers zod: specifier: ^3.25.76 version: 3.25.76 From 7afbffdb22b3156e8e997e043237c0fca3e17a4f Mon Sep 17 00:00:00 2001 From: David Turnbull Date: Fri, 3 Oct 2025 15:32:32 +1000 Subject: [PATCH 08/20] chore: remove unused deps --- packages/cli/package.json | 11 ++++----- packages/compiler/package.json | 8 +++---- pnpm-lock.yaml | 43 ---------------------------------- 3 files changed, 7 insertions(+), 55 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index 148cf07ff..c571f94dd 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -124,10 +124,7 @@ "dependencies": { "@lingo.dev/config": "workspace:*", "@lingo.dev/providers": "workspace:*", - "@ai-sdk/anthropic": "^1.2.11", - "@ai-sdk/google": "^1.2.19", - "@ai-sdk/mistral": "^1.2.8", - "@ai-sdk/openai": "^1.3.22", + "@babel/generator": "^7.27.1", "@babel/parser": "^7.27.1", "@babel/traverse": "^7.27.4", @@ -145,7 +142,7 @@ "@lingo.dev/_spec": "workspace:*", "@markdoc/markdoc": "^0.5.4", "@modelcontextprotocol/sdk": "^1.5.0", - "@openrouter/ai-sdk-provider": "^0.7.1", + "@paralleldrive/cuid2": "^2.2.2", "@types/ejs": "^3.1.5", "ai": "^4.3.15", @@ -190,7 +187,7 @@ "node-webvtt": "^1.9.4", "object-hash": "^3.0.0", "octokit": "^4.0.2", - "ollama-ai-provider": "^1.2.0", + "open": "^10.2.0", "ora": "^8.1.1", "p-limit": "^6.2.0", @@ -228,7 +225,7 @@ "@types/figlet": "^1.7.0", "@types/gettext-parser": "^4.0.4", "@types/glob": "^8.1.0", - "@types/ini": "^4.1.1", + "@types/is-url": "^1.2.32", "@types/jsdom": "^21.1.7", "@types/lodash": "^4.17.16", diff --git a/packages/compiler/package.json b/packages/compiler/package.json index e4bc9dc5e..66a82d495 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -39,16 +39,14 @@ }, "dependencies": { "@lingo.dev/providers": "workspace:*", - "@ai-sdk/google": "^1.2.19", - "@ai-sdk/groq": "^1.2.3", - "@ai-sdk/mistral": "^1.2.8", + "@babel/generator": "^7.26.5", "@babel/parser": "^7.26.7", "@babel/traverse": "^7.27.4", "@babel/types": "^7.26.7", "@lingo.dev/_sdk": "workspace:*", "@lingo.dev/_spec": "workspace:*", - "@openrouter/ai-sdk-provider": "^0.7.1", + "@prettier/sync": "^0.6.1", "ai": "^4.2.10", "dedent": "^1.6.0", @@ -57,7 +55,7 @@ "ini": "^5.0.0", "lodash": "^4.17.21", "object-hash": "^3.0.0", - "ollama-ai-provider": "^1.2.0", + "prettier": "^3.4.2", "unplugin": "^2.1.2", "zod": "^3.25.76", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index be333eba5..747ce0d0a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -341,18 +341,6 @@ importers: packages/cli: dependencies: - '@ai-sdk/anthropic': - specifier: ^1.2.11 - version: 1.2.11(zod@3.25.76) - '@ai-sdk/google': - specifier: ^1.2.19 - version: 1.2.19(zod@3.25.76) - '@ai-sdk/mistral': - specifier: ^1.2.8 - version: 1.2.8(zod@3.25.76) - '@ai-sdk/openai': - specifier: ^1.3.22 - version: 1.3.22(zod@3.25.76) '@babel/generator': specifier: ^7.27.1 version: 7.27.1 @@ -404,9 +392,6 @@ importers: '@modelcontextprotocol/sdk': specifier: ^1.5.0 version: 1.5.0 - '@openrouter/ai-sdk-provider': - specifier: ^0.7.1 - version: 0.7.1(ai@4.3.15(react@18.3.1)(zod@3.25.76))(zod@3.25.76) '@paralleldrive/cuid2': specifier: ^2.2.2 version: 2.2.2 @@ -542,9 +527,6 @@ importers: octokit: specifier: ^4.0.2 version: 4.1.2 - ollama-ai-provider: - specifier: ^1.2.0 - version: 1.2.0(zod@3.25.76) open: specifier: ^10.2.0 version: 10.2.0 @@ -651,9 +633,6 @@ importers: '@types/glob': specifier: ^8.1.0 version: 8.1.0 - '@types/ini': - specifier: ^4.1.1 - version: 4.1.1 '@types/is-url': specifier: ^1.2.32 version: 1.2.32 @@ -696,15 +675,6 @@ importers: packages/compiler: dependencies: - '@ai-sdk/google': - specifier: ^1.2.19 - version: 1.2.19(zod@3.25.76) - '@ai-sdk/groq': - specifier: ^1.2.3 - version: 1.2.9(zod@3.25.76) - '@ai-sdk/mistral': - specifier: ^1.2.8 - version: 1.2.8(zod@3.25.76) '@babel/generator': specifier: ^7.26.5 version: 7.27.1 @@ -726,9 +696,6 @@ importers: '@lingo.dev/providers': specifier: workspace:* version: link:../providers - '@openrouter/ai-sdk-provider': - specifier: ^0.7.1 - version: 0.7.1(ai@4.3.15(react@19.1.0)(zod@3.25.76))(zod@3.25.76) '@prettier/sync': specifier: ^0.6.1 version: 0.6.1(prettier@3.4.2) @@ -756,9 +723,6 @@ importers: object-hash: specifier: ^3.0.0 version: 3.0.0 - ollama-ai-provider: - specifier: ^1.2.0 - version: 1.2.0(zod@3.25.76) posthog-node: specifier: ^5.5.1 version: 5.5.1 @@ -13396,13 +13360,6 @@ snapshots: '@octokit/request-error': 6.1.7 '@octokit/webhooks-methods': 5.1.1 - '@openrouter/ai-sdk-provider@0.7.1(ai@4.3.15(react@18.3.1)(zod@3.25.76))(zod@3.25.76)': - dependencies: - '@ai-sdk/provider': 1.1.3 - '@ai-sdk/provider-utils': 2.2.8(zod@3.25.76) - ai: 4.3.15(react@18.3.1)(zod@3.25.76) - zod: 3.25.76 - '@openrouter/ai-sdk-provider@0.7.1(ai@4.3.15(react@19.1.0)(zod@3.25.76))(zod@3.25.76)': dependencies: '@ai-sdk/provider': 1.1.3 From a784c9c55f91a31b454819150a1c42bcd46cdf10 Mon Sep 17 00:00:00 2001 From: David Turnbull Date: Fri, 3 Oct 2025 16:58:10 +1000 Subject: [PATCH 09/20] chore: remove PRD --- PRD.md | 426 --------------------------------------------------------- 1 file changed, 426 deletions(-) delete mode 100644 PRD.md diff --git a/PRD.md b/PRD.md deleted file mode 100644 index 85196283a..000000000 --- a/PRD.md +++ /dev/null @@ -1,426 +0,0 @@ -# PRD: Unified Provider Architecture - -## Problem Statement - -There is significant drift between `packages/cli` and `packages/compiler` in their LLM provider implementations: - -### Current State - -**packages/cli supports:** -- OpenAI, Anthropic, Google, OpenRouter, Ollama, Mistral - -**packages/compiler supports:** -- Groq, Google, OpenRouter, Ollama, Mistral, Lingo.dev - -**packages/spec config schema defines:** -- `["openai", "anthropic", "google", "ollama", "openrouter", "mistral"]` - -### Drift Issues - -1. **Incomplete provider coverage** - CLI missing Groq; Compiler missing OpenAI/Anthropic -2. **Inconsistent implementations** - Different patterns for API key resolution, error handling, client creation -3. **No shared constants** - Provider IDs, metadata, env var names duplicated -4. **Config schema mismatch** - Spec doesn't list Groq as valid provider -5. **Maintenance burden** - Bug fixes must be applied twice; new providers require dual implementation - -## Goals - -1. **Eliminate drift** - Make it architecturally impossible for packages to diverge -2. **Support all 7 providers** - Groq, OpenAI, Anthropic, Google, OpenRouter, Ollama, Mistral -3. **Single source of truth** - One implementation of provider logic shared by both packages -4. **Consistent UX** - Same error messages, behavior, key resolution across all tooling -5. **Future-proof** - Easy to add new providers; other packages can consume provider logic - -## Architecture Decisions - -### Package Structure - -Create two new shared packages: - -``` -packages/config → .lingodotdevrc reading (user credentials) -packages/providers → Provider registry, API key resolution, client factories -packages/spec → i18n.json schema (CLI-only, existing) -packages/cli → Uses spec, config, providers -packages/compiler → Uses config, providers (NOT spec) -``` - -### Dependency Graph - -``` -packages/config (standalone, no deps) - ↓ -packages/providers (depends on config) - ↓ -packages/cli (depends on spec, config, providers) -packages/compiler (depends on config, providers) -``` - -**Key principle:** Compiler never depends on CLI-specific concerns (i18n.json schema in spec) - -### Design Decisions - -| Decision | Choice | Rationale | -|----------|--------|-----------| -| **Provider consolidation** | All 7 providers in both packages | Complete feature parity | -| **Abstraction level** | Maximal - full client factories | Eliminates all duplication, enforces consistency | -| **Error handling** | Structured error classes | Maximum optionality - consumers format as needed | -| **RC file reading** | Providers owns it (Phase 1), extracted to config (Phase 2) | Self-contained initially, shared later | -| **API design** | Layered (4 layers) | Flexibility - escape hatches for custom scenarios | - -## Implementation Plan - -### Phase 1: Create `packages/providers` (with duplicated rc reading) - -**Goal:** Establish provider package with complete functionality - -**Tasks:** -1. Create package scaffold with dependencies -2. Implement 4-layer API (see API Design section) -3. Port provider logic from CLI + compiler -4. Add all 7 providers with full support -5. Write comprehensive tests -6. Document API - -### Phase 2: Migrate CLI to use `packages/providers` - -**Goal:** Validate provider package works in production CLI - -**Tasks:** -1. Update CLI dependencies -2. Replace `cli/localizer/explicit.ts` with provider imports -3. Update `cli/utils/settings.ts` to use provider constants -4. Remove duplicated provider code -5. Ensure tests pass -6. Verify CLI functionality unchanged - -### Phase 3: Migrate compiler to use `packages/providers` - -**Goal:** Validate provider package works in production compiler - -**Tasks:** -1. Update compiler dependencies -2. Replace `compiler/lib/lcp/api/index.ts` provider logic with imports -3. Remove `compiler/utils/llm-api-key.ts` -4. Remove `compiler/lib/lcp/api/provider-details.ts` -5. Ensure tests pass -6. Verify compiler functionality unchanged - -### Phase 4: Extract rc reading to `packages/config` - -**Goal:** Eliminate remaining duplication in config reading - -**Tasks:** -1. Create `packages/config` package -2. Move `.lingodotdevrc` schema + reading logic from providers -3. Update providers to depend on config -4. Update CLI to use config for auth settings -5. Remove duplicated rc reading from CLI - -### Phase 5: Update `packages/spec` provider enum - -**Goal:** Ensure config schema matches implementation - -**Tasks:** -1. Add "groq" to provider ID enum in spec -2. Update config version (1.9 → 2.0) -3. Update JSON schema generation -4. Document migration path - -## API Design: `packages/providers` - -### Layer 1: Constants & Metadata - -```typescript -/** - * All supported provider IDs - */ -export const SUPPORTED_PROVIDERS = [ - 'groq', - 'openai', - 'anthropic', - 'google', - 'openrouter', - 'ollama', - 'mistral', -] as const; - -export type ProviderId = typeof SUPPORTED_PROVIDERS[number]; - -/** - * Metadata for each provider - */ -export interface ProviderMetadata { - name: string; // Display name (e.g., "Groq", "OpenAI") - apiKeyEnvVar?: string; // Environment variable name (e.g., "GROQ_API_KEY") - apiKeyConfigKey?: string; // RC file key path (e.g., "llm.groqApiKey") - getKeyLink: string; // Link to get API key - docsLink: string; // Link to API docs -} - -export const PROVIDER_METADATA: Record; -``` - -### Layer 2: Utilities - -```typescript -/** - * Get API key for provider from environment or rc file - * Returns undefined if not found - */ -export function getProviderApiKey(providerId: ProviderId): string | undefined; - -/** - * Resolve API key with custom sources - * @throws ProviderKeyMissingError if not found and required=true - */ -export function resolveProviderApiKey( - providerId: ProviderId, - options?: { - sources?: KeySources; - required?: boolean; - } -): string | undefined; - -/** - * Key sources for resolution - */ -export interface KeySources { - env?: Record; - rc?: RcData; -} - -/** - * Structured error: API key not found - */ -export class ProviderKeyMissingError extends Error { - constructor( - public providerId: ProviderId, - public envVar?: string, - public configKey?: string - ); -} - -/** - * Structured error: Authentication failed - */ -export class ProviderAuthFailedError extends Error { - constructor( - public providerId: ProviderId, - public originalError: Error - ); -} - -/** - * Structured error: Unsupported provider - */ -export class UnsupportedProviderError extends Error { - constructor(public providerId: string); -} -``` - -### Layer 3: Factory - -```typescript -/** - * Create AI SDK LanguageModel client for provider - * @throws ProviderKeyMissingError if key required but not found - * @throws UnsupportedProviderError if provider not supported - */ -export function createProviderClient( - providerId: ProviderId, - modelId: string, - options?: ClientOptions -): LanguageModel; - -export interface ClientOptions { - apiKey?: string; // Override API key (skip resolution) - baseUrl?: string; // Custom base URL for provider API - skipAuth?: boolean; // Skip authentication check (for Ollama) -} -``` - -### Layer 4: High-level - -```typescript -/** - * Get provider client + metadata in one call - * Convenience wrapper around createProviderClient + PROVIDER_METADATA - */ -export function getProvider( - providerId: ProviderId, - modelId: string, - options?: ClientOptions -): { - client: LanguageModel; - metadata: ProviderMetadata; -}; -``` - -## API Design: `packages/config` - -```typescript -/** - * Read and parse .lingodotdevrc file from home directory - */ -export function getRcConfig(): RcConfig; - -/** - * RC file data structure - */ -export interface RcConfig { - auth?: { - apiKey?: string; - apiUrl?: string; - webUrl?: string; - }; - llm?: { - groqApiKey?: string; - openaiApiKey?: string; - anthropicApiKey?: string; - googleApiKey?: string; - openrouterApiKey?: string; - mistralApiKey?: string; - }; -} - -/** - * Zod schema for validation - */ -export const rcConfigSchema: ZodType; -``` - -## Migration Examples - -### Before (CLI) - -```typescript -// packages/cli/src/cli/localizer/explicit.ts -switch (provider.id) { - case "openai": - return createAiSdkLocalizer({ - factory: (params) => createOpenAI(params).languageModel(provider.model), - id: provider.id, - prompt: provider.prompt, - apiKeyName: "OPENAI_API_KEY", - baseUrl: provider.baseUrl, - }); - // ... 5 more cases -} -``` - -### After (CLI) - -```typescript -// packages/cli/src/cli/localizer/explicit.ts -import { createProviderClient } from "@lingo.dev/providers"; - -const client = createProviderClient(provider.id, provider.model, { - baseUrl: provider.baseUrl, -}); -``` - -### Before (Compiler) - -```typescript -// packages/compiler/src/lib/lcp/api/index.ts (lines 291-392) -switch (providerId) { - case "groq": { - if (isRunningInCIOrDocker()) { - const groqFromEnv = getGroqKeyFromEnv(); - if (!groqFromEnv) { - this._failMissingLLMKeyCi(providerId); - } - } - const groqKey = getGroqKey(); - if (!groqKey) { - throw new Error("⚠️ GROQ API key not found..."); - } - return createGroq({ apiKey: groqKey })(modelId); - } - // ... 4 more cases -} -``` - -### After (Compiler) - -```typescript -// packages/compiler/src/lib/lcp/api/index.ts -import { createProviderClient, ProviderKeyMissingError } from "@lingo.dev/providers"; - -try { - return createProviderClient(providerId, modelId); -} catch (error) { - if (error instanceof ProviderKeyMissingError) { - // Custom error formatting for compiler context - this._failMissingLLMKeyCi(error.providerId); - } - throw error; -} -``` - -## Success Criteria - -### Functionality -- [ ] All 7 providers work in CLI -- [ ] All 7 providers work in compiler -- [ ] API key resolution works (env + rc file) -- [ ] Error handling preserves existing behavior -- [ ] No breaking changes for end users - -### Code Quality -- [ ] Zero duplication of provider logic -- [ ] All provider constants shared -- [ ] Comprehensive test coverage (>90%) -- [ ] Full API documentation -- [ ] Type safety maintained - -### Architecture -- [ ] Compiler doesn't depend on CLI concerns -- [ ] Clean dependency graph (no cycles) -- [ ] Layered API provides escape hatches -- [ ] Easy to add new providers (single location) - -### Validation -- [ ] CLI tests pass -- [ ] Compiler tests pass -- [ ] Integration tests pass -- [ ] Manual smoke testing complete - -## Risks & Mitigations - -| Risk | Impact | Mitigation | -|------|--------|------------| -| Breaking changes in migration | High | Phased rollout, comprehensive testing, feature flags | -| API key resolution differs subtly | Medium | Unit test all edge cases, validate against existing behavior | -| Dependency conflicts with AI SDK packages | Medium | Lock versions, test compatibility | -| RC file format assumptions differ | Low | Extract and validate schema early (Phase 4) | - -## Future Enhancements - -Once core architecture is in place: - -1. **Rate limiting** - Add rate limit handling to provider clients -2. **Retries** - Built-in retry logic with exponential backoff -3. **Observability** - Structured logging, metrics, tracing -4. **Caching** - Cache provider clients, metadata -5. **Validation** - Runtime validation of API keys, connectivity checks -6. **Provider plugins** - Allow custom provider implementations -7. **Cost tracking** - Track token usage per provider - -## Timeline Estimate - -- **Phase 1** (packages/providers): 2-3 days -- **Phase 2** (CLI migration): 1 day -- **Phase 3** (compiler migration): 1 day -- **Phase 4** (packages/config): 1 day -- **Phase 5** (spec update): 0.5 days -- **Testing & validation**: 1 day - -**Total: ~1 week** - -## Open Questions - -None - all architectural decisions have been made. - -Ready to begin implementation with Phase 1. From 21ccf0697f7b6e0a7822f4a6238fa95eb50714e0 Mon Sep 17 00:00:00 2001 From: David Turnbull Date: Fri, 3 Oct 2025 18:26:46 +1000 Subject: [PATCH 10/20] feat: dynamic llm keys in settings --- packages/cli/src/cli/utils/settings.ts | 88 ++++++++++++-------------- packages/config/src/index.spec.ts | 7 ++ packages/config/src/index.ts | 19 +----- 3 files changed, 50 insertions(+), 64 deletions(-) create mode 100644 packages/config/src/index.spec.ts diff --git a/packages/cli/src/cli/utils/settings.ts b/packages/cli/src/cli/utils/settings.ts index 0c7ac8c1f..68bea67bd 100644 --- a/packages/cli/src/cli/utils/settings.ts +++ b/packages/cli/src/cli/utils/settings.ts @@ -17,6 +17,18 @@ export function getSettings(explicitApiKey: string | undefined): CliSettings { _envVarsInfo(); + const llm: Record = {}; + for (const meta of Object.values(PROVIDER_METADATA)) { + const envVar = meta.apiKeyEnvVar; + const cfgKey = meta.apiKeyConfigKey; + if (!envVar || !cfgKey) continue; + const suffix = cfgKey.startsWith("llm.") ? cfgKey.slice(4) : undefined; + if (!suffix) continue; + const envVal = (env as any)[envVar] as string | undefined; + const rcVal = (systemFile.llm as any)?.[suffix] as string | undefined; + llm[suffix] = envVal || rcVal; + } + return { auth: { apiKey: @@ -33,15 +45,7 @@ export function getSettings(explicitApiKey: string | undefined): CliSettings { systemFile.auth?.webUrl || defaults.auth.webUrl, }, - llm: { - openaiApiKey: env.OPENAI_API_KEY || systemFile.llm?.openaiApiKey, - anthropicApiKey: env.ANTHROPIC_API_KEY || systemFile.llm?.anthropicApiKey, - groqApiKey: env.GROQ_API_KEY || systemFile.llm?.groqApiKey, - googleApiKey: env.GOOGLE_API_KEY || systemFile.llm?.googleApiKey, - openrouterApiKey: - env.OPENROUTER_API_KEY || systemFile.llm?.openrouterApiKey, - mistralApiKey: env.MISTRAL_API_KEY || systemFile.llm?.mistralApiKey, - }, + llm, }; } @@ -53,35 +57,28 @@ export function loadSystemSettings() { return _loadSystemFile(); } -const flattenZodObject = (schema: Z.ZodObject, prefix = ""): string[] => { - return Object.entries(schema.shape).flatMap(([key, value]) => { - const newPrefix = prefix ? `${prefix}.${key}` : key; - if (value instanceof Z.ZodObject) { - return flattenZodObject(value, newPrefix); - } - return [newPrefix]; - }); -}; - const SettingsSchema = Z.object({ auth: Z.object({ apiKey: Z.string(), apiUrl: Z.string(), webUrl: Z.string(), }), - llm: Z.object({ - openaiApiKey: Z.string().optional(), - anthropicApiKey: Z.string().optional(), - groqApiKey: Z.string().optional(), - googleApiKey: Z.string().optional(), - openrouterApiKey: Z.string().optional(), - mistralApiKey: Z.string().optional(), - }), + // Allow dynamic llm provider keys + llm: Z.record(Z.string().optional()), }); -export const SETTINGS_KEYS = flattenZodObject( - SettingsSchema, -) as readonly string[]; +function _providerConfigKeys(): string[] { + return Object.values(PROVIDER_METADATA) + .map((m) => m.apiKeyConfigKey) + .filter((v): v is string => Boolean(v)); +} + +export const SETTINGS_KEYS = [ + "auth.apiKey", + "auth.apiUrl", + "auth.webUrl", + ..._providerConfigKeys(), +] as string[]; // Private @@ -120,14 +117,8 @@ function _loadSystemFile() { apiUrl: Z.string().optional(), webUrl: Z.string().optional(), }).optional(), - llm: Z.object({ - openaiApiKey: Z.string().optional(), - anthropicApiKey: Z.string().optional(), - groqApiKey: Z.string().optional(), - googleApiKey: Z.string().optional(), - openrouterApiKey: Z.string().optional(), - mistralApiKey: Z.string().optional(), - }).optional(), + // Accept any llm provider key from rc file + llm: Z.record(Z.string().optional()).optional(), }) .passthrough() .parse(data); @@ -135,6 +126,16 @@ function _loadSystemFile() { function _saveSystemFile(settings: CliSettings) { const settingsFilePath = _getSettingsFilePath(); + const llmEntries: string[] = []; + for (const meta of Object.values(PROVIDER_METADATA)) { + const cfgKey = meta.apiKeyConfigKey; + if (!cfgKey) continue; + const suffix = cfgKey.startsWith("llm.") ? cfgKey.slice(4) : undefined; + if (!suffix) continue; + const value = (settings.llm as any)?.[suffix]; + if (value) llmEntries.push(`${suffix}=${value}`); + } + const content = [ `[auth]`, `apiKey=${settings.auth.apiKey}`, @@ -142,16 +143,7 @@ function _saveSystemFile(settings: CliSettings) { `webUrl=${settings.auth.webUrl}`, ``, `[llm]`, - settings.llm.openaiApiKey ? `openaiApiKey=${settings.llm.openaiApiKey}` : ``, - settings.llm.anthropicApiKey - ? `anthropicApiKey=${settings.llm.anthropicApiKey}` - : ``, - settings.llm.groqApiKey ? `groqApiKey=${settings.llm.groqApiKey}` : ``, - settings.llm.googleApiKey ? `googleApiKey=${settings.llm.googleApiKey}` : ``, - settings.llm.openrouterApiKey - ? `openrouterApiKey=${settings.llm.openrouterApiKey}` - : ``, - settings.llm.mistralApiKey ? `mistralApiKey=${settings.llm.mistralApiKey}` : ``, + ...llmEntries, ``, ] .filter(Boolean) diff --git a/packages/config/src/index.spec.ts b/packages/config/src/index.spec.ts new file mode 100644 index 000000000..4b3e3af1a --- /dev/null +++ b/packages/config/src/index.spec.ts @@ -0,0 +1,7 @@ +import { describe, it, expect } from "vitest"; + +describe("config package", () => { + it("loads without tests", () => { + expect(true).toBe(true); + }); +}); diff --git a/packages/config/src/index.ts b/packages/config/src/index.ts index 2c0b645b7..376c8df20 100644 --- a/packages/config/src/index.ts +++ b/packages/config/src/index.ts @@ -10,14 +10,7 @@ export interface RcConfig { apiUrl?: string; webUrl?: string; }; - llm?: { - groqApiKey?: string; - openaiApiKey?: string; - anthropicApiKey?: string; - googleApiKey?: string; - openrouterApiKey?: string; - mistralApiKey?: string; - }; + llm?: Record; [key: string]: any; } @@ -27,14 +20,8 @@ export const rcConfigSchema = Z.object({ apiUrl: Z.string().optional(), webUrl: Z.string().optional(), }).optional(), - llm: Z.object({ - groqApiKey: Z.string().optional(), - openaiApiKey: Z.string().optional(), - anthropicApiKey: Z.string().optional(), - googleApiKey: Z.string().optional(), - openrouterApiKey: Z.string().optional(), - mistralApiKey: Z.string().optional(), - }).optional(), + // Allow any llm provider keys and preserve them + llm: Z.record(Z.string().optional()).optional(), }) .passthrough() .transform((v) => v as RcConfig); From 4bd29981949e32b5d41c026db3faa2aae3958cda Mon Sep 17 00:00:00 2001 From: David Turnbull Date: Fri, 3 Oct 2025 18:36:33 +1000 Subject: [PATCH 11/20] fix: rc config load --- packages/compiler/package.json | 1 + packages/compiler/src/index.ts | 4 +-- packages/compiler/src/lib/lcp/api/index.ts | 6 ++-- packages/compiler/src/utils/observability.ts | 4 +-- packages/compiler/src/utils/rc.spec.ts | 32 -------------------- packages/compiler/src/utils/rc.ts | 15 --------- 6 files changed, 8 insertions(+), 54 deletions(-) delete mode 100644 packages/compiler/src/utils/rc.spec.ts delete mode 100644 packages/compiler/src/utils/rc.ts diff --git a/packages/compiler/package.json b/packages/compiler/package.json index 66a82d495..504b97139 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -39,6 +39,7 @@ }, "dependencies": { "@lingo.dev/providers": "workspace:*", + "@lingo.dev/config": "workspace:*", "@babel/generator": "^7.26.5", "@babel/parser": "^7.26.7", diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index a5d80819b..dcfdf89ef 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -11,7 +11,7 @@ import { isRunningInCIOrDocker } from "./utils/env"; import { loadDictionary, transformComponent } from "./_loader-utils"; import trackEvent from "./utils/observability"; import { PROVIDER_METADATA, resolveProviderApiKey } from "@lingo.dev/providers"; -import { getRc } from "./utils/rc"; +import { getRcConfig } from "@lingo.dev/config"; function getProviderDetails(providerId: string) { const meta = (PROVIDER_METADATA as any)[providerId]; @@ -373,7 +373,7 @@ function validateLLMKeyDetails(configuredProviders: string[]): void { })(); const foundInRc = (() => { - const rc = getRc(); + const rc = getRcConfig(); if (providerId === "lingo.dev") { return typeof _.get(rc, "auth.apiKey") === "string"; } diff --git a/packages/compiler/src/lib/lcp/api/index.ts b/packages/compiler/src/lib/lcp/api/index.ts index b06a873e7..bd415ede2 100644 --- a/packages/compiler/src/lib/lcp/api/index.ts +++ b/packages/compiler/src/lib/lcp/api/index.ts @@ -17,7 +17,7 @@ import { } from "@lingo.dev/providers"; import * as dotenv from "dotenv"; import path from "path"; -import { getRc } from "../../../utils/rc"; +import { getRcConfig } from "@lingo.dev/config"; export class LCPAPI { static async translate( @@ -150,8 +150,8 @@ export class LCPAPI { } const apiKey = getEnvWithDotenv("LINGODOTDEV_API_KEY") || - ((): string | undefined => { - const rc = getRc(); + (() => { + const rc = getRcConfig(); const val = _.get(rc, "auth.apiKey"); return typeof val === "string" ? val : undefined; })(); diff --git a/packages/compiler/src/utils/observability.ts b/packages/compiler/src/utils/observability.ts index 07935e7c8..adedec2a2 100644 --- a/packages/compiler/src/utils/observability.ts +++ b/packages/compiler/src/utils/observability.ts @@ -1,5 +1,5 @@ import { machineId } from "node-machine-id"; -import { getRc } from "./rc"; +import { getRcConfig } from "@lingo.dev/config"; export default async function trackEvent( event: string, @@ -44,7 +44,7 @@ export default async function trackEvent( } async function getActualId() { - const rc = getRc(); + const rc = getRcConfig(); const apiKey = process.env.LINGODOTDEV_API_KEY || rc?.auth?.apiKey; const apiUrl = process.env.LINGODOTDEV_API_URL || diff --git a/packages/compiler/src/utils/rc.spec.ts b/packages/compiler/src/utils/rc.spec.ts deleted file mode 100644 index 106a44fbb..000000000 --- a/packages/compiler/src/utils/rc.spec.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { describe, it, expect, vi, beforeEach } from "vitest"; -import { getRc } from "./rc"; - -vi.mock("os", () => ({ default: { homedir: () => "/home/test" } })); -vi.mock("fs", () => { - const mockFs = { - existsSync: vi.fn(() => false), - readFileSync: vi.fn(() => ""), - } as any; - return { ...mockFs, default: mockFs }; -}); - -import fsAny from "fs"; - -describe("getRc", () => { - beforeEach(() => { - (fsAny as any).existsSync.mockReset().mockReturnValue(false); - (fsAny as any).readFileSync.mockReset().mockReturnValue(""); - }); - - it("returns empty object when rc file missing", () => { - const data = getRc(); - expect(data).toEqual({}); - }); - - it("parses ini file when present", () => { - (fsAny as any).existsSync.mockReturnValue(true); - (fsAny as any).readFileSync.mockReturnValue("[auth]\napiKey=abc\n"); - const data = getRc(); - expect(data).toHaveProperty("auth.apiKey", "abc"); - }); -}); diff --git a/packages/compiler/src/utils/rc.ts b/packages/compiler/src/utils/rc.ts deleted file mode 100644 index fb4f9d92c..000000000 --- a/packages/compiler/src/utils/rc.ts +++ /dev/null @@ -1,15 +0,0 @@ -import os from "os"; -import path from "path"; -import fs from "fs"; -import Ini from "ini"; - -export function getRc() { - const settingsFile = ".lingodotdevrc"; - const homedir = os.homedir(); - const settingsFilePath = path.join(homedir, settingsFile); - const content = fs.existsSync(settingsFilePath) - ? fs.readFileSync(settingsFilePath, "utf-8") - : ""; - const data = Ini.parse(content); - return data; -} From eeddb1213c86ad8eae8e476476e1b557b44d25cd Mon Sep 17 00:00:00 2001 From: David Turnbull Date: Fri, 3 Oct 2025 18:44:48 +1000 Subject: [PATCH 12/20] chore: smoke tests --- .../cli/processor/providers-routing.spec.ts | 44 ++++++++++++ .../src/lib/lcp/api/providers-routing.spec.ts | 68 +++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 packages/cli/src/cli/processor/providers-routing.spec.ts create mode 100644 packages/compiler/src/lib/lcp/api/providers-routing.spec.ts diff --git a/packages/cli/src/cli/processor/providers-routing.spec.ts b/packages/cli/src/cli/processor/providers-routing.spec.ts new file mode 100644 index 000000000..5862cffe2 --- /dev/null +++ b/packages/cli/src/cli/processor/providers-routing.spec.ts @@ -0,0 +1,44 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +// Mock providers factory to observe routing +vi.mock("@lingo.dev/providers", async (importActual) => { + const actual = await importActual(); + return { + ...actual, + createProviderClient: vi.fn(() => ({} as any)), + }; +}); + +describe("processor routes providers via factory", () => { + beforeEach(async () => { + const mod = await import("@lingo.dev/providers"); + vi.mocked(mod.createProviderClient as any).mockClear(); + }); + + it("accepts every SUPPORTED_PROVIDERS and calls createProviderClient", async () => { + const { SUPPORTED_PROVIDERS, createProviderClient } = await import( + "@lingo.dev/providers" + ); + const createProcessor = (await import("./index")).default; + + for (const providerId of SUPPORTED_PROVIDERS) { + vi.mocked(createProviderClient as any).mockClear(); + + const processor = createProcessor( + { + id: providerId as any, + model: "test-model", + prompt: "test", + } as any, + { apiUrl: "http://localhost" }, + ); + + expect(typeof processor).toBe("function"); + expect(createProviderClient).toHaveBeenCalledWith( + providerId, + "test-model", + expect.any(Object), + ); + } + }); +}); diff --git a/packages/compiler/src/lib/lcp/api/providers-routing.spec.ts b/packages/compiler/src/lib/lcp/api/providers-routing.spec.ts new file mode 100644 index 000000000..586109649 --- /dev/null +++ b/packages/compiler/src/lib/lcp/api/providers-routing.spec.ts @@ -0,0 +1,68 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +// Mock providers factory to observe routing +vi.mock("@lingo.dev/providers", async (importActual) => { + const actual = await importActual(); + return { + ...actual, + createProviderClient: vi.fn(() => ({} as any)), + }; +}); + +// Mock AI SDK generateText to return the last message content (XML) +vi.mock("ai", () => { + return { + generateText: vi.fn((args: any) => { + const last = args.messages[args.messages.length - 1]; + return { text: last.content } as any; + }), + // Provide a dummy export to satisfy named import in source + LanguageModel: class {}, + }; +}); + +describe("LCPAPI routes providers via factory", () => { + beforeEach(async () => { + const mod = await import("@lingo.dev/providers"); + vi.mocked(mod.createProviderClient as any).mockClear(); + }); + + it("accepts every SUPPORTED_PROVIDERS and calls createProviderClient", async () => { + const { SUPPORTED_PROVIDERS, createProviderClient } = await import( + "@lingo.dev/providers" + ); + const { LCPAPI } = await import("./index"); + + for (const providerId of SUPPORTED_PROVIDERS) { + vi.mocked(createProviderClient as any).mockClear(); + + const models = { "*:*": `${providerId}:dummy-model` } as Record< + string, + string + >; + + // Minimal dictionary + const dictionary = { + version: 0.1, + locale: "en", + files: { + "a.json": { entries: { hello: "Hello" } }, + }, + } as const; + + const result = await LCPAPI.translate( + models, + dictionary as any, + "en", + "es", + null, + ); + + expect(result).toBeTruthy(); + // In compiler LCP path, factory is called with (providerId, model) only + const call = vi.mocked(createProviderClient as any).mock.calls[0]; + expect(call[0]).toBe(providerId); + expect(call[1]).toBe("dummy-model"); + } + }); +}); From a3545bbdbd1462b04b5cb6aaa1bf68e1133817e2 Mon Sep 17 00:00:00 2001 From: David Turnbull Date: Fri, 3 Oct 2025 22:54:30 +1000 Subject: [PATCH 13/20] fix: duplicate API key --- packages/cli/src/cli/utils/settings.ts | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/packages/cli/src/cli/utils/settings.ts b/packages/cli/src/cli/utils/settings.ts index 68bea67bd..449532e96 100644 --- a/packages/cli/src/cli/utils/settings.ts +++ b/packages/cli/src/cli/utils/settings.ts @@ -73,6 +73,12 @@ function _providerConfigKeys(): string[] { .filter((v): v is string => Boolean(v)); } +function _providerEnvVarKeys(): string[] { + return Object.values(PROVIDER_METADATA) + .map((m) => m.apiKeyEnvVar) + .filter((v): v is string => Boolean(v)); +} + export const SETTINGS_KEYS = [ "auth.apiKey", "auth.apiUrl", @@ -94,19 +100,15 @@ function _loadDefaults(): CliSettings { } function _loadEnv() { - return Z.object({ + const shape: Record = { LINGODOTDEV_API_KEY: Z.string().optional(), LINGODOTDEV_API_URL: Z.string().optional(), LINGODOTDEV_WEB_URL: Z.string().optional(), - OPENAI_API_KEY: Z.string().optional(), - ANTHROPIC_API_KEY: Z.string().optional(), - GROQ_API_KEY: Z.string().optional(), - GOOGLE_API_KEY: Z.string().optional(), - OPENROUTER_API_KEY: Z.string().optional(), - MISTRAL_API_KEY: Z.string().optional(), - }) - .passthrough() - .parse(process.env); + }; + for (const envVar of _providerEnvVarKeys()) { + shape[envVar] = Z.string().optional(); + } + return Z.object(shape).passthrough().parse(process.env); } function _loadSystemFile() { From eb5961a6193064e02d3e15884a21cda9627fb327 Mon Sep 17 00:00:00 2001 From: David Turnbull Date: Thu, 9 Oct 2025 10:38:56 +1100 Subject: [PATCH 14/20] fix: things --- packages/cli/src/cli/utils/settings.ts | 16 ++++++++++------ packages/compiler/src/lib/lcp/api/index.ts | 21 +++++++++++++-------- packages/providers/src/constants.ts | 17 +++++++---------- packages/providers/src/metadata.ts | 8 ++++---- 4 files changed, 34 insertions(+), 28 deletions(-) diff --git a/packages/cli/src/cli/utils/settings.ts b/packages/cli/src/cli/utils/settings.ts index 449532e96..c3ed0bee9 100644 --- a/packages/cli/src/cli/utils/settings.ts +++ b/packages/cli/src/cli/utils/settings.ts @@ -68,15 +68,19 @@ const SettingsSchema = Z.object({ }); function _providerConfigKeys(): string[] { - return Object.values(PROVIDER_METADATA) - .map((m) => m.apiKeyConfigKey) - .filter((v): v is string => Boolean(v)); + const keys: string[] = []; + for (const m of Object.values(PROVIDER_METADATA)) { + if (m.apiKeyConfigKey) keys.push(m.apiKeyConfigKey); + } + return keys; } function _providerEnvVarKeys(): string[] { - return Object.values(PROVIDER_METADATA) - .map((m) => m.apiKeyEnvVar) - .filter((v): v is string => Boolean(v)); + const keys: string[] = []; + for (const m of Object.values(PROVIDER_METADATA)) { + if (m.apiKeyEnvVar) keys.push(m.apiKeyEnvVar); + } + return keys; } export const SETTINGS_KEYS = [ diff --git a/packages/compiler/src/lib/lcp/api/index.ts b/packages/compiler/src/lib/lcp/api/index.ts index bd415ede2..d8a47cec4 100644 --- a/packages/compiler/src/lib/lcp/api/index.ts +++ b/packages/compiler/src/lib/lcp/api/index.ts @@ -17,6 +17,7 @@ import { } from "@lingo.dev/providers"; import * as dotenv from "dotenv"; import path from "path"; +import fs from "fs"; import { getRcConfig } from "@lingo.dev/config"; export class LCPAPI { @@ -132,14 +133,18 @@ export class LCPAPI { private static _createLingoDotDevEngine() { const getEnvWithDotenv = (name: string): string | undefined => { if (process.env[name]) return process.env[name]; - const result = dotenv.config({ - path: [ - path.resolve(process.cwd(), ".env"), - path.resolve(process.cwd(), ".env.local"), - path.resolve(process.cwd(), ".env.development"), - ], - }); - return result?.parsed?.[name]; + const candidates = [ + path.resolve(process.cwd(), ".env"), + path.resolve(process.cwd(), ".env.local"), + path.resolve(process.cwd(), ".env.development"), + ]; + for (const file of candidates) { + if (!fs.existsSync(file)) continue; + const result = dotenv.config({ path: file }); + if (process.env[name]) return process.env[name]; + if (result?.parsed?.[name]) return result.parsed[name]; + } + return undefined; }; if (isRunningInCIOrDocker()) { diff --git a/packages/providers/src/constants.ts b/packages/providers/src/constants.ts index 707fdf686..4659e60e0 100644 --- a/packages/providers/src/constants.ts +++ b/packages/providers/src/constants.ts @@ -1,11 +1,8 @@ -export const SUPPORTED_PROVIDERS = [ - "groq", - "openai", - "anthropic", - "google", - "openrouter", - "ollama", - "mistral", -] as const; +import { PROVIDER_METADATA } from "./metadata"; -export type ProviderId = (typeof SUPPORTED_PROVIDERS)[number]; +export type { ProviderId } from "./metadata"; + +// Derive supported providers from metadata keys to prevent drift +export const SUPPORTED_PROVIDERS = Object.freeze( + Object.keys(PROVIDER_METADATA), +) as readonly string[] as readonly import("./metadata").ProviderId[]; diff --git a/packages/providers/src/metadata.ts b/packages/providers/src/metadata.ts index b9d570643..033f0a631 100644 --- a/packages/providers/src/metadata.ts +++ b/packages/providers/src/metadata.ts @@ -1,5 +1,3 @@ -import { ProviderId } from "./constants"; - export interface ProviderMetadata { name: string; apiKeyEnvVar?: string; @@ -8,7 +6,7 @@ export interface ProviderMetadata { docsLink: string; } -export const PROVIDER_METADATA: Record = { +export const PROVIDER_METADATA = { groq: { name: "Groq", apiKeyEnvVar: "GROQ_API_KEY", @@ -58,4 +56,6 @@ export const PROVIDER_METADATA: Record = { getKeyLink: "https://console.mistral.ai", docsLink: "https://docs.mistral.ai", }, -}; +} as const satisfies Record; + +export type ProviderId = keyof typeof PROVIDER_METADATA; From c90487c37ee1d20b4694642abb1f7702d9371cbf Mon Sep 17 00:00:00 2001 From: David Turnbull Date: Thu, 16 Oct 2025 14:04:58 +1100 Subject: [PATCH 15/20] chore: refactor --- packages/config/src/index.ts | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/packages/config/src/index.ts b/packages/config/src/index.ts index 376c8df20..66c54361a 100644 --- a/packages/config/src/index.ts +++ b/packages/config/src/index.ts @@ -4,16 +4,6 @@ import fs from "fs"; import Ini from "ini"; import Z from "zod"; -export interface RcConfig { - auth?: { - apiKey?: string; - apiUrl?: string; - webUrl?: string; - }; - llm?: Record; - [key: string]: any; -} - export const rcConfigSchema = Z.object({ auth: Z.object({ apiKey: Z.string().optional(), @@ -23,13 +13,18 @@ export const rcConfigSchema = Z.object({ // Allow any llm provider keys and preserve them llm: Z.record(Z.string().optional()).optional(), }) - .passthrough() - .transform((v) => v as RcConfig); + .passthrough(); + +export type RcConfig = Z.infer; + +const SETTINGS_FILE = ".lingodotdevrc"; + +function getSettingsFilePath(): string { + return path.join(os.homedir(), SETTINGS_FILE); +} export function getRcConfig(): RcConfig { - const settingsFile = ".lingodotdevrc"; - const homedir = os.homedir(); - const settingsFilePath = path.join(homedir, settingsFile); + const settingsFilePath = getSettingsFilePath(); const content = fs.existsSync(settingsFilePath) ? fs.readFileSync(settingsFilePath, "utf-8") : ""; From c2c87904b60513c044df978899de57a0ac174549 Mon Sep 17 00:00:00 2001 From: David Turnbull Date: Thu, 16 Oct 2025 14:21:33 +1100 Subject: [PATCH 16/20] chore: remove dead code --- packages/providers/src/index.ts | 1 - packages/providers/src/keys.ts | 4 ---- 2 files changed, 5 deletions(-) diff --git a/packages/providers/src/index.ts b/packages/providers/src/index.ts index d466e15b4..20fbbdfdc 100644 --- a/packages/providers/src/index.ts +++ b/packages/providers/src/index.ts @@ -4,7 +4,6 @@ export { PROVIDER_METADATA, } from "./metadata"; export { - getProviderApiKey, resolveProviderApiKey, type KeySources, } from "./keys"; diff --git a/packages/providers/src/keys.ts b/packages/providers/src/keys.ts index 486846bac..ce96a2d80 100644 --- a/packages/providers/src/keys.ts +++ b/packages/providers/src/keys.ts @@ -32,10 +32,6 @@ export interface KeySources { rc?: RcConfig; } -export function getProviderApiKey(providerId: ProviderId): string | undefined { - return resolveProviderApiKey(providerId, { required: false }); -} - export function resolveProviderApiKey( providerId: ProviderId, options?: { sources?: KeySources; required?: boolean }, From d43d5fa730592d76cd7e2ed1b57e99d240d55700 Mon Sep 17 00:00:00 2001 From: David Turnbull Date: Thu, 16 Oct 2025 14:35:28 +1100 Subject: [PATCH 17/20] chore: remove skipAuth repetition --- packages/cli/src/cli/localizer/explicit.ts | 4 ---- packages/cli/src/cli/processor/index.ts | 2 -- packages/providers/src/factory.ts | 4 +++- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/cli/src/cli/localizer/explicit.ts b/packages/cli/src/cli/localizer/explicit.ts index 6dcf2a62a..882b90906 100644 --- a/packages/cli/src/cli/localizer/explicit.ts +++ b/packages/cli/src/cli/localizer/explicit.ts @@ -34,17 +34,14 @@ export default function createExplicitLocalizer( ); } - const skipAuth = provider.id === "ollama"; try { const model = createProviderClient(provider.id as ProviderId, provider.model, { baseUrl: provider.baseUrl, - skipAuth, }); return createLocalizerFromModel({ model, id: provider.id, prompt: provider.prompt, - skipAuth, }); } catch (error: unknown) { if (error instanceof ProviderKeyMissingError) { @@ -80,7 +77,6 @@ function createLocalizerFromModel(params: { model: LanguageModel; id: NonNullable["id"]; prompt: string; - skipAuth?: boolean; }): ILocalizer { const { model } = params; diff --git a/packages/cli/src/cli/processor/index.ts b/packages/cli/src/cli/processor/index.ts index 696a359be..c7e39d22e 100644 --- a/packages/cli/src/cli/processor/index.ts +++ b/packages/cli/src/cli/processor/index.ts @@ -75,11 +75,9 @@ function getPureModelProvider(provider: I18nConfig["provider"]) { throw new Error(createUnsupportedProviderErrorMessage(provider?.id)); } - const skipAuth = provider?.id === "ollama"; try { return createProviderClient(provider!.id as ProviderId, provider!.model, { baseUrl: provider!.baseUrl, - skipAuth, }); } catch (error: unknown) { if (error instanceof ProviderKeyMissingError) { diff --git a/packages/providers/src/factory.ts b/packages/providers/src/factory.ts index 59a2abcf4..e0e8bab12 100644 --- a/packages/providers/src/factory.ts +++ b/packages/providers/src/factory.ts @@ -1,6 +1,7 @@ import { LanguageModel } from "ai"; import { ProviderId } from "./constants"; import { resolveProviderApiKey } from "./keys"; +import { PROVIDER_METADATA } from "./metadata"; import { UnsupportedProviderError } from "./errors"; import { createOpenAI } from "@ai-sdk/openai"; import { createAnthropic } from "@ai-sdk/anthropic"; @@ -21,7 +22,8 @@ export function createProviderClient( modelId: string, options?: ClientOptions, ): LanguageModel { - const skipAuth = options?.skipAuth === true || providerId === "ollama"; + const skipAuth = + options?.skipAuth === true || !PROVIDER_METADATA[providerId]?.apiKeyEnvVar; const apiKey = options?.apiKey ?? resolveProviderApiKey(providerId, { required: !skipAuth }); switch (providerId) { From 6f3970b85b98e2f180b7363f17def6ef9b5771ad Mon Sep 17 00:00:00 2001 From: David Turnbull Date: Thu, 16 Oct 2025 14:43:06 +1100 Subject: [PATCH 18/20] chore: use ini stringify --- packages/cli/src/cli/utils/settings.ts | 27 +----------- packages/config/src/index.spec.ts | 59 ++++++++++++++++++++++++-- packages/config/src/index.ts | 17 ++++++++ 3 files changed, 75 insertions(+), 28 deletions(-) diff --git a/packages/cli/src/cli/utils/settings.ts b/packages/cli/src/cli/utils/settings.ts index c3ed0bee9..3955cb12d 100644 --- a/packages/cli/src/cli/utils/settings.ts +++ b/packages/cli/src/cli/utils/settings.ts @@ -3,7 +3,7 @@ import path from "path"; import _ from "lodash"; import Z from "zod"; import fs from "fs"; -import { getRcConfig } from "@lingo.dev/config"; +import { getRcConfig, saveRcConfig } from "@lingo.dev/config"; import { PROVIDER_METADATA } from "@lingo.dev/providers"; export type CliSettings = Z.infer; @@ -131,30 +131,7 @@ function _loadSystemFile() { } function _saveSystemFile(settings: CliSettings) { - const settingsFilePath = _getSettingsFilePath(); - const llmEntries: string[] = []; - for (const meta of Object.values(PROVIDER_METADATA)) { - const cfgKey = meta.apiKeyConfigKey; - if (!cfgKey) continue; - const suffix = cfgKey.startsWith("llm.") ? cfgKey.slice(4) : undefined; - if (!suffix) continue; - const value = (settings.llm as any)?.[suffix]; - if (value) llmEntries.push(`${suffix}=${value}`); - } - - const content = [ - `[auth]`, - `apiKey=${settings.auth.apiKey}`, - `apiUrl=${settings.auth.apiUrl}`, - `webUrl=${settings.auth.webUrl}`, - ``, - `[llm]`, - ...llmEntries, - ``, - ] - .filter(Boolean) - .join("\n"); - fs.writeFileSync(settingsFilePath, content); + saveRcConfig(settings); } function _getSettingsFilePath(): string { diff --git a/packages/config/src/index.spec.ts b/packages/config/src/index.spec.ts index 4b3e3af1a..73720dbd2 100644 --- a/packages/config/src/index.spec.ts +++ b/packages/config/src/index.spec.ts @@ -1,7 +1,60 @@ -import { describe, it, expect } from "vitest"; +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { saveRcConfig, getRcConfig } from "./index"; +import fs from "fs"; +import os from "os"; +import path from "path"; + +vi.mock("fs"); +vi.mock("os"); describe("config package", () => { - it("loads without tests", () => { - expect(true).toBe(true); + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(os.homedir).mockReturnValue("/mock/home"); + }); + + it("saveRcConfig uses Ini.stringify", () => { + const mockWrite = vi.mocked(fs.writeFileSync); + + saveRcConfig({ + auth: { + apiKey: "test-key", + apiUrl: "https://test.com", + webUrl: "https://web.test.com", + }, + llm: { + openaiApiKey: "openai-key", + }, + }); + + expect(mockWrite).toHaveBeenCalledWith( + path.join("/mock/home", ".lingodotdevrc"), + expect.stringContaining("[auth]") + ); + expect(mockWrite).toHaveBeenCalledWith( + path.join("/mock/home", ".lingodotdevrc"), + expect.stringContaining("[llm]") + ); + }); + + it("saveRcConfig handles undefined values gracefully", () => { + const mockWrite = vi.mocked(fs.writeFileSync); + + saveRcConfig({ + auth: { + apiKey: "test-key", + }, + llm: { + openaiApiKey: undefined, + anthropicApiKey: "defined-key", + }, + }); + + expect(mockWrite).toHaveBeenCalled(); + const writtenContent = mockWrite.mock.calls[0][1] as string; + + // Ini.stringify should omit undefined values, not write "undefined" + expect(writtenContent).not.toContain("undefined"); + expect(writtenContent).toContain("anthropicApiKey"); }); }); diff --git a/packages/config/src/index.ts b/packages/config/src/index.ts index 66c54361a..34083fff2 100644 --- a/packages/config/src/index.ts +++ b/packages/config/src/index.ts @@ -31,3 +31,20 @@ export function getRcConfig(): RcConfig { const data = Ini.parse(content); return rcConfigSchema.parse(data); } + +function removeUndefined(obj: T): T { + if (obj === null || typeof obj !== "object") return obj; + if (Array.isArray(obj)) return obj.map(removeUndefined) as T; + return Object.fromEntries( + Object.entries(obj) + .filter(([_, v]) => v !== undefined) + .map(([k, v]) => [k, removeUndefined(v)]) + ) as T; +} + +export function saveRcConfig(config: RcConfig): void { + const settingsFilePath = getSettingsFilePath(); + // Ini.stringify writes literal "undefined" strings, so filter them out first + const content = Ini.stringify(removeUndefined(config)); + fs.writeFileSync(settingsFilePath, content); +} From 09e1d7e39d646a89bc422e40f46d073339153a99 Mon Sep 17 00:00:00 2001 From: David Turnbull Date: Thu, 16 Oct 2025 14:51:27 +1100 Subject: [PATCH 19/20] chore: restructure files --- packages/providers/src/constants.ts | 8 -------- packages/providers/src/errors.ts | 2 +- packages/providers/src/factory.ts | 2 +- packages/providers/src/get-provider.ts | 14 -------------- packages/providers/src/index.ts | 4 ++-- packages/providers/src/keys.ts | 2 +- packages/providers/src/metadata.ts | 5 +++++ 7 files changed, 10 insertions(+), 27 deletions(-) delete mode 100644 packages/providers/src/constants.ts delete mode 100644 packages/providers/src/get-provider.ts diff --git a/packages/providers/src/constants.ts b/packages/providers/src/constants.ts deleted file mode 100644 index 4659e60e0..000000000 --- a/packages/providers/src/constants.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { PROVIDER_METADATA } from "./metadata"; - -export type { ProviderId } from "./metadata"; - -// Derive supported providers from metadata keys to prevent drift -export const SUPPORTED_PROVIDERS = Object.freeze( - Object.keys(PROVIDER_METADATA), -) as readonly string[] as readonly import("./metadata").ProviderId[]; diff --git a/packages/providers/src/errors.ts b/packages/providers/src/errors.ts index 8b061793f..082018d61 100644 --- a/packages/providers/src/errors.ts +++ b/packages/providers/src/errors.ts @@ -1,4 +1,4 @@ -import { ProviderId } from "./constants"; +import { ProviderId } from "./metadata"; export class ProviderKeyMissingError extends Error { constructor( diff --git a/packages/providers/src/factory.ts b/packages/providers/src/factory.ts index e0e8bab12..853499118 100644 --- a/packages/providers/src/factory.ts +++ b/packages/providers/src/factory.ts @@ -1,5 +1,5 @@ import { LanguageModel } from "ai"; -import { ProviderId } from "./constants"; +import { ProviderId } from "./metadata"; import { resolveProviderApiKey } from "./keys"; import { PROVIDER_METADATA } from "./metadata"; import { UnsupportedProviderError } from "./errors"; diff --git a/packages/providers/src/get-provider.ts b/packages/providers/src/get-provider.ts deleted file mode 100644 index f5f9e77c0..000000000 --- a/packages/providers/src/get-provider.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { ProviderId } from "./constants"; -import { ProviderMetadata, PROVIDER_METADATA } from "./metadata"; -import { createProviderClient, ClientOptions } from "./factory"; -import { LanguageModel } from "ai"; - -export function getProvider( - providerId: ProviderId, - modelId: string, - options?: ClientOptions, -): { client: LanguageModel; metadata: ProviderMetadata } { - const client = createProviderClient(providerId, modelId, options); - const metadata = PROVIDER_METADATA[providerId]; - return { client, metadata }; -} diff --git a/packages/providers/src/index.ts b/packages/providers/src/index.ts index 20fbbdfdc..5831abd6c 100644 --- a/packages/providers/src/index.ts +++ b/packages/providers/src/index.ts @@ -1,5 +1,6 @@ -export { SUPPORTED_PROVIDERS, type ProviderId } from "./constants"; export { + SUPPORTED_PROVIDERS, + type ProviderId, type ProviderMetadata, PROVIDER_METADATA, } from "./metadata"; @@ -16,4 +17,3 @@ export { createProviderClient, type ClientOptions, } from "./factory"; -export { getProvider } from "./get-provider"; diff --git a/packages/providers/src/keys.ts b/packages/providers/src/keys.ts index ce96a2d80..7dbded5da 100644 --- a/packages/providers/src/keys.ts +++ b/packages/providers/src/keys.ts @@ -1,7 +1,7 @@ import path from "path"; import fs from "fs"; import dotenv from "dotenv"; -import { ProviderId } from "./constants"; +import { ProviderId } from "./metadata"; import { PROVIDER_METADATA } from "./metadata"; import { getRcConfig, type RcConfig } from "@lingo.dev/config"; import { ProviderKeyMissingError } from "./errors"; diff --git a/packages/providers/src/metadata.ts b/packages/providers/src/metadata.ts index 033f0a631..5a6394270 100644 --- a/packages/providers/src/metadata.ts +++ b/packages/providers/src/metadata.ts @@ -59,3 +59,8 @@ export const PROVIDER_METADATA = { } as const satisfies Record; export type ProviderId = keyof typeof PROVIDER_METADATA; + +// Derive supported providers from metadata keys to prevent drift +export const SUPPORTED_PROVIDERS = Object.freeze( + Object.keys(PROVIDER_METADATA), +) as readonly string[] as readonly ProviderId[]; From c1a1c2d0d17c556f65c948722885baeb6687fc30 Mon Sep 17 00:00:00 2001 From: David Turnbull Date: Fri, 17 Oct 2025 16:00:11 +1100 Subject: [PATCH 20/20] fix: migration --- packages/spec/src/config.ts | 55 +++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/packages/spec/src/config.ts b/packages/spec/src/config.ts index f1af7a187..80c38d873 100644 --- a/packages/spec/src/config.ts +++ b/packages/spec/src/config.ts @@ -290,16 +290,15 @@ export const configV1_4Definition = extendConfigDefinition( // v1.4 -> v1.5 // Changes: add "provider" field to the config -const providerIdEnumValues = - SUPPORTED_PROVIDERS as unknown as [ - (typeof SUPPORTED_PROVIDERS)[number], - ...((typeof SUPPORTED_PROVIDERS)[number])[], - ]; - const providerSchema = Z.object({ - id: Z.enum(providerIdEnumValues).describe( - "Identifier of the translation provider service.", - ), + id: Z.enum([ + "openai", + "anthropic", + "google", + "ollama", + "openrouter", + "mistral", + ]).describe("Identifier of the translation provider service."), model: Z.string().describe("Model name to use for translations."), prompt: Z.string().describe( "Prompt template used when requesting translations.", @@ -487,25 +486,47 @@ export const configV1_10Definition = extendConfigDefinition( }, ); -// exports -// v1.10 -> v2.0 -// Changes: Use SUPPORTED_PROVIDERS from providers package for provider enum -export const configV2_0Definition = extendConfigDefinition( +// v1.10 -> v1.11 +// Changes: Add "groq" to provider enum +const providerSchemaV1_11 = Z.object({ + id: Z.enum([ + "openai", + "anthropic", + "google", + "ollama", + "openrouter", + "mistral", + "groq", + ]).describe("Identifier of the translation provider service."), + model: Z.string().describe("Model name to use for translations."), + prompt: Z.string().describe( + "Prompt template used when requesting translations.", + ), + baseUrl: Z.string() + .optional() + .describe("Custom base URL for the provider API (optional)."), + settings: modelSettingsSchema, +}).describe("Configuration for the machine-translation provider."); + +export const configV1_11Definition = extendConfigDefinition( configV1_10Definition, { - createSchema: (baseSchema) => baseSchema, + createSchema: (baseSchema) => + baseSchema.extend({ + provider: providerSchemaV1_11.optional(), + }), createDefaultValue: (baseDefaultValue) => ({ ...baseDefaultValue, - version: 2.0, + version: "1.11", }), createUpgrader: (oldConfig) => ({ ...oldConfig, - version: 2.0, + version: "1.11", }), }, ); -export const LATEST_CONFIG_DEFINITION = configV2_0Definition; +export const LATEST_CONFIG_DEFINITION = configV1_11Definition; export type I18nConfig = Z.infer<(typeof LATEST_CONFIG_DEFINITION)["schema"]>;