diff --git a/src/routes/chat/utils/handle-websocket-message.ts b/src/routes/chat/utils/handle-websocket-message.ts index b16164f1..413bf0f9 100644 --- a/src/routes/chat/utils/handle-websocket-message.ts +++ b/src/routes/chat/utils/handle-websocket-message.ts @@ -152,10 +152,10 @@ export function createWebSocketMessageHandler(deps: HandleMessageDeps) { })); break; } - case 'cf_agent_state': { - const { state } = message; - logger.debug('🔄 Agent state update received:', state); - + case 'agent_connected': { + const { state, templateDetails } = message; + console.log('Agent connected', state, templateDetails); + if (!isInitialStateRestored) { logger.debug('📥 Performing initial state restoration'); @@ -168,11 +168,11 @@ export function createWebSocketMessageHandler(deps: HandleMessageDeps) { setQuery(state.query); } - if (state.templateDetails?.allFiles && bootstrapFiles.length === 0) { - const files = Object.entries(state.templateDetails.allFiles).map(([filePath, fileContents]) => ({ + if (templateDetails?.allFiles && bootstrapFiles.length === 0) { + const files = Object.entries(templateDetails.allFiles).map(([filePath, fileContents]) => ({ filePath, fileContents, - })); + })).filter((file) => templateDetails.importantFiles.includes(file.filePath)); logger.debug('📥 Restoring bootstrap files:', files); loadBootstrapFiles(files); } @@ -254,6 +254,11 @@ export function createWebSocketMessageHandler(deps: HandleMessageDeps) { sendWebSocketMessage(websocket, 'preview'); } } + break; + } + case 'cf_agent_state': { + const { state } = message; + logger.debug('🔄 Agent state update received:', state); if (state.shouldBeGenerating) { logger.debug('🔄 shouldBeGenerating=true detected, auto-resuming generation'); diff --git a/worker/agents/assistants/projectsetup.ts b/worker/agents/assistants/projectsetup.ts index 534c816c..f261b3d1 100644 --- a/worker/agents/assistants/projectsetup.ts +++ b/worker/agents/assistants/projectsetup.ts @@ -127,15 +127,10 @@ ${error}`); context: this.inferenceContext, modelName: error? AIModels.GEMINI_2_5_FLASH : undefined, }); - if (!results || typeof results !== 'string') { - this.logger.info(`Failed to generate setup commands, results: `, { results }); - return { commands: [] }; - } - this.logger.info(`Generated setup commands: ${results}`); - this.save([createAssistantMessage(results)]); - return { commands: extractCommands(results) }; + this.save([createAssistantMessage(results.string)]); + return { commands: extractCommands(results.string) }; } catch (error) { this.logger.error("Error generating setup commands:", error); throw error; diff --git a/worker/agents/core/simpleGeneratorAgent.ts b/worker/agents/core/simpleGeneratorAgent.ts index 700bd2c2..90915f28 100644 --- a/worker/agents/core/simpleGeneratorAgent.ts +++ b/worker/agents/core/simpleGeneratorAgent.ts @@ -1,4 +1,4 @@ -import { Agent, AgentContext, Connection } from 'agents'; +import { Agent, AgentContext, Connection, ConnectionContext } from 'agents'; import { Blueprint, PhaseConceptGenerationSchemaType, @@ -12,7 +12,7 @@ import { GitHubExportResult } from '../../services/github/types'; import { CodeGenState, CurrentDevState, MAX_PHASES } from './state'; import { AllIssues, AgentSummary, AgentInitArgs, PhaseExecutionResult, UserContext } from './types'; import { PREVIEW_EXPIRED_ERROR, WebSocketMessageResponses } from '../constants'; -import { broadcastToConnections, handleWebSocketClose, handleWebSocketMessage } from './websocket'; +import { broadcastToConnections, handleWebSocketClose, handleWebSocketMessage, sendToConnection } from './websocket'; import { createObjectLogger, StructuredLogger } from '../../logger'; import { ProjectSetupAssistant } from '../assistants/projectsetup'; import { UserConversationProcessor, RenderToolCall } from '../operations/UserConversationProcessor'; @@ -33,7 +33,7 @@ import { WebSocketMessageData, WebSocketMessageType } from '../../api/websocketT import { InferenceContext, AgentActionKey } from '../inferutils/config.types'; import { AGENT_CONFIG } from '../inferutils/config'; import { ModelConfigService } from '../../database/services/ModelConfigService'; -import { FileFetcher, fixProjectIssues } from '../../services/code-fixer'; +import { fixProjectIssues } from '../../services/code-fixer'; import { FastCodeFixerOperation } from '../operations/PostPhaseCodeFixer'; import { looksLikeCommand } from '../utils/common'; import { generateBlueprint } from '../planning/blueprint'; @@ -82,6 +82,7 @@ export class SimpleCodeGeneratorAgent extends Agent { protected deploymentManager!: DeploymentManager; private previewUrlCache: string = ''; + private templateDetailsCache: TemplateDetails | null = null; // In-memory storage for user-uploaded images (not persisted in DO state) private pendingUserImages: ProcessedImageAttachment[] = [] @@ -121,7 +122,7 @@ export class SimpleCodeGeneratorAgent extends Agent { } getAgentId() { - return this.state.inferenceContext.agentId + return this.state.inferenceContext.agentId; } initialState: CodeGenState = { @@ -131,7 +132,7 @@ export class SimpleCodeGeneratorAgent extends Agent { generatedFilesMap: {}, agentMode: 'deterministic', sandboxInstanceId: undefined, - templateDetails: {} as TemplateDetails, + templateName: '', commandsHistory: [], lastPackageJson: '', pendingUserInputs: [], @@ -160,7 +161,7 @@ export class SimpleCodeGeneratorAgent extends Agent { ); // Initialize FileManager - this.fileManager = new FileManager(this.stateManager); + this.fileManager = new FileManager(this.stateManager, () => this.getTemplateDetails()); // Initialize DeploymentManager first (manages sandbox client caching) // DeploymentManager will use its own getClient() override for caching @@ -185,7 +186,8 @@ export class SimpleCodeGeneratorAgent extends Agent { ..._args: unknown[] ): Promise { - const { query, language, frameworks, hostname, inferenceContext, templateInfo, sandboxSessionId } = initArgs; + const { query, language, frameworks, hostname, inferenceContext, templateInfo } = initArgs; + const sandboxSessionId = inferenceContext.agentId; // Let the initial sessionId be the agentId this.initLogger(inferenceContext.agentId, sandboxSessionId, inferenceContext.userId); // Generate a blueprint @@ -211,12 +213,14 @@ export class SimpleCodeGeneratorAgent extends Agent { }) const packageJson = templateInfo.templateDetails?.allFiles['package.json']; + + this.templateDetailsCache = templateInfo.templateDetails; this.setState({ ...this.initialState, query, blueprint, - templateDetails: templateInfo.templateDetails, + templateName: templateInfo.templateDetails.name, sandboxInstanceId: undefined, generatedPhases: [], commandsHistory: [], @@ -252,7 +256,59 @@ export class SimpleCodeGeneratorAgent extends Agent { async isInitialized() { return this.getAgentId() ? true : false - } + } + + async onStart(_props?: Record | undefined): Promise { + this.logger().info(`Agent ${this.getAgentId()} session: ${this.state.sessionId} onStart`); + // Ignore if agent not initialized + if (!this.state.templateName?.trim()) { + this.logger().info(`Agent ${this.getAgentId()} session: ${this.state.sessionId} not initialized, ignoring onStart`); + return; + } + this.logger().info(`Agent ${this.getAgentId()} session: ${this.state.sessionId} onStart being processed, template name: ${this.state.templateName}`); + // Fill the template cache + await this.ensureTemplateDetails(); + this.logger().info(`Agent ${this.getAgentId()} session: ${this.state.sessionId} onStart processed successfully`); + } + + onStateUpdate(_state: CodeGenState, _source: "server" | Connection) {} + + setState(state: CodeGenState): void { + try { + super.setState(state); + } catch (error) { + this.broadcastError("Error setting state", error); + this.logger().error("State details:", { + originalState: JSON.stringify(this.state, null, 2), + newState: JSON.stringify(state, null, 2) + }); + } + } + + onConnect(connection: Connection, ctx: ConnectionContext) { + this.logger().info(`Agent connected for agent ${this.getAgentId()}`, { connection, ctx }); + sendToConnection(connection, 'agent_connected', { + state: this.state, + templateDetails: this.getTemplateDetails() + }); + } + + async ensureTemplateDetails() { + if (!this.templateDetailsCache) { + this.logger().info(`Template details being cached for template: ${this.state.templateName}`); + const results = await BaseSandboxService.getTemplateDetails(this.state.templateName); + if (!results.success || !results.templateDetails) { + throw new Error(`Failed to get template details for template: ${this.state.templateName}`); + } + this.templateDetailsCache = results.templateDetails; + this.logger().info(`Template details for template: ${this.state.templateName} cached successfully`); + } + return this.templateDetailsCache; + } + + private getTemplateDetails() { + return this.templateDetailsCache!; + } /* * Each DO has 10 gb of sqlite storage. However, the way agents sdk works, it stores the 'state' object of the agent as a single row @@ -332,20 +388,6 @@ export class SimpleCodeGeneratorAgent extends Agent { }); this.logger().info(`Agent initialized successfully for agent ${this.state.inferenceContext.agentId}`); } - - onStateUpdate(_state: CodeGenState, _source: "server" | Connection) {} - - setState(state: CodeGenState): void { - try { - super.setState(state); - } catch (error) { - this.broadcastError("Error setting state", error); - this.logger().error("State details:", { - originalState: JSON.stringify(this.state, null, 2), - newState: JSON.stringify(state, null, 2) - }); - } - } getPreviewUrlCache() { return this.previewUrlCache; @@ -358,7 +400,7 @@ export class SimpleCodeGeneratorAgent extends Agent { agentId: this.getAgentId(), query: this.state.query, blueprint: this.state.blueprint, - template: this.state.templateDetails, + template: this.getTemplateDetails(), inferenceContext: this.state.inferenceContext }); } @@ -403,7 +445,7 @@ export class SimpleCodeGeneratorAgent extends Agent { return { env: this.env, agentId: this.getAgentId(), - context: GenerationContext.from(this.state, this.logger()), + context: GenerationContext.from(this.state, this.getTemplateDetails(), this.logger()), logger: this.logger(), inferenceContext: this.getInferenceContext(), agent: this.codingAgent @@ -1352,7 +1394,7 @@ export class SimpleCodeGeneratorAgent extends Agent { return; } const issues = staticAnalysis.typecheck.issues.concat(staticAnalysis.lint.issues); - const allFiles = this.fileManager.getAllFiles(); + const allFiles = this.fileManager.getAllRelevantFiles(); const fastCodeFixer = await this.operations.fastCodeFixer.execute({ query: this.state.query, @@ -1392,35 +1434,13 @@ export class SimpleCodeGeneratorAgent extends Agent { this.logger().info(`Attempting to fix ${typeCheckIssues.length} TypeScript issues using deterministic code fixer`); const allFiles = this.fileManager.getAllFiles(); - // Create file fetcher callback - const fileFetcher: FileFetcher = async (filePath: string) => { - // Fetch a single file from the instance - try { - const result = await this.getSandboxServiceClient().getFiles(this.state.sandboxInstanceId!, [filePath]); - if (result.success && result.files.length > 0) { - this.logger().info(`Successfully fetched file: ${filePath}`); - return { - filePath: filePath, - fileContents: result.files[0].fileContents, - filePurpose: `Fetched file: ${filePath}` - }; - } else { - this.logger().debug(`File not found: ${filePath}`); - } - } catch (error) { - this.logger().debug(`Failed to fetch file ${filePath}: ${error instanceof Error ? error.message : 'Unknown error'}`); - } - return null; - }; - - const fixResult = await fixProjectIssues( + const fixResult = fixProjectIssues( allFiles.map(file => ({ filePath: file.filePath, fileContents: file.fileContents, filePurpose: '' })), - typeCheckIssues, - fileFetcher + typeCheckIssues ); this.broadcast(WebSocketMessageResponses.DETERMINISTIC_CODE_FIX_COMPLETED, { diff --git a/worker/agents/core/state.ts b/worker/agents/core/state.ts index 18716357..6ef7e1db 100644 --- a/worker/agents/core/state.ts +++ b/worker/agents/core/state.ts @@ -1,7 +1,6 @@ import type { Blueprint, PhaseConceptType , FileOutputType, } from '../schemas'; -import type { TemplateDetails } from '../../services/sandbox/sandboxTypes'; // import type { ScreenshotData } from './types'; import type { ConversationMessage } from '../inferutils/common'; import type { InferenceContext } from '../inferutils/config.types'; @@ -36,7 +35,7 @@ export interface CodeGenState { generatedPhases: PhaseState[]; commandsHistory?: string[]; // History of commands run lastPackageJson?: string; // Last package.json file contents - templateDetails: TemplateDetails; // TODO: Remove this from state and rely on directly fetching from sandbox + templateName: string; sandboxInstanceId?: string; shouldBeGenerating: boolean; // Persistent flag indicating generation should be active diff --git a/worker/agents/core/stateMigration.ts b/worker/agents/core/stateMigration.ts index fd62b7a9..b71a313f 100644 --- a/worker/agents/core/stateMigration.ts +++ b/worker/agents/core/stateMigration.ts @@ -1,10 +1,14 @@ import { CodeGenState, FileState } from './state'; import { StructuredLogger } from '../../logger'; +import { TemplateDetails } from 'worker/services/sandbox/sandboxTypes'; export class StateMigration { static migrateIfNeeded(state: CodeGenState, logger: StructuredLogger): CodeGenState | null { let needsMigration = false; + //------------------------------------------------------------------------------------ + // Migrate files from old schema + //------------------------------------------------------------------------------------ const migrateFile = (file: any): any => { const hasOldFormat = 'file_path' in file || 'file_contents' in file || 'file_purpose' in file; @@ -34,27 +38,9 @@ export class StateMigration { } } - let migratedTemplateDetails = state.templateDetails; - if ('files' in migratedTemplateDetails && migratedTemplateDetails?.files) { - const migratedTemplateFiles = (migratedTemplateDetails.files as Array).map(file => { - const migratedFile = migrateFile(file); - return migratedFile; - }); - - const allFiles = migratedTemplateFiles.reduce((acc, file) => { - acc[file.filePath] = file; - return acc; - }, {} as Record); - - migratedTemplateDetails = { - ...migratedTemplateDetails, - allFiles - }; - - // Remove 'files' property - delete (migratedTemplateDetails as any).files; - needsMigration = true; - } + //------------------------------------------------------------------------------------ + // Migrate conversations cleanups and internal memos + //------------------------------------------------------------------------------------ let migratedConversationMessages = state.conversationMessages; const MIN_MESSAGES_FOR_CLEANUP = 25; @@ -132,6 +118,9 @@ export class StateMigration { } } + //------------------------------------------------------------------------------------ + // Migrate inference context from old schema + //------------------------------------------------------------------------------------ let migratedInferenceContext = state.inferenceContext; if (migratedInferenceContext && 'userApiKeys' in migratedInferenceContext) { migratedInferenceContext = { @@ -142,6 +131,9 @@ export class StateMigration { needsMigration = true; } + //------------------------------------------------------------------------------------ + // Migrate deprecated props + //------------------------------------------------------------------------------------ const stateHasDeprecatedProps = 'latestScreenshot' in (state as any); if (stateHasDeprecatedProps) { needsMigration = true; @@ -151,11 +143,22 @@ export class StateMigration { if (!stateHasProjectUpdatesAccumulator) { needsMigration = true; } + + //------------------------------------------------------------------------------------ + // Migrate Template Details -> remove template details and instead use template name + //------------------------------------------------------------------------------------ + const hasTemplateDetails = 'templateDetails' in (state as any); + if (hasTemplateDetails) { + needsMigration = true; + const templateDetails = (state as any).templateDetails; + const templateName = (templateDetails as TemplateDetails).name; + delete (state as any).templateDetails; + (state as any).templateName = templateName; + } if (needsMigration) { logger.info('Migrating state: schema format, conversation cleanup, and security fixes', { generatedFilesCount: Object.keys(migratedFilesMap).length, - templateFilesCount: migratedTemplateDetails?.allFiles?.length || 0, finalConversationCount: migratedConversationMessages?.length || 0, removedUserApiKeys: state.inferenceContext && 'userApiKeys' in state.inferenceContext }); @@ -163,7 +166,6 @@ export class StateMigration { const newState = { ...state, generatedFilesMap: migratedFilesMap, - templateDetails: migratedTemplateDetails, conversationMessages: migratedConversationMessages, inferenceContext: migratedInferenceContext, projectUpdatesAccumulator: [] diff --git a/worker/agents/core/types.ts b/worker/agents/core/types.ts index 55aee1d6..af7ecc27 100644 --- a/worker/agents/core/types.ts +++ b/worker/agents/core/types.ts @@ -18,7 +18,6 @@ export interface AgentInitArgs { templateDetails: TemplateDetails; selection: TemplateSelection; } - sandboxSessionId: string images?: ProcessedImageAttachment[]; onBlueprintChunk: (chunk: string) => void; } diff --git a/worker/agents/domain/pure/FileProcessing.ts b/worker/agents/domain/pure/FileProcessing.ts index b3b809d7..d900169f 100644 --- a/worker/agents/domain/pure/FileProcessing.ts +++ b/worker/agents/domain/pure/FileProcessing.ts @@ -3,7 +3,7 @@ import type { StructuredLogger } from '../../../logger'; import { TemplateDetails } from '../../../services/sandbox/sandboxTypes'; import { applyUnifiedDiff } from '../../output-formats/diff-formats'; import { FileState } from 'worker/agents/core/state'; -import { getTemplateFiles } from 'worker/services/sandbox/utils'; +import { getTemplateFiles, getTemplateImportantFiles } from 'worker/services/sandbox/utils'; /** * File processing utilities @@ -88,14 +88,22 @@ export class FileProcessing { } /** - * Get all files combining template and generated files + * Get all relevant files combining template (important) and generated files * Template files are overridden by generated files with same path */ - static getAllFiles( + static getAllRelevantFiles( templateDetails: TemplateDetails | undefined, generatedFilesMap: Record ): FileState[] { - const templateFiles = templateDetails?.allFiles ? getTemplateFiles(templateDetails) : []; + return this.getAllFiles(templateDetails, generatedFilesMap, true); + } + + static getAllFiles( + templateDetails: TemplateDetails | undefined, + generatedFilesMap: Record, + onlyImportantFiles: boolean = false + ): FileState[] { + const templateFiles = templateDetails?.allFiles ? (onlyImportantFiles ? getTemplateImportantFiles(templateDetails) : getTemplateFiles(templateDetails)) : []; // Filter out template files that have been overridden by generated files const nonOverriddenTemplateFiles = templateFiles.filter( diff --git a/worker/agents/domain/values/GenerationContext.ts b/worker/agents/domain/values/GenerationContext.ts index 2d7f4014..39176ab9 100644 --- a/worker/agents/domain/values/GenerationContext.ts +++ b/worker/agents/domain/values/GenerationContext.ts @@ -30,22 +30,22 @@ export class GenerationContext { /** * Create context from current state */ - static from(state: CodeGenState, logger?: Pick): GenerationContext { + static from(state: CodeGenState, templateDetails: TemplateDetails, logger?: Pick): GenerationContext { const dependencies = DependencyManagement.mergeDependencies( - state.templateDetails?.deps || {}, + templateDetails.deps || {}, state.lastPackageJson, logger ); - const allFiles = FileProcessing.getAllFiles( - state.templateDetails, + const allFiles = FileProcessing.getAllRelevantFiles( + templateDetails, state.generatedFilesMap ); return new GenerationContext( state.query, state.blueprint, - state.templateDetails, + templateDetails, dependencies, allFiles, state.generatedPhases, diff --git a/worker/agents/index.ts b/worker/agents/index.ts index e5b445d3..db66b2df 100644 --- a/worker/agents/index.ts +++ b/worker/agents/index.ts @@ -7,10 +7,10 @@ import { StructuredLogger } from '../logger'; import { InferenceContext } from './inferutils/config.types'; import { SandboxSdkClient } from '../services/sandbox/sandboxSdkClient'; import { selectTemplate } from './planning/templateSelector'; -import { getSandboxService } from '../services/sandbox/factory'; import { TemplateDetails } from '../services/sandbox/sandboxTypes'; import { TemplateSelection } from './schemas'; import type { ImageAttachment } from '../types/image-attachment'; +import { BaseSandboxService } from 'worker/services/sandbox/BaseSandboxService'; export async function getAgentStub(env: Env, agentId: string, searchInOtherJurisdictions: boolean = false, logger: StructuredLogger) : Promise> { if (searchInOtherJurisdictions) { @@ -77,46 +77,39 @@ export async function getTemplateForQuery( query: string, images: ImageAttachment[] | undefined, logger: StructuredLogger, -) : Promise<{sandboxSessionId: string, templateDetails: TemplateDetails, selection: TemplateSelection}> { +) : Promise<{templateDetails: TemplateDetails, selection: TemplateSelection}> { // Fetch available templates const templatesResponse = await SandboxSdkClient.listTemplates(); if (!templatesResponse || !templatesResponse.success) { throw new Error(`Failed to fetch templates from sandbox service, ${templatesResponse.error}`); } - - const sandboxSessionId = generateId(); - - const [analyzeQueryResponse, sandboxClient] = await Promise.all([ - selectTemplate({ - env: env, - inferenceContext, - query, - availableTemplates: templatesResponse.templates, - images, - }), - getSandboxService(sandboxSessionId, 'default') - ]); - logger.info('Selected template', { selectedTemplate: analyzeQueryResponse }); + const analyzeQueryResponse = await selectTemplate({ + env, + inferenceContext, + query, + availableTemplates: templatesResponse.templates, + images, + }); + + logger.info('Selected template', { selectedTemplate: analyzeQueryResponse }); - // Find the selected template by name in the available templates - if (!analyzeQueryResponse.selectedTemplateName) { - logger.error('No suitable template found for code generation'); - throw new Error('No suitable template found for code generation'); - } + if (!analyzeQueryResponse.selectedTemplateName) { + logger.error('No suitable template found for code generation'); + throw new Error('No suitable template found for code generation'); + } - const selectedTemplate = templatesResponse.templates.find(template => template.name === analyzeQueryResponse.selectedTemplateName); - if (!selectedTemplate) { - logger.error('Selected template not found'); - throw new Error('Selected template not found'); - } - // Now fetch all the files from the instance - const templateDetailsResponse = await sandboxClient.getTemplateDetails(selectedTemplate.name); - if (!templateDetailsResponse.success || !templateDetailsResponse.templateDetails) { - logger.error('Failed to fetch files', { templateDetailsResponse }); - throw new Error('Failed to fetch files'); - } + const selectedTemplate = templatesResponse.templates.find(template => template.name === analyzeQueryResponse.selectedTemplateName); + if (!selectedTemplate) { + logger.error('Selected template not found'); + throw new Error('Selected template not found'); + } + const templateDetailsResponse = await BaseSandboxService.getTemplateDetails(selectedTemplate.name); + if (!templateDetailsResponse.success || !templateDetailsResponse.templateDetails) { + logger.error('Failed to fetch files', { templateDetailsResponse }); + throw new Error('Failed to fetch files'); + } - const templateDetails = templateDetailsResponse.templateDetails; - return { sandboxSessionId, templateDetails, selection: analyzeQueryResponse }; + const templateDetails = templateDetailsResponse.templateDetails; + return { templateDetails, selection: analyzeQueryResponse }; } \ No newline at end of file diff --git a/worker/agents/planning/blueprint.ts b/worker/agents/planning/blueprint.ts index 726320f7..5f3148f9 100644 --- a/worker/agents/planning/blueprint.ts +++ b/worker/agents/planning/blueprint.ts @@ -9,7 +9,7 @@ import { TemplateRegistry } from '../inferutils/schemaFormatters'; import z from 'zod'; import { imagesToBase64 } from 'worker/utils/images'; import { ProcessedImageAttachment } from 'worker/types/image-attachment'; -import { getTemplateFiles } from 'worker/services/sandbox/utils'; +import { getTemplateImportantFiles } from 'worker/services/sandbox/utils'; const logger = createLogger('Blueprint'); @@ -188,7 +188,7 @@ export async function generateBlueprint({ env, inferenceContext, query, language // --------------------------------------------------------------------------- const filesText = TemplateRegistry.markdown.serialize( - { files: getTemplateFiles(templateDetails).filter(f => !f.filePath.includes('package.json')) }, + { files: getTemplateImportantFiles(templateDetails).filter(f => !f.filePath.includes('package.json')) }, z.object({ files: z.array(TemplateFileSchema) }) ); diff --git a/worker/agents/services/implementations/DeploymentManager.ts b/worker/agents/services/implementations/DeploymentManager.ts index 29988211..7ebe4432 100644 --- a/worker/agents/services/implementations/DeploymentManager.ts +++ b/worker/agents/services/implementations/DeploymentManager.ts @@ -517,7 +517,7 @@ export class DeploymentManager extends BaseAgentService implements IDeploymentMa */ private async createNewInstance(): Promise { const state = this.getState(); - const templateName = state.templateDetails?.name || 'scratch'; + const templateName = state.templateName || 'scratch'; // Generate unique project name let prefix = (state.blueprint?.projectName || templateName) @@ -532,7 +532,7 @@ export class DeploymentManager extends BaseAgentService implements IDeploymentMa // Add AI proxy vars if AI template let localEnvVars: Record = {}; - if (state.templateDetails?.name?.includes('agents')) { + if (state.templateName?.includes('agents')) { localEnvVars = { "CF_AI_BASE_URL": generateAppProxyUrl(this.env), "CF_AI_API_KEY": await generateAppProxyToken( diff --git a/worker/agents/services/implementations/FileManager.ts b/worker/agents/services/implementations/FileManager.ts index 60d72bde..ef023842 100644 --- a/worker/agents/services/implementations/FileManager.ts +++ b/worker/agents/services/implementations/FileManager.ts @@ -2,9 +2,9 @@ import * as Diff from 'diff'; import { IFileManager } from '../interfaces/IFileManager'; import { IStateManager } from '../interfaces/IStateManager'; import { FileOutputType } from '../../schemas'; -// import { TemplateDetails } from '../../../services/sandbox/sandboxTypes'; import { FileProcessing } from '../../domain/pure/FileProcessing'; import { FileState } from 'worker/agents/core/state'; +import { TemplateDetails } from '../../../services/sandbox/sandboxTypes'; /** * Manages file operations for code generation @@ -12,7 +12,8 @@ import { FileState } from 'worker/agents/core/state'; */ export class FileManager implements IFileManager { constructor( - private stateManager: IStateManager + private stateManager: IStateManager, + private getTemplateDetailsFunc: () => TemplateDetails ) {} getGeneratedFile(path: string): FileOutputType | null { @@ -20,9 +21,19 @@ export class FileManager implements IFileManager { return state.generatedFilesMap[path] || null; } + /** + * Get all files combining template and generated files + * Template files are overridden by generated files with same path + * @returns Array of all files. Only returns important template files, not all! + */ + getAllRelevantFiles(): FileOutputType[] { + const state = this.stateManager.getState(); + return FileProcessing.getAllRelevantFiles(this.getTemplateDetailsFunc(), state.generatedFilesMap); + } + getAllFiles(): FileOutputType[] { const state = this.stateManager.getState(); - return FileProcessing.getAllFiles(state.templateDetails, state.generatedFilesMap); + return FileProcessing.getAllFiles(this.getTemplateDetailsFunc(), state.generatedFilesMap); } saveGeneratedFile(file: FileOutputType): FileState { @@ -86,6 +97,7 @@ export class FileManager implements IFileManager { generatedFilesMap: newFilesMap }); } + fileExists(path: string): boolean { return !!this.getGeneratedFile(path) } @@ -104,4 +116,27 @@ export class FileManager implements IFileManager { const state = this.stateManager.getState(); return Object.values(state.generatedFilesMap); } + + getTemplateFile(filePath: string) : FileOutputType | null { + const templateDetails = this.getTemplateDetailsFunc(); + const fileContents = templateDetails.allFiles[filePath]; + if (!fileContents) { + return null; + } + return { + filePath, + fileContents, + filePurpose: 'Bootstrapped template file', + } + } + + getFile(filePath: string) : FileOutputType | null { + // First search generated files + const generatedFile = this.getGeneratedFile(filePath); + if (generatedFile) { + return generatedFile; + } + // Then search template files + return this.getTemplateFile(filePath); + } } \ No newline at end of file diff --git a/worker/agents/services/interfaces/IFileManager.ts b/worker/agents/services/interfaces/IFileManager.ts index 58e8e5f1..02b9af01 100644 --- a/worker/agents/services/interfaces/IFileManager.ts +++ b/worker/agents/services/interfaces/IFileManager.ts @@ -12,7 +12,12 @@ export interface IFileManager { getGeneratedFile(path: string): FileOutputType | null; /** - * Get all files (template + generated) + * Get all relevant files (template (important) + generated) + */ + getAllRelevantFiles(): FileOutputType[]; + + /** + * Get all files (template (important) + generated) */ getAllFiles(): FileOutputType[]; diff --git a/worker/api/controllers/agent/controller.ts b/worker/api/controllers/agent/controller.ts index 2f4326cb..71275cdd 100644 --- a/worker/api/controllers/agent/controller.ts +++ b/worker/api/controllers/agent/controller.ts @@ -111,7 +111,7 @@ export class CodingAgentController extends BaseController { modelConfigsCount: Object.keys(userModelConfigs).length, }); - const { sandboxSessionId, templateDetails, selection } = await getTemplateForQuery(env, inferenceContext, query, body.images, this.logger); + const { templateDetails, selection } = await getTemplateForQuery(env, inferenceContext, query, body.images, this.logger); const websocketUrl = `${url.protocol === 'https:' ? 'wss:' : 'ws:'}//${url.host}/api/agent/${agentId}/ws`; const httpStatusUrl = `${url.origin}/api/agent/${agentId}`; @@ -145,7 +145,6 @@ export class CodingAgentController extends BaseController { writer.write({chunk}); }, templateInfo: { templateDetails, selection }, - sandboxSessionId }, body.agentMode || defaultCodeGenArgs.agentMode) as Promise; agentPromise.then(async (_state: CodeGenState) => { writer.write("terminate"); diff --git a/worker/api/websocketTypes.ts b/worker/api/websocketTypes.ts index 3208d768..76057985 100644 --- a/worker/api/websocketTypes.ts +++ b/worker/api/websocketTypes.ts @@ -1,7 +1,7 @@ import type { CodeReviewOutputType, FileConceptType, FileOutputType } from "../agents/schemas"; import type { CodeGenState } from "../agents/core/state"; import type { ConversationState } from "../agents/inferutils/common"; -import type { CodeIssue, RuntimeError, StaticAnalysisResponse } from "../services/sandbox/sandboxTypes"; +import type { CodeIssue, RuntimeError, StaticAnalysisResponse, TemplateDetails } from "../services/sandbox/sandboxTypes"; import type { CodeFixResult } from "../services/code-fixer"; import { IssueReport } from "../agents/domain/values/IssueReport"; import type { RateLimitExceededError } from 'shared/types/errors'; @@ -16,6 +16,12 @@ type StateMessage = { state: CodeGenState; }; +type AgentConnectedMessage = { + type: 'agent_connected'; + state: CodeGenState; + templateDetails: TemplateDetails; +}; + type ConversationStateMessage = { type: 'conversation_state'; state: ConversationState; @@ -408,6 +414,7 @@ type ServerLogMessage = { export type WebSocketMessage = | StateMessage + | AgentConnectedMessage | ConversationStateMessage | GenerationStartedMessage | FileGeneratingMessage diff --git a/worker/services/code-fixer/fixers/ts2304.ts b/worker/services/code-fixer/fixers/ts2304.ts index 25b8733a..c1de2bd9 100644 --- a/worker/services/code-fixer/fixers/ts2304.ts +++ b/worker/services/code-fixer/fixers/ts2304.ts @@ -14,15 +14,14 @@ import { handleFixerError } from '../utils/helpers'; * Fix TS2304 "Cannot find name" errors * Preserves exact logic from working DeclarationFixer.fixUndefinedName */ -export async function fixUndefinedName( +export function fixUndefinedName( context: FixerContext, issues: CodeIssue[] -): Promise { +): FixResult { const fixedIssues: FixedIssue[] = []; const unfixableIssues: UnfixableIssue[] = []; const modifiedFilesMap = new Map(); const newFiles: FileObject[] = []; - const fetchedFiles = new Set(context.fetchedFiles); // Group issues by file to handle multiple undefined names in the same file const issuesByFile = new Map(); @@ -35,11 +34,9 @@ export async function fixUndefinedName( // Process each file's issues together for (const [filePath, fileIssues] of issuesByFile) { try { - const fileContent = await getFileContent( + const fileContent = getFileContent( filePath, - context.files, - context.fileFetcher, - fetchedFiles + context.files ); if (!fileContent) { diff --git a/worker/services/code-fixer/fixers/ts2305.ts b/worker/services/code-fixer/fixers/ts2305.ts index dc2501c5..3f38e162 100644 --- a/worker/services/code-fixer/fixers/ts2305.ts +++ b/worker/services/code-fixer/fixers/ts2305.ts @@ -24,10 +24,10 @@ const logger = createObjectLogger({ name: 'TS2305Fixer' }, 'TS2305Fixer'); * Fix TS2305 "Module has no exported member" errors * Adds missing exports as stubs to the target file */ -export async function fixMissingExportedMember( +export function fixMissingExportedMember( context: FixerContext, issues: CodeIssue[] -): Promise { +): FixResult { const logs = createFixerLogMessages('TS2305Fixer', issues.length); logger.info(logs.start); @@ -49,7 +49,7 @@ export async function fixMissingExportedMember( } // Get source and target files using DRY helper - const filesResult = await getSourceAndTargetFiles(issue, context); + const filesResult = getSourceAndTargetFiles(issue, context); if (!filesResult) { logger.warn(`Failed to get source and target files for ${issue.filePath}`); unfixableIssues.push(createUnfixableIssue( diff --git a/worker/services/code-fixer/fixers/ts2307.ts b/worker/services/code-fixer/fixers/ts2307.ts index 2a37d550..c1c3c9fb 100644 --- a/worker/services/code-fixer/fixers/ts2307.ts +++ b/worker/services/code-fixer/fixers/ts2307.ts @@ -20,24 +20,23 @@ const logger = createObjectLogger({ name: 'TS2307Fixer' }, 'TS2307Fixer'); * Fix TS2307 "Cannot find module" errors * Preserves exact logic from working ImportExportFixer.fixModuleNotFound */ -export async function fixModuleNotFound( +export function fixModuleNotFound( context: FixerContext, issues: CodeIssue[] -): Promise { +): FixResult { logger.info(`Starting TS2307 fixer with ${issues.length} issues`); const fixedIssues: FixedIssue[] = []; const unfixableIssues = []; const modifiedFiles = []; const newFiles = []; - const fetchedFiles = new Set(context.fetchedFiles); for (const issue of issues) { logger.info(`Processing TS2307 issue: ${issue.message} at ${issue.filePath}:${issue.line}`); try { logger.info(`Getting AST for file: ${issue.filePath}`); - const ast = await getFileAST(issue.filePath, context.files, context.fileFetcher, fetchedFiles); + const ast = getFileAST(issue.filePath, context.files); if (!ast) { logger.warn(`Failed to get AST for ${issue.filePath}`); unfixableIssues.push({ @@ -81,12 +80,10 @@ export async function fixModuleNotFound( logger.info(`Searching for local module file: ${moduleSpecifier}`); // Try to find existing file with fuzzy matching - const foundFile = await findModuleFile( + const foundFile = findModuleFile( moduleSpecifier, issue.filePath, - context.files, - context.fileFetcher, - fetchedFiles + context.files ); logger.info(`Module file search result: ${foundFile || 'NOT FOUND'}`); @@ -124,11 +121,9 @@ export async function fixModuleNotFound( logger.info(`Resolved stub file path: "${targetFilePath}"`); logger.info(`Generating stub content for import: ${importInfo.defaultImport ? 'default: ' + importInfo.defaultImport + ', ' : ''}named: [${importInfo.namedImports.join(', ')}]`); - const stubContent = await generateStubFileContent( + const stubContent = generateStubFileContent( importInfo, - context.files, - context.fileFetcher, - fetchedFiles + context.files ); logger.info(`Generated stub content (${stubContent.length} characters)`); diff --git a/worker/services/code-fixer/fixers/ts2613.ts b/worker/services/code-fixer/fixers/ts2613.ts index dba6fcf3..77ceeaed 100644 --- a/worker/services/code-fixer/fixers/ts2613.ts +++ b/worker/services/code-fixer/fixers/ts2613.ts @@ -19,24 +19,23 @@ const logger = createObjectLogger({ name: 'TS2613Fixer' }, 'TS2613Fixer'); * Fix TS2613 "Module is not a module" errors * Preserves exact logic from working ImportExportFixer.fixModuleIsNotModule */ -export async function fixModuleIsNotModule( +export function fixModuleIsNotModule( context: FixerContext, issues: CodeIssue[] -): Promise { +): FixResult { logger.info(`Starting TS2613 fixer with ${issues.length} issues`); const fixedIssues: FixedIssue[] = []; const unfixableIssues: UnfixableIssue[] = []; const modifiedFiles: FileObject[] = []; const newFiles: FileObject[] = []; - const fetchedFiles = new Set(context.fetchedFiles); for (const issue of issues) { logger.info(`Processing TS2613 issue: ${issue.message} at ${issue.filePath}:${issue.line}`); try { logger.info(`Getting AST for file: ${issue.filePath}`); - const ast = await getFileAST(issue.filePath, context.files, context.fileFetcher, fetchedFiles); + const ast = getFileAST(issue.filePath, context.files); if (!ast) { logger.warn(`Failed to get AST for ${issue.filePath}`); unfixableIssues.push({ @@ -71,12 +70,10 @@ export async function fixModuleIsNotModule( const moduleSpecifier = importInfo ? importInfo.moduleSpecifier : (namespaceImport?.moduleSpecifier || ''); logger.info(`Searching for target file: ${moduleSpecifier}`); - const targetFile = await findModuleFile( + const targetFile = findModuleFile( moduleSpecifier, issue.filePath, - context.files, - context.fileFetcher, - fetchedFiles + context.files ); if (!targetFile) { @@ -94,10 +91,7 @@ export async function fixModuleIsNotModule( logger.info(`Found target file: ${targetFile}`); logger.info(`Getting AST for target file: ${targetFile}`); - logger.info(`Files in context: ${Array.from(context.files.keys()).join(', ')}`); - logger.info(`FetchedFiles: ${Array.from(fetchedFiles).join(', ')}`); - logger.info(`FileFetcher available: ${!!context.fileFetcher}`); - const targetAST = await getFileAST(targetFile, context.files, context.fileFetcher, fetchedFiles); + const targetAST = getFileAST(targetFile, context.files); logger.info(`getFileAST result for ${targetFile}: ${!!targetAST}`); if (!targetAST) { logger.warn(`Failed to parse target file: ${targetFile}`); diff --git a/worker/services/code-fixer/fixers/ts2614.ts b/worker/services/code-fixer/fixers/ts2614.ts index 41b71076..880ac143 100644 --- a/worker/services/code-fixer/fixers/ts2614.ts +++ b/worker/services/code-fixer/fixers/ts2614.ts @@ -18,17 +18,16 @@ const logger = createObjectLogger({ name: 'TS2614Fixer' }, 'TS2614Fixer'); * Fix TS2614 "Module has no exported member" errors (import/export mismatch) * Corrects import statements to match actual export types */ -export async function fixImportExportTypeMismatch( +export function fixImportExportTypeMismatch( context: FixerContext, issues: CodeIssue[] -): Promise { +): FixResult { logger.info(`Starting TS2614 fixer with ${issues.length} issues`); const fixedIssues: FixedIssue[] = []; const unfixableIssues: UnfixableIssue[] = []; const modifiedFiles: FileObject[] = []; const newFiles: FileObject[] = []; - const fetchedFiles = new Set(context.fetchedFiles); for (const issue of issues) { logger.info(`Processing TS2614 issue: ${issue.message} at ${issue.filePath}:${issue.line}`); @@ -36,7 +35,7 @@ export async function fixImportExportTypeMismatch( try { // Get AST for the file with the import issue logger.info(`Getting AST for source file: ${issue.filePath}`); - const sourceAST = await getFileAST(issue.filePath, context.files, context.fileFetcher, fetchedFiles); + const sourceAST = getFileAST(issue.filePath, context.files); if (!sourceAST) { logger.warn(`Failed to get AST for ${issue.filePath}`); unfixableIssues.push({ @@ -70,12 +69,10 @@ export async function fixImportExportTypeMismatch( // Find the target file logger.info(`Searching for target file: ${importInfo.moduleSpecifier}`); - const targetFile = await findModuleFile( + const targetFile = findModuleFile( importInfo.moduleSpecifier, issue.filePath, - context.files, - context.fileFetcher, - fetchedFiles + context.files ); if (!targetFile) { @@ -94,7 +91,7 @@ export async function fixImportExportTypeMismatch( // Get AST for target file to analyze actual exports logger.info(`Getting AST for target file: ${targetFile}`); - const targetAST = await getFileAST(targetFile, context.files, context.fileFetcher, fetchedFiles); + const targetAST = getFileAST(targetFile, context.files); if (!targetAST) { logger.warn(`Failed to parse target file: ${targetFile}`); unfixableIssues.push({ diff --git a/worker/services/code-fixer/fixers/ts2724.ts b/worker/services/code-fixer/fixers/ts2724.ts index bf8f6058..31114064 100644 --- a/worker/services/code-fixer/fixers/ts2724.ts +++ b/worker/services/code-fixer/fixers/ts2724.ts @@ -27,10 +27,10 @@ const logger = createObjectLogger({ name: 'TS2724Fixer' }, 'TS2724Fixer'); * Fix TS2724 "Incorrect named import" errors * Replaces incorrect named imports with the suggested correct ones from TypeScript */ -export async function fixIncorrectNamedImport( +export function fixIncorrectNamedImport( context: FixerContext, issues: CodeIssue[] -): Promise { +): FixResult { const logs = createFixerLogMessages('TS2724Fixer', issues.length); logger.info(logs.start); @@ -51,11 +51,9 @@ export async function fixIncorrectNamedImport( for (const [filePath, fileIssues] of issuesByFile) { try { // Get source file AST once - const sourceAST = await getFileAST( + const sourceAST = getFileAST( filePath, - context.files, - context.fileFetcher, - context.fetchedFiles as Set + context.files ); if (!sourceAST) { @@ -114,22 +112,18 @@ export async function fixIncorrectNamedImport( // Verify the suggested export actually exists in the target module // Resolve the module path const resolvedPath = resolvePathAlias(moduleSpecifier); - const targetFile = await findModuleFile( + const targetFile = findModuleFile( resolvedPath, filePath, - context.files, - context.fileFetcher, - context.fetchedFiles as Set + context.files ); if (targetFile) { // Get exports from the target file - const targetAST = await getFileAST( + const targetAST = getFileAST( targetFile, - context.files, - context.fileFetcher, - context.fetchedFiles as Set + context.files ); if (targetAST) { diff --git a/worker/services/code-fixer/index.ts b/worker/services/code-fixer/index.ts index 2cfd82d1..033f990a 100644 --- a/worker/services/code-fixer/index.ts +++ b/worker/services/code-fixer/index.ts @@ -7,7 +7,6 @@ import { FileObject } from './types'; import { CodeIssue } from '../sandbox/sandboxTypes'; import { CodeFixResult, - FileFetcher, FixerContext, FileMap, ProjectFile, @@ -35,14 +34,12 @@ import { fixIncorrectNamedImport } from './fixers/ts2724'; * * @param allFiles - Initial files to work with * @param issues - TypeScript compilation issues to fix - * @param fileFetcher - Optional callback to fetch additional files on-demand * @returns Promise containing fix results with modified/new files */ -export async function fixProjectIssues( +export function fixProjectIssues( allFiles: FileObject[], - issues: CodeIssue[], - fileFetcher?: FileFetcher -): Promise { + issues: CodeIssue[] +): CodeFixResult { try { // Build file map (mutable for caching fetched files) const fileMap = createFileMap(allFiles); @@ -51,7 +48,6 @@ export async function fixProjectIssues( const fetchedFiles = new Set(); const context: FixerContext = { files: fileMap, - fileFetcher, fetchedFiles }; @@ -65,7 +61,7 @@ export async function fixProjectIssues( const sortedIssues = sortFixOrder(fixableIssues); // Apply fixes sequentially, updating context after each - const results = await applyFixesSequentially( + const results = applyFixesSequentially( context, sortedIssues, fixerRegistry @@ -209,11 +205,11 @@ function sortFixOrder(issues: CodeIssue[]): CodeIssue[] { /** * Apply fixes sequentially, updating context after each fix */ -async function applyFixesSequentially( +function applyFixesSequentially( context: FixerContext, sortedIssues: CodeIssue[], fixerRegistry: FixerRegistry -): Promise { +): CodeFixResult { const fixedIssues: any[] = []; const unfixableIssues: any[] = []; const modifiedFiles = new Map(); @@ -259,7 +255,7 @@ async function applyFixesSequentially( try { // Apply fixer - const result = await fixer(context, issues); + const result = fixer(context, issues); // Collect results fixedIssues.push(...result.fixedIssues); @@ -354,7 +350,6 @@ export type { FixedIssue, UnfixableIssue, FileObject, - FileFetcher, FixerContext, FileMap, ProjectFile diff --git a/worker/services/code-fixer/types.ts b/worker/services/code-fixer/types.ts index 72db5ad4..c1e9b249 100644 --- a/worker/services/code-fixer/types.ts +++ b/worker/services/code-fixer/types.ts @@ -70,12 +70,6 @@ export interface CodeFixResult { // ============================================================================ // FILE AND AST MANAGEMENT // ============================================================================ - -/** - * File fetcher callback type for dynamically loading files not in the initial set - */ -export type FileFetcher = (filePath: string) => Promise; - /** * Represents a file in the project with its content and cached AST */ @@ -96,8 +90,6 @@ export type FileMap = Map; export interface FixerContext { /** Map of all files in the project (mutable for caching fetched files) */ files: FileMap; - /** Optional callback to fetch additional files */ - readonly fileFetcher?: FileFetcher; /** Cache of fetched files to prevent duplicate requests */ readonly fetchedFiles: ReadonlySet; } @@ -174,7 +166,7 @@ export interface FixResult { export type FixerFunction = ( context: FixerContext, issues: CodeIssue[] -) => Promise; +) => FixResult; /** * Registry of fixer functions by issue code diff --git a/worker/services/code-fixer/utils/helpers.ts b/worker/services/code-fixer/utils/helpers.ts index e27aafd0..88ae6590 100644 --- a/worker/services/code-fixer/utils/helpers.ts +++ b/worker/services/code-fixer/utils/helpers.ts @@ -17,19 +17,17 @@ import { resolveModuleFile, validateModuleOperation } from './modules'; * Standard pattern: Get source file AST and import info * Used by TS2305, TS2613, TS2614 fixers */ -export async function getSourceFileAndImport( +export function getSourceFileAndImport( issue: CodeIssue, context: FixerContext -): Promise<{ +): { sourceAST: t.File; importInfo: { moduleSpecifier: string; defaultImport?: string; namedImports: string[]; specifier?: string }; -} | null> { +} | null { // Get AST for the source file - const sourceAST = await getFileAST( + const sourceAST = getFileAST( issue.filePath, context.files, - context.fileFetcher, - context.fetchedFiles as Set ); if (!sourceAST) { @@ -49,14 +47,14 @@ export async function getSourceFileAndImport( * Standard pattern: Get target file for a module specifier * Used by TS2305, TS2613, TS2614 fixers */ -export async function getTargetFileAndAST( +export function getTargetFileAndAST( moduleSpecifier: string, fromFilePath: string, context: FixerContext -): Promise<{ +): { targetFilePath: string; targetAST: t.File; -} | null> { +} | null { // Validate the module operation first const validation = validateModuleOperation(moduleSpecifier, null); if (!validation.valid) { @@ -64,7 +62,7 @@ export async function getTargetFileAndAST( } // Resolve the target file - const targetFilePath = await resolveModuleFile(moduleSpecifier, fromFilePath, context); + const targetFilePath = resolveModuleFile(moduleSpecifier, fromFilePath, context); if (!targetFilePath) { return null; } @@ -76,11 +74,9 @@ export async function getTargetFileAndAST( } // Get AST for target file - const targetAST = await getFileAST( + const targetAST = getFileAST( targetFilePath, context.files, - context.fileFetcher, - context.fetchedFiles as Set ); if (!targetAST) { @@ -94,17 +90,17 @@ export async function getTargetFileAndAST( * Combined pattern: Get both source and target files * Used by import/export fixers that need both files */ -export async function getSourceAndTargetFiles( +export function getSourceAndTargetFiles( issue: CodeIssue, context: FixerContext -): Promise<{ +): { sourceAST: t.File; importInfo: { moduleSpecifier: string; defaultImport?: string; namedImports: string[]; specifier?: string }; targetFilePath: string; targetAST: t.File; -} | null> { +} | null { // Get source file and import info - const sourceResult = await getSourceFileAndImport(issue, context); + const sourceResult = getSourceFileAndImport(issue, context); if (!sourceResult) { return null; } @@ -112,7 +108,7 @@ export async function getSourceAndTargetFiles( const { sourceAST, importInfo } = sourceResult; // Get target file and AST - const targetResult = await getTargetFileAndAST( + const targetResult = getTargetFileAndAST( importInfo.moduleSpecifier, issue.filePath, context diff --git a/worker/services/code-fixer/utils/imports.ts b/worker/services/code-fixer/utils/imports.ts index 896d7cb6..4ca1dae7 100644 --- a/worker/services/code-fixer/utils/imports.ts +++ b/worker/services/code-fixer/utils/imports.ts @@ -4,8 +4,8 @@ */ import * as t from '@babel/types'; -import { ImportInfo, ExportInfo, ImportUsage, FileMap, FileFetcher } from '../types'; -import { parseCode, traverseAST, isScriptFile } from './ast'; +import { ImportInfo, ExportInfo, ImportUsage, FileMap } from '../types'; +import { parseCode, traverseAST } from './ast'; import { createObjectLogger } from '../../../logger'; const logger = createObjectLogger({ name: 'ImportUtils' }, 'ImportUtils'); @@ -371,12 +371,10 @@ export function analyzeNameUsage(ast: t.File, name: string): ImportUsage | null /** * Get file content from FileMap or fetch it if not available */ -export async function getFileContent( +export function getFileContent( filePath: string, files: FileMap, - fileFetcher?: FileFetcher, - fetchedFiles?: Set -): Promise { +): string | null { logger.info(`ImportUtils: Getting content for file: ${filePath}`); const file = files.get(filePath); @@ -384,32 +382,8 @@ export async function getFileContent( logger.info(`ImportUtils: Found file in context: ${filePath}`); return file.content; } - - // Try to fetch if not available and we have a fetcher - if (fileFetcher && fetchedFiles && !fetchedFiles.has(filePath)) { - try { - logger.info(`ImportUtils: Fetching file: ${filePath}`); - fetchedFiles.add(filePath); // Mark as attempted - const result = await fileFetcher(filePath); - - if (result && isScriptFile(result.filePath)) { - logger.info(`ImportUtils: Successfully fetched ${filePath}, storing in files map`); - // Store the fetched file in the mutable files map - files.set(filePath, { - filePath: filePath, - content: result.fileContents, - ast: undefined - }); - return result.fileContents; - } else { - logger.info(`ImportUtils: File ${filePath} was fetched but is not a script file or result is null`); - } - } catch (error) { - logger.warn(`ImportUtils: Failed to fetch file ${filePath}: ${error instanceof Error ? error.message : 'Unknown error'}`); - } - } else { - logger.info(`ImportUtils: Not fetching ${filePath} - fileFetcher: ${!!fileFetcher}, fetchedFiles: ${!!fetchedFiles}, alreadyFetched: ${fetchedFiles?.has(filePath)}`); - } + + logger.info(`ImportUtils: File not found in context: ${filePath}`); return null; } @@ -417,12 +391,10 @@ export async function getFileContent( /** * Get file AST from FileMap with caching, or parse it if needed */ -export async function getFileAST( +export function getFileAST( filePath: string, files: FileMap, - fileFetcher?: FileFetcher, - fetchedFiles?: Set -): Promise { +): t.File | null { logger.info(`ImportUtils: Getting AST for file: ${filePath}`); const file = files.get(filePath); @@ -432,7 +404,7 @@ export async function getFileAST( return file.ast; } - const content = await getFileContent(filePath, files, fileFetcher, fetchedFiles); + const content = getFileContent(filePath, files); if (!content) { logger.info(`ImportUtils: No content available for ${filePath}`); return null; diff --git a/worker/services/code-fixer/utils/modules.ts b/worker/services/code-fixer/utils/modules.ts index 2248be62..699c97a7 100644 --- a/worker/services/code-fixer/utils/modules.ts +++ b/worker/services/code-fixer/utils/modules.ts @@ -59,43 +59,39 @@ export function canModifyFile(filePath: string): boolean { * Resolve a module specifier to an actual file path within the project * Unified resolution logic used by all fixers */ -export async function resolveModuleFile( +export function resolveModuleFile( moduleSpecifier: string, fromFilePath: string, context: FixerContext -): Promise { +): string | null { // Skip external modules - we cannot modify them if (isExternalModule(moduleSpecifier)) { return null; } // Use existing findModuleFile logic for internal modules - return await findModuleFile( + return findModuleFile( moduleSpecifier, fromFilePath, - context.files, - context.fileFetcher, - context.fetchedFiles as Set + context.files ); } /** * Check if a target file exists and can be modified */ -export async function canModifyTargetFile( +export function canModifyTargetFile( targetFilePath: string, context: FixerContext -): Promise { +): boolean { if (!canModifyFile(targetFilePath)) { return false; } try { - const content = await getFileContent( + const content = getFileContent( targetFilePath, - context.files, - context.fileFetcher, - context.fetchedFiles as Set + context.files ); return content !== null; } catch { diff --git a/worker/services/code-fixer/utils/paths.ts b/worker/services/code-fixer/utils/paths.ts index 7fb6be38..be4c6ba1 100644 --- a/worker/services/code-fixer/utils/paths.ts +++ b/worker/services/code-fixer/utils/paths.ts @@ -3,7 +3,7 @@ * Extracted from working ImportExportAnalyzer to preserve exact functionality */ -import { FileMap, FileFetcher } from '../types'; +import { FileMap } from '../types'; import { isScriptFile } from './ast'; import { getFileContent } from './imports'; @@ -31,13 +31,11 @@ export function resolvePathAlias(importSpecifier: string): string { * Resolve relative import paths to absolute paths within the project * Preserves exact logic from working implementation */ -export async function resolveImportPath( +export function resolveImportPath( importSpecifier: string, currentFilePath: string, - files: FileMap, - fileFetcher?: FileFetcher, - fetchedFiles?: Set -): Promise { + files: FileMap +): string { if (importSpecifier.startsWith('./') || importSpecifier.startsWith('../')) { // Relative import - resolve relative to current file directory const currentDirParts = currentFilePath.split('/').slice(0, -1); @@ -63,7 +61,7 @@ export async function resolveImportPath( if (!resolvedPath.endsWith(ext)) { const withExt = resolvedPath + ext; try { - const fileContent = await getFileContent(withExt, files, fileFetcher, fetchedFiles); + const fileContent = getFileContent(withExt, files); if (fileContent) return withExt; } catch { // File doesn't exist or can't be fetched, try next extension @@ -86,18 +84,16 @@ export async function resolveImportPath( * Find a module file using fuzzy matching and file fetching * Preserves exact logic from working ImportExportAnalyzer.findModuleFile */ -export async function findModuleFile( +export function findModuleFile( importSpecifier: string, currentFilePath: string, - files: FileMap, - fileFetcher?: FileFetcher, - fetchedFiles?: Set -): Promise { + files: FileMap +): string | null { // Handle path aliases like @/components/ui/button const resolvedSpecifier = resolvePathAlias(importSpecifier); // Try exact match first (relative/absolute paths) - const exactMatch = await resolveImportPath(resolvedSpecifier, currentFilePath, files, fileFetcher, fetchedFiles); + const exactMatch = resolveImportPath(resolvedSpecifier, currentFilePath, files); if (exactMatch) { // Check if file exists in files Map after potential fetching const allFiles = Array.from(files.keys()); @@ -127,7 +123,7 @@ export async function findModuleFile( try { // Try to get file content (this will trigger fetching if available) - const content = await getFileContent(candidatePath, files, fileFetcher, fetchedFiles); + const content = getFileContent(candidatePath, files); if (content) { return candidatePath; } diff --git a/worker/services/code-fixer/utils/stubs.ts b/worker/services/code-fixer/utils/stubs.ts index 2c91a6fa..c5e6da4b 100644 --- a/worker/services/code-fixer/utils/stubs.ts +++ b/worker/services/code-fixer/utils/stubs.ts @@ -5,7 +5,7 @@ */ import * as t from '@babel/types'; -import { ImportInfo, ImportUsage, FileMap, FileFetcher } from '../types'; +import { ImportInfo, ImportUsage, FileMap } from '../types'; import { createFileAST, shouldUseJSXExtension, generateCode, parseCode } from './ast'; import { analyzeImportUsage, getFileAST } from './imports'; @@ -17,13 +17,11 @@ import { analyzeImportUsage, getFileAST } from './imports'; * Analyze how imports are used to generate appropriate stubs * Preserves exact logic from working implementation */ -export async function analyzeImportUsageForStub( +export function analyzeImportUsageForStub( importInfo: ImportInfo, - files: FileMap, - fileFetcher?: FileFetcher, - fetchedFiles?: Set -): Promise { - const sourceAST = await getFileAST(importInfo.filePath, files, fileFetcher, fetchedFiles); + files: FileMap +): ImportUsage[] { + const sourceAST = getFileAST(importInfo.filePath, files); if (!sourceAST) return []; const importNames = [ @@ -94,13 +92,11 @@ export function generateStubFileAST( /** * Generate stub file content as a string */ -export async function generateStubFileContent( +export function generateStubFileContent( importInfo: ImportInfo, files: FileMap, - fileFetcher?: FileFetcher, - fetchedFiles?: Set -): Promise { - const usageAnalysis = await analyzeImportUsageForStub(importInfo, files, fileFetcher, fetchedFiles); +): string { + const usageAnalysis = analyzeImportUsageForStub(importInfo, files); const stubAST = generateStubFileAST(importInfo, usageAnalysis); const generated = generateCode(stubAST); diff --git a/worker/services/sandbox/BaseSandboxService.ts b/worker/services/sandbox/BaseSandboxService.ts index 9410a53a..5690df1e 100644 --- a/worker/services/sandbox/BaseSandboxService.ts +++ b/worker/services/sandbox/BaseSandboxService.ts @@ -29,38 +29,43 @@ import { ListInstancesResponse, GitHubPushRequest, GitHubPushResponse, + TemplateDetails, } from './sandboxTypes'; import { createObjectLogger, StructuredLogger } from '../../logger'; import { env } from 'cloudflare:workers' import { FileOutputType } from 'worker/agents/schemas'; +import { ZipExtractor } from './zipExtractor'; +import { FileTreeBuilder } from './fileTreeBuilder'; - /** - * Streaming event for enhanced command execution - */ - export interface StreamEvent { +/** + * Streaming event for enhanced command execution + */ +export interface StreamEvent { type: 'stdout' | 'stderr' | 'exit' | 'error'; data?: string; code?: number; error?: string; timestamp: Date; - } - - export interface TemplateInfo { - name: string; - language?: string; - frameworks?: string[]; - description: { - selection: string; - usage: string; - }; - } - - /** - * Abstract base class providing complete RunnerService API compatibility - * All implementations MUST support every method defined here - */ - export abstract class BaseSandboxService { +} + +export interface TemplateInfo { + name: string; + language?: string; + frameworks?: string[]; + description: { + selection: string; + usage: string; + }; +} + +const templateDetailsCache: Record = {}; + +/** + * Abstract base class providing complete RunnerService API compatibility + * All implementations MUST support every method defined here +*/ +export abstract class BaseSandboxService { protected logger: StructuredLogger; protected sandboxId: string; @@ -113,11 +118,98 @@ import { FileOutputType } from 'worker/agents/schemas'; } /** - * Get details for a specific template including files and structure + * Get details for a specific template - fully in-memory, no sandbox operations + * Downloads zip from R2, extracts in memory, and returns all files with metadata * Returns: { success: boolean, templateDetails?: {...}, error?: string } */ - abstract getTemplateDetails(templateName: string): Promise; - + static async getTemplateDetails(templateName: string, downloadDir?: string): Promise { + try { + if (templateDetailsCache[templateName]) { + console.log(`Template details for template: ${templateName} found in cache`); + return { + success: true, + templateDetails: templateDetailsCache[templateName] + }; + } + // Download template zip from R2 + const downloadUrl = downloadDir ? `${downloadDir}/${templateName}.zip` : `${templateName}.zip`; + const r2Object = await env.TEMPLATES_BUCKET.get(downloadUrl); + + if (!r2Object) { + throw new Error(`Template '${templateName}' not found in bucket`); + } + + const zipData = await r2Object.arrayBuffer(); + + // Extract all files in memory + const allFiles = ZipExtractor.extractFiles(zipData); + + // Build file tree + const fileTree = FileTreeBuilder.buildFromTemplateFiles(allFiles, { rootPath: '.' }); + + // Extract dependencies from package.json + const packageJsonFile = allFiles.find(f => f.filePath === 'package.json'); + const packageJson = packageJsonFile ? JSON.parse(packageJsonFile.fileContents) : null; + const dependencies = packageJson?.dependencies || {}; + + // Parse metadata files + const dontTouchFile = allFiles.find(f => f.filePath === '.donttouch_files.json'); + const dontTouchFiles = dontTouchFile ? JSON.parse(dontTouchFile.fileContents) : []; + + const redactedFile = allFiles.find(f => f.filePath === '.redacted_files.json'); + const redactedFiles = redactedFile ? JSON.parse(redactedFile.fileContents) : []; + + const importantFile = allFiles.find(f => f.filePath === '.important_files.json'); + const importantFiles = importantFile ? JSON.parse(importantFile.fileContents) : []; + + // Get template info from catalog + const catalogResponse = await BaseSandboxService.listTemplates(); + const catalogInfo = catalogResponse.success + ? catalogResponse.templates.find(t => t.name === templateName) + : null; + + // Remove metadata files and convert to map for efficient lookups + const filteredFiles = allFiles.filter(f => + !f.filePath.startsWith('.') || + (!f.filePath.endsWith('.json') && !f.filePath.startsWith('.git')) + ); + + // Convert array to map: filePath -> fileContents + const filesMap: Record = {}; + for (const file of filteredFiles) { + filesMap[file.filePath] = file.fileContents; + } + + const templateDetails: TemplateDetails = { + name: templateName, + description: { + selection: catalogInfo?.description.selection || '', + usage: catalogInfo?.description.usage || '' + }, + fileTree, + allFiles: filesMap, + language: catalogInfo?.language, + deps: dependencies, + importantFiles, + dontTouchFiles, + redactedFiles, + frameworks: catalogInfo?.frameworks || [] + }; + + templateDetailsCache[templateName] = templateDetails; + + return { + success: true, + templateDetails + }; + } catch (error) { + return { + success: false, + error: `Failed to get template details: ${error instanceof Error ? error.message : 'Unknown error'}` + }; + } + } + // ========================================== // INSTANCE LIFECYCLE (Required) // ========================================== @@ -226,4 +318,4 @@ import { FileOutputType } from 'worker/agents/schemas'; * Push instance files to existing GitHub repository */ abstract pushToGitHub(instanceId: string, request: GitHubPushRequest, files: FileOutputType[]): Promise - } \ No newline at end of file +} \ No newline at end of file diff --git a/worker/services/sandbox/remoteSandboxService.ts b/worker/services/sandbox/remoteSandboxService.ts index 69acc93f..42f06085 100644 --- a/worker/services/sandbox/remoteSandboxService.ts +++ b/worker/services/sandbox/remoteSandboxService.ts @@ -1,5 +1,4 @@ import { - TemplateDetailsResponse, BootstrapResponse, GetInstanceResponse, BootstrapStatusResponse, @@ -14,7 +13,6 @@ import { DeploymentResult, GetLogsResponse, ListInstancesResponse, - TemplateDetailsResponseSchema, BootstrapResponseSchema, BootstrapRequest, GetInstanceResponseSchema, @@ -115,12 +113,6 @@ export class RemoteSandboxServiceClient extends BaseSandboxService{ }; } } - /** - * Get details for a specific template. - */ - async getTemplateDetails(templateName: string): Promise { - return this.makeRequest(`/templates/${templateName}`, 'GET', TemplateDetailsResponseSchema); - } /** * Create a new runner instance. diff --git a/worker/services/sandbox/sandboxSdkClient.ts b/worker/services/sandbox/sandboxSdkClient.ts index 2f21e956..d0638222 100644 --- a/worker/services/sandbox/sandboxSdkClient.ts +++ b/worker/services/sandbox/sandboxSdkClient.ts @@ -1,7 +1,6 @@ import { getSandbox, Sandbox, parseSSEStream, LogEvent, ExecResult } from '@cloudflare/sandbox'; import { - TemplateDetailsResponse, BootstrapResponse, GetInstanceResponse, BootstrapStatusResponse, @@ -24,8 +23,6 @@ import { GetLogsResponse, ListInstancesResponse, StoredError, - TemplateInfo, - TemplateDetails, } from './sandboxTypes'; import { createObjectLogger } from '../../logger'; @@ -40,7 +37,7 @@ import { import { createAssetManifest } from '../deployer/utils/index'; -import { CodeFixResult, FileFetcher, fixProjectIssues } from '../code-fixer'; +import { CodeFixResult, fixProjectIssues } from '../code-fixer'; import { FileObject } from '../code-fixer/types'; import { generateId } from '../../utils/idGenerator'; import { ResourceProvisioner } from './resourceProvisioner'; @@ -393,96 +390,6 @@ export class SandboxSdkClient extends BaseSandboxService { this.logger.info(`Template already exists`); } } - - - async getTemplateDetails(templateName: string): Promise { - try { - this.logger.info('Retrieving template details', { templateName }); - - await this.ensureTemplateExists(templateName); - - this.logger.info('Template setup complete'); - - const [fileTree, catalogInfo, dontTouchFiles, redactedFiles] = await Promise.all([ - this.buildFileTree(templateName), - this.getTemplateFromCatalog(templateName), - this.fetchDontTouchFiles(templateName), - this.fetchRedactedFiles(templateName) - ]); - - if (!fileTree) { - throw new Error(`Failed to build file tree for template ${templateName}`); - } - - const filesResponse = await this.getFiles(templateName, undefined, true, redactedFiles); // Use template name as directory - - this.logger.info('Template files retrieved'); - - // Parse package.json for dependencies - let dependencies: Record = {}; - try { - const packageJsonFile = filesResponse.files.find(file => file.filePath === 'package.json'); - if (!packageJsonFile) { - throw new Error('package.json not found'); - } - const packageJson = JSON.parse(packageJsonFile.fileContents) as { - dependencies?: Record; - devDependencies?: Record; - }; - dependencies = { - ...packageJson.dependencies || {}, - ...packageJson.devDependencies || {} - }; - } catch { - this.logger.info('No package.json found', { templateName }); - } - - const allFiles = filesResponse.files.reduce((acc, file) => { - acc[file.filePath] = file.fileContents; - return acc; - }, {} as Record); - const templateDetails: TemplateDetails = { - name: templateName, - description: { - selection: catalogInfo?.description.selection || '', - usage: catalogInfo?.description.usage || '' - }, - fileTree, - allFiles, - importantFiles: filesResponse.files.map(file => file.filePath), - language: catalogInfo?.language, - deps: dependencies, - dontTouchFiles, - redactedFiles, - frameworks: catalogInfo?.frameworks || [] - }; - - this.logger.info('Template files retrieved', { templateName, fileCount: filesResponse.files.length }); - - return { - success: true, - templateDetails - }; - } catch (error) { - this.logger.error('getTemplateDetails', error, { templateName }); - return { - success: false, - error: `Failed to get template details: ${error instanceof Error ? error.message : 'Unknown error'}` - }; - } - } - - private async getTemplateFromCatalog(templateName: string): Promise { - try { - const templatesResponse = await SandboxSdkClient.listTemplates(); - if (templatesResponse.success) { - return templatesResponse.templates.find(t => t.name === templateName) || null; - } - return null; - } catch { - return null; - } - } private async buildFileTree(instanceId: string): Promise { try { @@ -1809,35 +1716,14 @@ export class SandboxSdkClient extends BaseSandboxService { // Create file fetcher callback const session = await this.getInstanceSession(instanceId); - const fileFetcher: FileFetcher = async (filePath: string) => { - // Fetch a single file from the instance - try { - const result = await session.readFile(`/workspace/${instanceId}/${filePath}`); - if (result.success) { - this.logger.info(`Successfully fetched file: ${filePath}`); - return { - filePath: filePath, - fileContents: result.content, - filePurpose: `Fetched file: ${filePath}` - }; - } else { - this.logger.debug(`File not found: ${filePath}`); - } - } catch (error) { - this.logger.debug(`Failed to fetch file ${filePath}: ${error instanceof Error ? error.message : 'Unknown error'}`); - } - return null; - }; - // Use the new functional API - const fixResult = await fixProjectIssues( + const fixResult = fixProjectIssues( files.map(file => ({ filePath: file.filePath, fileContents: file.fileContents, filePurpose: '' })), analysisResult.typecheck.issues, - fileFetcher ); for (const file of fixResult.modifiedFiles) { await session.writeFile(`/workspace/${instanceId}/${file.filePath}`, file.fileContents); diff --git a/worker/services/sandbox/utils.ts b/worker/services/sandbox/utils.ts index 18bd6519..4ff4e248 100644 --- a/worker/services/sandbox/utils.ts +++ b/worker/services/sandbox/utils.ts @@ -1,10 +1,10 @@ import { TemplateDetails, TemplateFile } from "./sandboxTypes"; -export function getTemplateImportantFiles(templateDetails: TemplateDetails): TemplateFile[] { +export function getTemplateImportantFiles(templateDetails: TemplateDetails, filterRedacted: boolean = true): TemplateFile[] { return templateDetails.importantFiles.map(filePath => ({ filePath, - fileContents: templateDetails.allFiles[filePath], - })); + fileContents: filterRedacted && templateDetails.redactedFiles.includes(filePath) ? 'REDACTED' : templateDetails.allFiles[filePath] + })).filter(f => f.fileContents); } export function getTemplateFiles(templateDetails: TemplateDetails): TemplateFile[] { diff --git a/worker/services/sandbox/zipExtractor.ts b/worker/services/sandbox/zipExtractor.ts index b81d5170..e67eba7b 100644 --- a/worker/services/sandbox/zipExtractor.ts +++ b/worker/services/sandbox/zipExtractor.ts @@ -9,6 +9,18 @@ export class ZipExtractor { // Max uncompressed size (50MB) private static readonly MAX_UNCOMPRESSED_SIZE = 50 * 1024 * 1024; + // Known binary file extensions - skip UTF-8 decode attempt + private static readonly BINARY_EXTENSIONS = new Set([ + '.png', '.jpg', '.jpeg', '.gif', '.ico', '.webp', '.bmp', + '.woff', '.woff2', '.ttf', '.otf', '.eot', + '.zip', '.tar', '.gz', '.pdf', + '.mp3', '.mp4', '.webm', '.ogg', + '.bin', '.exe', '.dll', '.so' + ]); + + // TextDecoder + private static readonly utf8Decoder = new TextDecoder('utf-8', { fatal: true, ignoreBOM: false }); + /** * Extracts all files from a zip archive * @@ -43,21 +55,20 @@ export class ZipExtractor { let fileContents: string; - // Attempt UTF-8 decoding - try { - const decoder = new TextDecoder('utf-8'); - fileContents = decoder.decode(fileData); - - // Replacement character indicates invalid UTF-8 sequence (binary data) - if (fileContents.includes('\uFFFD')) { - throw new Error('Contains replacement characters'); + // Check if file extension is known binary + const isBinary = this.isBinaryExtension(filePath); + + if (isBinary) { + // Skip UTF-8 decode attempt, go straight to base64 + fileContents = `base64:${this.base64Encode(fileData)}`; + } else { + // Attempt UTF-8 decoding with fatal mode (throws on invalid) + try { + fileContents = this.utf8Decoder.decode(fileData); + } catch { + // Binary file detected, encode as base64 + fileContents = `base64:${this.base64Encode(fileData)}`; } - } catch (error) { - // Binary file detected, encode as base64 - const binaryString = Array.from(fileData) - .map(byte => String.fromCharCode(byte)) - .join(''); - fileContents = `base64:${btoa(binaryString)}`; } files.push({ @@ -109,4 +120,31 @@ export class ZipExtractor { return fileContents.startsWith('base64:'); } + /** + * check if file extension is known binary type + */ + private static isBinaryExtension(filePath: string): boolean { + const lastDot = filePath.lastIndexOf('.'); + if (lastDot === -1) return false; + + const ext = filePath.substring(lastDot).toLowerCase(); + return this.BINARY_EXTENSIONS.has(ext); + } + + /** + * base64 encoding + */ + private static base64Encode(data: Uint8Array): string { + let binaryString = ''; + const len = data.length; + + // Process in chunks + const chunkSize = 8192; + for (let i = 0; i < len; i += chunkSize) { + const chunk = data.subarray(i, Math.min(i + chunkSize, len)); + binaryString += String.fromCharCode(...chunk); + } + + return btoa(binaryString); + } }