v3: no bundle builds for the dev CLI (#923)

* WIP no bundling

* Convert dev CLI to use an unbundled build process to support otel instrumentation

* Fix pnpm lock file

* A couple of fixes to get typechecking to work

Author: ericallam

apps/kubernetes-provider/package.json

@@ -10,8 +10,7 @@
     "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.cjs --platform=node",
     "build:image": "docker build -f Containerfile . -t kubernetes-provider",
     "dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts",
-    "start": "tsx src/index.ts",
-    "typecheck": "tsc --noEmit"
+    "start": "tsx src/index.ts"
   },
   "keywords": [],
   "author": "",
@@ -28,4 +27,4 @@
     "tsx": "^4.7.0",
     "typescript": "^5.3.3"
   }
-}
+}

packages/cli-v3/package.json

@@ -38,10 +38,12 @@
     "@trigger.dev/core": "workspace:*",
     "@trigger.dev/tsconfig": "workspace:*",
     "@types/gradient-string": "^1.1.2",
+    "@types/jsonlines": "^0.1.5",
     "@types/mock-fs": "^4.13.1",
     "@types/node": "18",
     "@types/object-hash": "^3.0.6",
     "@types/react": "^18.2.48",
+    "@types/semver": "^7.3.13",
     "@types/ws": "^8.5.3",
     "cpy-cli": "^5.0.0",
     "npm-run-all": "^4.1.5",
@@ -59,7 +61,7 @@
     "build:prod-containerfile": "src/Containerfile.prod"
   },
   "scripts": {
-    "typecheck": "tsc",
+    "typecheck": "tsc -p tsconfig.check.json",
     "build": "npm run clean && run-p build:**",
     "build:main": "tsup",
     "build:facade": "tsup --config tsup.facade.config.ts",
@@ -95,6 +97,7 @@
     "@opentelemetry/semantic-conventions": "^1.21.0",
     "@traceloop/instrumentation-openai": "^0.3.9",
     "@trigger.dev/core": "workspace:*",
+    "@trigger.dev/core-apps": "workspace:*",
     "@types/degit": "^2.8.3",
     "chalk": "^5.2.0",
     "chokidar": "^3.5.3",
@@ -104,11 +107,12 @@
     "dotenv": "^16.4.4",
     "esbuild": "^0.19.11",
     "evt": "^2.4.13",
-    "execa": "^7.0.0",
+    "execa": "^8.0.0",
     "find-up": "^7.0.0",
     "gradient-string": "^2.0.2",
     "import-meta-resolve": "^4.0.0",
     "ink": "^4.4.1",
+    "jsonlines": "^0.1.1",
     "liquidjs": "^10.9.2",
     "mock-fs": "^5.2.0",
     "nanoid": "^4.0.2",
@@ -121,6 +125,7 @@
     "proxy-agent": "^6.3.0",
     "react": "^18.2.0",
     "react-error-boundary": "^4.0.12",
+    "semver": "^7.5.0",
     "simple-git": "^3.19.0",
     "socket.io-client": "^4.7.4",
     "source-map-support": "^0.5.21",

packages/cli-v3/src/commands/dev.tsx

@@ -10,7 +10,7 @@ import {
 import chalk from "chalk";
 import { watch } from "chokidar";
 import { Command } from "commander";
-import { BuildContext, context } from "esbuild";
+import { BuildContext, Metafile, context } from "esbuild";
 import { resolve as importResolve } from "import-meta-resolve";
 import { Box, Text, render, useApp, useInput } from "ink";
 import { createHash } from "node:crypto";
@@ -23,14 +23,14 @@ import React, { Suspense, useEffect } from "react";
 import { ClientOptions, WebSocket as wsWebSocket } from "ws";
 import { z } from "zod";
 import * as packageJson from "../../package.json";
+import { CliApiClient } from "../apiClient";
+import { CommonCommandOptions } from "../cli/common.js";
 import { BackgroundWorker, BackgroundWorkerCoordinator } from "../dev/backgroundWorker.js";
+import { getConfigPath, readConfig } from "../utilities/configFiles";
 import { printStandloneInitialBanner } from "../utilities/initialBanner.js";
 import { logger } from "../utilities/logger.js";
 import { isLoggedIn } from "../utilities/session.js";
-import { CommonCommandOptions } from "../cli/common.js";
-import { getConfigPath, readConfig } from "../utilities/configFiles";
 import { createTaskFileImports, gatherTaskFiles } from "../utilities/taskFiles";
-import { CliApiClient } from "../apiClient";
 
 let apiClient: CliApiClient | undefined;
 
@@ -298,10 +298,14 @@ function useDev({
         new URL(importResolve("./worker-facade.js", import.meta.url)).href.replace("file://", ""),
         "utf-8"
       );
-      const entryPointContents = workerFacade.replace(
-        "__TASKS__",
-        createTaskFileImports(taskFiles)
-      );
+
+      const registerTracingPath = new URL(
+        importResolve("./register-tracing.js", import.meta.url)
+      ).href.replace("file://", "");
+
+      const entryPointContents = workerFacade
+        .replace("__TASKS__", createTaskFileImports(taskFiles))
+        .replace("__REGISTER_TRACING__", `import { tracingSDK } from "${registerTracingPath}";`);
 
       let firstBuild = true;
 
@@ -318,17 +322,15 @@ function useDev({
         write: false,
         minify: false,
         sourcemap: "external", // does not set the //# sourceMappingURL= comment in the file, we handle it ourselves
+        packages: "external", // https://esbuild.github.io/api/#packages
         logLevel: "warning",
         platform: "node",
-        format: "esm",
+        format: "cjs", // This is needed to support opentelemetry instrumentation that uses module patching
         target: ["node18", "es2020"],
         outdir: "out",
         define: {
           TRIGGER_API_URL: `"${config.triggerUrl}"`,
         },
-        banner: {
-          js: `import { createRequire } from 'module';const require = createRequire(import.meta.url);`,
-        },
         plugins: [
           {
             name: "trigger.dev v3",
@@ -379,7 +381,7 @@ function useDev({
                 }
 
                 // Create a file at join(dir, ".trigger", path) with the fileContents
-                const fullPath = join(config.projectDir, ".trigger", `${contentHash}.mjs`);
+                const fullPath = join(config.projectDir, ".trigger", `${contentHash}.js`);
                 const sourceMapPath = `${fullPath}.map`;
 
                 const outputFileWithSourceMap = `${
@@ -389,6 +391,10 @@ function useDev({
                 await fs.promises.mkdir(dirname(fullPath), { recursive: true });
                 await fs.promises.writeFile(fullPath, outputFileWithSourceMap);
 
+                logger.debug(`Wrote background worker to ${fullPath}`);
+
+                const dependencies = gatherRequiredDependencies(metaOutput);
+
                 if (sourceMapFile) {
                   const sourceMapPath = `${fullPath}.map`;
                   await fs.promises.writeFile(sourceMapPath, sourceMapFile.text);
@@ -399,6 +405,7 @@ function useDev({
 
                 const backgroundWorker = new BackgroundWorker(fullPath, {
                   projectDir: config.projectDir,
+                  dependencies,
                   env: {
                     TRIGGER_API_URL: apiUrl,
                     TRIGGER_API_KEY: apiKey,
@@ -608,3 +615,48 @@ function WebsocketFactory(apiKey: string) {
     }
   };
 }
+
+// Returns the dependencies that are required by the output that are found in output and the CLI package dependencies
+// Returns the dependency names and the version to use (taken from the CLI deps package.json)
+function gatherRequiredDependencies(outputMeta: Metafile["outputs"][string]) {
+  const dependencies: Record<string, string> = {};
+
+  for (const file of outputMeta.imports) {
+    if (file.kind !== "require-call" || !file.external) {
+      continue;
+    }
+
+    const packageName = detectPackageNameFromImportPath(file.path);
+
+    if (dependencies[packageName]) {
+      continue;
+    }
+
+    const internalDependencyVersion = (packageJson.dependencies as Record<string, string>)[
+      packageName
+    ];
+
+    if (internalDependencyVersion) {
+      dependencies[packageName] = internalDependencyVersion;
+    }
+  }
+
+  return dependencies;
+}
+
+// Expects path to be in the format:
+//  - source-map-support/register.js
+//  - @opentelemetry/api
+//  - zod
+//
+// With the result being:
+//  - source-map-support
+//  - @opentelemetry/api
+//  - zod
+function detectPackageNameFromImportPath(path: string): string {
+  if (path.startsWith("@")) {
+    return path.split("/").slice(0, 2).join("/");
+  } else {
+    return path.split("/")[0] as string;
+  }
+}

packages/cli-v3/src/dev/backgroundWorker.ts

@@ -20,10 +20,11 @@ import chalk from "chalk";
 import dotenv from "dotenv";
 import { Evt } from "evt";
 import { ChildProcess, fork } from "node:child_process";
-import { resolve } from "node:path";
+import { dirname, resolve } from "node:path";
 import terminalLink from "terminal-link";
-import { logger } from "../utilities/logger.js";
 import { safeDeleteFileSync } from "../utilities/fileSystem.js";
+import { installPackages } from "../utilities/installPackages.js";
+import { logger } from "../utilities/logger.js";
 
 export type CurrentWorkers = BackgroundWorkerCoordinator["currentWorkers"];
 export class BackgroundWorkerCoordinator {
@@ -202,6 +203,7 @@ class CleanupProcessError extends Error {
 
 export type BackgroundWorkerParams = {
   env: Record<string, string>;
+  dependencies?: Record<string, string>;
   projectDir: string;
   debuggerOn: boolean;
   debugOtel?: boolean;
@@ -253,6 +255,11 @@ export class BackgroundWorker {
       throw new Error("Worker already initialized");
     }
 
+    // Install the dependencies in dirname(this.path) using npm and child_process
+    if (this.params.dependencies) {
+      await installPackages(this.params.dependencies, { cwd: dirname(this.path) });
+    }
+
     let resolved = false;
 
     this.tasks = await new Promise<Array<TaskMetadataWithFilePath>>((resolve, reject) => {
@@ -469,6 +476,7 @@ class TaskRunProcess {
   async initialize() {
     this._child = fork(this.path, {
       stdio: [/*stdin*/ "ignore", /*stdout*/ "pipe", /*stderr*/ "pipe", "ipc"],
+      cwd: dirname(this.path),
       env: {
         ...this.env,
         OTEL_RESOURCE_ATTRIBUTES: JSON.stringify({

packages/cli-v3/src/dev/register-tracing.ts

@@ -0,0 +1,16 @@
+import { Resource } from "@opentelemetry/resources";
+import { OpenAIInstrumentation } from "@traceloop/instrumentation-openai";
+import {
+  SemanticInternalAttributes,
+  TracingDiagnosticLogLevel,
+  TracingSDK,
+} from "@trigger.dev/core/v3";
+
+export const tracingSDK = new TracingSDK({
+  url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
+  resource: new Resource({
+    [SemanticInternalAttributes.CLI_VERSION]: "3.0.0",
+  }),
+  instrumentations: [new OpenAIInstrumentation()],
+  diagLogLevel: (process.env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none",
+});

packages/cli-v3/src/worker-facade.ts renamed to packages/cli-v3/src/dev/worker-facade.ts

@@ -1,20 +1,9 @@
-import "source-map-support/register";
-import { TracingSDK } from "@trigger.dev/core/v3/otel";
-// import { OpenAIInstrumentation } from "@traceloop/instrumentation-openai";
-
-// IMPORTANT: this needs to be the first import to work properly
-// WARNING: [WARNING] Constructing "ImportInTheMiddle" will crash at run-time because it's an import namespace object, not a constructor [call-import-namespace]
-// TODO: https://github.com/open-telemetry/opentelemetry-js/issues/3954
-const tracingSDK = new TracingSDK({
-  url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
-  resource: new Resource({
-    [SemanticInternalAttributes.CLI_VERSION]: packageJson.version,
-  }),
-  instrumentations: [
-    // new OpenAIInstrumentation(),
-  ],
-  diagLogLevel: (process.env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none",
-});
+import "source-map-support/register.js";
+import { type TracingSDK } from "@trigger.dev/core/v3";
+
+__REGISTER_TRACING__;
+declare const __REGISTER_TRACING__: unknown;
+declare const tracingSDK: TracingSDK;
 
 const otelTracer = tracingSDK.getTracer("trigger-dev-worker", packageJson.version);
 const otelLogger = tracingSDK.getLogger("trigger-dev-worker", packageJson.version);
@@ -42,13 +31,13 @@ import {
   taskContextManager,
   workerToChildMessages,
   type BackgroundWorkerProperties,
+  type TracingDiagnosticLogLevel,
 } from "@trigger.dev/core/v3";
-import * as packageJson from "../package.json";
+import * as packageJson from "../../package.json";
 
 import { Resource } from "@opentelemetry/resources";
 import { flattenAttributes } from "@trigger.dev/core/v3";
-import { TaskMetadataWithFunctions } from "./types.js";
-import { TracingDiagnosticLogLevel } from "@trigger.dev/core/v3/otel/tracingSDK";
+import { TaskMetadataWithFunctions } from "../types.js";
 
 const tracer = new TriggerTracer({ tracer: otelTracer, logger: otelLogger });
 const consoleInterceptor = new ConsoleInterceptor(otelLogger);

packages/cli-v3/src/prod-facade.ts

@@ -1,5 +1,5 @@
 // import "source-map-support/register";
-import { TracingSDK } from "@trigger.dev/core/v3/otel";
+import { TracingSDK } from "@trigger.dev/core/v3";
 // import { OpenAIInstrumentation } from "@traceloop/instrumentation-openai";
 
 // IMPORTANT: this needs to be the first import to work properly
@@ -48,7 +48,7 @@ import * as packageJson from "../package.json";
 import { Resource } from "@opentelemetry/resources";
 import { flattenAttributes } from "@trigger.dev/core/v3";
 import { TaskMetadataWithFunctions } from "./types";
-import { TracingDiagnosticLogLevel } from "@trigger.dev/core/v3/otel/tracingSDK";
+import { TracingDiagnosticLogLevel } from "@trigger.dev/core/v3";
 
 const tracer = new TriggerTracer({ tracer: otelTracer, logger: otelLogger });
 const consoleInterceptor = new ConsoleInterceptor(otelLogger);