feat(webapp): trace deployment outcomes and compute template creation (#4015)

Deploy success/failure wasn't easily observable: compute template
creation only logged, and the terminal deployment statuses (deployed /
failed / timed-out) weren't traced — so deploy health couldn't be seen
without querying the database.

This adds two spans:

- `compute.template.create` around template creation at finalize,
tagging the resolved mode and per-preset outcome. `resolveMode` now
returns its decision (mode + reason) so the span can record why a mode
was chosen.
- `deployment.outcome` via a small shared helper
(`recordDeploymentOutcome`) emitted at every terminal-status write —
finalize (deployed), fail / index-failed / background-worker (failed),
and timeout (timed out) — so deploy success/failure is queryable by
status and reason.

The helper is best-effort (org/project/env enrichment where cheaply
available) and never throws, so telemetry can't break a deploy.

Author: nicktrn

apps/webapp/app/v3/services/computeTemplateCreation.server.ts

@@ -13,9 +13,26 @@ import { isOrgMigrated } from "~/runEngine/concerns/computeMigration.server";
 import { backingForQueue, workerRegionRegistry } from "~/v3/workerRegions.server";
 import { globalFlagsRegistry } from "~/v3/globalFlagsRegistry.server";
 import { getEntitlement } from "~/services/platform.v3.server";
+import { startActiveSpan, attributesFromAuthenticatedEnv } from "~/v3/tracer.server";
 
 type TemplateCreationMode = "required" | "shadow" | "skip";
 
+// Why the mode was chosen — slices the compute.template.create span by path.
+type TemplateModeReason =
+  | "no-client"
+  | "no-project"
+  | "microvm-native"
+  | "migrated"
+  | "compute-access"
+  | "rollout"
+  | "none";
+
+type ResolvedTemplateMode = {
+  mode: TemplateCreationMode;
+  migrated: boolean;
+  reason: TemplateModeReason;
+};
+
 type ResolvedPreset = {
   name: MachinePresetName;
   cpu: number;
@@ -60,89 +77,116 @@ export class ComputeTemplateCreationService {
     prisma: PrismaClientOrTransaction;
     writer?: WritableStreamDefaultWriter;
   }): Promise<void> {
-    const mode = await this.resolveMode(options.projectId, options.prisma);
+    return startActiveSpan("compute.template.create", async (span) => {
+      const { mode, migrated, reason } = await this.resolveMode(
+        options.projectId,
+        options.prisma
+      );
 
-    if (mode === "skip") {
-      return;
-    }
+      span.setAttributes({
+        ...attributesFromAuthenticatedEnv(options.authenticatedEnv),
+        "compute.template.mode": mode,
+        "compute.template.migrated": migrated,
+        "compute.template.reason": reason,
+        "compute.template.deployment_id": options.deploymentFriendlyId,
+        "compute.template.presets_total": this.presets.length,
+        "compute.template.presets_required": this.requiredPresets.size,
+      });
 
-    if (mode === "shadow") {
-      this.createTemplate(options.imageReference, { background: true })
-        .then((outcome) => {
-          if (outcome.error) {
-            logger.error("Shadow template creation failed", {
+      if (mode === "skip") {
+        span.setAttribute("compute.template.result", "skipped");
+        return;
+      }
+
+      if (mode === "shadow") {
+        // Shadow is fire-and-forget (background build), so the span only records
+        // that it was dispatched — the build outcome lands server-side later.
+        span.setAttribute("compute.template.result", "shadow_dispatched");
+        this.createTemplate(options.imageReference, { background: true })
+          .then((outcome) => {
+            if (outcome.error) {
+              logger.error("Shadow template creation failed", {
+                id: options.deploymentFriendlyId,
+                imageReference: options.imageReference,
+                error: outcome.error,
+              });
+            }
+          })
+          .catch((error) => {
+            logger.error("Shadow template creation threw unexpectedly", {
               id: options.deploymentFriendlyId,
               imageReference: options.imageReference,
-              error: outcome.error,
+              error: error instanceof Error ? error.message : String(error),
             });
-          }
-        })
-        .catch((error) => {
-          logger.error("Shadow template creation threw unexpectedly", {
-            id: options.deploymentFriendlyId,
-            imageReference: options.imageReference,
-            error: error instanceof Error ? error.message : String(error),
           });
-        });
-      return;
-    }
-
-    // Required mode
-    if (options.writer) {
-      try {
-        await options.writer.write(
-          `event: log\ndata: ${JSON.stringify({ message: "Building compute template..." })}\n\n`
-        );
-      } catch {
-        // Stream may be closed if client disconnected - continue with template creation
+        return;
       }
-    }
-
-    logger.info("Creating compute template (required mode)", {
-      id: options.deploymentFriendlyId,
-      imageReference: options.imageReference,
-      presets: this.presets.map((p) => p.name),
-      requiredPresets: [...this.requiredPresets],
-    });
 
-    const outcome = await this.createTemplate(options.imageReference);
-    const failureMessage = this.failureMessageForRequiredMode(
-      outcome,
-      options.deploymentFriendlyId,
-      options.imageReference
-    );
+      // Required mode
+      if (options.writer) {
+        try {
+          await options.writer.write(
+            `event: log\ndata: ${JSON.stringify({ message: "Building compute template..." })}\n\n`
+          );
+        } catch {
+          // Stream may be closed if client disconnected - continue with template creation
+        }
+      }
 
-    if (failureMessage) {
-      logger.error("Compute template creation failed", {
+      logger.info("Creating compute template (required mode)", {
         id: options.deploymentFriendlyId,
         imageReference: options.imageReference,
-        error: failureMessage,
+        presets: this.presets.map((p) => p.name),
+        requiredPresets: [...this.requiredPresets],
       });
 
-      const failService = new FailDeploymentService();
-      await failService.call(options.authenticatedEnv, options.deploymentFriendlyId, {
-        error: {
-          name: "TemplateCreationFailed",
-          message: `Failed to create compute template: ${failureMessage}`,
-        },
-      });
+      const outcome = await this.createTemplate(options.imageReference);
+      span.setAttribute("compute.template.presets_built", outcome.results.length);
 
-      throw new ServiceValidationError(`Compute template creation failed: ${failureMessage}`);
-    }
+      const failureMessage = this.failureMessageForRequiredMode(
+        outcome,
+        options.deploymentFriendlyId,
+        options.imageReference
+      );
+
+      if (failureMessage) {
+        span.setAttributes({
+          "compute.template.result": "failed",
+          "compute.template.failure": failureMessage,
+        });
 
-    logger.info("Compute template created", {
-      id: options.deploymentFriendlyId,
-      imageReference: options.imageReference,
-      results: outcome.results.length,
+        logger.error("Compute template creation failed", {
+          id: options.deploymentFriendlyId,
+          imageReference: options.imageReference,
+          error: failureMessage,
+        });
+
+        const failService = new FailDeploymentService();
+        await failService.call(options.authenticatedEnv, options.deploymentFriendlyId, {
+          error: {
+            name: "TemplateCreationFailed",
+            message: `Failed to create compute template: ${failureMessage}`,
+          },
+        });
+
+        throw new ServiceValidationError(`Compute template creation failed: ${failureMessage}`);
+      }
+
+      span.setAttribute("compute.template.result", "created");
+      logger.info("Compute template created", {
+        id: options.deploymentFriendlyId,
+        imageReference: options.imageReference,
+        results: outcome.results.length,
+      });
     });
   }
 
   async resolveMode(
     projectId: string,
     prisma: PrismaClientOrTransaction
-  ): Promise<TemplateCreationMode> {
+  ): Promise<ResolvedTemplateMode> {
     if (!this.client) {
-      return "skip";
+      return { mode: "skip", migrated: false, reason: "no-client" };
     }
 
     const project = await prisma.project.findFirst({
@@ -158,11 +202,11 @@ export class ComputeTemplateCreationService {
     });
 
     if (!project) {
-      return "skip";
+      return { mode: "skip", migrated: false, reason: "no-project" };
     }
 
     if (project.defaultWorkerGroup?.workloadType === "MICROVM") {
-      return "required";
+      return { mode: "required", migrated: false, reason: "microvm-native" };
     }
 
     // Migrated orgs route runs to the compute backing even though their stored
@@ -194,22 +238,26 @@ export class ComputeTemplateCreationService {
       }
       if (migrated) {
         // required => template built at deploy (deploy fails on error); off => shadow.
-        return decision.flags?.computeMigrationRequireTemplate ? "required" : "shadow";
+        return {
+          mode: decision.flags?.computeMigrationRequireTemplate ? "required" : "shadow",
+          migrated: true,
+          reason: "migrated",
+        };
       }
     }
 
     const hasComputeAccess = await resolveComputeAccess(prisma, project.organization.featureFlags);
 
     if (hasComputeAccess) {
-      return "shadow";
+      return { mode: "shadow", migrated: false, reason: "compute-access" };
     }
 
     const rolloutPct = Number(env.COMPUTE_TEMPLATE_SHADOW_ROLLOUT_PCT ?? "0");
     if (rolloutPct > 0 && Math.random() * 100 < rolloutPct) {
-      return "shadow";
+      return { mode: "shadow", migrated: false, reason: "rollout" };
     }
 
-    return "skip";
+    return { mode: "skip", migrated: false, reason: "none" };
   }
 
   async createTemplate(

apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts

@@ -15,6 +15,7 @@ import {
 } from "./createBackgroundWorker.server";
 import { findOrCreateBackgroundWorker } from "./createDeploymentBackgroundWorkerV4/findOrCreateBackgroundWorker.server";
 import { TimeoutDeploymentService } from "./timeoutDeployment.server";
+import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server";
 import { env } from "~/env.server";
 
 export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService {
@@ -111,7 +112,7 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService {
         if (findOrCreateError instanceof ServiceValidationError) {
           // `#failBackgroundWorkerDeployment` already throws its argument; the
           // outer `throw` covers the non-SVE branch.
-          await this.#failBackgroundWorkerDeployment(deployment, findOrCreateError);
+          await this.#failBackgroundWorkerDeployment(deployment, findOrCreateError, environment);
         }
         throw findOrCreateError;
       }
@@ -144,7 +145,7 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService {
 
         const serviceError = new ServiceValidationError("Error creating background worker files");
 
-        await this.#failBackgroundWorkerDeployment(deployment, serviceError);
+        await this.#failBackgroundWorkerDeployment(deployment, serviceError, environment);
 
         throw serviceError;
       }
@@ -167,7 +168,7 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService {
             error: resourcesError.message,
           });
 
-          await this.#failBackgroundWorkerDeployment(deployment, resourcesError);
+          await this.#failBackgroundWorkerDeployment(deployment, resourcesError, environment);
           throw resourcesError;
         }
 
@@ -179,7 +180,7 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService {
           "Error creating background worker resources"
         );
 
-        await this.#failBackgroundWorkerDeployment(deployment, serviceError);
+        await this.#failBackgroundWorkerDeployment(deployment, serviceError, environment);
 
         throw serviceError;
       }
@@ -206,7 +207,7 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService {
             error: schedulesError.message,
           });
 
-          await this.#failBackgroundWorkerDeployment(deployment, schedulesError);
+          await this.#failBackgroundWorkerDeployment(deployment, schedulesError, environment);
           throw schedulesError;
         }
 
@@ -220,7 +221,7 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService {
 
         const serviceError = new ServiceValidationError("Error syncing declarative schedules");
 
-        await this.#failBackgroundWorkerDeployment(deployment, serviceError);
+        await this.#failBackgroundWorkerDeployment(deployment, serviceError, environment);
 
         throw serviceError;
       }
@@ -264,7 +265,11 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService {
     });
   }
 
-  async #failBackgroundWorkerDeployment(deployment: WorkerDeployment, error: Error) {
+  async #failBackgroundWorkerDeployment(
+    deployment: WorkerDeployment,
+    error: Error,
+    environment: AuthenticatedEnvironment
+  ) {
     // Guarded BUILDING → FAILED transition, symmetric with the BUILDING → DEPLOYING
     // transition in `call()`. With idempotent retries, two attempts can run side-by-side;
     // without the predicate, one attempt's failure could downgrade the deployment after
@@ -297,6 +302,16 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService {
       // sibling attempt may have just enqueued it as part of a successful
       // BUILDING → DEPLOYING transition.
       await TimeoutDeploymentService.dequeue(deployment.id, this._prisma);
+
+      recordDeploymentOutcome({
+        status: "FAILED",
+        deploymentFriendlyId: deployment.friendlyId,
+        organizationId: environment.organizationId,
+        projectId: environment.projectId,
+        environmentId: environment.id,
+        environmentType: environment.type,
+        reason: error.message,
+      });
     }
 
     throw error;

apps/webapp/app/v3/services/deploymentIndexFailed.server.ts

@@ -3,6 +3,7 @@ import { BaseService } from "./baseService.server";
 import { logger } from "~/services/logger.server";
 import { type WorkerDeploymentStatus } from "@trigger.dev/database";
 import { DeploymentService } from "./deployment.server";
+import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server";
 
 const FINAL_DEPLOYMENT_STATUSES: WorkerDeploymentStatus[] = [
   "CANCELED",
@@ -74,6 +75,16 @@ export class DeploymentIndexFailed extends BaseService {
       },
     });
 
+    recordDeploymentOutcome({
+      status: "FAILED",
+      deploymentFriendlyId: deployment.friendlyId,
+      organizationId: deployment.environment.project.organizationId,
+      projectId: deployment.environment.projectId,
+      environmentId: deployment.environmentId,
+      environmentType: deployment.environment.type,
+      reason: error.message,
+    });
+
     const deploymentService = new DeploymentService();
     await deploymentService
       .appendToEventLog(deployment.environment.project, failedDeployment, [

apps/webapp/app/v3/services/failDeployment.server.ts

@@ -5,6 +5,7 @@ import { type WorkerDeploymentStatus } from "@trigger.dev/database";
 import { type FailDeploymentRequestBody } from "@trigger.dev/core/v3/schemas";
 import { type AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { DeploymentService } from "./deployment.server";
+import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server";
 
 export const FINAL_DEPLOYMENT_STATUSES: WorkerDeploymentStatus[] = [
   "CANCELED",
@@ -51,6 +52,16 @@ export class FailDeploymentService extends BaseService {
       },
     });
 
+    recordDeploymentOutcome({
+      status: "FAILED",
+      deploymentFriendlyId: friendlyId,
+      organizationId: authenticatedEnv.organizationId,
+      projectId: authenticatedEnv.projectId,
+      environmentId: authenticatedEnv.id,
+      environmentType: authenticatedEnv.type,
+      reason: params.error.message,
+    });
+
     const deploymentService = new DeploymentService();
     await deploymentService
       .appendToEventLog(authenticatedEnv.project, failedDeployment, [