Skip to content

Commit 215b60c

Browse files
authored
v3: superjson output support (#971)
* superjson output support * Better support for superjson in the task events, limiting output attributes * Offload large outputs to object store (r2) * Finishing up the offloading of large outputs/payloads to an object store
1 parent 9c89ab4 commit 215b60c

File tree

39 files changed

+1152
-135
lines changed

39 files changed

+1152
-135
lines changed

.env.example

+5-1
Original file line numberDiff line numberDiff line change
@@ -59,4 +59,8 @@ COORDINATOR_SECRET=coordinator-secret # generate the actual secret with `openssl
5959
# CONTAINER_REGISTRY_ORIGIN=<Container registry origin e.g. https://registry.digitalocean.com>
6060
# CONTAINER_REGISTRY_USERNAME=<Container registry username e.g. Digital ocean email address>
6161
# CONTAINER_REGISTRY_PASSWORD=<Container registry password e.g. Digital ocean PAT>
62-
# DEV_OTEL_EXPORTER_OTLP_ENDPOINT="http://0.0.0.0:4318"
62+
# DEV_OTEL_EXPORTER_OTLP_ENDPOINT="http://0.0.0.0:4318"
63+
# These are needed for the object store (for handling large payloads/outputs)
64+
# OBJECT_STORE_BASE_URL="https://{bucket}.{accountId}.r2.cloudflarestorage.com"
65+
# OBJECT_STORE_ACCESS_KEY_ID=
66+
# OBJECT_STORE_SECRET_ACCESS_KEY=

.nvmrc

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
v18.18.0
1+
v20.11.1

apps/webapp/app/components/code/CodeBlock.tsx

+1-1
Original file line numberDiff line numberDiff line change
@@ -367,7 +367,7 @@ function Chrome({ title }: { title?: string }) {
367367
);
368368
}
369369

370-
function TitleRow({ title }: { title: string }) {
370+
export function TitleRow({ title }: { title: string }) {
371371
return (
372372
<div className="flex items-center justify-between px-4">
373373
<Paragraph variant="base/bright" className="w-full border-b border-grid-dimmed py-2.5">

apps/webapp/app/components/primitives/Buttons.tsx

+5-2
Original file line numberDiff line numberDiff line change
@@ -285,7 +285,7 @@ export const Button = forwardRef<HTMLButtonElement, ButtonPropsType>(
285285

286286
type LinkPropsType = Pick<
287287
LinkProps,
288-
"to" | "target" | "onClick" | "onMouseDown" | "onMouseEnter" | "onMouseLeave"
288+
"to" | "target" | "onClick" | "onMouseDown" | "onMouseEnter" | "onMouseLeave" | "download"
289289
> &
290290
React.ComponentProps<typeof ButtonContent>;
291291
export const LinkButton = ({
@@ -294,6 +294,7 @@ export const LinkButton = ({
294294
onMouseDown,
295295
onMouseEnter,
296296
onMouseLeave,
297+
download,
297298
...props
298299
}: LinkPropsType) => {
299300
const innerRef = useRef<HTMLAnchorElement>(null);
@@ -308,7 +309,7 @@ export const LinkButton = ({
308309
});
309310
}
310311

311-
if (to.toString().startsWith("http")) {
312+
if (to.toString().startsWith("http") || to.toString().startsWith("/resources")) {
312313
return (
313314
<ExtLink
314315
href={to.toString()}
@@ -318,6 +319,7 @@ export const LinkButton = ({
318319
onMouseDown={onMouseDown}
319320
onMouseEnter={onMouseEnter}
320321
onMouseLeave={onMouseLeave}
322+
download={download}
321323
>
322324
<ButtonContent {...props} />
323325
</ExtLink>
@@ -332,6 +334,7 @@ export const LinkButton = ({
332334
onMouseDown={onMouseDown}
333335
onMouseEnter={onMouseEnter}
334336
onMouseLeave={onMouseLeave}
337+
download={download}
335338
>
336339
<ButtonContent {...props} />
337340
</Link>

apps/webapp/app/env.server.ts

+3
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,9 @@ const EnvironmentSchema = z.object({
9696
CONTAINER_REGISTRY_PASSWORD: z.string().optional(),
9797
DEPLOY_REGISTRY_HOST: z.string().optional(),
9898
DEV_OTEL_EXPORTER_OTLP_ENDPOINT: z.string().optional(),
99+
OBJECT_STORE_BASE_URL: z.string().optional(),
100+
OBJECT_STORE_ACCESS_KEY_ID: z.string().optional(),
101+
OBJECT_STORE_SECRET_ACCESS_KEY: z.string().optional(),
99102
});
100103

101104
export type Environment = z.infer<typeof EnvironmentSchema>;

apps/webapp/app/presenters/v3/SpanPresenter.server.ts

+19-12
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,4 @@
1-
import { Attributes } from "@opentelemetry/api";
2-
import {
3-
ExceptionEventProperties,
4-
SemanticInternalAttributes,
5-
SpanEvent,
6-
SpanEvents,
7-
correctErrorStackTrace,
8-
isExceptionSpanEvent,
9-
} from "@trigger.dev/core/v3";
10-
import { z } from "zod";
1+
import { prettyPrintPacket } from "@trigger.dev/core/v3";
112
import { PrismaClient, prisma } from "~/db.server";
123
import { eventRepository } from "~/v3/eventRepository.server";
134

@@ -48,12 +39,28 @@ export class SpanPresenter {
4839
throw new Error("Event not found");
4940
}
5041

42+
const output =
43+
span.outputType === "application/store"
44+
? `/resources/packets/${span.environmentId}/${span.output}`
45+
: typeof span.output !== "undefined" && span.output !== null
46+
? prettyPrintPacket(span.output, span.outputType ?? undefined)
47+
: undefined;
48+
49+
const payload =
50+
span.payloadType === "application/store"
51+
? `/resources/packets/${span.environmentId}/${span.payload}`
52+
: typeof span.payload !== "undefined" && span.payload !== null
53+
? prettyPrintPacket(span.payload, span.payloadType ?? undefined)
54+
: undefined;
55+
5156
return {
5257
event: {
5358
...span,
5459
events: span.events,
55-
output: span.output ? JSON.stringify(span.output, null, 2) : undefined,
56-
payload: span.payload ? JSON.stringify(span.payload, null, 2) : undefined,
60+
output,
61+
outputType: span.outputType ?? "application/json",
62+
payload,
63+
payloadType: span.payloadType ?? "application/json",
5764
properties: span.properties ? JSON.stringify(span.properties, null, 2) : undefined,
5865
showActionBar: span.show?.actions === true,
5966
},

apps/webapp/app/presenters/v3/TestTaskPresenter.server.ts

+2
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,8 @@ export class TestTaskPresenter {
8686
taskr."runtimeEnvironmentId"
8787
FROM
8888
taskruns AS taskr
89+
WHERE
90+
taskr."payloadType" = 'application/json'
8991
ORDER BY
9092
taskr."createdAt" DESC;`;
9193

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route.tsx

+28-4
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { QueueListIcon, StopCircleIcon } from "@heroicons/react/20/solid";
1+
import { CloudArrowDownIcon, QueueListIcon, StopCircleIcon } from "@heroicons/react/20/solid";
22
import { useParams } from "@remix-run/react";
33
import { LoaderFunctionArgs } from "@remix-run/server-runtime";
44
import { formatDurationNanoseconds, nanosecondsToMilliseconds } from "@trigger.dev/core/v3";
@@ -12,7 +12,6 @@ import { Dialog, DialogTrigger } from "~/components/primitives/Dialog";
1212
import { Header2 } from "~/components/primitives/Headers";
1313
import { Paragraph } from "~/components/primitives/Paragraph";
1414
import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
15-
import { TextLink } from "~/components/primitives/TextLink";
1615
import { CancelRunDialog } from "~/components/runs/v3/CancelRunDialog";
1716
import { LiveTimer } from "~/components/runs/v3/LiveTimer";
1817
import { RunIcon } from "~/components/runs/v3/RunIcon";
@@ -149,10 +148,10 @@ export default function Page() {
149148

150149
{event.events !== undefined && <SpanEvents spanEvents={event.events} />}
151150
{event.payload !== undefined && (
152-
<CodeBlock rowTitle="Payload" code={event.payload} maxLines={20} />
151+
<PacketDisplay data={event.payload} dataType={event.payloadType} title="Payload" />
153152
)}
154153
{event.output !== undefined && (
155-
<CodeBlock rowTitle="Output" code={event.output} maxLines={20} />
154+
<PacketDisplay data={event.output} dataType={event.outputType} title="Output" />
156155
)}
157156
{event.properties !== undefined && (
158157
<CodeBlock rowTitle="Properties" code={event.properties} maxLines={20} />
@@ -204,6 +203,31 @@ export default function Page() {
204203
);
205204
}
206205

206+
function PacketDisplay({
207+
data,
208+
dataType,
209+
title,
210+
}: {
211+
data: string;
212+
dataType: string;
213+
title: string;
214+
}) {
215+
if (dataType === "application/store") {
216+
return (
217+
<div className="flex flex-col">
218+
<Paragraph variant="base/bright" className="w-full border-b border-grid-dimmed py-2.5">
219+
{title}
220+
</Paragraph>
221+
<LinkButton LeadingIcon={CloudArrowDownIcon} to={data} variant="tertiary/medium" download>
222+
Download
223+
</LinkButton>
224+
</div>
225+
);
226+
} else {
227+
return <CodeBlock rowTitle={title} code={data} maxLines={20} />;
228+
}
229+
}
230+
207231
type TimelineProps = {
208232
startTime: Date;
209233
duration: number;
+101
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
import type { ActionFunctionArgs } from "@remix-run/server-runtime";
2+
import { json } from "@remix-run/server-runtime";
3+
import { z } from "zod";
4+
import { env } from "~/env.server";
5+
import { authenticateApiRequest } from "~/services/apiAuth.server";
6+
import { logger } from "~/services/logger.server";
7+
import { r2 } from "~/v3/r2.server";
8+
9+
const ParamsSchema = z.object({
10+
"*": z.string(),
11+
});
12+
13+
export async function action({ request, params }: ActionFunctionArgs) {
14+
// Ensure this is a POST request
15+
if (request.method.toUpperCase() !== "PUT") {
16+
return { status: 405, body: "Method Not Allowed" };
17+
}
18+
19+
// Next authenticate the request
20+
const authenticationResult = await authenticateApiRequest(request);
21+
22+
if (!authenticationResult) {
23+
return json({ error: "Invalid or Missing API key" }, { status: 401 });
24+
}
25+
26+
const parsedParams = ParamsSchema.parse(params);
27+
const filename = parsedParams["*"];
28+
29+
if (!env.OBJECT_STORE_BASE_URL) {
30+
return json({ error: "Object store base URL is not set" }, { status: 500 });
31+
}
32+
33+
if (!r2) {
34+
return json({ error: "Object store credentials are not set" }, { status: 500 });
35+
}
36+
37+
const url = new URL(env.OBJECT_STORE_BASE_URL);
38+
url.pathname = `/packets/${authenticationResult.environment.project.externalRef}/${authenticationResult.environment.slug}/${filename}`;
39+
url.searchParams.set("X-Amz-Expires", "300"); // 5 minutes
40+
41+
const signed = await r2.sign(
42+
new Request(url, {
43+
method: "PUT",
44+
}),
45+
{
46+
aws: { signQuery: true },
47+
}
48+
);
49+
50+
logger.debug("Generated presigned URL", {
51+
url: signed.url,
52+
headers: Object.fromEntries(signed.headers),
53+
});
54+
55+
// Caller can now use this URL to upload to that object.
56+
return json({ presignedUrl: signed.url });
57+
}
58+
59+
export async function loader({ request, params }: ActionFunctionArgs) {
60+
// Next authenticate the request
61+
const authenticationResult = await authenticateApiRequest(request);
62+
63+
if (!authenticationResult) {
64+
return json({ error: "Invalid or Missing API key" }, { status: 401 });
65+
}
66+
67+
const parsedParams = ParamsSchema.parse(params);
68+
const filename = parsedParams["*"];
69+
70+
if (!env.OBJECT_STORE_BASE_URL) {
71+
return json({ error: "Object store base URL is not set" }, { status: 500 });
72+
}
73+
74+
if (!r2) {
75+
return json({ error: "Object store credentials are not set" }, { status: 500 });
76+
}
77+
78+
const url = new URL(env.OBJECT_STORE_BASE_URL);
79+
url.pathname = `/packets/${authenticationResult.environment.project.externalRef}/${authenticationResult.environment.slug}/${filename}`;
80+
url.searchParams.set("X-Amz-Expires", "300"); // 5 minutes
81+
82+
const signed = await r2.sign(
83+
new Request(url, {
84+
method: request.method,
85+
}),
86+
{
87+
aws: { signQuery: true },
88+
}
89+
);
90+
91+
logger.debug("Generated presigned URL", {
92+
url: signed.url,
93+
headers: Object.fromEntries(signed.headers),
94+
});
95+
96+
const getUrl = new URL(url.href);
97+
getUrl.searchParams.delete("X-Amz-Expires");
98+
99+
// Caller can now use this URL to upload to that object.
100+
return json({ presignedUrl: signed.url });
101+
}

apps/webapp/app/routes/otel.v1.logs.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ export async function action({ request }: ActionFunctionArgs) {
99
if (contentType === "application/json") {
1010
const body = await request.json();
1111

12-
const exportResponse = await otlpExporter.exportLogs(body as ExportLogsServiceRequest, true);
12+
const exportResponse = await otlpExporter.exportLogs(body as ExportLogsServiceRequest, false);
1313

1414
return json(exportResponse, { status: 200 });
1515
} else if (contentType === "application/x-protobuf") {

apps/webapp/app/routes/otel.v1.traces.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ export async function action({ request }: ActionFunctionArgs) {
1111

1212
const exportResponse = await otlpExporter.exportTraces(
1313
body as ExportTraceServiceRequest,
14-
true
14+
false
1515
);
1616

1717
return json(exportResponse, { status: 200 });
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
import { LoaderFunctionArgs } from "@remix-run/node";
2+
import { basename } from "node:path";
3+
import { z } from "zod";
4+
import { prisma } from "~/db.server";
5+
import { env } from "~/env.server";
6+
import { requireUserId } from "~/services/session.server";
7+
import { r2 } from "~/v3/r2.server";
8+
9+
const ParamSchema = z.object({
10+
environmentId: z.string(),
11+
"*": z.string(),
12+
});
13+
14+
export async function loader({ request, params }: LoaderFunctionArgs) {
15+
const userId = await requireUserId(request);
16+
const { environmentId, "*": filename } = ParamSchema.parse(params);
17+
18+
const environment = await prisma.runtimeEnvironment.findFirst({
19+
where: {
20+
id: environmentId,
21+
organization: {
22+
members: {
23+
some: {
24+
userId,
25+
},
26+
},
27+
},
28+
},
29+
include: {
30+
project: true,
31+
},
32+
});
33+
34+
if (!environment) {
35+
return new Response("Not found", { status: 404 });
36+
}
37+
38+
if (!env.OBJECT_STORE_BASE_URL) {
39+
return new Response("Object store base URL is not set", { status: 500 });
40+
}
41+
42+
if (!r2) {
43+
return new Response("Object store credentials are not set", { status: 500 });
44+
}
45+
46+
const url = new URL(env.OBJECT_STORE_BASE_URL);
47+
url.pathname = `/packets/${environment.project.externalRef}/${environment.slug}/${filename}`;
48+
url.searchParams.set("X-Amz-Expires", "30"); // 30 seconds
49+
50+
const signed = await r2.sign(
51+
new Request(url, {
52+
method: "GET",
53+
}),
54+
{
55+
aws: { signQuery: true },
56+
}
57+
);
58+
59+
const response = await fetch(signed.url, {
60+
headers: signed.headers,
61+
});
62+
63+
return new Response(response.body, {
64+
status: 200,
65+
headers: {
66+
"Content-Type": "application/octet-stream",
67+
"Content-Disposition": `attachment; filename="${basename(url.pathname)}"`,
68+
},
69+
});
70+
}

0 commit comments

Comments
 (0)