feat(pdf): Support PDF attachments (#2709)

* feat(pdf): Support PDF attachments

* Add tests and other review feedback

* Fix doc

* More renaming and cleanup

* refactor: extract Vercel content part schemas to types/chat.ts for reuse

Move inline Zod schemas from chatDataStream.ts and message-parts.ts into
types/chat.ts as shared, exported schemas. This eliminates duplicate
definitions and makes schema management easier.

Co-authored-by: Andrew Mikofalvy <amikofalvy@users.noreply.github.com>
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: claude[bot] <41898282+claude[bot]@users.noreply.github.com>
Co-authored-by: Andrew Mikofalvy <amikofalvy@users.noreply.github.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: 4 people

agents-api/__snapshots__/openapi.json

@@ -39186,6 +39186,33 @@
                                 ],
                                 "type": "object"
                               },
+                              {
+                                "properties": {
+                                  "filename": {
+                                    "type": "string"
+                                  },
+                                  "mediaType": {
+                                    "type": "string"
+                                  },
+                                  "mimeType": {
+                                    "type": "string"
+                                  },
+                                  "text": {
+                                    "type": "string"
+                                  },
+                                  "type": {
+                                    "enum": [
+                                      "file"
+                                    ],
+                                    "type": "string"
+                                  }
+                                },
+                                "required": [
+                                  "type",
+                                  "text"
+                                ],
+                                "type": "object"
+                              },
                               {
                                 "properties": {
                                   "text": {
@@ -39196,8 +39223,7 @@
                                       {
                                         "enum": [
                                           "audio",
-                                          "video",
-                                          "file"
+                                          "video"
                                         ],
                                         "type": "string"
                                       },
@@ -39851,6 +39877,36 @@
                                       "image_url"
                                     ],
                                     "type": "object"
+                                  },
+                                  {
+                                    "properties": {
+                                      "file": {
+                                        "properties": {
+                                          "file_data": {
+                                            "pattern": "^data:application\\/pdf;base64,",
+                                            "type": "string"
+                                          },
+                                          "filename": {
+                                            "type": "string"
+                                          }
+                                        },
+                                        "required": [
+                                          "file_data"
+                                        ],
+                                        "type": "object"
+                                      },
+                                      "type": {
+                                        "enum": [
+                                          "file"
+                                        ],
+                                        "type": "string"
+                                      }
+                                    },
+                                    "required": [
+                                      "type",
+                                      "file"
+                                    ],
+                                    "type": "object"
                                   }
                                 ]
                               },

agents-api/src/__tests__/run/agents/Agent.test.ts

@@ -262,7 +262,7 @@ vi.mock('../../../domains/run/session/AgentSession.js', () => ({
   },
 }));
 
-vi.mock('../../../domains/run/services/blob-storage/image-upload-helpers', () => ({
+vi.mock('../../../domains/run/services/blob-storage/file-upload-helpers', () => ({
   buildPersistedMessageContent: buildPersistedMessageContentMock,
 }));
 
@@ -1512,6 +1512,40 @@ describe('Agent Image Support', () => {
       })
     );
   });
+
+  test('passes inline PDF data to generateText as file content', async () => {
+    const agent = new Agent(mockAgentConfig, mockExecutionContext);
+    const pdfBytes = Buffer.from(
+      '%PDF-1.4\n1 0 obj\n<< /Type /Catalog >>\nendobj\n',
+      'utf8'
+    ).toString('base64');
+
+    await agent.generate([
+      { kind: 'text', text: 'Summarize this PDF' },
+      {
+        kind: 'file',
+        file: {
+          bytes: pdfBytes,
+          mimeType: 'application/pdf',
+        },
+      },
+    ]);
+
+    const { generateText } = await import('ai');
+    expect(generateText).toHaveBeenCalledWith(
+      expect.objectContaining({
+        messages: expect.arrayContaining([
+          expect.objectContaining({
+            role: 'user',
+            content: expect.arrayContaining([
+              expect.objectContaining({ type: 'text' }),
+              expect.objectContaining({ type: 'file', mediaType: 'application/pdf' }),
+            ]),
+          }),
+        ]),
+      })
+    );
+  });
 });
 
 describe('Agent tool result persistence', () => {

agents-api/src/__tests__/run/routes/chat.test.ts

@@ -258,6 +258,34 @@ describe('Chat Routes', () => {
       expect(response.headers.get('content-type')).toBe('text/event-stream');
     });
 
+    it('should accept inline PDF content item in OpenAI-style messages', async () => {
+      const response = await makeRequest('/run/v1/chat/completions', {
+        method: 'POST',
+        body: JSON.stringify({
+          model: 'claude-3-sonnet',
+          messages: [
+            {
+              role: 'user',
+              content: [
+                { type: 'text', text: 'Summarize this document' },
+                {
+                  type: 'file',
+                  file: {
+                    file_data: 'data:application/pdf;base64,JVBERi0xLjQK',
+                    filename: 'document.pdf',
+                  },
+                },
+              ],
+            },
+          ],
+          conversationId: 'conv-123',
+        }),
+      });
+
+      expect(response.status).toBe(200);
+      expect(response.headers.get('content-type')).toBe('text/event-stream');
+    });
+
     it('should handle conversation creation', async () => {
       const response = await makeRequest('/run/v1/chat/completions', {
         method: 'POST',

agents-api/src/__tests__/run/routes/chat/dataChat.test.ts

@@ -193,6 +193,34 @@ describe('Chat Data Stream Route', () => {
     expect(text).toMatch(/response/);
   });
 
+  it('should accept inline PDF file part in Vercel messages format', async () => {
+    const body = {
+      messages: [
+        {
+          role: 'user',
+          content: 'Summarize this PDF',
+          parts: [
+            { type: 'text', text: 'Summarize this PDF' },
+            {
+              type: 'file',
+              text: 'data:application/pdf;base64,JVBERi0xLjQK',
+              mediaType: 'application/pdf',
+              filename: 'doc.pdf',
+            },
+          ],
+        },
+      ],
+    };
+
+    const res = await makeRequest('/run/api/chat', {
+      method: 'POST',
+      body: JSON.stringify(body),
+    });
+
+    expect(res.status).toBe(200);
+    expect(res.headers.get('x-vercel-ai-data-stream')).toBe('v2');
+  });
+
   it('should stream approval UI events published to ToolApprovalUiBus (simulating delegated agent approval)', async () => {
     // Ensure deterministic requestId inside route subscription (chatds-${Date.now()})
     const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(12345);

agents-api/src/domains/run/agents/agent-types.ts

@@ -33,7 +33,14 @@ export type AiSdkImagePart = {
   experimental_providerMetadata?: { openai?: { imageDetail?: ImageDetail } };
 };
 
-export type AiSdkContentPart = AiSdkTextPart | AiSdkImagePart;
+export type AiSdkFilePart = {
+  type: 'file';
+  data: string | URL;
+  mediaType: string;
+  filename?: string;
+};
+
+export type AiSdkContentPart = AiSdkTextPart | AiSdkImagePart | AiSdkFilePart;
 
 /**
  * Creates a stopWhen condition that stops when any tool call name starts with the given prefix

agents-api/src/domains/run/agents/generation/conversation-history.ts

@@ -1,4 +1,6 @@
 import type { FilePart } from '@inkeep/agents-core';
+import { normalizeMimeType } from '@inkeep/agents-core/constants/allowed-file-formats';
+import { getLogger } from '../../../../logger';
 import {
   createDefaultConversationHistoryConfig,
   getConversationHistoryWithCompression,
@@ -11,6 +13,36 @@ import {
 import type { AgentRunContext, AiSdkContentPart } from '../agent-types';
 import { getPrimaryModel, getSummarizerModel } from './model-config';
 
+const PDF_MEDIA_TYPE = 'application/pdf';
+const logger = getLogger('conversation-history');
+
+function mapFileToAiSdkContentPart(
+  fileValue: string | URL,
+  mimeType: string,
+  metadata: { detail?: string; filename?: string } | undefined
+): AiSdkContentPart | null {
+  if (mimeType.startsWith('image/')) {
+    return {
+      type: 'image',
+      image: fileValue,
+      ...(metadata?.detail && {
+        experimental_providerMetadata: { openai: { imageDetail: metadata.detail as any } },
+      }),
+    };
+  }
+
+  if (mimeType === PDF_MEDIA_TYPE) {
+    return {
+      type: 'file',
+      data: fileValue,
+      mediaType: PDF_MEDIA_TYPE,
+      ...(metadata?.filename ? { filename: metadata.filename } : {}),
+    };
+  }
+
+  return null;
+}
+
 export async function buildConversationHistory(
   ctx: AgentRunContext,
   contextId: string,
@@ -81,7 +113,7 @@ export function buildInitialMessages(
   systemPrompt: string,
   conversationHistory: string,
   userMessage: string,
-  imageParts?: FilePart[]
+  fileParts?: FilePart[]
 ): any[] {
   const messages: any[] = [];
   messages.push({ role: 'system', content: systemPrompt });
@@ -90,7 +122,7 @@ export function buildInitialMessages(
     messages.push({ role: 'user', content: conversationHistory });
   }
 
-  const userContent = buildUserMessageContent(userMessage, imageParts);
+  const userContent = buildUserMessageContent(userMessage, fileParts);
   messages.push({
     role: 'user',
     content: userContent,
@@ -101,30 +133,31 @@ export function buildInitialMessages(
 
 export function buildUserMessageContent(
   text: string,
-  imageParts?: FilePart[]
+  fileParts?: FilePart[]
 ): string | AiSdkContentPart[] {
-  if (!imageParts || imageParts.length === 0) {
+  if (!fileParts || fileParts.length === 0) {
     return text;
   }
 
   const content: AiSdkContentPart[] = [{ type: 'text', text }];
 
-  for (const part of imageParts) {
+  for (const part of fileParts) {
     const file = part.file;
-    const imageValue =
+    const fileValue =
       'uri' in file && file.uri
         ? new URL(file.uri)
-        : `data:${file.mimeType || 'image/*'};base64,${file.bytes}`;
-
-    const imagePart: AiSdkContentPart = {
-      type: 'image',
-      image: imageValue,
-      ...(part.metadata?.detail && {
-        experimental_providerMetadata: { openai: { imageDetail: part.metadata.detail } },
-      }),
-    };
-
-    content.push(imagePart);
+        : `data:${file.mimeType || ''};base64,${file.bytes}`;
+    const mimeType = normalizeMimeType(file.mimeType ?? '');
+    const mappedPart = mapFileToAiSdkContentPart(fileValue, mimeType, {
+      detail: typeof part.metadata?.detail === 'string' ? part.metadata.detail : undefined,
+      filename: typeof part.metadata?.filename === 'string' ? part.metadata.filename : undefined,
+    });
+
+    if (mappedPart) {
+      content.push(mappedPart);
+    } else {
+      logger.warn({ mimeType, source: 'user-message' }, 'Dropping unsupported file content part');
+    }
   }
 
   return content;

agents-api/src/domains/run/agents/generation/generate.ts

@@ -149,10 +149,7 @@ export async function runGenerate(
           .join('')
       : '';
   const userMessage = `${textParts}${dataContext}`;
-  const imageParts = userParts.filter(
-    (part): part is FilePart =>
-      part.kind === 'file' && part.file.mimeType?.startsWith('image/') === true
-  );
+  const fileParts = userParts.filter((part): part is FilePart => part.kind === 'file');
   const conversationIdForSpan = runtimeContext?.metadata?.conversationId;
 
   return tracer.startActiveSpan(
@@ -200,14 +197,21 @@ export async function runGenerate(
 
         const { primaryModelSettings, modelSettings, hasStructuredOutput, timeoutMs } =
           configureModelSettings(ctx);
+        const inlinePdfFileCount = fileParts.filter(
+          (part) => part.file.mimeType?.toLowerCase().startsWith('application/pdf') === true
+        ).length;
+        span.setAttributes({
+          'input.file_count': fileParts.length,
+          'input.pdf_file_count': inlinePdfFileCount,
+        });
         let response: ResolvedGenerationResponse;
         let textResponse: string;
 
         const messages = buildInitialMessages(
           systemPrompt,
           conversationHistory,
           userMessage,
-          imageParts
+          fileParts
         );
 
         const { originalMessageCount, compressor } = setupCompression(

agents-api/src/domains/run/agents/generation/tool-result-for-conversation-history.ts

@@ -1,6 +1,6 @@
 import type { MessageContent, Part } from '@inkeep/agents-core';
-import { makeMessageContentParts } from '../../services/blob-storage/image-upload';
-import { buildPersistedMessageContent } from '../../services/blob-storage/image-upload-helpers';
+import { makeMessageContentParts } from '../../services/blob-storage/file-upload';
+import { buildPersistedMessageContent } from '../../services/blob-storage/file-upload-helpers';
 import { isToolResultDenied } from '../../utils/tool-result';
 import type { AgentRunContext } from '../agent-types';
 

agents-api/src/domains/run/routes/chat.ts

@@ -20,11 +20,11 @@ import { flushBatchProcessor } from '../../../instrumentation';
 import { getLogger } from '../../../logger';
 import { contextValidationMiddleware, handleContextResolution } from '../context';
 import { ExecutionHandler } from '../handlers/executionHandler';
-import { buildPersistedMessageContent } from '../services/blob-storage/image-upload-helpers';
+import { buildPersistedMessageContent } from '../services/blob-storage/file-upload-helpers';
 import { toolApprovalUiBus } from '../session/ToolApprovalUiBus';
 import { createSSEStreamHelper } from '../stream/stream-helpers';
 import type { Message } from '../types/chat';
-import { ImageContentItemSchema } from '../types/chat';
+import { FileContentItemSchema, ImageContentItemSchema } from '../types/chat';
 import { errorOp } from '../utils/agent-operations';
 import { extractTextFromParts, getMessagePartsFromOpenAIContent } from '../utils/message-parts';
 
@@ -68,6 +68,7 @@ const chatCompletionsRoute = createProtectedRoute({
                             text: z.string(),
                           }),
                           ImageContentItemSchema,
+                          FileContentItemSchema,
                         ])
                       ),
                     ])