Add video support UserMessage

docs/services/chat-service.md

@@ -626,7 +626,7 @@ AssistantMessage assistant = chatService.chat(messages).toAssistantMessage();
 messages.add(assistant);
 
 while (assistant.hasToolCalls()) {
-    messages.addAll(assistant.processTools(toolRegistry::execute));
+    messages.addAll(assistant.processTools(toolRegistry));
     assistant = chatService.chat(messages).toAssistantMessage();
     messages.add(assistant);
 }

modules/watsonx-ai/src/main/java/com/ibm/watsonx/ai/chat/ToolRegistry.java

@@ -38,7 +38,7 @@
  *
  * List<ChatMessages> messages = ...
  * var assistantMessage = chatService.chat(messages).toAssistantMessage();
- * assistantMessage.processTools(toolRegistry::execute);
+ * assistantMessage.processTools(toolRegistry);
  * }</pre>
  *
  * @see ExecutableTool

modules/watsonx-ai/src/main/java/com/ibm/watsonx/ai/chat/model/UserMessage.java

@@ -151,6 +151,49 @@ public static UserMessage image(String text, InputStream is, String mimetype) {
         }
     }
 
+    /**
+     * Creates a new {@link UserMessage} containing text instructions and a video from a file.
+     *
+     * @param text the text instructions or prompt describing what to do with the video
+     * @param file the file containing the video data
+     * @return a new {@link UserMessage} containing {@link TextContent} and {@link VideoContent}
+     */
+    public static UserMessage video(String text, File file) {
+        return video(text, file.toPath());
+    }
+
+    /**
+     * Creates a new {@link UserMessage} containing text instructions and a video from a path.
+     *
+     * @param text the text instructions or prompt describing what to do with the video
+     * @param path the path to the file containing the video data
+     * @return a new {@link UserMessage} containing {@link TextContent} and {@link VideoContent}
+     */
+    public static UserMessage video(String text, Path path) {
+        try (var is = Files.newInputStream(path)) {
+            var mimeType = Files.probeContentType(path);
+            return video(text, is, mimeType);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Creates a new {@link UserMessage} containing text instructions and a video from an input stream.
+     *
+     * @param text the text instructions or prompt describing what to do with the video
+     * @param is the input stream containing the video data
+     * @param mimetype the MIME type of the video (e.g., "video/mp4", "video/mpeg")
+     * @return a new {@link UserMessage} containing {@link TextContent} and {@link VideoContent}
+     */
+    public static UserMessage video(String text, InputStream is, String mimetype) {
+        try {
+            return of(null, List.of(TextContent.of(text), VideoContent.from(is, mimetype)));
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     /**
      * Extracts the text content.
      * <p>

modules/watsonx-ai/src/test/java/com/ibm/watsonx/ai/DeploymentServiceTest.java

@@ -2101,7 +2101,7 @@ public void onCompleteToolCall(CompletedToolCall completeToolCall) {
 
         ChatResponse chatResponse = assertDoesNotThrow(() -> future.get());
         AssistantMessage assistantMessage = chatResponse.toAssistantMessage();
-        assistantMessage.processTools(toolRegistry::execute);
+        assistantMessage.processTools(toolRegistry);
         assertEquals(2, processedToolCallCount.get());
         assertEquals(2, completedToolCallCount.get());
         assertEquals("tool_calls", chatResponse.finishReason().value());

modules/watsonx-ai/src/test/java/com/ibm/watsonx/ai/chat/ChatMessageTest.java

@@ -18,6 +18,7 @@
 import com.ibm.watsonx.ai.chat.model.ImageContent;
 import com.ibm.watsonx.ai.chat.model.TextContent;
 import com.ibm.watsonx.ai.chat.model.UserMessage;
+import com.ibm.watsonx.ai.chat.model.VideoContent;
 
 public class ChatMessageTest {
 
@@ -67,4 +68,27 @@ void should_throw_runtime_exeception_when_image_file_is_corrupted() throws IOExc
         when(is.readAllBytes()).thenThrow(new IOException());
         assertThrows(RuntimeException.class, () -> UserMessage.image("Describe this image", is, "image/jpeg"));
     }
+
+    @Test
+    void should_create_an_user_message_with_text_and_video_contents() throws IOException, URISyntaxException {
+        var file = new File(ClassLoader.getSystemResource("cat.mp4").toURI());
+        var userMessage = UserMessage.video("Describe this video", file);
+        assertEquals(2, userMessage.content().size());
+        assertEquals("Describe this video", userMessage.content().get(0).toString());
+        var videoContent = ((VideoContent) userMessage.content().get(1)).videoUrl();
+        assertTrue(videoContent.url().startsWith("data:video/mp4;base64,"));
+    }
+
+    @Test
+    void should_throw_runtime_exception_when_video_file_does_not_exist() {
+        var file = new File("non-existent-file.mp4");
+        assertThrows(RuntimeException.class, () -> UserMessage.video("Describe this video", file));
+    }
+
+    @Test
+    void should_throw_runtime_exeception_when_video_file_is_corrupted() throws IOException, URISyntaxException {
+        var is = mock(InputStream.class);
+        when(is.readAllBytes()).thenThrow(new IOException());
+        assertThrows(RuntimeException.class, () -> UserMessage.video("Describe this image", is, "video/mp4"));
+    }
 }

modules/watsonx-ai/src/test/java/com/ibm/watsonx/ai/chat/ChatServiceStreamingTest.java

@@ -3880,7 +3880,7 @@ public void onCompleteToolCall(CompletedToolCall completeToolCall) {
 
         ChatResponse chatResponse = assertDoesNotThrow(() -> future.get());
         AssistantMessage assistantMessage = chatResponse.toAssistantMessage();
-        assistantMessage.processTools(toolRegistry::execute);
+        assistantMessage.processTools(toolRegistry);
         assertEquals(2, processedToolCallCount.get());
         assertEquals(2, completedToolCallCount.get());
         assertEquals("tool_calls", chatResponse.finishReason().value());

modules/watsonx-ai/src/test/java/com/ibm/watsonx/ai/it/DeploymentServiceIT.java

@@ -11,6 +11,7 @@
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.Duration;
 import java.util.HashMap;
@@ -59,6 +60,7 @@ public class DeploymentServiceIT {
     static final String API_KEY = System.getenv("WATSONX_API_KEY");
     static final String DEPLOYMENT_ID = System.getenv("WATSONX_DEPLOYMENT_ID");
     static final String URL = System.getenv("WATSONX_URL");
+    static final String NVIDIA_DEPLOYMENT_ID = System.getenv("WATSONX_NVIDIA_DEPLOYMENT_ID");
 
     static final Authenticator authentication = IBMCloudAuthenticator.builder()
         .apiKey(API_KEY)
@@ -346,6 +348,30 @@ void should_force_tool_execution_when_tool_choice_option_is_set_to_required() {
             assertNotNull(assistantMessage.toolCalls());
             assertEquals(1, assistantMessage.toolCalls().size());
         }
+
+        @Test
+        @EnabledIfEnvironmentVariable(named = "WATSONX_NVIDIA_DEPLOYMENT_ID", matches = ".+")
+        void should_return_description_when_video_is_sent_in_chat() {
+
+            var video = assertDoesNotThrow(() -> Path.of(getClass().getClassLoader().getResource("cat.mp4").toURI()));
+
+            var deploymentService = DeploymentService.builder()
+                .baseUrl(URL)
+                .authenticator(authentication)
+                .logRequests(true)
+                .logResponses(true)
+                .build();
+
+            ChatRequest request = ChatRequest.builder()
+                .deploymentId(NVIDIA_DEPLOYMENT_ID)
+                .messages(UserMessage.video("Tell me more about this video", video))
+                .build();
+
+            var chatResponse = assertDoesNotThrow(() -> deploymentService.chat(request));
+            var assistantMessage = chatResponse.toAssistantMessage();
+            assertFalse(assistantMessage.content().isBlank());
+            assertTrue(assistantMessage.content().contains("cat"));
+        }
     }
 
     @Nested

modules/watsonx-ai/src/test/java/com/ibm/watsonx/ai/it/ToolServiceIT.java

@@ -298,7 +298,7 @@ void should_use_rag_query_tool() {
         messages.add(assistantMessage);
 
         if (assistantMessage.hasToolCalls()) {
-            var toolMessage = assistantMessage.processTools(toolRegistry::execute);
+            var toolMessage = assistantMessage.processTools(toolRegistry);
             messages.addAll(toolMessage);
             assistantMessage = chatService.chat(messages).toAssistantMessage();
         }