From fe2a5fac76d86b296705d22d9c41786fda55a7f3 Mon Sep 17 00:00:00 2001
From: Luke Schlangen <lukeschlangen@gmail.com>
Date: Mon, 5 Feb 2024 15:10:48 -0600
Subject: [PATCH 1/5] chore: remove .preview from generative ai methods

chore: update ai package versions

fix: fix broken tests

fix: update dependency to use github over npm

feat: change assertion to more common output

added function calling examples

removed double await, fix linter
---
 generative-ai/snippets/countTokens.js         |  4 +-
 .../snippets/functionCallingStreamChat.js     | 99 +++++++++++++++++++
 .../snippets/functionCallingStreamContent.js  | 85 ++++++++++++++++
 generative-ai/snippets/nonStreamingChat.js    | 17 ++--
 generative-ai/snippets/nonStreamingContent.js |  2 +-
 .../snippets/nonStreamingMultipartContent.js  |  4 +-
 generative-ai/snippets/package.json           |  2 +-
 generative-ai/snippets/safetySettings.js      |  5 +-
 .../snippets/sendMultiModalPromptWithImage.js |  2 +-
 .../snippets/sendMultiModalPromptWithVideo.js |  2 +-
 generative-ai/snippets/streamChat.js          |  2 +-
 generative-ai/snippets/streamContent.js       |  2 +-
 .../snippets/streamMultipartContent.js        |  4 +-
 .../test/functionCallingStreamChat.test.js    | 36 +++++++
 .../test/functionCallingStreamContent.test.js | 36 +++++++
 .../sendMultiModalPromptWithImage.test.js     |  2 +-
 .../sendMultiModalPromptWithVideo.test.js     |  2 +-
 17 files changed, 283 insertions(+), 23 deletions(-)
 create mode 100644 generative-ai/snippets/functionCallingStreamChat.js
 create mode 100644 generative-ai/snippets/functionCallingStreamContent.js
 create mode 100644 generative-ai/snippets/test/functionCallingStreamChat.test.js
 create mode 100644 generative-ai/snippets/test/functionCallingStreamContent.test.js

diff --git a/generative-ai/snippets/countTokens.js b/generative-ai/snippets/countTokens.js
index 492be74da3..40ce023064 100644
--- a/generative-ai/snippets/countTokens.js
+++ b/generative-ai/snippets/countTokens.js
@@ -24,10 +24,10 @@ async function countTokens(
   model = 'gemini-pro'
 ) {
   // Initialize Vertex with your Cloud project and location
-  const vertex_ai = new VertexAI({project: projectId, location: location});
+  const vertexAI = new VertexAI({project: projectId, location: location});
 
   // Instantiate the model
-  const generativeModel = vertex_ai.preview.getGenerativeModel({
+  const generativeModel = vertexAI.getGenerativeModel({
     model: model,
   });
 
diff --git a/generative-ai/snippets/functionCallingStreamChat.js b/generative-ai/snippets/functionCallingStreamChat.js
new file mode 100644
index 0000000000..e1a2dabcc4
--- /dev/null
+++ b/generative-ai/snippets/functionCallingStreamChat.js
@@ -0,0 +1,99 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// [START aiplatform_gemini_function_calling_chat]
+const {
+    VertexAI,
+    FunctionDeclarationSchemaType
+} = require('@google-cloud/vertexai');
+
+const functionDeclarations = [
+  {
+    function_declarations: [
+      {
+        name: 'get_current_weather',
+        description: 'get weather in a given location',
+        parameters: {
+          type: FunctionDeclarationSchemaType.OBJECT,
+          properties: {
+            location: { type: FunctionDeclarationSchemaType.STRING },
+            unit: {
+              type: FunctionDeclarationSchemaType.STRING,
+              enum: ['celsius', 'fahrenheit'],
+            },
+          },
+          required: ['location'],
+        },
+      },
+    ],
+  },
+];
+
+const functionResponseParts = [
+  {
+    functionResponse: {
+      name: 'get_current_weather',
+      response:
+        { name: 'get_current_weather', content: { weather: 'super nice' } },
+    },
+  },
+];
+
+/**
+ * TODO(developer): Update these variables before running the sample.
+ */
+async function functionCallingStreamChat(
+  projectId = 'PROJECT_ID',
+  location = 'us-central1',
+  model = 'gemini-pro'
+) {
+  // Initialize Vertex with your Cloud project and location
+  const vertexAI = new VertexAI({ project: projectId, location: location });
+
+  // Instantiate the model
+  const generativeModel = vertexAI.preview.getGenerativeModel({
+    model: model,
+  });
+
+  // Create a chat session and pass your function declarations
+  const chat = generativeModel.startChat({
+    tools: functionDeclarations,
+  });
+
+  const chatInput1 = 'What is the weather in Boston?';
+
+  // This should include a functionCall response from the model
+  const result1 = await chat.sendMessageStream(chatInput1);
+  for await (const item of result1.stream) {
+    console.log(item.candidates[0]);
+  }
+  const response1 = await result1.response;
+
+  // Send a follow up message with a FunctionResponse
+  const result2 = await chat.sendMessageStream(functionResponseParts);
+  for await (const item of result2.stream) {
+    console.log(item.candidates[0]);
+  }
+
+  // This should include a text response from the model using the response content
+  // provided above
+  const response2 = await result2.response;
+  console.log(response2.candidates[0].content.parts[0].text);
+}
+// [END aiplatform_gemini_function_calling_chat]
+
+functionCallingStreamChat(...process.argv.slice(2)).catch(err => {
+  console.error(err.message);
+  process.exitCode = 1;
+});
diff --git a/generative-ai/snippets/functionCallingStreamContent.js b/generative-ai/snippets/functionCallingStreamContent.js
new file mode 100644
index 0000000000..dead7b088b
--- /dev/null
+++ b/generative-ai/snippets/functionCallingStreamContent.js
@@ -0,0 +1,85 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// [START aiplatform_gemini_function_calling_content]
+const { VertexAI, FunctionDeclarationSchemaType } = require('@google-cloud/vertexai');
+
+const functionDeclarations = [
+    {
+      function_declarations: [
+        {
+          name: 'get_current_weather',
+          description: 'get weather in a given location',
+          parameters: {
+            type: FunctionDeclarationSchemaType.OBJECT,
+            properties: {
+              location: {type: FunctionDeclarationSchemaType.STRING},
+              unit: {
+                type: FunctionDeclarationSchemaType.STRING,
+                enum: ['celsius', 'fahrenheit'],
+              },
+            },
+            required: ['location'],
+          },
+        },
+      ],
+    },
+  ];
+  
+  const functionResponseParts = [
+    {
+      functionResponse: {
+        name: 'get_current_weather',
+        response:
+            {name: 'get_current_weather', content: {weather: 'super nice'}},
+      },
+    },
+  ];
+
+/**
+ * TODO(developer): Update these variables before running the sample.
+ */
+async function functionCallingStreamChat(
+    projectId = 'PROJECT_ID',
+    location = 'us-central1',
+    model = 'gemini-pro'
+) {
+    // Initialize Vertex with your Cloud project and location
+    const vertexAI = new VertexAI({ project: projectId, location: location });
+
+    // Instantiate the model
+    const generativeModel = vertexAI.preview.getGenerativeModel({
+        model: model,
+    });
+
+    const request = {
+        contents: [
+          {role: 'user', parts: [{text: 'What is the weather in Boston?'}]},
+          {role: 'model', parts: [{functionCall: {name: 'get_current_weather', args: {'location': 'Boston'}}}]},
+          {role: 'function', parts: functionResponseParts}
+        ],
+        tools: functionDeclarations,
+      };
+      const streamingResp =
+          await generativeModel.generateContentStream(request);
+      for await (const item of streamingResp.stream) {
+        console.log(item.candidates[0].content.parts[0].text);
+      }    
+}
+// [END aiplatform_gemini_function_calling_content]
+
+functionCallingStreamChat(...process.argv.slice(2)).catch(err => {
+    console.error(err.message);
+    process.exitCode = 1;
+});
diff --git a/generative-ai/snippets/nonStreamingChat.js b/generative-ai/snippets/nonStreamingChat.js
index 0199074f44..8d2af6609b 100644
--- a/generative-ai/snippets/nonStreamingChat.js
+++ b/generative-ai/snippets/nonStreamingChat.js
@@ -27,7 +27,7 @@ async function createNonStreamingChat(
   const vertexAI = new VertexAI({project: projectId, location: location});
 
   // Instantiate the model
-  const generativeModel = vertexAI.preview.getGenerativeModel({
+  const generativeModel = vertexAI.getGenerativeModel({
     model: model,
   });
 
@@ -37,20 +37,23 @@ async function createNonStreamingChat(
   console.log(`User: ${chatInput1}`);
 
   const result1 = await chat.sendMessage(chatInput1);
-  const response1 = result1.response.candidates[0].content.parts[0].text;
-  console.log('Chat bot: ', response1);
+  const response1 = result1.response;
+  const text1 = response1.candidates[0].content.parts[0].text;
+  console.log('Chat bot: ', text1);
 
   const chatInput2 = 'Can you tell me a scientific fun fact?';
   console.log(`User: ${chatInput2}`);
   const result2 = await chat.sendMessage(chatInput2);
-  const response2 = result2.response.candidates[0].content.parts[0].text;
-  console.log('Chat bot: ', response2);
+  const response2 = await result2.response;
+  const text2 = response2.candidates[0].content.parts[0].text;
+  console.log('Chat bot: ', text2);
 
   const chatInput3 = 'How can I learn more about that?';
   console.log(`User: ${chatInput3}`);
   const result3 = await chat.sendMessage(chatInput3);
-  const response3 = result3.response.candidates[0].content.parts[0].text;
-  console.log('Chat bot: ', response3);
+  const response3 = await result3.response;
+  const text3 = response3.candidates[0].content.parts[0].text;
+  console.log('Chat bot: ', text3);
 }
 // [END aiplatform_gemini_multiturn_chat_nonstreaming]
 
diff --git a/generative-ai/snippets/nonStreamingContent.js b/generative-ai/snippets/nonStreamingContent.js
index 1d82a7d7bb..65b4dc80ea 100644
--- a/generative-ai/snippets/nonStreamingContent.js
+++ b/generative-ai/snippets/nonStreamingContent.js
@@ -27,7 +27,7 @@ async function createNonStreamingContent(
   const vertexAI = new VertexAI({project: projectId, location: location});
 
   // Instantiate the model
-  const generativeModel = vertexAI.preview.getGenerativeModel({
+  const generativeModel = vertexAI.getGenerativeModel({
     model: model,
   });
 
diff --git a/generative-ai/snippets/nonStreamingMultipartContent.js b/generative-ai/snippets/nonStreamingMultipartContent.js
index d281919313..962339fd8e 100644
--- a/generative-ai/snippets/nonStreamingMultipartContent.js
+++ b/generative-ai/snippets/nonStreamingMultipartContent.js
@@ -29,7 +29,7 @@ async function createNonStreamingMultipartContent(
   const vertexAI = new VertexAI({project: projectId, location: location});
 
   // Instantiate the model
-  const generativeVisionModel = vertexAI.preview.getGenerativeModel({
+  const generativeVisionModel = vertexAI.getGenerativeModel({
     model: model,
   });
 
@@ -50,7 +50,7 @@ async function createNonStreamingMultipartContent(
   };
 
   console.log('Prompt Text:');
-  console.log(request.contents[0].parts[0].text);
+  console.log(request.contents[0].parts[1].text);
 
   console.log('Non-Streaming Response Text:');
   // Create the response stream
diff --git a/generative-ai/snippets/package.json b/generative-ai/snippets/package.json
index 29a62f5f37..59bb0948c1 100644
--- a/generative-ai/snippets/package.json
+++ b/generative-ai/snippets/package.json
@@ -13,7 +13,7 @@
     "test": "c8 mocha -p -j 2 --timeout 2400000 test/*.test.js"
   },
   "dependencies": {
-    "@google-cloud/aiplatform": "^3.0.0",
+    "@google-cloud/aiplatform": "^3.12.0",
     "@google-cloud/vertexai": "github:googleapis/nodejs-vertexai",
     "axios": "^1.6.2",
     "supertest": "^6.3.3"
diff --git a/generative-ai/snippets/safetySettings.js b/generative-ai/snippets/safetySettings.js
index ac4509f646..306fa54db5 100644
--- a/generative-ai/snippets/safetySettings.js
+++ b/generative-ai/snippets/safetySettings.js
@@ -31,7 +31,7 @@ async function setSafetySettings(
   const vertexAI = new VertexAI({project: projectId, location: location});
 
   // Instantiate the model
-  const generativeModel = vertexAI.preview.getGenerativeModel({
+  const generativeModel = vertexAI.getGenerativeModel({
     model: model,
     // The following parameters are optional
     // They can also be passed to individual content generation requests
@@ -59,6 +59,7 @@ async function setSafetySettings(
   for await (const item of responseStream.stream) {
     if (item.candidates[0].finishReason === 'SAFETY') {
       console.log('This response stream terminated due to safety concerns.');
+      break;
     } else {
       process.stdout.write(item.candidates[0].content.parts[0].text);
     }
@@ -66,7 +67,7 @@ async function setSafetySettings(
 }
 // [END aiplatform_gemini_safety_settings]
 
-setSafetySettings(...process.argv.slice(3)).catch(err => {
+setSafetySettings(...process.argv.slice(2)).catch(err => {
   console.error(err.message);
   process.exitCode = 1;
 });
diff --git a/generative-ai/snippets/sendMultiModalPromptWithImage.js b/generative-ai/snippets/sendMultiModalPromptWithImage.js
index b6dcb09aa7..acc4514b28 100644
--- a/generative-ai/snippets/sendMultiModalPromptWithImage.js
+++ b/generative-ai/snippets/sendMultiModalPromptWithImage.js
@@ -43,7 +43,7 @@ async function sendMultiModalPromptWithImage(
   // Initialize Vertex with your Cloud project and location
   const vertexAI = new VertexAI({project: projectId, location: location});
 
-  const generativeVisionModel = vertexAI.preview.getGenerativeModel({
+  const generativeVisionModel = vertexAI.getGenerativeModel({
     model: model,
   });
 
diff --git a/generative-ai/snippets/sendMultiModalPromptWithVideo.js b/generative-ai/snippets/sendMultiModalPromptWithVideo.js
index cd777cbf12..a6c4dad1cc 100644
--- a/generative-ai/snippets/sendMultiModalPromptWithVideo.js
+++ b/generative-ai/snippets/sendMultiModalPromptWithVideo.js
@@ -26,7 +26,7 @@ async function sendMultiModalPromptWithVideo(
   // Initialize Vertex with your Cloud project and location
   const vertexAI = new VertexAI({project: projectId, location: location});
 
-  const generativeVisionModel = vertexAI.preview.getGenerativeModel({
+  const generativeVisionModel = vertexAI.getGenerativeModel({
     model: model,
   });
 
diff --git a/generative-ai/snippets/streamChat.js b/generative-ai/snippets/streamChat.js
index bf6edcfa40..7400774dc8 100644
--- a/generative-ai/snippets/streamChat.js
+++ b/generative-ai/snippets/streamChat.js
@@ -27,7 +27,7 @@ async function createStreamChat(
   const vertexAI = new VertexAI({project: projectId, location: location});
 
   // Instantiate the model
-  const generativeModel = vertexAI.preview.getGenerativeModel({
+  const generativeModel = vertexAI.getGenerativeModel({
     model: model,
   });
 
diff --git a/generative-ai/snippets/streamContent.js b/generative-ai/snippets/streamContent.js
index 9004c6b515..cfaf156e5d 100644
--- a/generative-ai/snippets/streamContent.js
+++ b/generative-ai/snippets/streamContent.js
@@ -27,7 +27,7 @@ async function createStreamContent(
   const vertexAI = new VertexAI({project: projectId, location: location});
 
   // Instantiate the model
-  const generativeModel = vertexAI.preview.getGenerativeModel({
+  const generativeModel = vertexAI.getGenerativeModel({
     model: model,
   });
 
diff --git a/generative-ai/snippets/streamMultipartContent.js b/generative-ai/snippets/streamMultipartContent.js
index b336258c29..670a0b13c2 100644
--- a/generative-ai/snippets/streamMultipartContent.js
+++ b/generative-ai/snippets/streamMultipartContent.js
@@ -29,7 +29,7 @@ async function createStreamMultipartContent(
   const vertexAI = new VertexAI({project: projectId, location: location});
 
   // Instantiate the model
-  const generativeVisionModel = vertexAI.preview.getGenerativeModel({
+  const generativeVisionModel = vertexAI.getGenerativeModel({
     model: model,
   });
 
@@ -50,7 +50,7 @@ async function createStreamMultipartContent(
   };
 
   console.log('Prompt Text:');
-  console.log(request.contents[0].parts[0].text);
+  console.log(request.contents[0].parts[1].text);
   console.log('Streaming Response Text:');
 
   // Create the response stream
diff --git a/generative-ai/snippets/test/functionCallingStreamChat.test.js b/generative-ai/snippets/test/functionCallingStreamChat.test.js
new file mode 100644
index 0000000000..482d79bd40
--- /dev/null
+++ b/generative-ai/snippets/test/functionCallingStreamChat.test.js
@@ -0,0 +1,36 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict';
+
+const {assert} = require('chai');
+const {describe, it} = require('mocha');
+const cp = require('child_process');
+
+const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'});
+
+describe('Generative AI Function Calling Stream Chat', () => {
+  const project = 'cloud-llm-preview1';
+  const location = 'us-central1';
+  const model = 'gemini-pro';
+
+  it('should create stream chat and begin the conversation the same in each instance', async () => {
+    const output = execSync(
+      `node ./functionCallingStreamChat.js ${project} ${location} ${model}`
+    );
+
+    // Assert that the response is what we expect
+    assert(output.match(/The weather in Boston is super nice./));
+  });
+});
diff --git a/generative-ai/snippets/test/functionCallingStreamContent.test.js b/generative-ai/snippets/test/functionCallingStreamContent.test.js
new file mode 100644
index 0000000000..f80ea029d8
--- /dev/null
+++ b/generative-ai/snippets/test/functionCallingStreamContent.test.js
@@ -0,0 +1,36 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict';
+
+const {assert} = require('chai');
+const {describe, it} = require('mocha');
+const cp = require('child_process');
+
+const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'});
+
+describe('Generative AI Function Calling Stream Content', () => {
+  const project = 'cloud-llm-preview1';
+  const location = 'us-central1';
+  const model = 'gemini-pro';
+
+  it('should create stream chat and begin the conversation the same in each instance', async () => {
+    const output = execSync(
+      `node ./functionCallingStreamContent.js ${project} ${location} ${model}`
+    );
+
+    // Assert that the response is what we expect
+    assert(output.match(/The weather in Boston is super nice./));
+  });
+});
diff --git a/generative-ai/snippets/test/sendMultiModalPromptWithImage.test.js b/generative-ai/snippets/test/sendMultiModalPromptWithImage.test.js
index a421893228..414fff580f 100644
--- a/generative-ai/snippets/test/sendMultiModalPromptWithImage.test.js
+++ b/generative-ai/snippets/test/sendMultiModalPromptWithImage.test.js
@@ -30,6 +30,6 @@ describe('Generative AI Stream MultiModal with Image', () => {
       `node ./sendMultiModalPromptWithImage.js ${project} ${location} ${model}`
     );
     // Ensure that the conversation is what we expect for these images
-    assert(output.match(/Paris/));
+    assert(output.match(/city: Rio de Janeiro, Landmark: Christ the Redeemer/));
   });
 });
diff --git a/generative-ai/snippets/test/sendMultiModalPromptWithVideo.test.js b/generative-ai/snippets/test/sendMultiModalPromptWithVideo.test.js
index 91d6fedb56..17e3e07bef 100644
--- a/generative-ai/snippets/test/sendMultiModalPromptWithVideo.test.js
+++ b/generative-ai/snippets/test/sendMultiModalPromptWithVideo.test.js
@@ -30,6 +30,6 @@ describe('Generative AI Stream MultiModal with Video', () => {
       `node ./sendMultiModalPromptWithVideo.js ${project} ${location} ${model}`
     );
     // Ensure that the conversation is what we expect for these images
-    assert(output.match(/advertisement/));
+    assert(output.match(/Zootopia/));
   });
 });

From 7af8d9de8efcef09df46fb049e85a2adc06a74ff Mon Sep 17 00:00:00 2001
From: Eric Dong <itseric@google.com>
Date: Wed, 7 Feb 2024 14:16:16 -0500
Subject: [PATCH 2/5] feat: readd GenAI embedding code samples (#3616)

Co-authored-by: Patti Shin <pattishin@users.noreply.github.com>
---
 .../predict-image-from-image-and-text.js      | 96 +++++++++++++++++++
 .../snippets/predict-image-from-text.js       | 81 ++++++++++++++++
 .../predict-image-from-image-and-text.test.js | 55 +++++++++++
 .../test/predict-image-from-text.test.js      | 48 ++++++++++
 4 files changed, 280 insertions(+)
 create mode 100644 ai-platform/snippets/predict-image-from-image-and-text.js
 create mode 100644 ai-platform/snippets/predict-image-from-text.js
 create mode 100644 ai-platform/snippets/test/predict-image-from-image-and-text.test.js
 create mode 100644 ai-platform/snippets/test/predict-image-from-text.test.js

diff --git a/ai-platform/snippets/predict-image-from-image-and-text.js b/ai-platform/snippets/predict-image-from-image-and-text.js
new file mode 100644
index 0000000000..1b16df13dc
--- /dev/null
+++ b/ai-platform/snippets/predict-image-from-image-and-text.js
@@ -0,0 +1,96 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+async function main(
+  project,
+  location = 'us-central1',
+  baseImagePath,
+  textPrompt
+) {
+  // [START aiplatform_sdk_text_image_embedding]
+  /**
+   * TODO(developer): Uncomment these variables before running the sample.\
+   * (Not necessary if passing values as arguments)
+   */
+  // const project = 'YOUR_PROJECT_ID';
+  // const location = 'YOUR_PROJECT_LOCATION';
+  // const bastImagePath = "YOUR_BASED_IMAGE_PATH"
+  // const textPrompt = 'YOUR_TEXT_PROMPT';
+  const aiplatform = require('@google-cloud/aiplatform');
+
+  // Imports the Google Cloud Prediction service client
+  const {PredictionServiceClient} = aiplatform.v1;
+
+  // Import the helper module for converting arbitrary protobuf.Value objects.
+  const {helpers} = aiplatform;
+
+  // Specifies the location of the api endpoint
+  const clientOptions = {
+    apiEndpoint: 'us-central1-aiplatform.googleapis.com',
+  };
+  const publisher = 'google';
+  const model = 'multimodalembedding@001';
+
+  // Instantiates a client
+  const predictionServiceClient = new PredictionServiceClient(clientOptions);
+
+  async function predictImageFromImageAndText() {
+    // Configure the parent resource
+    const endpoint = `projects/${project}/locations/${location}/publishers/${publisher}/models/${model}`;
+
+    const fs = require('fs');
+    const imageFile = fs.readFileSync(baseImagePath);
+
+    // Convert the image data to a Buffer and base64 encode it.
+    const encodedImage = Buffer.from(imageFile).toString('base64');
+
+    const prompt = {
+      text: textPrompt,
+      image: {
+        bytesBase64Encoded: encodedImage,
+      },
+    };
+    const instanceValue = helpers.toValue(prompt);
+    const instances = [instanceValue];
+
+    const parameter = {
+      sampleCount: 1,
+    };
+    const parameters = helpers.toValue(parameter);
+
+    const request = {
+      endpoint,
+      instances,
+      parameters,
+    };
+
+    // Predict request
+    const [response] = await predictionServiceClient.predict(request);
+    console.log('Get image embedding response');
+    const predictions = response.predictions;
+    console.log('\tPredictions :');
+    for (const prediction of predictions) {
+      console.log(`\t\tPrediction : ${JSON.stringify(prediction)}`);
+    }
+  }
+
+  await predictImageFromImageAndText();
+  // [END aiplatform_sdk_text_image_embedding]
+}
+
+exports.predictImageFromImageAndText = main;
diff --git a/ai-platform/snippets/predict-image-from-text.js b/ai-platform/snippets/predict-image-from-text.js
new file mode 100644
index 0000000000..c12c97399e
--- /dev/null
+++ b/ai-platform/snippets/predict-image-from-text.js
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+async function main(project, location = 'us-central1', textPrompt) {
+  // [START aiplatform_sdk_text_embedding]
+  /**
+   * TODO(developer): Uncomment these variables before running the sample.\
+   * (Not necessary if passing values as arguments)
+   */
+  // const project = 'YOUR_PROJECT_ID';
+  // const location = 'YOUR_PROJECT_LOCATION';
+  // const textPrompt = 'YOUR_TEXT_PROMPT';
+  const aiplatform = require('@google-cloud/aiplatform');
+
+  // Imports the Google Cloud Prediction service client
+  const {PredictionServiceClient} = aiplatform.v1;
+
+  // Import the helper module for converting arbitrary protobuf.Value objects.
+  const {helpers} = aiplatform;
+
+  // Specifies the location of the api endpoint
+  const clientOptions = {
+    apiEndpoint: 'us-central1-aiplatform.googleapis.com',
+  };
+  const publisher = 'google';
+  const model = 'multimodalembedding@001';
+
+  // Instantiates a client
+  const predictionServiceClient = new PredictionServiceClient(clientOptions);
+
+  async function predictImageFromText() {
+    // Configure the parent resource
+    const endpoint = `projects/${project}/locations/${location}/publishers/${publisher}/models/${model}`;
+
+    const prompt = {
+      text: textPrompt,
+    };
+    const instanceValue = helpers.toValue(prompt);
+    const instances = [instanceValue];
+
+    const parameter = {
+      sampleCount: 1,
+    };
+    const parameters = helpers.toValue(parameter);
+
+    const request = {
+      endpoint,
+      instances,
+      parameters,
+    };
+
+    // Predict request
+    const [response] = await predictionServiceClient.predict(request);
+    console.log('Get image embedding response');
+    const predictions = response.predictions;
+    console.log('\tPredictions :');
+    for (const prediction of predictions) {
+      console.log(`\t\tPrediction : ${JSON.stringify(prediction)}`);
+    }
+  }
+
+  await predictImageFromText();
+  // [END aiplatform_sdk_text_embedding]
+}
+
+exports.predictImageFromText = main;
diff --git a/ai-platform/snippets/test/predict-image-from-image-and-text.test.js b/ai-platform/snippets/test/predict-image-from-image-and-text.test.js
new file mode 100644
index 0000000000..ff1eeb2997
--- /dev/null
+++ b/ai-platform/snippets/test/predict-image-from-image-and-text.test.js
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+const {assert} = require('chai');
+const {describe, it} = require('mocha');
+const sinon = require('sinon');
+
+const {
+  predictImageFromImageAndText,
+} = require('../predict-image-from-image-and-text');
+
+const project = process.env.CAIP_PROJECT_ID;
+const LOCATION = 'us-central1';
+const baseImagePath = 'resources/daisy.jpg';
+const textPrompt = 'an impressionist painting';
+
+describe('AI platform generates image from image and text', async () => {
+  const stubConsole = function () {
+    sinon.stub(console, 'error');
+    sinon.stub(console, 'log');
+  };
+
+  const restoreConsole = function () {
+    console.log.restore();
+    console.error.restore();
+  };
+
+  beforeEach(stubConsole);
+  afterEach(restoreConsole);
+
+  it('should make predictions using a large language model', async () => {
+    await predictImageFromImageAndText(
+      project,
+      LOCATION,
+      baseImagePath,
+      textPrompt
+    );
+    assert.include(console.log.firstCall.args, 'Get image embedding response');
+  });
+});
diff --git a/ai-platform/snippets/test/predict-image-from-text.test.js b/ai-platform/snippets/test/predict-image-from-text.test.js
new file mode 100644
index 0000000000..4cc161447f
--- /dev/null
+++ b/ai-platform/snippets/test/predict-image-from-text.test.js
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+const {assert} = require('chai');
+const {describe, it} = require('mocha');
+const sinon = require('sinon');
+
+const {predictImageFromText} = require('../predict-image-from-text');
+
+const project = process.env.CAIP_PROJECT_ID;
+const LOCATION = 'us-central1';
+const textPrompt =
+  'small red boat on water in the morning watercolor illustration muted colors';
+
+describe('AI platform generates image from text', async () => {
+  const stubConsole = function () {
+    sinon.stub(console, 'error');
+    sinon.stub(console, 'log');
+  };
+
+  const restoreConsole = function () {
+    console.log.restore();
+    console.error.restore();
+  };
+
+  beforeEach(stubConsole);
+  afterEach(restoreConsole);
+
+  it('should make predictions using a large language model', async () => {
+    await predictImageFromText(project, LOCATION, textPrompt);
+    assert.include(console.log.firstCall.args, 'Get image embedding response');
+  });
+});

From c8d11eaa7e26a1418b2c83f2858171ff3d8381d7 Mon Sep 17 00:00:00 2001
From: Patti Shin <pattishin@users.noreply.github.com>
Date: Wed, 14 Feb 2024 14:46:00 -0500
Subject: [PATCH 3/5] fix: adding in type:module in package.docs.json (#3624)

chore: attempt to fix linting errors

chore: attempt to fix linting errors

chore: attempt to lint functionCallingStreamChat

chore: lint functionCallingStreamContent

chore: lint functionCallingStreamContent

chore: lint functionCallingStreamContent

chore: lint functionCallingStreamContent

chore: lint functionCallingStreamContent
---
 .../snippets/functionCallingStreamChat.js     |  13 +--
 .../snippets/functionCallingStreamContent.js  | 110 ++++++++++--------
 run/helloworld/package.docs.json              |   1 +
 3 files changed, 68 insertions(+), 56 deletions(-)

diff --git a/generative-ai/snippets/functionCallingStreamChat.js b/generative-ai/snippets/functionCallingStreamChat.js
index e1a2dabcc4..62caf5d12c 100644
--- a/generative-ai/snippets/functionCallingStreamChat.js
+++ b/generative-ai/snippets/functionCallingStreamChat.js
@@ -14,8 +14,8 @@
 
 // [START aiplatform_gemini_function_calling_chat]
 const {
-    VertexAI,
-    FunctionDeclarationSchemaType
+  VertexAI,
+  FunctionDeclarationSchemaType,
 } = require('@google-cloud/vertexai');
 
 const functionDeclarations = [
@@ -27,7 +27,7 @@ const functionDeclarations = [
         parameters: {
           type: FunctionDeclarationSchemaType.OBJECT,
           properties: {
-            location: { type: FunctionDeclarationSchemaType.STRING },
+            location: {type: FunctionDeclarationSchemaType.STRING},
             unit: {
               type: FunctionDeclarationSchemaType.STRING,
               enum: ['celsius', 'fahrenheit'],
@@ -44,8 +44,7 @@ const functionResponseParts = [
   {
     functionResponse: {
       name: 'get_current_weather',
-      response:
-        { name: 'get_current_weather', content: { weather: 'super nice' } },
+      response: {name: 'get_current_weather', content: {weather: 'super nice'}},
     },
   },
 ];
@@ -59,7 +58,7 @@ async function functionCallingStreamChat(
   model = 'gemini-pro'
 ) {
   // Initialize Vertex with your Cloud project and location
-  const vertexAI = new VertexAI({ project: projectId, location: location });
+  const vertexAI = new VertexAI({project: projectId, location: location});
 
   // Instantiate the model
   const generativeModel = vertexAI.preview.getGenerativeModel({
@@ -78,7 +77,7 @@ async function functionCallingStreamChat(
   for await (const item of result1.stream) {
     console.log(item.candidates[0]);
   }
-  const response1 = await result1.response;
+  await result1.response;
 
   // Send a follow up message with a FunctionResponse
   const result2 = await chat.sendMessageStream(functionResponseParts);
diff --git a/generative-ai/snippets/functionCallingStreamContent.js b/generative-ai/snippets/functionCallingStreamContent.js
index dead7b088b..c0b87624f5 100644
--- a/generative-ai/snippets/functionCallingStreamContent.js
+++ b/generative-ai/snippets/functionCallingStreamContent.js
@@ -13,73 +13,85 @@
 // limitations under the License.
 
 // [START aiplatform_gemini_function_calling_content]
-const { VertexAI, FunctionDeclarationSchemaType } = require('@google-cloud/vertexai');
+const {
+  VertexAI,
+  FunctionDeclarationSchemaType,
+} = require('@google-cloud/vertexai');
 
 const functionDeclarations = [
-    {
-      function_declarations: [
-        {
-          name: 'get_current_weather',
-          description: 'get weather in a given location',
-          parameters: {
-            type: FunctionDeclarationSchemaType.OBJECT,
-            properties: {
-              location: {type: FunctionDeclarationSchemaType.STRING},
-              unit: {
-                type: FunctionDeclarationSchemaType.STRING,
-                enum: ['celsius', 'fahrenheit'],
-              },
+  {
+    function_declarations: [
+      {
+        name: 'get_current_weather',
+        description: 'get weather in a given location',
+        parameters: {
+          type: FunctionDeclarationSchemaType.OBJECT,
+          properties: {
+            location: {type: FunctionDeclarationSchemaType.STRING},
+            unit: {
+              type: FunctionDeclarationSchemaType.STRING,
+              enum: ['celsius', 'fahrenheit'],
             },
-            required: ['location'],
           },
+          required: ['location'],
         },
-      ],
-    },
-  ];
-  
-  const functionResponseParts = [
-    {
-      functionResponse: {
-        name: 'get_current_weather',
-        response:
-            {name: 'get_current_weather', content: {weather: 'super nice'}},
       },
+    ],
+  },
+];
+
+const functionResponseParts = [
+  {
+    functionResponse: {
+      name: 'get_current_weather',
+      response: {name: 'get_current_weather', content: {weather: 'super nice'}},
     },
-  ];
+  },
+];
 
 /**
  * TODO(developer): Update these variables before running the sample.
  */
 async function functionCallingStreamChat(
-    projectId = 'PROJECT_ID',
-    location = 'us-central1',
-    model = 'gemini-pro'
+  projectId = 'PROJECT_ID',
+  location = 'us-central1',
+  model = 'gemini-pro'
 ) {
-    // Initialize Vertex with your Cloud project and location
-    const vertexAI = new VertexAI({ project: projectId, location: location });
+  // Initialize Vertex with your Cloud project and location
+  const vertexAI = new VertexAI({project: projectId, location: location});
 
-    // Instantiate the model
-    const generativeModel = vertexAI.preview.getGenerativeModel({
-        model: model,
-    });
+  // Instantiate the model
+  const generativeModel = vertexAI.preview.getGenerativeModel({
+    model: model,
+  });
 
-    const request = {
-        contents: [
-          {role: 'user', parts: [{text: 'What is the weather in Boston?'}]},
-          {role: 'model', parts: [{functionCall: {name: 'get_current_weather', args: {'location': 'Boston'}}}]},
-          {role: 'function', parts: functionResponseParts}
+  const request = {
+    contents: [
+      {role: 'user', parts: [{text: 'What is the weather in Boston?'}]},
+      {
+        role: 'model',
+        parts: [
+          {
+            functionCall: {
+              name: 'get_current_weather',
+              args: {'location': 'Boston'},
+            },
+          }
         ],
-        tools: functionDeclarations,
-      };
-      const streamingResp =
-          await generativeModel.generateContentStream(request);
-      for await (const item of streamingResp.stream) {
-        console.log(item.candidates[0].content.parts[0].text);
-      }    
+      },
+      {role: 'function', parts: functionResponseParts},
+    ],
+    tools: functionDeclarations,
+  };
+  const streamingResp =
+    await generativeModel.generateContentStream(request);
+  for await (const item of streamingResp.stream) {
+    console.log(item.candidates[0].content.parts[0].text);
+  }
 }
 // [END aiplatform_gemini_function_calling_content]
 
 functionCallingStreamChat(...process.argv.slice(2)).catch(err => {
-    console.error(err.message);
-    process.exitCode = 1;
+  console.error(err.message);
+  process.exitCode = 1;
 });
diff --git a/run/helloworld/package.docs.json b/run/helloworld/package.docs.json
index 44ee6235b2..8661600a08 100644
--- a/run/helloworld/package.docs.json
+++ b/run/helloworld/package.docs.json
@@ -4,6 +4,7 @@
   "version": "1.0.0",
   "private": true,
   "main": "index.js",
+  "type": "module",
   "scripts": {
     "start": "node index.js"
   },

From c9840ad50b115469e919e8681f2bdbfa040146df Mon Sep 17 00:00:00 2001
From: Guillaume Laforge <glaforge@gmail.com>
Date: Thu, 15 Feb 2024 18:39:40 +0100
Subject: [PATCH 4/5] fix: Attempt at pleasing the Linter (#3625)

* Attempt at pleasing the Linter

* fix(linter): trying to please CI

---------

Co-authored-by: Patti Shin <pattishin@users.noreply.github.com>

From 18d5a81242a3b76681c2fcbcfdab0d195826148e Mon Sep 17 00:00:00 2001
From: "pattishin@google.com" <pattishin@users.noreply.github.com>
Date: Thu, 15 Feb 2024 10:45:24 -0800
Subject: [PATCH 5/5] fix: lint

---
 generative-ai/snippets/functionCallingStreamContent.js | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/generative-ai/snippets/functionCallingStreamContent.js b/generative-ai/snippets/functionCallingStreamContent.js
index c0b87624f5..e0b71e2a29 100644
--- a/generative-ai/snippets/functionCallingStreamContent.js
+++ b/generative-ai/snippets/functionCallingStreamContent.js
@@ -74,17 +74,16 @@ async function functionCallingStreamChat(
           {
             functionCall: {
               name: 'get_current_weather',
-              args: {'location': 'Boston'},
+              args: {location: 'Boston'},
             },
-          }
+          },
         ],
       },
       {role: 'function', parts: functionResponseParts},
     ],
     tools: functionDeclarations,
   };
-  const streamingResp =
-    await generativeModel.generateContentStream(request);
+  const streamingResp = await generativeModel.generateContentStream(request);
   for await (const item of streamingResp.stream) {
     console.log(item.candidates[0].content.parts[0].text);
   }