Skip to content

Commit f377daa

Browse files
arbrownarbrown
authored
feat(generative-ai): Add basic samples for Gemini / VertexAI inference (#3670)
* feat(generative-ai): Add sample for basic text generation. * chore: fix lint issues * feat: Add basic multimodal example * chore: add header * feat: Add streaming inference example * feat: Add example for multimodal streaming response * fix: Fix some tests that were not running correctly * chore: Clarify test names * fix: Adjust cloud storage bucket and argurment order for consistency * chore: fix lint errors * chore: Update some tests and prompt text to be clearer * Update model to the correct multimodal model * Clarify multimodal prompt based on order * chore: Update function calling stream model to address flaky test.
1 parent dcfe8fa commit f377daa

11 files changed

Lines changed: 415 additions & 4 deletions

‎generative-ai/snippets/inference/nonStreamMultiModalityBasic.js‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
// Copyright 2024 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
// [START generativeaionvertexai_non_stream_multimodality_basic]
16+
const {VertexAI} = require('@google-cloud/vertexai');
17+
18+
/**
19+
* TODO(developer): Update these variables before running the sample.
20+
*/
21+
async function generateContent(
22+
projectId = 'PROJECT_ID',
23+
location = 'us-central1',
24+
model = 'gemini-1.5-pro-preview-0409'
25+
) {
26+
// Initialize Vertex AI
27+
const vertexAI = new VertexAI({project: projectId, location: location});
28+
const generativeModel = vertexAI.getGenerativeModel({model: model});
29+
30+
const request = {
31+
contents: [
32+
{
33+
role: 'user',
34+
parts: [
35+
{
36+
file_data: {
37+
file_uri: 'gs://cloud-samples-data/video/animals.mp4',
38+
mime_type: 'video/mp4',
39+
},
40+
},
41+
{
42+
file_data: {
43+
file_uri:
44+
'gs://cloud-samples-data/generative-ai/image/character.jpg',
45+
mime_type: 'image/jpeg',
46+
},
47+
},
48+
{text: 'Are this video and image correlated?'},
49+
],
50+
},
51+
],
52+
};
53+
54+
const result = await generativeModel.generateContent(request);
55+
56+
console.log(result.response.candidates[0].content.parts[0].text);
57+
}
58+
// [END generativeaionvertexai_non_stream_multimodality_basic]
59+
60+
generateContent(...process.argv.slice(2)).catch(err => {
61+
console.error(err.message);
62+
process.exitCode = 1;
63+
});

‎generative-ai/snippets/inference/nonStreamTextBasic.js‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
// Copyright 2024 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
// [START generativeaionvertexai_non_stream_text_basic]
16+
const {VertexAI} = require('@google-cloud/vertexai');
17+
18+
/**
19+
* TODO(developer): Update these variables before running the sample.
20+
*/
21+
async function generateContent(
22+
projectId = 'PROJECT_ID',
23+
location = 'us-central1',
24+
model = 'gemini-1.0-pro'
25+
) {
26+
// Initialize Vertex with your Cloud project and location
27+
const vertexAI = new VertexAI({project: projectId, location: location});
28+
29+
// Instantiate the model
30+
const generativeModel = vertexAI.getGenerativeModel({
31+
model: model,
32+
});
33+
34+
const request = {
35+
contents: [
36+
{
37+
role: 'user',
38+
parts: [
39+
{
40+
text: 'Write a story about a magic backpack.',
41+
},
42+
],
43+
},
44+
],
45+
};
46+
47+
console.log(JSON.stringify(request));
48+
49+
const result = await generativeModel.generateContent(request);
50+
51+
console.log(result.response.candidates[0].content.parts[0].text);
52+
}
53+
// [END generativeaionvertexai_non_stream_text_basic]
54+
55+
generateContent(...process.argv.slice(2)).catch(err => {
56+
console.error(err.message);
57+
process.exitCode = 1;
58+
});

‎generative-ai/snippets/inference/streamMultiModalityBasic.js‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
// Copyright 2024 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
// [START generativeaionvertexai_stream_multimodality_basic]
16+
const {VertexAI} = require('@google-cloud/vertexai');
17+
18+
/**
19+
* TODO(developer): Update these variables before running the sample.
20+
*/
21+
async function generateContent(
22+
projectId = 'PROJECT_ID',
23+
location = 'us-central1',
24+
model = 'gemini-1.5-pro-preview-0409'
25+
) {
26+
// Initialize Vertex AI
27+
const vertexAI = new VertexAI({project: projectId, location: location});
28+
const generativeModel = vertexAI.getGenerativeModel({model: model});
29+
30+
const request = {
31+
contents: [
32+
{
33+
role: 'user',
34+
parts: [
35+
{
36+
file_data: {
37+
file_uri: 'gs://cloud-samples-data/video/animals.mp4',
38+
mime_type: 'video/mp4',
39+
},
40+
},
41+
{
42+
file_data: {
43+
file_uri:
44+
'gs://cloud-samples-data/generative-ai/image/character.jpg',
45+
mime_type: 'image/jpeg',
46+
},
47+
},
48+
{text: 'Are this video and image correlated?'},
49+
],
50+
},
51+
],
52+
};
53+
54+
const result = await generativeModel.generateContentStream(request);
55+
56+
for await (const item of result.stream) {
57+
console.log(item.candidates[0].content.parts[0].text);
58+
}
59+
}
60+
// [END generativeaionvertexai_stream_multimodality_basic]
61+
62+
generateContent(...process.argv.slice(2)).catch(err => {
63+
console.error(err.message);
64+
process.exitCode = 1;
65+
});

‎generative-ai/snippets/inference/streamTextBasic.js‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
// Copyright 2024 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
// [START generativeaionvertexai_stream_text_basic]
16+
const {VertexAI} = require('@google-cloud/vertexai');
17+
18+
/**
19+
* TODO(developer): Update these variables before running the sample.
20+
*/
21+
async function generateContent(
22+
projectId = 'PROJECT_ID',
23+
location = 'us-central1',
24+
model = 'gemini-1.0-pro'
25+
) {
26+
// Initialize Vertex with your Cloud project and location
27+
const vertexAI = new VertexAI({project: projectId, location: location});
28+
29+
// Instantiate the model
30+
const generativeModel = vertexAI.getGenerativeModel({
31+
model: model,
32+
});
33+
34+
const request = {
35+
contents: [
36+
{
37+
role: 'user',
38+
parts: [
39+
{
40+
text: 'Write a story about a magic backpack.',
41+
},
42+
],
43+
},
44+
],
45+
};
46+
47+
console.log(JSON.stringify(request));
48+
49+
const result = await generativeModel.generateContentStream(request);
50+
for await (const item of result.stream) {
51+
console.log(item.candidates[0].content.parts[0].text);
52+
}
53+
}
54+
// [END generativeaionvertexai_stream_text_basic]
55+
56+
generateContent(...process.argv.slice(2)).catch(err => {
57+
console.error(err.message);
58+
process.exitCode = 1;
59+
});

‎generative-ai/snippets/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
"*.js"
1111
],
1212
"scripts": {
13-
"test": "c8 mocha -p -j 2 --timeout 2400000 test/*.test.js"
13+
"test": "c8 mocha -p -j 2 --timeout 2400000 test/*.test.js test/**/*.test.js"
1414
},
1515
"dependencies": {
1616
"@google-cloud/aiplatform": "^3.12.0",

‎generative-ai/snippets/test/functionCallingStreamChat.test.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'});
2121

2222
const projectId = process.env.CAIP_PROJECT_ID;
2323
const location = process.env.LOCATION;
24-
const model = 'gemini-1.0-pro';
24+
const model = 'gemini-1.5-pro-preview-0409';
2525

2626
describe('Generative AI Function Calling Stream Chat', () => {
2727
/**
@@ -30,7 +30,7 @@ describe('Generative AI Function Calling Stream Chat', () => {
3030
*/
3131
// const projectId = 'YOUR_PROJECT_ID';
3232
// const location = 'YOUR_LOCATION';
33-
// const model = 'gemini-1.0-pro';
33+
// const model = 'gemini-1.5-pro-preview-0409';
3434

3535
it('should create stream chat and begin the conversation the same in each instance', async () => {
3636
const output = execSync(

‎generative-ai/snippets/test/gemini-audio-summarization.test.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'});
2222
const projectId = process.env.CAIP_PROJECT_ID;
2323

2424
describe('Summarize audio', async () => {
25-
it('should summerize audio', async () => {
25+
it('should summarize audio', async () => {
2626
const output = execSync(
2727
`node ./gemini-audio-summarization.js ${projectId}`
2828
);

‎generative-ai/snippets/test/inference/nonStreamMultiModalityBasic.test.js‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
// Copyright 2024 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
'use strict';
16+
17+
const {assert} = require('chai');
18+
const {describe, it} = require('mocha');
19+
const cp = require('child_process');
20+
const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'});
21+
22+
const projectId = process.env.CAIP_PROJECT_ID;
23+
const location = process.env.LOCATION;
24+
const model = 'gemini-1.5-pro-preview-0409';
25+
26+
describe('Generative AI Multimodal Text Inference', () => {
27+
/**
28+
* TODO(developer): Uncomment these variables before running the sample.\
29+
* (Not necessary if passing values as arguments)
30+
*/
31+
// const projectId = 'YOUR_PROJECT_ID';
32+
// const location = 'YOUR_LOCATION';
33+
// const model = 'gemini-1.5-pro-preview-0409';
34+
35+
it('should generate text based on a prompt containing text, a video, and an image', async () => {
36+
const output = execSync(
37+
`node ./inference/nonStreamMultiModalityBasic.js ${projectId} ${location} ${model}`
38+
);
39+
assert(output.length > 0);
40+
});
41+
});

‎generative-ai/snippets/test/inference/nonStreamTextBasic.test.js‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
// Copyright 2024 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
'use strict';
16+
17+
const {assert} = require('chai');
18+
const {describe, it} = require('mocha');
19+
const cp = require('child_process');
20+
const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'});
21+
22+
const projectId = process.env.CAIP_PROJECT_ID;
23+
const location = process.env.LOCATION;
24+
const model = 'gemini-1.0-pro';
25+
26+
describe('Generative AI Basic Text Inference', () => {
27+
/**
28+
* TODO(developer): Uncomment these variables before running the sample.\
29+
* (Not necessary if passing values as arguments)
30+
*/
31+
// const projectId = 'YOUR_PROJECT_ID';
32+
// const location = 'YOUR_LOCATION';
33+
// const model = 'gemini-1.0-pro';
34+
35+
it('should create a generative text model and infer text from a prompt', async () => {
36+
const output = execSync(
37+
`node ./inference/nonStreamTextBasic.js ${projectId} ${location} ${model}`
38+
);
39+
40+
// Assert that the correct prompt was issued
41+
assert(output.match(/Write a story about a magic backpack/));
42+
});
43+
});

0 commit comments

Comments
 (0)