diff --git a/ai-platform/snippets/predict-image-from-image-and-text.js b/ai-platform/snippets/predict-image-from-image-and-text.js new file mode 100644 index 0000000000..1b16df13dc --- /dev/null +++ b/ai-platform/snippets/predict-image-from-image-and-text.js @@ -0,0 +1,96 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +async function main( + project, + location = 'us-central1', + baseImagePath, + textPrompt +) { + // [START aiplatform_sdk_text_image_embedding] + /** + * TODO(developer): Uncomment these variables before running the sample.\ + * (Not necessary if passing values as arguments) + */ + // const project = 'YOUR_PROJECT_ID'; + // const location = 'YOUR_PROJECT_LOCATION'; + // const bastImagePath = "YOUR_BASED_IMAGE_PATH" + // const textPrompt = 'YOUR_TEXT_PROMPT'; + const aiplatform = require('@google-cloud/aiplatform'); + + // Imports the Google Cloud Prediction service client + const {PredictionServiceClient} = aiplatform.v1; + + // Import the helper module for converting arbitrary protobuf.Value objects. + const {helpers} = aiplatform; + + // Specifies the location of the api endpoint + const clientOptions = { + apiEndpoint: 'us-central1-aiplatform.googleapis.com', + }; + const publisher = 'google'; + const model = 'multimodalembedding@001'; + + // Instantiates a client + const predictionServiceClient = new PredictionServiceClient(clientOptions); + + async function predictImageFromImageAndText() { + // Configure the parent resource + const endpoint = `projects/${project}/locations/${location}/publishers/${publisher}/models/${model}`; + + const fs = require('fs'); + const imageFile = fs.readFileSync(baseImagePath); + + // Convert the image data to a Buffer and base64 encode it. + const encodedImage = Buffer.from(imageFile).toString('base64'); + + const prompt = { + text: textPrompt, + image: { + bytesBase64Encoded: encodedImage, + }, + }; + const instanceValue = helpers.toValue(prompt); + const instances = [instanceValue]; + + const parameter = { + sampleCount: 1, + }; + const parameters = helpers.toValue(parameter); + + const request = { + endpoint, + instances, + parameters, + }; + + // Predict request + const [response] = await predictionServiceClient.predict(request); + console.log('Get image embedding response'); + const predictions = response.predictions; + console.log('\tPredictions :'); + for (const prediction of predictions) { + console.log(`\t\tPrediction : ${JSON.stringify(prediction)}`); + } + } + + await predictImageFromImageAndText(); + // [END aiplatform_sdk_text_image_embedding] +} + +exports.predictImageFromImageAndText = main; diff --git a/ai-platform/snippets/predict-image-from-text.js b/ai-platform/snippets/predict-image-from-text.js new file mode 100644 index 0000000000..c12c97399e --- /dev/null +++ b/ai-platform/snippets/predict-image-from-text.js @@ -0,0 +1,81 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +async function main(project, location = 'us-central1', textPrompt) { + // [START aiplatform_sdk_text_embedding] + /** + * TODO(developer): Uncomment these variables before running the sample.\ + * (Not necessary if passing values as arguments) + */ + // const project = 'YOUR_PROJECT_ID'; + // const location = 'YOUR_PROJECT_LOCATION'; + // const textPrompt = 'YOUR_TEXT_PROMPT'; + const aiplatform = require('@google-cloud/aiplatform'); + + // Imports the Google Cloud Prediction service client + const {PredictionServiceClient} = aiplatform.v1; + + // Import the helper module for converting arbitrary protobuf.Value objects. + const {helpers} = aiplatform; + + // Specifies the location of the api endpoint + const clientOptions = { + apiEndpoint: 'us-central1-aiplatform.googleapis.com', + }; + const publisher = 'google'; + const model = 'multimodalembedding@001'; + + // Instantiates a client + const predictionServiceClient = new PredictionServiceClient(clientOptions); + + async function predictImageFromText() { + // Configure the parent resource + const endpoint = `projects/${project}/locations/${location}/publishers/${publisher}/models/${model}`; + + const prompt = { + text: textPrompt, + }; + const instanceValue = helpers.toValue(prompt); + const instances = [instanceValue]; + + const parameter = { + sampleCount: 1, + }; + const parameters = helpers.toValue(parameter); + + const request = { + endpoint, + instances, + parameters, + }; + + // Predict request + const [response] = await predictionServiceClient.predict(request); + console.log('Get image embedding response'); + const predictions = response.predictions; + console.log('\tPredictions :'); + for (const prediction of predictions) { + console.log(`\t\tPrediction : ${JSON.stringify(prediction)}`); + } + } + + await predictImageFromText(); + // [END aiplatform_sdk_text_embedding] +} + +exports.predictImageFromText = main; diff --git a/ai-platform/snippets/test/predict-image-from-image-and-text.test.js b/ai-platform/snippets/test/predict-image-from-image-and-text.test.js new file mode 100644 index 0000000000..ff1eeb2997 --- /dev/null +++ b/ai-platform/snippets/test/predict-image-from-image-and-text.test.js @@ -0,0 +1,55 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const {assert} = require('chai'); +const {describe, it} = require('mocha'); +const sinon = require('sinon'); + +const { + predictImageFromImageAndText, +} = require('../predict-image-from-image-and-text'); + +const project = process.env.CAIP_PROJECT_ID; +const LOCATION = 'us-central1'; +const baseImagePath = 'resources/daisy.jpg'; +const textPrompt = 'an impressionist painting'; + +describe('AI platform generates image from image and text', async () => { + const stubConsole = function () { + sinon.stub(console, 'error'); + sinon.stub(console, 'log'); + }; + + const restoreConsole = function () { + console.log.restore(); + console.error.restore(); + }; + + beforeEach(stubConsole); + afterEach(restoreConsole); + + it('should make predictions using a large language model', async () => { + await predictImageFromImageAndText( + project, + LOCATION, + baseImagePath, + textPrompt + ); + assert.include(console.log.firstCall.args, 'Get image embedding response'); + }); +}); diff --git a/ai-platform/snippets/test/predict-image-from-text.test.js b/ai-platform/snippets/test/predict-image-from-text.test.js new file mode 100644 index 0000000000..4cc161447f --- /dev/null +++ b/ai-platform/snippets/test/predict-image-from-text.test.js @@ -0,0 +1,48 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const {assert} = require('chai'); +const {describe, it} = require('mocha'); +const sinon = require('sinon'); + +const {predictImageFromText} = require('../predict-image-from-text'); + +const project = process.env.CAIP_PROJECT_ID; +const LOCATION = 'us-central1'; +const textPrompt = + 'small red boat on water in the morning watercolor illustration muted colors'; + +describe('AI platform generates image from text', async () => { + const stubConsole = function () { + sinon.stub(console, 'error'); + sinon.stub(console, 'log'); + }; + + const restoreConsole = function () { + console.log.restore(); + console.error.restore(); + }; + + beforeEach(stubConsole); + afterEach(restoreConsole); + + it('should make predictions using a large language model', async () => { + await predictImageFromText(project, LOCATION, textPrompt); + assert.include(console.log.firstCall.args, 'Get image embedding response'); + }); +});