diff --git a/notebooks/community/model_garden/model_garden_pytorch_vilt_vqa.ipynb b/notebooks/community/model_garden/model_garden_pytorch_vilt_vqa.ipynb index 8fb31992a..9a7b36f4d 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_vilt_vqa.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_vilt_vqa.ipynb @@ -8,7 +8,7 @@ }, "outputs": [], "source": [ - "# Copyright 2023 Google LLC\n", + "# Copyright 2024 Google LLC\n", "#\n", "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", @@ -26,37 +26,29 @@ { "cell_type": "markdown", "metadata": { - "id": "2bd716bf3e39" + "id": "99c1c3fc2ca5" }, "source": [ "# Vertex AI Model Garden - ViLT VQA\n", "\n", - "\n", - "
\n", - " \n", - " \"Colab Run in Colab\n", + "\n", + " \n", - " \n", - " \n", - "
\n", + " \n", + " \"Google
Run in Colab Enterprise\n", " \n", " 		\n", + " \n", " \n", - " \"GitHub\n", - " View on GitHub\n", + " \"GitHub
View on GitHub\n", " \n", " 		\n", - " \n", - " \"Vertex\n", - "Open in Vertex AI Workbench\n", - " \n", - " (a Python-3 CPU notebook is recommended)\n", - "
" + "
" ] }, { "cell_type": "markdown", "metadata": { - "id": "d8cd12648da4" + "id": "3de7470326a2" }, "source": [ "## Overview\n", @@ -76,7 +68,7 @@ "* Vertex AI\n", "* Cloud Storage\n", "\n", - "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage." + "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage." ] }, { @@ -85,302 +77,217 @@ "id": "264c07757582" }, "source": [ - "## Setup environment\n", - "\n", - "**NOTE**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "d73ffa0c0b83" - }, - "source": [ - "### Colab only" + "## Run the notebook" ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "id": "2707b02ef5df" + "cellView": "form", + "id": "ioensNKM8ned" }, "outputs": [], "source": [ - "if \"google.colab\" in str(get_ipython()):\n", - " ! pip3 install --upgrade google-cloud-aiplatform\n", - " from google.colab import auth as google_auth\n", + "# @title Setup Google Cloud project\n", "\n", - " google_auth.authenticate_user()\n", + "# @markdown 1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n", "\n", - " # Restart the notebook kernel after installs.\n", - " import IPython\n", + "# @markdown 2. [Optional] [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs. Set the BUCKET_URI for the experiment environment. The specified Cloud Storage bucket (`BUCKET_URI`) should be located in the same region as where the notebook was launched. Note that a multi-region bucket (eg. \"us\") is not considered a match for a single region covered by the multi-region range (eg. \"us-central1\"). If not set, a unique GCS bucket will be created instead.\n", "\n", - " app = IPython.Application.instance()\n", - " app.kernel.do_shutdown(True)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "0f826ff482a2" - }, - "source": [ - "### Setup Google Cloud project\n", + "import os\n", + "import uuid\n", + "from datetime import datetime\n", "\n", - "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n", + "from google.cloud import aiplatform\n", "\n", - "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n", "\n", - "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n", + "# Get the default cloud project id.\n", + "PROJECT_ID = os.environ[\"GOOGLE_CLOUD_PROJECT\"]\n", "\n", - "1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "8958ebc71868" - }, - "source": [ - "Fill following variables for experiments environment:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "9db30f827a65" - }, - "outputs": [], - "source": [ - "# Cloud project id.\n", - "PROJECT_ID = \"\" # @param {type:\"string\"}\n", + "# Get the default region for launching jobs.\n", + "REGION = os.environ[\"GOOGLE_CLOUD_REGION\"]\n", "\n", - "# The region you want to launch jobs in.\n", - "REGION = \"us-central1\" # @param {type:\"string\"}\n", + "# Enable the Vertex AI API and Compute Engine API, if not already.\n", + "print(\"Enabling Vertex AI API and Compute Engine API.\")\n", + "! gcloud services enable aiplatform.googleapis.com compute.googleapis.com\n", "\n", - "# The Cloud Storage bucket for storing experiments output. Fill it without the 'gs://' prefix.\n", - "GCS_BUCKET = \"\" # @param {type:\"string\"}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "92f16e22c20b" - }, - "source": [ - "Initialize Vertex AI API:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "1680c257acfb" - }, - "outputs": [], - "source": [ - "from google.cloud import aiplatform\n", + "# Cloud Storage bucket for storing the experiment artifacts.\n", + "# A unique GCS bucket will be created for the purpose of this notebook. If you\n", + "# prefer using your own GCS bucket, change the value yourself below.\n", + "now = datetime.now().strftime(\"%Y%m%d%H%M%S\")\n", + "BUCKET_URI = \"gs://\" # @param {type:\"string\"}\n", + "BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", "\n", - "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=GCS_BUCKET)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "6ca48b699d17" - }, - "source": [ - "### Define constants" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "de9882ea89ea" - }, - "outputs": [], - "source": [ - "# The pre-built serving docker image. It contains serving scripts and models.\n", - "SERVE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-transformers-serve\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "10188266a5cd" - }, - "source": [ - "### Define common functions" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "cac4478ae098" - }, - "outputs": [], - "source": [ - "import base64\n", - "import os\n", - "from datetime import datetime\n", - "from io import BytesIO\n", - "\n", - "import requests\n", - "from google.cloud import aiplatform\n", - "from PIL import Image\n", + "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", + " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", + " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", + " ! gsutil mb -l {REGION} {BUCKET_URI}\n", + "else:\n", + " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", + " if bucket_region != REGION:\n", + " raise ValueError(\n", + " \"Bucket region %s is different from notebook region %s\"\n", + " % (bucket_region, REGION)\n", + " )\n", + "print(f\"Using this GCS Bucket: {BUCKET_URI}\")\n", "\n", + "STAGING_BUCKET = os.path.join(BUCKET_URI, \"temporal\")\n", + "MODEL_BUCKET = os.path.join(BUCKET_URI, \"vilt-vqa\")\n", "\n", - "def create_job_name(prefix):\n", - " user = os.environ.get(\"USER\")\n", - " now = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n", - " job_name = f\"{prefix}-{user}-{now}\"\n", - " return job_name\n", "\n", + "# Initialize Vertex AI API.\n", + "print(\"Initializing Vertex AI API.\")\n", + "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=STAGING_BUCKET)\n", "\n", - "def download_image(url):\n", - " response = requests.get(url)\n", - " return Image.open(BytesIO(response.content))\n", + "# Gets the default SERVICE_ACCOUNT.\n", + "shell_output = ! gcloud projects describe $PROJECT_ID\n", + "project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n", + "SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n", + "print(\"Using this default Service Account:\", SERVICE_ACCOUNT)\n", "\n", "\n", - "def image_to_base64(image, format=\"JPEG\"):\n", - " buffer = BytesIO()\n", - " image.save(buffer, format=format)\n", - " image_str = base64.b64encode(buffer.getvalue()).decode(\"utf-8\")\n", - " return image_str\n", + "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", "\n", + "! gcloud config set project $PROJECT_ID\n", "\n", - "def base64_to_image(image_str):\n", - " image = Image.open(BytesIO(base64.b64decode(image_str)))\n", - " return image\n", + "# The pre-built serving docker image. It contains serving scripts and models.\n", + "SERVE_DOCKER_URI = \"us-docker.pkg.dev/deeplearning-platform-release/gcr.io/huggingface-pytorch-inference-cu121.2-2.transformers.4-41.ubuntu2204.py311\"\n", "\n", + "models, endpoints = {}, {}\n", "\n", - "def image_grid(imgs, rows=2, cols=2):\n", - " w, h = imgs[0].size\n", - " grid = Image.new(\"RGB\", size=(cols * w, rows * h))\n", - " for i, img in enumerate(imgs):\n", - " grid.paste(img, box=(i % cols * w, i // cols * h))\n", - " return grid\n", "\n", + "def deploy_model(\n", + " model_id, task, machine_type=\"g2-standard-8\", accelerator_type=\"NVIDIA_L4\"\n", + "):\n", + " \"\"\"Create a Vertex AI Endpoint and deploy the specified model to the endpoint.\"\"\"\n", + " model_name = model_id\n", "\n", - "def deploy_model(model_id, task):\n", - " model_name = \"vilt-vqa\"\n", " endpoint = aiplatform.Endpoint.create(display_name=f\"{model_name}-endpoint\")\n", " serving_env = {\n", - " \"MODEL_ID\": model_id,\n", - " \"TASK\": task,\n", + " \"HF_MODEL_ID\": model_id,\n", + " \"HF_TASK\": task,\n", " \"DEPLOY_SOURCE\": \"notebook\",\n", " }\n", - " # If the model_id is a GCS path, use artifact_uri to pass it to serving docker.\n", - " artifact_uri = model_id if model_id.startswith(\"gs://\") else None\n", + "\n", " model = aiplatform.Model.upload(\n", " display_name=model_name,\n", " serving_container_image_uri=SERVE_DOCKER_URI,\n", " serving_container_ports=[7080],\n", - " serving_container_predict_route=\"/predictions/transformers_serving\",\n", + " serving_container_predict_route=\"/pred\",\n", " serving_container_health_route=\"/ping\",\n", " serving_container_environment_variables=serving_env,\n", - " artifact_uri=artifact_uri,\n", " )\n", + "\n", " model.deploy(\n", " endpoint=endpoint,\n", - " machine_type=\"n1-standard-8\",\n", - " accelerator_type=\"NVIDIA_TESLA_T4\",\n", + " machine_type=machine_type,\n", + " accelerator_type=accelerator_type,\n", " accelerator_count=1,\n", " deploy_request_timeout=1800,\n", + " service_account=SERVICE_ACCOUNT,\n", " )\n", " return model, endpoint" ] }, - { - "cell_type": "markdown", - "metadata": { - "id": "d2d72ecdb8c9" - }, - "source": [ - "## Upload and deploy models" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "9448c5f545fa" - }, - "source": [ - "This section uploads the pre-trained model to Model Registry and deploys it on the Endpoint with 1 T4 GPU.\n", - "\n", - "The model deployment step will take ~15 minutes to complete.\n", - "\n", - "Once deployed, you can send images and questions to get answers." - ] - }, { "cell_type": "code", "execution_count": null, "metadata": { - "id": "b4b46c28d8b1" + "cellView": "form", + "id": "2707b02ef5df" }, "outputs": [], "source": [ - "model, endpoint = deploy_model(\n", - " model_id=\"dandelin/vilt-b32-finetuned-vqa\", task=\"visual-question-answering\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "80b3fd2ace09" - }, - "source": [ - "NOTE: The model weights will be downloaded after the deployment succeeds. Thus additional 5 minutes of waiting time is needed **after** the above model deployment step succeeds and before you run the next step below. Otherwise you might see a `ServiceUnavailable: 503 502:Bad Gateway` error when you send requests to the endpoint." + "# @title Deploy the model to Vertex for online predictions\n", + "\n", + "# @markdown This section uploads the model to Model Registry and deploys it on the Endpoint. It takes ~15 minutes to finish.\n", + "\n", + "MODEL_ID = \"dandelin/vilt-b32-finetuned-vqa\"\n", + "task = \"visual-question-answering\"\n", + "\n", + "models[\"model\"], endpoints[\"endpoint\"] = deploy_model(model_id=MODEL_ID, task=task)\n", + "\n", + "print(\"endpoint_name:\", endpoints[\"endpoint\"].name)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "id": "6be655247cb1" + "cellView": "form", + "id": "bb7adab99e41" }, "outputs": [], "source": [ - "image = download_image(\"http://images.cocodataset.org/val2017/000000039769.jpg\")\n", - "display(image)\n", - "\n", - "question = \"Which cat is bigger?\"\n", - "instances = [\n", - " {\"image\": image_to_base64(image), \"text\": question},\n", - "]\n", - "preds = endpoint.predict(instances=instances).predictions\n", - "print(question)\n", - "print(preds)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "db7ffebdb4be" - }, - "source": [ - "### Clean up resources" + "# @title Predict\n", + "# @markdown Once deployment succeeds, you can send requests to the endpoint with images and questions.\n", + "\n", + "# @markdown Example: \n", + "\n", + "# @markdown ```\n", + "# @markdown {\n", + "# @markdown \"image\": \"http://images.cocodataset.org/val2017/000000039769.jpg\"\n", + "# @markdown \"question\": \"Which cat is bigger?\"\n", + "# @markdown }\n", + "# @markdown ```\n", + "\n", + "# @markdown Note that `image` should be an http uri (starting with \"http://\" or \"https://\"), a local path, or base64 encoded bytes.\n", + "\n", + "# Loads an existing endpoint instance using the endpoint name:\n", + "# - Using `endpoint_name = endpoint.name` allows us to get the endpoint name of\n", + "# the endpoint `endpoint` created in the cell above.\n", + "# - Alternatively, you can set `endpoint_name = \"1234567890123456789\"` to load\n", + "# an existing endpoint with the ID 1234567890123456789.\n", + "\n", + "# You may uncomment the code below to load an existing endpoint.\n", + "# endpoint_name = \"\" # @param {type:\"string\"}\n", + "# aip_endpoint_name = (\n", + "# f\"projects/{PROJECT_ID}/locations/{REGION}/endpoints/{endpoint_name}\"\n", + "# )\n", + "# endpoint = aiplatform.Endpoint(aip_endpoint_name)\n", + "# print(\"Using this existing endpoint from a different session: {aip_endpoint_name}\")\n", + "\n", + "image = \"http://images.cocodataset.org/val2017/000000039769.jpg\" # @param {type: \"string\"}\n", + "question = \"Which cat is bigger?\" # @param {type: \"string\"}\n", + "\n", + "instances = [{\n", + " \"image\": image,\n", + " \"question\": question,\n", + "}]\n", + "\n", + "response = endpoints[\"endpoint\"].predict(instances=instances)\n", + "for prediction in response.predictions:\n", + " print(prediction)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "id": "2ccf3714dbe9" + "cellView": "form", + "id": "6c460088b873" }, "outputs": [], "source": [ + "# @title Clean up resources\n", + "# @markdown Delete the experiment models and endpoints to recycle the resources\n", + "# @markdown and avoid unnecessary continuous charges that may incur.\n", + "\n", "# Undeploy model and delete endpoint.\n", - "endpoint.delete(force=True)\n", + "for endpoint in endpoints.values():\n", + " endpoint.delete(force=True)\n", "\n", "# Delete models.\n", - "model.delete()" + "for model in models.values():\n", + " model.delete()\n", + "\n", + "delete_bucket = False # @param {type:\"boolean\"}\n", + "if delete_bucket:\n", + " ! gsutil -m rm -r $BUCKET_NAME" ] } ],