custom-models/import_models/perf-test-imported_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "784b1f22-bed9-4c39-bc7d-07fd9906fec0",
   "metadata": {},
   "source": [
    "# Performance test Custom Model Import on Amazon Bedrock\n",
    "\n",
    "This notebook illustrates the process of performance testing the fine tuned model once it is hosted in Bedrock. You can view the process to import the model via [Custom Model Import]()\n",
    "\n",
    "\n",
    "\n",
    "### License Information\n",
    "\n",
    "In this notebook we are providing a sample of how to performance test. This is by no means a definetive guide on how to performance test your models. This can be used as a starting point for your testing. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea10d748-6125-4899-b5b0-d83afe7578d0",
   "metadata": {},
   "source": [
    "### Installing pre-requisites\n",
    "\n",
    "- Please un comment and run this cell to install the required libraries to run this notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "11130253-ccc6-4b22-af9d-9c7d75b716e3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#!pip install boto3 numpy --upgrade --quiet"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a28f1478-48ae-4298-be1e-0ed7193b9cf3",
   "metadata": {},
   "source": [
    "## Setup\n",
    "\n",
    "Loading the boto3 client we will need to access our model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c2d35b36-b132-4ee7-923c-98b71de61169",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "\n",
    "from io import StringIO\n",
    "import sys\n",
    "import textwrap\n",
    "import os\n",
    "from typing import Optional\n",
    "\n",
    "# External Dependencies:\n",
    "import boto3\n",
    "from botocore.config import Config\n",
    "\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "def print_ww(*args, width: int = 100, **kwargs):\n",
    "    \"\"\"Like print(), but wraps output to `width` characters (default 100)\"\"\"\n",
    "    buffer = StringIO()\n",
    "    try:\n",
    "        _stdout = sys.stdout\n",
    "        sys.stdout = buffer\n",
    "        print(*args, **kwargs)\n",
    "        output = buffer.getvalue()\n",
    "    finally:\n",
    "        sys.stdout = _stdout\n",
    "    for line in output.splitlines():\n",
    "        print(\"\\n\".join(textwrap.wrap(line, width=width)))\n",
    "        \n",
    "\n",
    "def get_boto_client_tmp_cred(\n",
    "    retry_config = None,\n",
    "    target_region: Optional[str] = None,\n",
    "    runtime: Optional[bool] = True,\n",
    "    service_name: Optional[str] = None,\n",
    "):\n",
    "\n",
    "    if not service_name:\n",
    "        if runtime:\n",
    "            service_name='bedrock-runtime'\n",
    "        else:\n",
    "            service_name='bedrock'\n",
    "\n",
    "    bedrock_client = boto3.client(\n",
    "        service_name=service_name,\n",
    "        config=retry_config,\n",
    "        aws_access_key_id=os.getenv(\"AWS_ACCESS_KEY_ID\"),\n",
    "        aws_secret_access_key=os.getenv(\"AWS_SECRET_ACCESS_KEY\"),\n",
    "        aws_session_token=os.getenv('AWS_SESSION_TOKEN',\"\"),\n",
    "\n",
    "    )\n",
    "    print(\"boto3 Bedrock client successfully created!\")\n",
    "    print(bedrock_client._endpoint)\n",
    "    return bedrock_client    \n",
    "\n",
    "def get_boto_client(\n",
    "    assumed_role: Optional[str] = None,\n",
    "    region: Optional[str] = None,\n",
    "    runtime: Optional[bool] = True,\n",
    "    service_name: Optional[str] = None,\n",
    "):\n",
    "    \"\"\"Create a boto3 client for Amazon Bedrock, with optional configuration overrides\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    assumed_role :\n",
    "        Optional ARN of an AWS IAM role to assume for calling the Bedrock service. If not\n",
    "        specified, the current active credentials will be used.\n",
    "    region :\n",
    "        Optional name of the AWS Region in which the service should be called (e.g. \"us-east-1\").\n",
    "        If not specified, AWS_REGION or AWS_DEFAULT_REGION environment variable will be used.\n",
    "    runtime :\n",
    "        Optional choice of getting different client to perform operations with the Amazon Bedrock service.\n",
    "    \"\"\"\n",
    "    if region is None:\n",
    "        target_region = os.environ.get(\"AWS_REGION\", os.environ.get(\"AWS_DEFAULT_REGION\"))\n",
    "    else:\n",
    "        target_region = region\n",
    "\n",
    "    print(f\"Create new client\\n  Using region: {target_region}\")\n",
    "    session_kwargs = {\"region_name\": target_region}\n",
    "    client_kwargs = {**session_kwargs}\n",
    "\n",
    "    profile_name = os.environ.get(\"AWS_PROFILE\", None)\n",
    "    retry_config = Config(\n",
    "        region_name=target_region,\n",
    "        signature_version = 'v4',\n",
    "        retries={\n",
    "            \"max_attempts\": 10,\n",
    "            \"mode\": \"standard\",\n",
    "        },\n",
    "    )\n",
    "    if profile_name:\n",
    "        print(f\"  Using profile: {profile_name}\")\n",
    "        session_kwargs[\"profile_name\"] = profile_name\n",
    "    else: # use temp credentials -- add to the client kwargs\n",
    "        print(f\"  Using temp credentials\")\n",
    "\n",
    "        return get_boto_client_tmp_cred(retry_config=retry_config,target_region=target_region, runtime=runtime, service_name=service_name)\n",
    "\n",
    "    session = boto3.Session(**session_kwargs)\n",
    "\n",
    "    if assumed_role:\n",
    "        print(f\"  Using role: {assumed_role}\", end='')\n",
    "        sts = session.client(\"sts\")\n",
    "        response = sts.assume_role(\n",
    "            RoleArn=str(assumed_role),\n",
    "            RoleSessionName=\"cmi-llm-1\"\n",
    "        )\n",
    "        print(\" ... successful!\")\n",
    "        client_kwargs[\"aws_access_key_id\"] = response[\"Credentials\"][\"AccessKeyId\"]\n",
    "        client_kwargs[\"aws_secret_access_key\"] = response[\"Credentials\"][\"SecretAccessKey\"]\n",
    "        client_kwargs[\"aws_session_token\"] = response[\"Credentials\"][\"SessionToken\"]\n",
    "\n",
    "    if not service_name:\n",
    "        if runtime:\n",
    "            service_name='bedrock-runtime'\n",
    "        else:\n",
    "            service_name='bedrock'\n",
    "\n",
    "    bedrock_client = session.client(\n",
    "        service_name=service_name,\n",
    "        config=retry_config,\n",
    "        **client_kwargs\n",
    "    )\n",
    "\n",
    "    print(\"boto3 Bedrock client successfully created!\")\n",
    "    print(bedrock_client._endpoint)\n",
    "    return bedrock_client"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fdd2fad9",
   "metadata": {},
   "source": [
    "### Boto3 client\n",
    "- Create the run time client which we will use to run through the various classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5d160878-db73-4fdd-9868-18bc2304be34",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Create new client\n",
      "  Using region: us-east-1\n",
      "  Using temp credentials\n",
      "boto3 Bedrock client successfully created!\n",
      "bedrock-runtime(https://bedrock-runtime.us-east-1.amazonaws.com)\n"
     ]
    }
   ],
   "source": [
    "#os.environ[\"AWS_PROFILE\"] = '<replace with your profile if you have that set up>'\n",
    "region_aws = 'us-east-1' #- replace with your region\n",
    "boto3_bedrock = get_boto_client(region=region_aws, runtime=True, service_name='bedrock-runtime')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c06a7e63",
   "metadata": {},
   "source": [
    "### Read the prompts from a file\n",
    "\n",
    "This is for the larger context sizes. we want to read the prompts from a file which allows us to customize easily\n",
    "\n",
    "- we have provided 2 files one has smaller 475 tokens approximately\n",
    "- The second file is a larger file which has close to 2000 tokens approximately\n",
    "- Fell free to use either or bring your own data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4af41d94",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "approx no of tokens ---- > 474.5 \n"
     ]
    }
   ],
   "source": [
    "prompt_test = \"Generate an article om economics\"\n",
    "with open(\"./perf_data/perf_test_small.txt\", \"r+\") as file1:\n",
    "    # Reading from a file\n",
    "    prompt_test = file1.read()\n",
    "    \n",
    "print(f\"approx no of tokens ---- > {len(prompt_test)/4} \")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9599f116",
   "metadata": {},
   "source": [
    "### Async method to invoke the model. Since this is a IO operation we are not impacted by the GIL\n",
    "\n",
    "- We cannot use the asyncio library because the boto3 is not a async function \n",
    "- We will resort to threading since this is a IO operation and not impacted by GIL.\n",
    "- However for your own perf testing we would recomend you to use multi processing assuming your CPU has enough cores or any performance testing library designed for parallel test\n",
    "- We create a Thread pool with max size determined by the `max_parallel_invocations` variable\n",
    "- We will create a batch of runs determined by the `batch_size` variable\n",
    "- this means say we want to run a batch size of 100 records and run 10 of them parallel at anything, we wil set the `max_parallel_invocations` to 10 and `batch_size` to 100\n",
    "\n",
    "This function takes in the following\n",
    "- it will use the run _id of the calling thread from the pool\n",
    "- it will create a new boto3 client and use for this run\n",
    "- The wrapper runs each 3 times and then returns the model time taken in sec and also reports the latency metrics from what the model returned back"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b28fc5b4-f6df-478f-8bba-f5f0414d4778",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import boto3\n",
    "import json\n",
    "import traceback\n",
    "import asyncio\n",
    "import time\n",
    "\n",
    "def invoke_custom_model(boto_client, model_arn, prompt, run_id, max_tokens=200, temperature=0, top_p=0.9):\n",
    "    response = \"\"\n",
    "    print(f\"starting run id --- > {run_id}\")\n",
    "    try:\n",
    "        response = boto_client.converse(\n",
    "            modelId=model_arn,\n",
    "            messages=[\n",
    "                {\n",
    "                    \"role\": \"user\",\n",
    "                    \"content\": [\n",
    "                        {\n",
    "                            \"text\": prompt\n",
    "                        }\n",
    "                    ]\n",
    "                }\n",
    "            ],\n",
    "            inferenceConfig={\n",
    "                \"temperature\": temperature,\n",
    "                \"maxTokens\": max_tokens,\n",
    "                \"topP\": top_p\n",
    "            }\n",
    "            #additionalModelRequestFields={\n",
    "            #}\n",
    "        )\n",
    "        #print(response)\n",
    "        #print(f\"returning results:: ending run id --- > {run_id}:: \", flush=True)\n",
    "    except :\n",
    "        print(traceback.format_exc())\n",
    "    try:\n",
    "        result = (\n",
    "            response['metrics']['latencyMs'],\n",
    "            response['usage']['inputTokens'], \n",
    "            response['usage']['outputTokens'],\n",
    "            f\"{len(response['output']['message']['content'][0]['text'])}\" \\\n",
    "            + '\\n--- Latency: ' + str(response['metrics']['latencyMs']) \\\n",
    "            + 'ms - Input tokens:' + str(response['usage']['inputTokens']) \\\n",
    "            + ' - Output tokens:' + str(response['usage']['outputTokens']) + ' ---\\n'\n",
    "        )\n",
    "    except:\n",
    "        print(traceback.format_exc())\n",
    "        result = -1,-1,-1, \"Output parsing error\"\n",
    "    \n",
    "\n",
    "    \n",
    "    return result\n",
    "\n",
    "\n",
    "def invoke_wrapper(model_arn, prompt, run_id, max_tokens=200, temperature=0, top_p=0.9):\n",
    "    boto_client = get_boto_client(region=region_aws, runtime=True, service_name='bedrock-runtime')\n",
    "    \n",
    "    start_time_sec = time.time()\n",
    "    for _ in range(3):\n",
    "        result = invoke_custom_model(boto_client, model_arn, prompt, run_id, max_tokens, temperature, top_p)\n",
    "        \n",
    "    time_diff = (time.time() - start_time_sec)/3\n",
    "    \n",
    "    return time_diff, result[0], result[1], result[2] # return a tuple of time diff and the latency metrics, total  Input tokens and total output tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "62075070-5431-4fbb-a706-70b17cacf2a9",
   "metadata": {},
   "source": [
    "#### Set your variables as described above\n",
    "\n",
    "- Set the Model id to be the [model arn](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html) you would get back after running your import process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "8db4580a",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_id = 'us.meta.llama3-2-1b-instruct-v1:0'\n",
    "#model_id = 'us.meta.llama3-2-11b-instruct-v1:0'\n",
    "#model_id = 'anthropic.claude-3-haiku-20240307-v1:0'\n",
    "\n",
    "#- how many total runs do we want to execute\n",
    "batch_size = 2\n",
    "\n",
    "#- how many of these runs should be executed in parallel\n",
    "max_parallel_invocations = 2\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a216d37",
   "metadata": {},
   "source": [
    "#### Run it via threading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "56023bf0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Create new client\n",
      "  Using region: us-east-1Create new client\n",
      "  Using region: us-east-1\n",
      "  Using temp credentials\n",
      "\n",
      "  Using temp credentials\n",
      "boto3 Bedrock client successfully created!\n",
      "bedrock-runtime(https://bedrock-runtime.us-east-1.amazonaws.com)\n",
      "starting run id --- > 1\n",
      "boto3 Bedrock client successfully created!\n",
      "bedrock-runtime(https://bedrock-runtime.us-east-1.amazonaws.com)\n",
      "starting run id --- > 0\n",
      "starting run id --- > 0\n",
      "starting run id --- > 1\n",
      "starting run id --- > 0\n",
      "starting run id --- > 1\n",
      "Result: (0.9432307084401449, 781, 486, 139)\n",
      "Result: (0.9755787054697672, 780, 486, 139)\n"
     ]
    }
   ],
   "source": [
    "import concurrent.futures \n",
    "\n",
    "result_metrics = []\n",
    "with concurrent.futures.ThreadPoolExecutor(max_workers = max_parallel_invocations) as executor:\n",
    "\n",
    "    # Submit tasks to the thread pool\n",
    "    futures_cmi = [executor.submit(invoke_wrapper, model_id, prompt_test,run_id ) for run_id in range(batch_size)]\n",
    "\n",
    "    # Get the results as they are completed\n",
    "    for future in concurrent.futures.as_completed(futures_cmi):\n",
    "        result = future.result()\n",
    "        result_metrics.append(result)\n",
    "        print(f\"Result: {result}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f333adef",
   "metadata": {},
   "source": [
    "### Here we tabulate the results\n",
    "\n",
    "- we will display Mean and P90 values\n",
    "- we display the end to end latency as measured by our clock \n",
    "- we will also display the latency reported by the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "3ce3d12d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Final Mean latency results --- > by Wall time:1.0329072078069053 secs::  by Model Latency metrics:803.0 ms\n",
      " Final P90 latency results --- > by Wall time:1.0379081169764202 secs ::  by Model Latency metrics:807.8 ms\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "time_end_end = [result[0] for result in result_metrics]\n",
    "time_model_latency = [result[1] for result in result_metrics]\n",
    "\n",
    "time_mean = np.mean(time_end_end)\n",
    "time_mean_by_model = np.mean(time_model_latency)\n",
    "\n",
    "\n",
    "p90 = np.percentile(time_end_end, 90)\n",
    "p90_by_model = np.percentile(time_model_latency, 90)\n",
    "\n",
    "print(f\" Final Mean latency results --- > by Wall time:{time_mean} secs::  by Model Latency metrics:{time_mean_by_model} ms\")\n",
    "print(f\" Final P90 latency results --- > by Wall time:{p90} secs ::  by Model Latency metrics:{p90_by_model} ms\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "620ac1db",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "availableInstances": [
   {
    "_defaultOrder": 0,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.t3.medium",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 1,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.t3.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 2,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.t3.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 3,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.t3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 4,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 5,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 6,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 7,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 8,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 9,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 10,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 11,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 12,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5d.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 13,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5d.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 14,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5d.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 15,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5d.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 16,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5d.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 17,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5d.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 18,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5d.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 19,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 20,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": true,
    "memoryGiB": 0,
    "name": "ml.geospatial.interactive",
    "supportedImageNames": [
     "sagemaker-geospatial-v1-0"
    ],
    "vcpuNum": 0
   },
   {
    "_defaultOrder": 21,
    "_isFastLaunch": true,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.c5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 22,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.c5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 23,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.c5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 24,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.c5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 25,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 72,
    "name": "ml.c5.9xlarge",
    "vcpuNum": 36
   },
   {
    "_defaultOrder": 26,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 96,
    "name": "ml.c5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 27,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 144,
    "name": "ml.c5.18xlarge",
    "vcpuNum": 72
   },
   {
    "_defaultOrder": 28,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.c5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 29,
    "_isFastLaunch": true,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g4dn.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 30,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g4dn.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 31,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g4dn.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 32,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g4dn.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 33,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g4dn.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 34,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g4dn.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 35,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 61,
    "name": "ml.p3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 36,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 244,
    "name": "ml.p3.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 37,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 488,
    "name": "ml.p3.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 38,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.p3dn.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 39,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.r5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 40,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.r5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 41,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.r5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 42,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.r5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 43,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.r5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 44,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.r5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 45,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.r5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 46,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.r5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 47,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 48,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 49,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 50,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 51,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 52,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 53,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.g5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 54,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.g5.48xlarge",
    "vcpuNum": 192
   },
   {
    "_defaultOrder": 55,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 56,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4de.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 57,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.trn1.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 58,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.trn1.32xlarge",
    "vcpuNum": 128
   },
   {
    "_defaultOrder": 59,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.trn1n.32xlarge",
    "vcpuNum": 128
   }
  ],
  "instance_type": "ml.t3.medium",
  "kernelspec": {
   "display_name": "trainenv",
   "language": "python",
   "name": "trainenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}