00-data-prep-eda/00c_get_wikidata_side_features.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "6490115c-e827-4a77-8493-df418960b0af",
   "metadata": {},
   "source": [
    "# Collect feature data on MovieLens movies from Wikidata.\n",
    "\n",
    "**references**\n",
    "\n",
    "* internal colab: [QueryWikiData.ipynb](https://source.corp.google.com/piper///depot/google3/learning/laser/experiments/dpwg/movielens/QueryWikiData.ipynb)\n",
    "* tf recommendations: [movielens utils](https://source.corp.google.com/piper///depot/google3/third_party/tensorflow_models/official/recommendation/movielens.py)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e4e80a25-8080-4ade-a496-f7ec2199a087",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PREFIX: rec-bandits-v2\n"
     ]
    }
   ],
   "source": [
    "VERSION        = \"v2\"                       # TODO\n",
    "PREFIX         = f'rec-bandits-{VERSION}'   # TODO\n",
    "\n",
    "print(f\"PREFIX: {PREFIX}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d24c184c-a6dd-4067-90ce-2c14cc591b05",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "PROJECT_ID               = \"hybrid-vertex\"\n",
      "PROJECT_NUM              = \"934903580331\"\n",
      "LOCATION                 = \"us-central1\"\n",
      "\n",
      "REGION                   = \"us-central1\"\n",
      "BQ_LOCATION              = \"US\"\n",
      "VPC_NETWORK_NAME         = \"ucaip-haystack-vpc-network\"\n",
      "\n",
      "VERTEX_SA                = \"934903580331-compute@developer.gserviceaccount.com\"\n",
      "\n",
      "PREFIX                   = \"rec-bandits-v2\"\n",
      "VERSION                  = \"v2\"\n",
      "\n",
      "BUCKET_NAME              = \"rec-bandits-v2-hybrid-vertex-bucket\"\n",
      "BUCKET_URI               = \"gs://rec-bandits-v2-hybrid-vertex-bucket\"\n",
      "DATA_GCS_PREFIX          = \"data\"\n",
      "DATA_PATH                = \"gs://rec-bandits-v2-hybrid-vertex-bucket/data\"\n",
      "VOCAB_SUBDIR             = \"vocabs\"\n",
      "VOCAB_FILENAME           = \"vocab_dict.pkl\"\n",
      "DATA_PATH_KFP_DEMO       = \"gs://rec-bandits-v2-hybrid-vertex-bucket/data/kfp_demo_data/u.data\"\n",
      "\n",
      "VPC_NETWORK_FULL         = \"projects/934903580331/global/networks/ucaip-haystack-vpc-network\"\n",
      "\n",
      "BIGQUERY_DATASET_NAME    = \"mvlens_rec_bandits_v2\"\n",
      "BIGQUERY_TABLE_NAME      = \"training_dataset\"\n",
      "\n",
      "REPOSITORY               = \"rl-movielens-rec-bandits-v2\"\n",
      "\n",
      "DOCKERNAME_01            = \"Dockerfile_train_my_perarm_env\"\n",
      "IMAGE_NAME_01            = \"train-my-perarm-env-v2\"\n",
      "IMAGE_URI_01             = \"gcr.io/hybrid-vertex/train-my-perarm-env-v2\"\n",
      "\n",
      "DOCKERNAME_02            = \"Dockerfile_perarm_feats\"\n",
      "IMAGE_NAME_02            = \"train-perarm-feats-v2\"\n",
      "IMAGE_URI_02             = \"gcr.io/hybrid-vertex/train-perarm-feats-v2\"\n",
      "\n",
      "DOCKERNAME_03            = \"Dockerfile_ranking_bandit\"\n",
      "IMAGE_NAME_03            = \"train-rank-bandit-v2\"\n",
      "IMAGE_URI_03             = \"gcr.io/hybrid-vertex/train-rank-bandit-v2\"\n",
      "\n",
      "DOCKERNAME_04            = \"Dockerfile_train_mab_e2e\"\n",
      "IMAGE_NAME_04            = \"train-mab-e2e-v2\"\n",
      "IMAGE_URI_04             = \"gcr.io/hybrid-vertex/train-mab-e2e-v2\"\n",
      "\n",
      "DOCKERNAME_04_pred       = \"Dockerfile_pred_mab_e2e\"\n",
      "IMAGE_NAME_04_pred       = \"pred-mab-e2e-v2\"\n",
      "IMAGE_URI_04_pred        = \"gcr.io/hybrid-vertex/pred-mab-e2e-v2\"\n",
      "\n",
      "REMOTE_IMAGE_NAME        = \"us-central1-docker.pkg.dev/hybrid-vertex/rl-movielens-rec-bandits-v2/local_docker_tfa\"\n",
      "\n",
      "REPO_DOCKER_PATH_PREFIX  = \"src\"\n",
      "RL_SUB_DIR               = \"per_arm_rl\"\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# staging GCS\n",
    "GCP_PROJECTS             = !gcloud config get-value project\n",
    "PROJECT_ID               = GCP_PROJECTS[0]\n",
    "\n",
    "# GCS bucket and paths\n",
    "BUCKET_NAME              = f'{PREFIX}-{PROJECT_ID}-bucket'\n",
    "BUCKET_URI               = f'gs://{BUCKET_NAME}'\n",
    "\n",
    "config = !gsutil cat {BUCKET_URI}/config/notebook_env.py\n",
    "print(config.n)\n",
    "exec(config.n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "2f004070-862c-441c-948a-e43bf9bf0198",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'gs://rec-bandits-v2-hybrid-vertex-bucket/data'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DATA_PATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "826dca8a-2501-41bb-a6e3-e57b9c063de2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# google cloud\n",
    "from google.cloud import aiplatform, storage\n",
    "\n",
    "# cloud storage client\n",
    "storage_client = storage.Client(project=PROJECT_ID)\n",
    "# bucket = storage_client.bucket(BUCKET_NAME)\n",
    "\n",
    "# Vertex client\n",
    "aiplatform.init(project=PROJECT_ID, location=LOCATION)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "172ab530-18c0-462e-a7a0-baafbc4a5572",
   "metadata": {},
   "outputs": [],
   "source": [
    "bucket = storage_client.bucket(BUCKET_NAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "dc36183e-a40f-4a60-8ff7-f1a442c2e92f",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "# !pip install sparqlwrapper\n",
    "from SPARQLWrapper import SPARQLWrapper, JSON\n",
    "\n",
    "import sys\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from joblib import Parallel, delayed\n",
    "from tqdm.notebook import tqdm\n",
    "import json\n",
    "import urllib.request\n",
    "import zipfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "35b5c1e7-7a47-4932-a37e-ca2b1c0fdf24",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(columns = ['sid'])\n",
    "df.empty"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "548b812d-5398-4c56-ab55-426538bb599b",
   "metadata": {},
   "source": [
    "## Query functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5610ef68-2e8a-4acc-83dc-7d8583ab7b75",
   "metadata": {},
   "outputs": [],
   "source": [
    "ENDPOINT_URL = \"https://query.wikidata.org/sparql\"\n",
    "\n",
    "def make_full_query(id):\n",
    "    \"\"\"\n",
    "    Takes an imbd ID and return a formatted sparql query.\n",
    "\n",
    "    Args:\n",
    "        id: str, imdb movie identifier\n",
    "\n",
    "    Returns:\n",
    "        query : str, sparql query\n",
    "    \"\"\"\n",
    "  # SPARQL views the database composed of triplets: ?subject ?predicate ?object\n",
    "  # Relevant subjects:\n",
    "  #   ?item: item ID\n",
    "  #   ?itemLabel: item name (e.g. film title)\n",
    "  #   ?cast: ID of cast member\n",
    "  #   ?castLabel:  name of cast member\n",
    "  #   ?property: ID of cast member type\n",
    "  #   ?propLabel: type of cast member (e.g. director)\n",
    "  #   ?genreID: ID of genre\n",
    "  #   ?genre: name of genre\n",
    "  #   ?topicID: ID of topic\n",
    "  #   ?topic: name of topic\n",
    "  # Relevant predicates\n",
    "  #   wdt:P31 - instance of (P for property)\n",
    "  #   wdt:P279 - subclass off\n",
    "  #   wtd:P345 - IMDb ID\n",
    "  #   wdt:P136 - genre\n",
    "  #   wtd:P921 - main subjecy\n",
    "  # Relevant objects\n",
    "  #   wd:Q11424 - film (entity)\n",
    "  #   wd:Q5 - human\n",
    "\n",
    "  # The query further limits responses to English responses to avoid duplicates\n",
    "  # of movie entries recorded in other languages. Foreign films will return\n",
    "  # corresponding data recorded in English.\n",
    "\n",
    "  # In plain language the query reads:\n",
    "  # \"Find the wikidata item that is either an instance of film or a subclass of\n",
    "  # a film and with IMDb id matching <id>. Further take all properties of this \n",
    "  # item (movie) which are instances of humans (cast) and record the type of\n",
    "  # cast member they are. Finally collect the genre and topic properties of the \n",
    "  # item and only maintain the version in English.\"\n",
    "\n",
    "\n",
    "    query = \"\"\"\n",
    "    SELECT DISTINCT ?item ?itemLabel ?cast ?castLabel ?property ?propLabel ?genreID ?genre ?topicID ?topic WHERE {\n",
    "    ?item wdt:P31/wdt:P279* wd:Q11424 . \\n\"\"\" + f\"    ?item wdt:P345 'tt{id}' .\\n\" +\"\"\"    \n",
    "    # take all claims on this movie\n",
    "    ?item ?property ?cast.\n",
    "    # that involve a human\n",
    "    ?cast wdt:P31 wd:Q5 .\n",
    "    ?prop wikibase:directClaim ?property .\n",
    "    ?item wdt:P136 ?genreID filter (lang(?genre) = \"en\").\n",
    "    ?genreID rdfs:label ?genre.\n",
    "    ?item wdt:P921 ?topicID filter (lang(?topic) = \"en\").\n",
    "    ?topicID rdfs:label ?topic.\n",
    "    SERVICE wikibase:label { bd:serviceParam wikibase:language \"en\". }}\n",
    "    \"\"\"\n",
    "    return(query)\n",
    "\n",
    "def make_genre_query(id):\n",
    "    query = \"\"\"SELECT DISTINCT ?item ?itemLabel ?genreID ?genre\n",
    "    WHERE {\n",
    "    ?item wdt:P31/wdt:P279* wd:Q11424 . \\n\"\"\" + f\"    ?item wdt:P345 'tt{id}' .\\n\" +\"\"\"\n",
    "    ?item wdt:P136 ?genreID filter (lang(?genre) = \"en\").\n",
    "    ?genreID rdfs:label ?genre.\n",
    "    SERVICE wikibase:label { bd:serviceParam wikibase:language \"en\". }\n",
    "    }\n",
    "    \"\"\"\n",
    "    return(query)\n",
    "\n",
    "def make_topic_query(id):\n",
    "    query = \"\"\"SELECT DISTINCT ?item ?itemLabel ?topicID ?topic\n",
    "    WHERE {\n",
    "    ?item wdt:P31/wdt:P279* wd:Q11424 . \\n\"\"\" + f\"    ?item wdt:P345 'tt{id}' .\\n\" +\"\"\"\n",
    "    ?item wdt:P921 ?topicID filter (lang(?topic) = \"en\").\n",
    "    ?topicID rdfs:label ?topic.\n",
    "    SERVICE wikibase:label { bd:serviceParam wikibase:language \"en\". }\n",
    "    }\n",
    "    \"\"\"\n",
    "    return(query)\n",
    "\n",
    "def make_cast_query(id):\n",
    "    query = \"\"\"SELECT DISTINCT ?item ?itemLabel ?cast ?castLabel ?property ?propLabel\n",
    "    WHERE {\n",
    "    ?item wdt:P31/wdt:P279* wd:Q11424 . \\n\"\"\" + f\"    ?item wdt:P345 'tt{id}' .\\n\" +\"\"\"\n",
    "    # take all claims on this movie\n",
    "    ?item ?property ?cast.\n",
    "    # that involve a human\n",
    "    ?cast wdt:P31 wd:Q5 .\n",
    "    ?prop wikibase:directClaim ?property .\n",
    "    SERVICE wikibase:label { bd:serviceParam wikibase:language \"en\". }\n",
    "    }\n",
    "    \"\"\"\n",
    "    return(query)\n",
    "\n",
    "def get_results(id, how='full', endpoint_url=ENDPOINT_URL):\n",
    "    python_version = f\"{sys.version_info[0]}.{sys.version_info[1]}\"\n",
    "    user_agent = f\"WDQS-example Python/{python_version}\"\n",
    "    sparql = SPARQLWrapper(endpoint_url, agent=user_agent)\n",
    "    if how == 'full':\n",
    "        format_q = make_full_query\n",
    "    if how == 'genre':\n",
    "        format_q = make_genre_query\n",
    "    if how == 'topic':\n",
    "        format_q = make_topic_query\n",
    "    if how == 'cast':\n",
    "        format_q = make_cast_query\n",
    "\n",
    "    def pad_id(id):\n",
    "        \"\"\"Helper function to add enough leading 0s until the id has 7 characters\"\"\"\n",
    "        str_id = str(id)\n",
    "        len_id = len(str_id)\n",
    "        return('0'*(7-len_id)+str_id)\n",
    "\n",
    "    padded_id = pad_id(id)\n",
    "    sparql.setQuery(format_q(padded_id))\n",
    "    sparql.setReturnFormat(JSON)\n",
    "    results = sparql.query().convert()\n",
    "    results = results[\"results\"][\"bindings\"]\n",
    "    if len(results) == 0:\n",
    "        print(f\"INFO: Movie ID {padded_id} was not found on WikiData\")\n",
    "    results = [{k:v['value'] for k, v in item.items()} for item in results]\n",
    "    results = [dict(item, **{'imdbID':padded_id}) for item in results]\n",
    "    \n",
    "    return results\n",
    "\n",
    "def flatten(list_of_lists):\n",
    "    \"\"\"Flattens a list of list of varying size into a single list\"\"\"\n",
    "    flat_list = [item for sublist in list_of_lists for item in sublist]\n",
    "    return flat_list\n",
    "\n",
    "def extract_entity(uri):\n",
    "    \"\"\"Given an URI returns the identifier of the Wiki Entity\"\"\"\n",
    "    uri_parts = uri.split('/')\n",
    "    return uri_parts[-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5d65e40-8259-4d40-89bf-a157a3fd451e",
   "metadata": {},
   "source": [
    "## Download MovieLens data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "432bce43-a982-4573-8ae8-ac7fe810730e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LOCAL_OUTPUT_DIRECTORY : movielens\n"
     ]
    }
   ],
   "source": [
    "data_size = '20m'\n",
    "assert data_size in ['100k', '1m', '20m','25m']\n",
    "\n",
    "LOCAL_OUTPUT_DIRECTORY = \"movielens\"\n",
    "print(f\"LOCAL_OUTPUT_DIRECTORY : {LOCAL_OUTPUT_DIRECTORY}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1f78ce81-c7af-4c1a-a0d0-465e1d7c32e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ! rm -rf ./$LOCAL_OUTPUT_DIRECTORY\n",
    "# ! mkdir ./$LOCAL_OUTPUT_DIRECTORY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1e789bde-1f35-41e0-92b6-2abb52a98a3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_url = f\"https://files.grouplens.org/datasets/movielens/ml-{data_size}.zip\"\n",
    "\n",
    "a = urllib.request.urlretrieve(data_url, \"data\")\n",
    "\n",
    "if not os.path.exists(f\"{LOCAL_OUTPUT_DIRECTORY}/\"):\n",
    "    os.makedirs(f\"{LOCAL_OUTPUT_DIRECTORY}/\")\n",
    "\n",
    "zip_ref = zipfile.ZipFile(\"data\", \"r\")\n",
    "zip_ref.extractall(f\"{LOCAL_OUTPUT_DIRECTORY}/ml_{data_size}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e44513c6-a59c-4f42-807a-fe26dfc8fa3f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[01;34mmovielens\u001b[00m\n",
      "└── \u001b[01;34mml_20m\u001b[00m\n",
      "    └── \u001b[01;34mml-20m\u001b[00m\n",
      "        ├── README.txt\n",
      "        ├── genome-scores.csv\n",
      "        ├── genome-tags.csv\n",
      "        ├── links.csv\n",
      "        ├── movies.csv\n",
      "        ├── ratings.csv\n",
      "        └── tags.csv\n",
      "\n",
      "2 directories, 7 files\n"
     ]
    }
   ],
   "source": [
    "! tree $LOCAL_OUTPUT_DIRECTORY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "45b740a6-6123-4c94-91fe-a23d41f02f75",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movieId</th>\n",
       "      <th>imdbId</th>\n",
       "      <th>tmdbId</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>114709</td>\n",
       "      <td>862.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>113497</td>\n",
       "      <td>8844.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>113228</td>\n",
       "      <td>15602.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>114885</td>\n",
       "      <td>31357.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>113041</td>\n",
       "      <td>11862.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   movieId  imdbId   tmdbId\n",
       "0        1  114709    862.0\n",
       "1        2  113497   8844.0\n",
       "2        3  113228  15602.0\n",
       "3        4  114885  31357.0\n",
       "4        5  113041  11862.0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read Links dataset\n",
    "datapath = os.path.join(LOCAL_OUTPUT_DIRECTORY, f\"ml_{data_size}\", f\"ml-{data_size}\")\n",
    "link_df = pd.read_csv(os.path.join(datapath, 'links.csv'))\n",
    "link_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "ade4a133-5162-44c3-97f4-2605dec3486b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'gs://rec-bandits-v2-hybrid-vertex-bucket/data/ml_20m'"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DEST_GCS_PATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "39a4fa08-c53a-43a3-a2f7-869d09354004",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copying file://movielens/ml_20m/ml-20m/links.csv [Content-Type=text/csv]...\n",
      "Copying file://movielens/ml_20m/ml-20m/README.txt [Content-Type=text/plain]...  \n",
      "Copying file://movielens/ml_20m/ml-20m/genome-scores.csv [Content-Type=text/csv]...\n",
      "==> NOTE: You are uploading one or more large file(s), which would run          \n",
      "significantly faster if you enable parallel composite uploads. This\n",
      "feature can be enabled by editing the\n",
      "\"parallel_composite_upload_threshold\" value in your .boto\n",
      "configuration file. However, note that if you do this large files will\n",
      "be uploaded as `composite objects\n",
      "<https://cloud.google.com/storage/docs/composite-objects>`_,which\n",
      "means that any user who downloads such objects will need to have a\n",
      "compiled crcmod installed (see \"gsutil help crcmod\"). This is because\n",
      "without a compiled crcmod, computing checksums on composite objects is\n",
      "so slow that gsutil disables downloads of composite objects.\n",
      "\n",
      "Copying file://movielens/ml_20m/ml-20m/ratings.csv [Content-Type=text/csv]...   \n",
      "- [4 files][817.8 MiB/817.8 MiB]   63.4 MiB/s                                   \n",
      "==> NOTE: You are performing a sequence of gsutil operations that may\n",
      "run significantly faster if you instead use gsutil -m cp ... Please\n",
      "see the -m section under \"gsutil help options\" for further information\n",
      "about when gsutil -m can be advantageous.\n",
      "\n",
      "Copying file://movielens/ml_20m/ml-20m/movies.csv [Content-Type=text/csv]...\n",
      "Copying file://movielens/ml_20m/ml-20m/tags.csv [Content-Type=text/csv]...      \n",
      "Copying file://movielens/ml_20m/ml-20m/genome-tags.csv [Content-Type=text/csv]...\n",
      "- [7 files][835.0 MiB/835.0 MiB]   60.4 MiB/s                                   \n",
      "Operation completed over 7 objects/835.0 MiB.                                    \n"
     ]
    }
   ],
   "source": [
    "!gsutil cp -r $LOCAL_OUTPUT_DIRECTORY $DEST_GCS_PATH"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fdcac574-13cd-4a39-a98e-8bc01e8b6669",
   "metadata": {},
   "source": [
    "## Send sparql queries to WikiData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d9e415fb-7e35-455c-bead-8a8b44dc27bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_feat_datapath = 'raw_features'\n",
    "if not os.path.exists(raw_feat_datapath):\n",
    "    os.makedirs(raw_feat_datapath)\n",
    "\n",
    "def save_json(result_dict, fname, save_path = raw_feat_datapath):\n",
    "    res = json.dumps(result_dict)\n",
    "    res = res.encode('utf-8')\n",
    "    with open(os.path.join(save_path, fname), 'w') as f:\n",
    "        f.write(str(res))\n",
    "        \n",
    "LOCAL_GENRE_FILE = 'genre.json'\n",
    "LOCAL_TOPIC_FILE = 'topic.json'\n",
    "LOCAL_CAST_FILE = 'cast.json'\n",
    "        \n",
    "DEST_GCS_PATH = f\"{DATA_PATH}/ml_{data_size}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f07ae435-535d-4fbc-8f46-86a2dad16dcc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# genre information\n",
    "result_genre = Parallel(n_jobs=4)(delayed(get_results)(i, 'genre') for i in tqdm(link_df['imdbId']))\n",
    "result_genre_flat = flatten(result_genre)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d7337143-d065-4f56-99de-29d8949edfe4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "LOCAL_GENRE_FILE = 'genre.json'\n",
    "\n",
    "# result_genre_flat\n",
    "save_json(result_genre_flat, LOCAL_GENRE_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "0db068bf-0122-44eb-bb27-22d9026bcab2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'gs://rec-bandits-v2-hybrid-vertex-bucket/data/ml_20m/raw_features/genre.json'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# LOCAL_FILE = f\"{raw_feat_datapath}/genre.json\"\n",
    "# DEST_BLOB = f\"{DATA_PATH}/ml_{data_size}/{LOCAL_FILE}\"\n",
    "\n",
    "# blob = bucket.blob(DEST_BLOB)\n",
    "# blob.upload_from_filename(LOCAL_FILE)\n",
    "\n",
    "! gsutil cp $raw_feat_datapath/$LOCAL_GENRE_FILE $DEST_GCS_PATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "235e03b6-437b-4a72-ac10-d76174b2fa7d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# topic information\n",
    "result_topic = Parallel(n_jobs=4)(delayed(get_results)(i, 'topic') for i in tqdm(link_df['imdbId']))\n",
    "result_topic_flat = flatten(result_topic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "56d17cb7-5080-4e9a-ab99-1bf16b7dc7c9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# cast information\n",
    "result_cast = Parallel(n_jobs=4)(delayed(get_results)(i, 'cast') for i in tqdm(link_df['imdbId']))\n",
    "result_cast_flat = flatten(result_cast)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b255b77-f380-45ef-8d1c-d73ea6091b58",
   "metadata": {},
   "source": [
    "## Save raw feature data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "08c3ccfa-4ffa-425c-9ef8-d15867c482ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "LOCAL_GENRE_FILE = 'genre.json'\n",
    "LOCAL_TOPIC_FILE = 'topic.json'\n",
    "LOCAL_CAST_FILE = 'cast.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "749bff80-b45b-4b07-83fb-fbc8d2fd5fca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copying file://raw_features/genre.json [Content-Type=application/json]...\n",
      "/ [1 files][ 10.5 MiB/ 10.5 MiB]                                                \n",
      "Operation completed over 1 objects/10.5 MiB.                                     \n"
     ]
    }
   ],
   "source": [
    "# save_json(result_genre_flat, LOCAL_GENRE_FILE)\n",
    "\n",
    "! gsutil cp $raw_feat_datapath/$LOCAL_GENRE_FILE $DEST_GCS_PATH/$raw_feat_datapath/$LOCAL_GENRE_FILE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "b5810de3-76f1-4656-9e34-a30b7b1b5794",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copying file://raw_features/topic.json [Content-Type=application/json]...\n",
      "/ [1 files][  2.1 MiB/  2.1 MiB]                                                \n",
      "Operation completed over 1 objects/2.1 MiB.                                      \n"
     ]
    }
   ],
   "source": [
    "save_json(result_topic_flat, LOCAL_TOPIC_FILE)\n",
    "\n",
    "! gsutil cp $raw_feat_datapath/$LOCAL_TOPIC_FILE $DEST_GCS_PATH/$raw_feat_datapath//$LOCAL_TOPIC_FILE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "a25e6c1e-84d7-4340-8e31-9ae56a1ea043",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copying file://raw_features/cast.json [Content-Type=application/json]...\n",
      "- [1 files][113.4 MiB/113.4 MiB]                                                \n",
      "Operation completed over 1 objects/113.4 MiB.                                    \n"
     ]
    }
   ],
   "source": [
    "save_json(result_cast_flat, LOCAL_CAST_FILE)\n",
    "\n",
    "! gsutil cp $raw_feat_datapath/$LOCAL_CAST_FILE $DEST_GCS_PATH/$raw_feat_datapath//$LOCAL_CAST_FILE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "635aafc2-d2b6-422f-b3d2-a3830606c2e6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gs://rec-bandits-v2-hybrid-vertex-bucket/data/ml_20m\n"
     ]
    }
   ],
   "source": [
    "!gsutil ls $DEST_GCS_PATH/"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e270fadc-c86f-43ba-a161-6b74963b241b",
   "metadata": {},
   "source": [
    "## Process data and save it as a csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "1b4b6649-9e05-492c-a4ce-cd111e2707cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "feat_datapath = 'processed_features'\n",
    "if not os.path.exists(feat_datapath):\n",
    "    os.makedirs(feat_datapath)\n",
    "\n",
    "def process_save_csv(\n",
    "    result_dict, \n",
    "    fname, \n",
    "    rename_cols = None, \n",
    "    extract_entity_cols = None, \n",
    "    save_path = feat_datapath\n",
    "):\n",
    "    result_df = pd.DataFrame(result_dict)\n",
    "    if rename_cols:\n",
    "        result_df.rename(columns = rename_cols, inplace = True)\n",
    "    if extract_entity_cols:\n",
    "        for col in extract_entity_cols:\n",
    "            result_df[col] = result_df.apply(lambda x: extract_entity(x[col]), axis = 1)\n",
    "    result_df.to_csv(os.path.join(save_path, fname), index=False)\n",
    "    \n",
    "LOCAL_GENRE_CSV = 'genre.csv'\n",
    "LOCAL_TOPIC_CSV = 'topic.csv'\n",
    "LOCAL_CAST_CSV  = 'cast.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "4a1d5ca1-1605-4d1d-a781-814904106b34",
   "metadata": {},
   "outputs": [],
   "source": [
    "process_save_csv(\n",
    "    result_genre_flat, 'genre.csv', \n",
    "    rename_cols = {\n",
    "        'item':'wikiItemID', \n",
    "        'itemLabel':'wikiItemLabel',\n",
    "        'genre':'genreLabel'\n",
    "    },\n",
    "    extract_entity_cols = ['wikiItemID', 'genreID']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "64f65565-5c3d-46a3-a07e-03ca4e96f693",
   "metadata": {},
   "outputs": [],
   "source": [
    "process_save_csv(\n",
    "    result_topic_flat, 'topic.csv',\n",
    "    rename_cols = {\n",
    "        'item':'wikiItemID', \n",
    "        'itemLabel':'wikiItemLabel',\n",
    "        'topic':'topicLabel'\n",
    "    },\n",
    "    extract_entity_cols = ['wikiItemID', 'topicID']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "822231c9-4dd6-4d95-8ee1-6dd0623660a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "process_save_csv(\n",
    "    result_cast_flat, 'cast.csv',\n",
    "    rename_cols = {\n",
    "        'item':'wikiItemID', \n",
    "        'itemLabel':'wikiItemLabel',\n",
    "        'cast':'personID',\n",
    "        'castLabel':'personLabel',\n",
    "        'property':'roleID',\n",
    "        'propLabel':'roleLabel'\n",
    "    },\n",
    "    extract_entity_cols = ['wikiItemID', 'personID', 'roleID']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "89f9bfd1-9de7-4419-9b0b-c77a16c48319",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cast.csv  genre.csv  topic.csv\n"
     ]
    }
   ],
   "source": [
    "!ls $feat_datapath"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "e1869a1c-623e-427e-bead-a6eaefea97cb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copying file://processed_features/genre.csv [Content-Type=text/csv]...\n",
      "/ [1 files][  3.2 MiB/  3.2 MiB]                                                \n",
      "Operation completed over 1 objects/3.2 MiB.                                      \n"
     ]
    }
   ],
   "source": [
    "! gsutil cp $feat_datapath/$LOCAL_GENRE_CSV $DEST_GCS_PATH/$feat_datapath/$LOCAL_GENRE_CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "072bea96-be0c-4895-b6a6-ca04fc316917",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copying file://processed_features/topic.csv [Content-Type=text/csv]...\n",
      "/ [1 files][622.2 KiB/622.2 KiB]                                                \n",
      "Operation completed over 1 objects/622.2 KiB.                                    \n"
     ]
    }
   ],
   "source": [
    "! gsutil cp $feat_datapath/$LOCAL_TOPIC_CSV $DEST_GCS_PATH/$feat_datapath/$LOCAL_TOPIC_CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "162b015b-f42d-47d7-a1b0-b0469c5a611a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copying file://processed_features/cast.csv [Content-Type=text/csv]...\n",
      "/ [1 files][ 30.7 MiB/ 30.7 MiB]                                                \n",
      "Operation completed over 1 objects/30.7 MiB.                                     \n"
     ]
    }
   ],
   "source": [
    "! gsutil cp $feat_datapath/$LOCAL_CAST_CSV $DEST_GCS_PATH/$feat_datapath/$LOCAL_CAST_CSV"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f9e9e6e-c3a5-4016-8cef-a69e2e6cebbb",
   "metadata": {},
   "source": [
    "# inspect files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "650f23ef-230d-4e06-a330-e0a35d5bf6d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gs://rec-bandits-v2-hybrid-vertex-bucket/data/ml_20m/processed_features/cast.csv\n",
      "gs://rec-bandits-v2-hybrid-vertex-bucket/data/ml_20m/processed_features/genre.csv\n",
      "gs://rec-bandits-v2-hybrid-vertex-bucket/data/ml_20m/processed_features/topic.csv\n"
     ]
    }
   ],
   "source": [
    "!gsutil ls $DEST_GCS_PATH/$feat_datapath"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "63904c20-b485-435f-bee8-73c3f68c0605",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cast_df shape : (436642, 7)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wikiItemID</th>\n",
       "      <th>roleID</th>\n",
       "      <th>personID</th>\n",
       "      <th>wikiItemLabel</th>\n",
       "      <th>personLabel</th>\n",
       "      <th>roleLabel</th>\n",
       "      <th>imdbID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q171048</td>\n",
       "      <td>P58</td>\n",
       "      <td>Q3397612</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>Joel Cohen</td>\n",
       "      <td>screenwriter</td>\n",
       "      <td>114709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q171048</td>\n",
       "      <td>P725</td>\n",
       "      <td>Q374093</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>John Ratzenberger</td>\n",
       "      <td>voice actor</td>\n",
       "      <td>114709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q171048</td>\n",
       "      <td>P58</td>\n",
       "      <td>Q269214</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>John Lasseter</td>\n",
       "      <td>screenwriter</td>\n",
       "      <td>114709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q171048</td>\n",
       "      <td>P57</td>\n",
       "      <td>Q269214</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>John Lasseter</td>\n",
       "      <td>director</td>\n",
       "      <td>114709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q171048</td>\n",
       "      <td>P170</td>\n",
       "      <td>Q269214</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>John Lasseter</td>\n",
       "      <td>creator</td>\n",
       "      <td>114709</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  wikiItemID roleID  personID wikiItemLabel        personLabel     roleLabel  \\\n",
       "0    Q171048    P58  Q3397612     Toy Story         Joel Cohen  screenwriter   \n",
       "1    Q171048   P725   Q374093     Toy Story  John Ratzenberger   voice actor   \n",
       "2    Q171048    P58   Q269214     Toy Story      John Lasseter  screenwriter   \n",
       "3    Q171048    P57   Q269214     Toy Story      John Lasseter      director   \n",
       "4    Q171048   P170   Q269214     Toy Story      John Lasseter       creator   \n",
       "\n",
       "   imdbID  \n",
       "0  114709  \n",
       "1  114709  \n",
       "2  114709  \n",
       "3  114709  \n",
       "4  114709  "
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cast_df = pd.read_csv(f\"{DEST_GCS_PATH}/{feat_datapath}/{LOCAL_CAST_CSV}\")\n",
    "\n",
    "print(f\"cast_df shape : {cast_df.shape}\")\n",
    "cast_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "62cdd92a-4ea1-402e-81f8-9381406def2e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "topic_df shape : (11773, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wikiItemID</th>\n",
       "      <th>topicID</th>\n",
       "      <th>wikiItemLabel</th>\n",
       "      <th>topicLabel</th>\n",
       "      <th>imdbID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q222939</td>\n",
       "      <td>Q182154</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>time travel</td>\n",
       "      <td>113497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q222939</td>\n",
       "      <td>Q3244175</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>tabletop game</td>\n",
       "      <td>113497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q782465</td>\n",
       "      <td>Q191089</td>\n",
       "      <td>Grumpier Old Men</td>\n",
       "      <td>old age</td>\n",
       "      <td>113228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1304560</td>\n",
       "      <td>Q11995</td>\n",
       "      <td>Father of the Bride Part II</td>\n",
       "      <td>human pregnancy</td>\n",
       "      <td>113041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q42198</td>\n",
       "      <td>Q214126</td>\n",
       "      <td>Heat</td>\n",
       "      <td>Los Angeles Police Department</td>\n",
       "      <td>113277</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  wikiItemID   topicID                wikiItemLabel  \\\n",
       "0    Q222939   Q182154                      Jumanji   \n",
       "1    Q222939  Q3244175                      Jumanji   \n",
       "2    Q782465   Q191089             Grumpier Old Men   \n",
       "3   Q1304560    Q11995  Father of the Bride Part II   \n",
       "4     Q42198   Q214126                         Heat   \n",
       "\n",
       "                      topicLabel  imdbID  \n",
       "0                    time travel  113497  \n",
       "1                  tabletop game  113497  \n",
       "2                        old age  113228  \n",
       "3                human pregnancy  113041  \n",
       "4  Los Angeles Police Department  113277  "
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_df = pd.read_csv(f\"{DEST_GCS_PATH}/{feat_datapath}/{LOCAL_TOPIC_CSV}\")\n",
    "\n",
    "print(f\"topic_df shape : {topic_df.shape}\")\n",
    "topic_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "ea0249fd-13d3-429f-a353-3d881d51ac86",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "genre_df shape : (58984, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wikiItemID</th>\n",
       "      <th>genreID</th>\n",
       "      <th>wikiItemLabel</th>\n",
       "      <th>genreLabel</th>\n",
       "      <th>imdbID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q171048</td>\n",
       "      <td>Q157394</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>fantasy film</td>\n",
       "      <td>114709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q171048</td>\n",
       "      <td>Q157443</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>comedy film</td>\n",
       "      <td>114709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q171048</td>\n",
       "      <td>Q319221</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>adventure film</td>\n",
       "      <td>114709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q171048</td>\n",
       "      <td>Q663106</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>buddy film</td>\n",
       "      <td>114709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q222939</td>\n",
       "      <td>Q157394</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>fantasy film</td>\n",
       "      <td>113497</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  wikiItemID  genreID wikiItemLabel      genreLabel  imdbID\n",
       "0    Q171048  Q157394     Toy Story    fantasy film  114709\n",
       "1    Q171048  Q157443     Toy Story     comedy film  114709\n",
       "2    Q171048  Q319221     Toy Story  adventure film  114709\n",
       "3    Q171048  Q663106     Toy Story      buddy film  114709\n",
       "4    Q222939  Q157394       Jumanji    fantasy film  113497"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "genre_df = pd.read_csv(f\"{DEST_GCS_PATH}/{feat_datapath}/{LOCAL_GENRE_CSV}\")\n",
    "\n",
    "print(f\"genre_df shape : {genre_df.shape}\")\n",
    "genre_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "de8f50d8-44fb-4abe-b8c5-cd7dfbdfe1fb",
   "metadata": {},
   "source": [
    "## Aggregate wiki features"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "804d3ab2-6742-4a8e-b00e-59065d8ee029",
   "metadata": {},
   "source": [
    "### Cast features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "73685b09-8844-44b5-9e7f-b2bcd112eab9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cast_df shape : (436642, 7)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wikiItemID</th>\n",
       "      <th>roleID</th>\n",
       "      <th>personID</th>\n",
       "      <th>wikiItemLabel</th>\n",
       "      <th>personLabel</th>\n",
       "      <th>roleLabel</th>\n",
       "      <th>imdbID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q171048</td>\n",
       "      <td>P58</td>\n",
       "      <td>Q3397612</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>Joel Cohen</td>\n",
       "      <td>screenwriter</td>\n",
       "      <td>114709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q171048</td>\n",
       "      <td>P725</td>\n",
       "      <td>Q374093</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>John Ratzenberger</td>\n",
       "      <td>voice actor</td>\n",
       "      <td>114709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q171048</td>\n",
       "      <td>P58</td>\n",
       "      <td>Q269214</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>John Lasseter</td>\n",
       "      <td>screenwriter</td>\n",
       "      <td>114709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q171048</td>\n",
       "      <td>P57</td>\n",
       "      <td>Q269214</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>John Lasseter</td>\n",
       "      <td>director</td>\n",
       "      <td>114709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q171048</td>\n",
       "      <td>P170</td>\n",
       "      <td>Q269214</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>John Lasseter</td>\n",
       "      <td>creator</td>\n",
       "      <td>114709</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  wikiItemID roleID  personID wikiItemLabel        personLabel     roleLabel  \\\n",
       "0    Q171048    P58  Q3397612     Toy Story         Joel Cohen  screenwriter   \n",
       "1    Q171048   P725   Q374093     Toy Story  John Ratzenberger   voice actor   \n",
       "2    Q171048    P58   Q269214     Toy Story      John Lasseter  screenwriter   \n",
       "3    Q171048    P57   Q269214     Toy Story      John Lasseter      director   \n",
       "4    Q171048   P170   Q269214     Toy Story      John Lasseter       creator   \n",
       "\n",
       "   imdbID  \n",
       "0  114709  \n",
       "1  114709  \n",
       "2  114709  \n",
       "3  114709  \n",
       "4  114709  "
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(f\"cast_df shape : {cast_df.shape}\")\n",
    "\n",
    "cast_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "0e4ead62-bc1c-43dd-bc70-9f5c93a24d0e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cast_df_v1 shape : (408479, 7)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wikiItemID</th>\n",
       "      <th>roleID</th>\n",
       "      <th>personID</th>\n",
       "      <th>wikiItemLabel</th>\n",
       "      <th>personLabel</th>\n",
       "      <th>roleLabel</th>\n",
       "      <th>imdbID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q171048</td>\n",
       "      <td>P58</td>\n",
       "      <td>Q3397612</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>Joel Cohen</td>\n",
       "      <td>screenwriter</td>\n",
       "      <td>114709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q171048</td>\n",
       "      <td>P725</td>\n",
       "      <td>Q374093</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>John Ratzenberger</td>\n",
       "      <td>voice actor</td>\n",
       "      <td>114709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q171048</td>\n",
       "      <td>P170</td>\n",
       "      <td>Q269214</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>John Lasseter</td>\n",
       "      <td>creator</td>\n",
       "      <td>114709</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  wikiItemID roleID  personID wikiItemLabel        personLabel     roleLabel  \\\n",
       "0    Q171048    P58  Q3397612     Toy Story         Joel Cohen  screenwriter   \n",
       "1    Q171048   P725   Q374093     Toy Story  John Ratzenberger   voice actor   \n",
       "2    Q171048   P170   Q269214     Toy Story      John Lasseter       creator   \n",
       "\n",
       "   imdbID  \n",
       "0  114709  \n",
       "1  114709  \n",
       "2  114709  "
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cast_df_v1 = cast_df.drop_duplicates(\n",
    "    subset = ['imdbID', 'personID'],\n",
    "    keep = 'last'\n",
    ").reset_index(drop = True)\n",
    "\n",
    "print(f\"cast_df_v1 shape : {cast_df_v1.shape}\")\n",
    "\n",
    "cast_df_v1.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "aa5bc4a2-0f03-44ac-86e5-f52c47744787",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cast_list_agg_df shape : (25771, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>imdbID</th>\n",
       "      <th>personLabel</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>[William Kennedy Dickson, Charles Kayser, Will...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8</td>\n",
       "      <td>[Fred Ott, William Kennedy Dickson, William He...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10</td>\n",
       "      <td>[Louis Lumière]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   imdbID                                        personLabel\n",
       "0       5  [William Kennedy Dickson, Charles Kayser, Will...\n",
       "1       8  [Fred Ott, William Kennedy Dickson, William He...\n",
       "2      10                                    [Louis Lumière]"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cast_list_agg_df = cast_df_v1.groupby('imdbID')['personLabel'].apply(list).reset_index()\n",
    "print(f\"cast_list_agg_df shape : {cast_list_agg_df.shape}\")\n",
    "\n",
    "cast_list_agg_df.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80a5a858-29da-4c76-87b8-1c9de81ae571",
   "metadata": {},
   "source": [
    "### Topic features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "69aa6f0c-9595-46dd-8061-ead8686a377c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "topic_df shape : (11773, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wikiItemID</th>\n",
       "      <th>topicID</th>\n",
       "      <th>wikiItemLabel</th>\n",
       "      <th>topicLabel</th>\n",
       "      <th>imdbID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q222939</td>\n",
       "      <td>Q182154</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>time travel</td>\n",
       "      <td>113497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q222939</td>\n",
       "      <td>Q3244175</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>tabletop game</td>\n",
       "      <td>113497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q782465</td>\n",
       "      <td>Q191089</td>\n",
       "      <td>Grumpier Old Men</td>\n",
       "      <td>old age</td>\n",
       "      <td>113228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q1304560</td>\n",
       "      <td>Q11995</td>\n",
       "      <td>Father of the Bride Part II</td>\n",
       "      <td>human pregnancy</td>\n",
       "      <td>113041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q42198</td>\n",
       "      <td>Q214126</td>\n",
       "      <td>Heat</td>\n",
       "      <td>Los Angeles Police Department</td>\n",
       "      <td>113277</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  wikiItemID   topicID                wikiItemLabel  \\\n",
       "0    Q222939   Q182154                      Jumanji   \n",
       "1    Q222939  Q3244175                      Jumanji   \n",
       "2    Q782465   Q191089             Grumpier Old Men   \n",
       "3   Q1304560    Q11995  Father of the Bride Part II   \n",
       "4     Q42198   Q214126                         Heat   \n",
       "\n",
       "                      topicLabel  imdbID  \n",
       "0                    time travel  113497  \n",
       "1                  tabletop game  113497  \n",
       "2                        old age  113228  \n",
       "3                human pregnancy  113041  \n",
       "4  Los Angeles Police Department  113277  "
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(f\"topic_df shape : {topic_df.shape}\")\n",
    "\n",
    "topic_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "377c2eec-4477-4003-8c8a-33a567ac3a97",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "topic_list_agg_df shape : (6449, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>imdbID</th>\n",
       "      <th>topicLabel</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1527</td>\n",
       "      <td>[adultery]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2381</td>\n",
       "      <td>[organized crime]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4008</td>\n",
       "      <td>[dinosaur]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   imdbID         topicLabel\n",
       "0    1527         [adultery]\n",
       "1    2381  [organized crime]\n",
       "2    4008         [dinosaur]"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_list_agg_df = topic_df.groupby('imdbID')['topicLabel'].apply(list).reset_index()\n",
    "print(f\"topic_list_agg_df shape : {topic_list_agg_df.shape}\")\n",
    "\n",
    "topic_list_agg_df.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "88be7f1d-eb98-407e-8d23-b70aa92a5672",
   "metadata": {},
   "source": [
    "### genre features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "dba87f27-3b49-4fcc-8d7b-9672cd3f83cb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "genre_df shape : (58984, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wikiItemID</th>\n",
       "      <th>genreID</th>\n",
       "      <th>wikiItemLabel</th>\n",
       "      <th>genreLabel</th>\n",
       "      <th>imdbID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q171048</td>\n",
       "      <td>Q157394</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>fantasy film</td>\n",
       "      <td>114709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q171048</td>\n",
       "      <td>Q157443</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>comedy film</td>\n",
       "      <td>114709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q171048</td>\n",
       "      <td>Q319221</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>adventure film</td>\n",
       "      <td>114709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q171048</td>\n",
       "      <td>Q663106</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>buddy film</td>\n",
       "      <td>114709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q222939</td>\n",
       "      <td>Q157394</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>fantasy film</td>\n",
       "      <td>113497</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  wikiItemID  genreID wikiItemLabel      genreLabel  imdbID\n",
       "0    Q171048  Q157394     Toy Story    fantasy film  114709\n",
       "1    Q171048  Q157443     Toy Story     comedy film  114709\n",
       "2    Q171048  Q319221     Toy Story  adventure film  114709\n",
       "3    Q171048  Q663106     Toy Story      buddy film  114709\n",
       "4    Q222939  Q157394       Jumanji    fantasy film  113497"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(f\"genre_df shape : {genre_df.shape}\")\n",
    "\n",
    "genre_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "26a17462-9bb1-4cf5-92f3-a20e4e74b3d1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "genre_list_agg_df shape : (25434, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>imdbID</th>\n",
       "      <th>genreLabel</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>[docufiction film, documentary film, silent film]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8</td>\n",
       "      <td>[silent film]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10</td>\n",
       "      <td>[documentary film, silent film]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   imdbID                                         genreLabel\n",
       "0       5  [docufiction film, documentary film, silent film]\n",
       "1       8                                      [silent film]\n",
       "2      10                    [documentary film, silent film]"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "genre_list_agg_df = genre_df.groupby('imdbID')['genreLabel'].apply(list).reset_index()\n",
    "print(f\"genre_list_agg_df shape : {genre_list_agg_df.shape}\")\n",
    "\n",
    "genre_list_agg_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "229011dd-8c05-416f-8d73-05a03b1e75ad",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "environment": {
   "kernel": "python3",
   "name": "tf2-gpu.2-13.m112",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-13:m112"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}