00-data-prep-eda/00a_data_download_prep.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "118ebdf5-5d6d-4b59-8e10-9c3ae7f635f1",
   "metadata": {},
   "source": [
    "# Movielens download\n",
    "\n",
    "1. download [MovieLens 1M](https://www.tensorflow.org/datasets/catalog/movielens#movielens1m-movies) public dataset\n",
    "2. Write datasets to TF-Records\n",
    "3. Generate dataset vocabulary and look-up dictionaries"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1c4aa8e-a5c9-4682-8621-b4da69d1c5e7",
   "metadata": {},
   "source": [
    "## Notebook config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0b75c6b6-0854-483c-95b3-b0b9a0675d4e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PREFIX: rec-bandits-v2\n"
     ]
    }
   ],
   "source": [
    "VERSION        = \"v2\"                       # TODO\n",
    "PREFIX         = f'rec-bandits-{VERSION}'   # TODO\n",
    "\n",
    "print(f\"PREFIX: {PREFIX}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "16eacb02-8026-4a99-9e54-48b39a7955cd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "PROJECT_ID               = \"hybrid-vertex\"\n",
      "PROJECT_NUM              = \"934903580331\"\n",
      "LOCATION                 = \"us-central1\"\n",
      "\n",
      "REGION                   = \"us-central1\"\n",
      "BQ_LOCATION              = \"US\"\n",
      "VPC_NETWORK_NAME         = \"ucaip-haystack-vpc-network\"\n",
      "\n",
      "VERTEX_SA                = \"934903580331-compute@developer.gserviceaccount.com\"\n",
      "\n",
      "PREFIX                   = \"rec-bandits-v2\"\n",
      "VERSION                  = \"v2\"\n",
      "\n",
      "BUCKET_NAME              = \"rec-bandits-v2-hybrid-vertex-bucket\"\n",
      "BUCKET_URI               = \"gs://rec-bandits-v2-hybrid-vertex-bucket\"\n",
      "DATA_GCS_PREFIX          = \"data\"\n",
      "DATA_PATH                = \"gs://rec-bandits-v2-hybrid-vertex-bucket/data\"\n",
      "VOCAB_SUBDIR             = \"vocabs\"\n",
      "VOCAB_FILENAME           = \"vocab_dict.pkl\"\n",
      "DATA_PATH_KFP_DEMO       = \"gs://rec-bandits-v2-hybrid-vertex-bucket/data/kfp_demo_data/u.data\"\n",
      "\n",
      "VPC_NETWORK_FULL         = \"projects/934903580331/global/networks/ucaip-haystack-vpc-network\"\n",
      "\n",
      "BIGQUERY_DATASET_NAME    = \"mvlens_rec_bandits_v2\"\n",
      "BIGQUERY_TABLE_NAME      = \"training_dataset\"\n",
      "\n",
      "REPOSITORY               = \"rl-movielens-rec-bandits-v2\"\n",
      "\n",
      "DOCKERNAME_01            = \"Dockerfile_train_my_perarm_env\"\n",
      "IMAGE_NAME_01            = \"train-my-perarm-env-v2\"\n",
      "IMAGE_URI_01             = \"gcr.io/hybrid-vertex/train-my-perarm-env-v2\"\n",
      "\n",
      "DOCKERNAME_02            = \"Dockerfile_perarm_feats\"\n",
      "IMAGE_NAME_02            = \"train-perarm-feats-v2\"\n",
      "IMAGE_URI_02             = \"gcr.io/hybrid-vertex/train-perarm-feats-v2\"\n",
      "\n",
      "DOCKERNAME_03            = \"Dockerfile_ranking_bandit\"\n",
      "IMAGE_NAME_03            = \"train-rank-bandit-v2\"\n",
      "IMAGE_URI_03             = \"gcr.io/hybrid-vertex/train-rank-bandit-v2\"\n",
      "\n",
      "DOCKERNAME_04            = \"Dockerfile_train_mab_e2e\"\n",
      "IMAGE_NAME_04            = \"train-mab-e2e-v2\"\n",
      "IMAGE_URI_04             = \"gcr.io/hybrid-vertex/train-mab-e2e-v2\"\n",
      "\n",
      "DOCKERNAME_04_pred       = \"Dockerfile_pred_mab_e2e\"\n",
      "IMAGE_NAME_04_pred       = \"pred-mab-e2e-v2\"\n",
      "IMAGE_URI_04_pred        = \"gcr.io/hybrid-vertex/pred-mab-e2e-v2\"\n",
      "\n",
      "REMOTE_IMAGE_NAME        = \"us-central1-docker.pkg.dev/hybrid-vertex/rl-movielens-rec-bandits-v2/local_docker_tfa\"\n",
      "\n",
      "REPO_DOCKER_PATH_PREFIX  = \"src\"\n",
      "RL_SUB_DIR               = \"per_arm_rl\"\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# staging GCS\n",
    "GCP_PROJECTS             = !gcloud config get-value project\n",
    "PROJECT_ID               = GCP_PROJECTS[0]\n",
    "\n",
    "# GCS bucket and paths\n",
    "BUCKET_NAME              = f'{PREFIX}-{PROJECT_ID}-bucket'\n",
    "BUCKET_URI               = f'gs://{BUCKET_NAME}'\n",
    "\n",
    "config = !gsutil cat {BUCKET_URI}/config/notebook_env.py\n",
    "print(config.n)\n",
    "exec(config.n)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "742cc37f-e105-4b8c-8e0c-21259c4a8b82",
   "metadata": {},
   "source": [
    "## imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "dea59394-f0f3-4545-8f12-1e164cf373c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'\n",
    "\n",
    "from pprint import pprint\n",
    "import pickle as pkl\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# tensorflow\n",
    "import tensorflow as tf\n",
    "\n",
    "# google cloud\n",
    "from google.cloud import aiplatform, storage\n",
    "\n",
    "# cloud storage client\n",
    "storage_client = storage.Client(project=PROJECT_ID)\n",
    "# bucket = storage_client.bucket(BUCKET_NAME)\n",
    "\n",
    "# Vertex client\n",
    "aiplatform.init(project=PROJECT_ID, location=LOCATION)\n",
    "\n",
    "sys.path.append(\"..\")\n",
    "from src.data import data_utils as data_utils"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9389269a-f574-4404-97e2-5e18a52f2c4b",
   "metadata": {},
   "source": [
    "### specify data tag"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "77cd0224-34a5-4f85-a94e-de0f466e4384",
   "metadata": {},
   "source": [
    "## Training data preperation\n",
    "\n",
    "The examples generation process performs the following steps:\n",
    "\n",
    "* Downloads movielens dataset\n",
    "* Groups movie rating records by `user`, and orders per-user movie rating records by `timestamp`.\n",
    "* Generates TensorFlow examples with features:\n",
    "  \n",
    "```\n",
    " feature_description = {\n",
    "    # context sequence item features\n",
    "    'context_movie_id': tf.io.FixedLenFeature(shape=(MAX_CONTEXT_LENGTH), dtype=tf.string),\n",
    "    'context_movie_rating': tf.io.FixedLenFeature(shape=(MAX_CONTEXT_LENGTH), dtype=tf.float32),\n",
    "    'context_rating_timestamp': tf.io.FixedLenFeature(shape=(MAX_CONTEXT_LENGTH), dtype=tf.int64),\n",
    "    'context_movie_genre': tf.io.FixedLenFeature(shape=(MAX_GENRE_LENGTH), dtype=tf.string),\n",
    "    'context_movie_year': tf.io.FixedLenFeature(shape=(MAX_CONTEXT_LENGTH), dtype=tf.int64),\n",
    "    'context_movie_title': tf.io.FixedLenFeature(shape=(MAX_CONTEXT_LENGTH), dtype=tf.string),\n",
    "\n",
    "    # target/label item features\n",
    "    'target_movie_id': tf.io.FixedLenFeature(shape=(), dtype=tf.string),\n",
    "    'target_movie_rating': tf.io.FixedLenFeature(shape=(), dtype=tf.float32),\n",
    "    'target_rating_timestamp': tf.io.FixedLenFeature(shape=(), dtype=tf.int64),\n",
    "    'target_movie_genres': tf.io.FixedLenFeature(shape=(data_config.MAX_GENRE_LENGTH), dtype=tf.string),\n",
    "    'target_movie_year': tf.io.FixedLenFeature(shape=(), dtype=tf.int64),\n",
    "    'target_movie_title': tf.io.FixedLenFeature(shape=(), dtype=tf.string),\n",
    "\n",
    "    # user - global context features\n",
    "    'user_id': tf.io.FixedLenFeature(shape=(), dtype=tf.string),\n",
    "    'user_gender': tf.io.FixedLenFeature(shape=(), dtype=tf.string),\n",
    "    'user_age': tf.io.FixedLenFeature(shape=(), dtype=tf.int64),\n",
    "    'user_occupation_text': tf.io.FixedLenFeature(shape=(), dtype=tf.string),\n",
    "    'user_zip_code': tf.io.FixedLenFeature(shape=(), dtype=tf.string),\n",
    "}\n",
    "```\n",
    "\n",
    "There's case that one user activity will have multiple values for a single feature. For example, the movie genre feature in movielens dataset, each movie can have multiple genres. For this case, we suggest to concatenate all movies' genres for the activity sequence. Let's look at one example, if the user activity sequence is:\n",
    "\n",
    "```\n",
    "Star Wars: Episode IV - A New Hope (1977), Genres: Action|Adventure|Fantasy\n",
    "Terminator 2: Judgment Day (1991), Genres: Action|Sci-Fi|Thriller\n",
    "Jurassic Park (1993), Genres: Action|Adventure|Sci-Fi\n",
    "```\n",
    "The context_movie_genre feature will be\n",
    "\n",
    "> \"Action, Adventure, Fantasy, Action, Sci-Fi, Thriller, Action, Adventure, Sci-Fi\"\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0af8bd37-0629-4e4e-96e1-764480a62cf7",
   "metadata": {},
   "source": [
    "## set download and prep config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9005edd6-ec07-4826-8079-900ddd9c3d32",
   "metadata": {},
   "outputs": [],
   "source": [
    "EXAMPLE_GEN_GCS_PATH = \"data/movielens/m1m\"\n",
    "LOCAL_OUTPUT_DIR     = \"local_data/examples\"\n",
    "LOCAL_EXTRACT_DIR    = \"local_data/raw\"\n",
    "TF_RECORD_PREFIX     = \"ml-1m-gen\"\n",
    "\n",
    "MAX_CONTEXT_LENGTH   = 10\n",
    "MAX_GENRE_LENGTH     = 10\n",
    "MIN_TIMELINE_LENGTH  = 3\n",
    "MIN_RATING           = 1\n",
    "\n",
    "NUM_TRAIN_RECORDS    = 8\n",
    "NUM_VAL_RECORDS      = 3\n",
    "\n",
    "BUILD_VOCAB          = True"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad7f2324-5174-4f62-8004-42c84ecc83f2",
   "metadata": {},
   "source": [
    "## copy these commands to command line terminal\n",
    "\n",
    "> either navigate to `src/data` directory or add appropriate prefix to `example_gen_movielens` command"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4ec8dffe-b58d-45a7-81f5-bad5797e81cf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "copy these commands into terminal:\n",
      "\n",
      "export PROJECT_ID=hybrid-vertex\n",
      "export BUCKET_NAME=rec-bandits-v2-hybrid-vertex-bucket\n",
      "export EXAMPLE_GEN_GCS_PATH=data/movielens/m1m\n",
      "export TF_RECORD_PREFIX=ml-1m-gen\n",
      "export LOCAL_EXTRACT_DIR=local_data/raw\n",
      "export LOCAL_OUTPUT_DIR=local_data/examples\n",
      "export MAX_GENRE_LENGTH=10\n",
      "export MAX_CONTEXT_LENGTH=10\n",
      "export MIN_TIMELINE_LENGTH=3\n",
      "export MIN_RATING=1\n",
      "export BUILD_VOCAB=True\n",
      "export NUM_TRAIN_RECORDS=8\n",
      "export NUM_VAL_RECORDS=3\n"
     ]
    }
   ],
   "source": [
    "print(\"copy these commands into terminal:\\n\")\n",
    "print(f\"export PROJECT_ID={PROJECT_ID}\")\n",
    "print(f\"export BUCKET_NAME={BUCKET_NAME}\")\n",
    "print(f\"export EXAMPLE_GEN_GCS_PATH={EXAMPLE_GEN_GCS_PATH}\")\n",
    "print(f\"export TF_RECORD_PREFIX={TF_RECORD_PREFIX}\")\n",
    "print(f\"export LOCAL_EXTRACT_DIR={LOCAL_EXTRACT_DIR}\")\n",
    "print(f\"export LOCAL_OUTPUT_DIR={LOCAL_OUTPUT_DIR}\")\n",
    "print(f\"export MAX_GENRE_LENGTH={MAX_GENRE_LENGTH}\")\n",
    "print(f\"export MAX_CONTEXT_LENGTH={MAX_CONTEXT_LENGTH}\")\n",
    "print(f\"export MIN_TIMELINE_LENGTH={MIN_TIMELINE_LENGTH}\")\n",
    "print(f\"export MIN_RATING={MIN_RATING}\")\n",
    "print(f\"export BUILD_VOCAB={BUILD_VOCAB}\")\n",
    "print(f\"export NUM_TRAIN_RECORDS={NUM_TRAIN_RECORDS}\")\n",
    "print(f\"export NUM_VAL_RECORDS={NUM_VAL_RECORDS}\")\n",
    "# print(f\"chmod +x example_gen_movielens.py\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bd276027-4a67-44b2-9b38-1156ca4f6cb9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "python example_gen_movielens.py --project_id=$PROJECT_ID --gcs_bucket_name=$BUCKET_NAME --gcs_data_path_prefix=$EXAMPLE_GEN_GCS_PATH --tfrecord_prefix=$TF_RECORD_PREFIX --local_data_dir=$LOCAL_EXTRACT_DIR --local_output_dir=$LOCAL_OUTPUT_DIR --min_timeline_length=$MIN_TIMELINE_LENGTH --max_context_length=$MAX_CONTEXT_LENGTH --max_context_movie_genre_length=$MAX_GENRE_LENGTH --min_rating=$MIN_RATING --train_data_fraction=0.9 --build_vocabs=$BUILD_VOCAB --num_train_tfrecords=$NUM_TRAIN_RECORDS --num_test_tfrecords=$NUM_VAL_RECORDS \n"
     ]
    }
   ],
   "source": [
    "data_gen_command = \"\"\"python example_gen_movielens.py --project_id=$PROJECT_ID --gcs_bucket_name=$BUCKET_NAME \\\n",
    "--gcs_data_path_prefix=$EXAMPLE_GEN_GCS_PATH \\\n",
    "--tfrecord_prefix=$TF_RECORD_PREFIX \\\n",
    "--local_data_dir=$LOCAL_EXTRACT_DIR \\\n",
    "--local_output_dir=$LOCAL_OUTPUT_DIR \\\n",
    "--min_timeline_length=$MIN_TIMELINE_LENGTH \\\n",
    "--max_context_length=$MAX_CONTEXT_LENGTH \\\n",
    "--max_context_movie_genre_length=$MAX_GENRE_LENGTH \\\n",
    "--min_rating=$MIN_RATING \\\n",
    "--train_data_fraction=0.9 \\\n",
    "--build_vocabs=$BUILD_VOCAB \\\n",
    "--num_train_tfrecords=$NUM_TRAIN_RECORDS \\\n",
    "--num_test_tfrecords=$NUM_VAL_RECORDS \\\n",
    "\"\"\"\n",
    "print(data_gen_command)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ecbf81ec-0ed1-436d-874f-dae59a38f704",
   "metadata": {},
   "source": [
    "### Create `data_config.py`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2d38742e-18d6-4e77-bcf2-4f90b03935ca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROJECT_ID           = \"hybrid-vertex\"\n",
      "EXAMPLE_GEN_GCS_PATH = \"data/movielens/m1m\"\n",
      "TF_RECORD_PREFIX     = \"ml-1m-gen\"\n",
      "MAX_CONTEXT_LENGTH   = 10\n",
      "MAX_GENRE_LENGTH     = 10\n",
      "\n"
     ]
    }
   ],
   "source": [
    "data_config = f\"\"\"PROJECT_ID           = \"{PROJECT_ID}\"\n",
    "EXAMPLE_GEN_GCS_PATH = \"{EXAMPLE_GEN_GCS_PATH}\"\n",
    "TF_RECORD_PREFIX     = \"{TF_RECORD_PREFIX}\"\n",
    "MAX_CONTEXT_LENGTH   = {MAX_CONTEXT_LENGTH}\n",
    "MAX_GENRE_LENGTH     = {MAX_GENRE_LENGTH}\n",
    "\"\"\"\n",
    "print(data_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "76b8b898-3469-4636-a1da-406bbd1a9ae3",
   "metadata": {},
   "outputs": [],
   "source": [
    "LOCAL_DATA_CONFIG_FILE = f\"../src/data/data_config.py\"\n",
    "\n",
    "with open(LOCAL_DATA_CONFIG_FILE, 'w') as f:\n",
    "    f.write(data_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "970a1c04-7df0-404b-9bdf-97c2592533d4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROJECT_ID_v1           = hybrid-vertex\n",
      "EXAMPLE_GEN_GCS_PATH_v1 = data/movielens/m1m\n",
      "TF_RECORD_PREFIX_v1     = ml-1m-gen\n",
      "MAX_CONTEXT_LENGTH_v1   = 10\n",
      "MAX_GENRE_LENGTH_v1     = 10\n"
     ]
    }
   ],
   "source": [
    "sys.path.append(\"..\")\n",
    "from src.data import data_config as data_config\n",
    "\n",
    "PROJECT_ID_v1           = data_config.PROJECT_ID\n",
    "EXAMPLE_GEN_GCS_PATH_v1 = data_config.EXAMPLE_GEN_GCS_PATH\n",
    "TF_RECORD_PREFIX_v1     = data_config.TF_RECORD_PREFIX\n",
    "MAX_CONTEXT_LENGTH_v1   = data_config.MAX_CONTEXT_LENGTH\n",
    "MAX_GENRE_LENGTH_v1     = data_config.MAX_GENRE_LENGTH\n",
    "\n",
    "print(f\"PROJECT_ID_v1           = {PROJECT_ID_v1}\")\n",
    "print(f\"EXAMPLE_GEN_GCS_PATH_v1 = {EXAMPLE_GEN_GCS_PATH_v1}\")\n",
    "print(f\"TF_RECORD_PREFIX_v1     = {TF_RECORD_PREFIX_v1}\")\n",
    "print(f\"MAX_CONTEXT_LENGTH_v1   = {MAX_CONTEXT_LENGTH_v1}\")\n",
    "print(f\"MAX_GENRE_LENGTH_v1     = {MAX_GENRE_LENGTH_v1}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93a6528a-ff87-4db8-ae33-b4271a85260b",
   "metadata": {},
   "source": [
    "Once complete, your local `data` dir should resemble this:\n",
    "\n",
    "```\n",
    "├── __init__.py\n",
    "├── local_data\n",
    "│   ├── examples\n",
    "│   │   ├── csvs\n",
    "│   │   ├── train\n",
    "│   │   │   ├── ml-1m-gen-001-of-008.tfrecord\n",
    "│   │   │   ├── ml-1m-gen-002-of-008.tfrecord\n",
    "│   │   │   ├── ml-1m-gen-003-of-008.tfrecord\n",
    "│   │   │   ├── ml-1m-gen-004-of-008.tfrecord\n",
    "│   │   │   ├── ml-1m-gen-005-of-008.tfrecord\n",
    "│   │   │   ├── ml-1m-gen-006-of-008.tfrecord\n",
    "│   │   │   ├── ml-1m-gen-007-of-008.tfrecord\n",
    "│   │   │   └── ml-1m-gen-008-of-008.tfrecord\n",
    "│   │   ├── val\n",
    "│   │   │   ├── ml-1m-gen-001-of-003.tfrecord\n",
    "│   │   │   ├── ml-1m-gen-002-of-003.tfrecord\n",
    "│   │   │   └── ml-1m-gen-003-of-003.tfrecord\n",
    "│   │   └── vocabs\n",
    "│   │       └── vocab_dict.pkl\n",
    "│   └── raw\n",
    "│       └── datasets\n",
    "│           ├── ml-1m\n",
    "│           │   ├── README\n",
    "│           │   ├── movies.dat\n",
    "│           │   ├── ratings.dat\n",
    "│           │   └── users.dat\n",
    "│           └── ml-1m.zip\n",
    "├── data_config.py\n",
    "├── data_utils.py\n",
    "├── example_gen_movielens.py\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ebd840a6-58a6-4593-9ad0-7dd2264cb879",
   "metadata": {},
   "source": [
    "**(optional) delete local**\n",
    "\n",
    "> paste the following in command line terminal"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e2fa4827-4f36-4405-9691-2a3604034166",
   "metadata": {},
   "source": [
    "# Confim tfrecords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d5765ed3-39b0-4e61-af1a-63f2039fb3cf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GCS_DATA_PATH : gs://rec-bandits-v2-hybrid-vertex-bucket/data/movielens/m1m\n",
      "gs://rec-bandits-v2-hybrid-vertex-bucket/data/movielens/m1m/train/\n",
      "gs://rec-bandits-v2-hybrid-vertex-bucket/data/movielens/m1m/val/\n",
      "gs://rec-bandits-v2-hybrid-vertex-bucket/data/movielens/m1m/vocabs/\n"
     ]
    }
   ],
   "source": [
    "GCS_DATA_PATH = f\"{BUCKET_URI}/{EXAMPLE_GEN_GCS_PATH}\"\n",
    "\n",
    "print(f\"GCS_DATA_PATH : {GCS_DATA_PATH}\")\n",
    "\n",
    "! gsutil ls $GCS_DATA_PATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ae22bd7a-8543-4ac2-8556-221c500de826",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['gs://rec-bandits-v2-hybrid-vertex-bucket/data/movielens/m1m/train/ml-1m-gen-001-of-008.tfrecord',\n",
       " 'gs://rec-bandits-v2-hybrid-vertex-bucket/data/movielens/m1m/train/ml-1m-gen-002-of-008.tfrecord',\n",
       " 'gs://rec-bandits-v2-hybrid-vertex-bucket/data/movielens/m1m/train/ml-1m-gen-003-of-008.tfrecord',\n",
       " 'gs://rec-bandits-v2-hybrid-vertex-bucket/data/movielens/m1m/train/ml-1m-gen-004-of-008.tfrecord',\n",
       " 'gs://rec-bandits-v2-hybrid-vertex-bucket/data/movielens/m1m/train/ml-1m-gen-005-of-008.tfrecord',\n",
       " 'gs://rec-bandits-v2-hybrid-vertex-bucket/data/movielens/m1m/train/ml-1m-gen-006-of-008.tfrecord',\n",
       " 'gs://rec-bandits-v2-hybrid-vertex-bucket/data/movielens/m1m/train/ml-1m-gen-007-of-008.tfrecord',\n",
       " 'gs://rec-bandits-v2-hybrid-vertex-bucket/data/movielens/m1m/train/ml-1m-gen-008-of-008.tfrecord']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_files = []\n",
    "for blob in storage_client.list_blobs(\n",
    "    f\"{BUCKET_NAME}\", \n",
    "    prefix=f'{EXAMPLE_GEN_GCS_PATH}/train/', \n",
    "    # delimiter='/'\n",
    "):\n",
    "    if '.tfrecord' in blob.name:\n",
    "        all_files.append(blob.public_url.replace(\"https://storage.googleapis.com/\", \"gs://\"))\n",
    "        \n",
    "all_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b5ac98a6-de95-4a88-b673-ef4cc3a4fe05",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['gs://rec-bandits-v2-hybrid-vertex-bucket/data/movielens/m1m/train/ml-1m-gen-001-of-008.tfrecord',\n",
       " 'gs://rec-bandits-v2-hybrid-vertex-bucket/data/movielens/m1m/train/ml-1m-gen-002-of-008.tfrecord',\n",
       " 'gs://rec-bandits-v2-hybrid-vertex-bucket/data/movielens/m1m/train/ml-1m-gen-003-of-008.tfrecord',\n",
       " 'gs://rec-bandits-v2-hybrid-vertex-bucket/data/movielens/m1m/train/ml-1m-gen-004-of-008.tfrecord',\n",
       " 'gs://rec-bandits-v2-hybrid-vertex-bucket/data/movielens/m1m/train/ml-1m-gen-005-of-008.tfrecord',\n",
       " 'gs://rec-bandits-v2-hybrid-vertex-bucket/data/movielens/m1m/train/ml-1m-gen-006-of-008.tfrecord',\n",
       " 'gs://rec-bandits-v2-hybrid-vertex-bucket/data/movielens/m1m/train/ml-1m-gen-007-of-008.tfrecord',\n",
       " 'gs://rec-bandits-v2-hybrid-vertex-bucket/data/movielens/m1m/train/ml-1m-gen-008-of-008.tfrecord',\n",
       " 'gs://rec-bandits-v2-hybrid-vertex-bucket/data/movielens/m1m/val/ml-1m-gen-001-of-003.tfrecord',\n",
       " 'gs://rec-bandits-v2-hybrid-vertex-bucket/data/movielens/m1m/val/ml-1m-gen-002-of-003.tfrecord',\n",
       " 'gs://rec-bandits-v2-hybrid-vertex-bucket/data/movielens/m1m/val/ml-1m-gen-003-of-003.tfrecord']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for blob in storage_client.list_blobs(\n",
    "    f\"{BUCKET_NAME}\", \n",
    "    prefix=f'{EXAMPLE_GEN_GCS_PATH}/val/', \n",
    "    # delimiter='/'\n",
    "):\n",
    "    if '.tfrecord' in blob.name:\n",
    "        all_files.append(blob.public_url.replace(\"https://storage.googleapis.com/\", \"gs://\"))\n",
    "        \n",
    "all_files"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a46dd3e2-cec1-407f-8ef1-6fb6dd749284",
   "metadata": {},
   "source": [
    "## non-sequence dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0cf0e2e8-883c-4e0c-9599-0dfbe90c59fe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'target_movie_genres': <tf.Tensor: shape=(1, 10), dtype=string, numpy=\n",
      "array([[b'Comedy', b'UNK', b'UNK', b'UNK', b'UNK', b'UNK', b'UNK',\n",
      "        b'UNK', b'UNK', b'UNK']], dtype=object)>,\n",
      " 'target_movie_id': <tf.Tensor: shape=(1,), dtype=string, numpy=array([b'2701'], dtype=object)>,\n",
      " 'target_movie_rating': <tf.Tensor: shape=(1,), dtype=float32, numpy=array([2.], dtype=float32)>,\n",
      " 'target_movie_title': <tf.Tensor: shape=(1,), dtype=string, numpy=array([b'Bowfinger (1999)'], dtype=object)>,\n",
      " 'target_movie_year': <tf.Tensor: shape=(1,), dtype=int64, numpy=array([1999])>,\n",
      " 'target_rating_timestamp': <tf.Tensor: shape=(1,), dtype=int64, numpy=array([974655587])>,\n",
      " 'user_age': <tf.Tensor: shape=(1,), dtype=int64, numpy=array([18])>,\n",
      " 'user_gender': <tf.Tensor: shape=(1,), dtype=string, numpy=array([b'M'], dtype=object)>,\n",
      " 'user_id': <tf.Tensor: shape=(1,), dtype=string, numpy=array([b'2092'], dtype=object)>,\n",
      " 'user_occupation_text': <tf.Tensor: shape=(1,), dtype=string, numpy=array([b'other'], dtype=object)>,\n",
      " 'user_zip_code': <tf.Tensor: shape=(1,), dtype=string, numpy=array([b'48322'], dtype=object)>}\n"
     ]
    }
   ],
   "source": [
    "mv_dataset = tf.data.TFRecordDataset(all_files)\n",
    "mv_dataset = mv_dataset.map(data_utils._parse_function)\n",
    "\n",
    "for x in mv_dataset.batch(1).take(1):\n",
    "    pprint(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc9dbdeb-1c7a-439e-9dce-ab27a9b39ed9",
   "metadata": {},
   "outputs": [],
   "source": [
    "x[0][\"target_movie_genres\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4fbce514-aac1-4dc0-9fed-06ff2bbb22ee",
   "metadata": {},
   "source": [
    "## sequence dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a83b68fd-cba7-429c-8692-fa78fcaa5253",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_seq_dataset = train_dataset.map(data_utils._parse_seq_function)\n",
    "\n",
    "# see train example\n",
    "for x in train_seq_dataset.skip(5).take(1):\n",
    "    pprint(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bad59306-2d27-4bbf-b04d-654cd54dccc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "x[0][\"context_movie_title\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9bbf774d-3d94-49c4-a5c8-727b688495d2",
   "metadata": {},
   "source": [
    "## Validate vocab file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5181f5a5-a0f6-4ed2-97ae-c283df69df24",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'vocab_dict.pkl'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "VOCAB_FILENAME"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "df496893-ee39-4151-9259-fce6f482e140",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading vocab...\n",
      "Downloaded vocab from: gs://rec-bandits-v2-hybrid-vertex-bucket/data/movielens/m1m/vocabs/vocab_dict.pkl\n",
      "\n",
      "'movie_id'\n",
      "'movie_year'\n",
      "'movie_genre'\n",
      "'movie_title'\n",
      "'user_id'\n",
      "'user_gender_vocab'\n",
      "'user_age_vocab'\n",
      "'user_occ_vocab'\n",
      "'user_zip_vocab'\n",
      "'min_timestamp'\n",
      "'max_timestamp'\n",
      "'timestamp_buckets'\n"
     ]
    }
   ],
   "source": [
    "EXISTING_VOCAB_FILE = f'gs://{BUCKET_NAME}/{EXAMPLE_GEN_GCS_PATH}/vocabs/{VOCAB_FILENAME}'\n",
    "print(f\"Downloading vocab...\")\n",
    "\n",
    "os.system(f'gsutil -q cp {EXISTING_VOCAB_FILE} .')\n",
    "print(f\"Downloaded vocab from: {EXISTING_VOCAB_FILE}\\n\")\n",
    "\n",
    "filehandler = open(VOCAB_FILENAME, 'rb')\n",
    "vocab_dict = pkl.load(filehandler)\n",
    "filehandler.close()\n",
    "\n",
    "for key in vocab_dict.keys():\n",
    "    pprint(key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "68352062-7366-40d3-b584-baec7beb8ea8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# vocab_dict['movie_year']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "55054fff-1825-4364-952d-4d289930bbe4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MOVIELENS_NUM_MOVIES : 3884\n",
      "MOVIELENS_NUM_USERS  : 6041\n"
     ]
    }
   ],
   "source": [
    "MOVIELENS_NUM_MOVIES = len(vocab_dict['movie_id'])\n",
    "MOVIELENS_NUM_USERS = len(vocab_dict['user_id'])\n",
    "\n",
    "print(f\"MOVIELENS_NUM_MOVIES : {MOVIELENS_NUM_MOVIES}\")\n",
    "print(f\"MOVIELENS_NUM_USERS  : {MOVIELENS_NUM_USERS}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dd65d9be-9ca4-42f6-8d4c-d4382bfe559f",
   "metadata": {},
   "source": [
    "# Create look-up dictionaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "1a2e5d64-c6fe-4208-a3e3-fae323c57dcc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "USER_AGE_DIM: 7\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{1: 0, 35: 1, 45: 2, 18: 3, 50: 4, 56: 5, 25: 6}"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "USER_AGE_LOOKUP = data_utils.get_dictionary_lookup_by_tf_data_key(\n",
    "    key = 'user_age'\n",
    "    , dataset = mv_dataset\n",
    ")\n",
    "\n",
    "USER_AGE_DIM = len(USER_AGE_LOOKUP)\n",
    "print(f\"USER_AGE_DIM: {USER_AGE_DIM}\")\n",
    "\n",
    "# USER_AGE_LOOKUP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "6e3c81cf-4410-45f3-9ee0-ff04380483b3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "USER_OCC_DIM: 21\n"
     ]
    }
   ],
   "source": [
    "USER_OCC_LOOKUP = data_utils.get_dictionary_lookup_by_tf_data_key(\n",
    "    key = 'user_occupation_text'\n",
    "    , dataset= mv_dataset\n",
    ")\n",
    "USER_OCC_LOOKUP[b''] = 21\n",
    "USER_OCC_DIM = len(USER_OCC_LOOKUP)\n",
    "print(f\"USER_OCC_DIM: {USER_OCC_DIM}\")\n",
    "\n",
    "USER_OCC_LOOKUP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "67825a09-76c6-4274-afe7-697b910b7073",
   "metadata": {},
   "outputs": [],
   "source": [
    "config = f'''\n",
    "USER_AGE_LOOKUP       = {USER_AGE_LOOKUP}\n",
    "USER_AGE_DIM          = {USER_AGE_DIM}\n",
    "USER_OCC_LOOKUP       = {USER_OCC_LOOKUP}\n",
    "USER_OCC_DIM          = {USER_OCC_DIM}\n",
    "MOVIELENS_NUM_MOVIES  = {MOVIELENS_NUM_MOVIES}\n",
    "MOVIELENS_NUM_USERS   = {MOVIELENS_NUM_USERS}\n",
    "'''\n",
    "    \n",
    "with open(f'../{REPO_DOCKER_PATH_PREFIX}/data/mv_lookup_dicts.py', 'w') as f:\n",
    "    f.write(config)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78438723-5de0-493e-b1b0-3e2d07eacdc2",
   "metadata": {},
   "source": [
    "# Clean-up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54ccf6d7-985e-4ed2-ba1a-8314a588d966",
   "metadata": {},
   "outputs": [],
   "source": [
    "# (optional) delete local\n",
    "# cd src/data\n",
    "# rm -rf local_data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98553d63-bb32-4dda-afee-ad8000764839",
   "metadata": {},
   "source": [
    "### move files with `gsutil`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "1225818b-bef1-4221-847b-247f020256a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !gsutil cp gs://$BUCKET_NAME/data/movielens/movielens-1m/vocab_dict.pkl gs://$BUCKET_NAME/data/movielens/movielens-1m-gen/vocab_dict.pkl"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "280dabf0-5671-4932-babe-33ad925f7ee5",
   "metadata": {},
   "source": [
    "**Finished**"
   ]
  }
 ],
 "metadata": {
  "environment": {
   "kernel": "conda-base-py",
   "name": "workbench-notebooks.m126",
   "type": "gcloud",
   "uri": "us-docker.pkg.dev/deeplearning-platform-release/gcr.io/workbench-notebooks:m126"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel) (Local)",
   "language": "python",
   "name": "conda-base-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}