03-train-data-exploration.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import gc\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from glob import glob\n",
    "from joblib import delayed, Parallel\n",
    "import matplotlib.pyplot as plt\n",
    "# import dask.dataframe as dd\n",
    "# import dask.array as da\n",
    "%matplotlib inline\n",
    "pd.set_option('display.max_columns', 500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_files = sorted(glob('./data/training_set/*.csv.gz'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check if the training data files are sorted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_if_file_is_sorted(x):\n",
    "    ts = pd.read_csv(x)\n",
    "    original_index = ts.index.values.copy()\n",
    "    ts.sort_values(['session_id', 'session_position'], inplace=True)\n",
    "    new_index = ts.index.values.copy()\n",
    "    is_sorted = np.all(original_index == new_index)\n",
    "    ts['ts'] = pd.to_datetime(ts.date + ' ' + ts.hour_of_day.astype('str') + ':00:00')\n",
    "    min_ts = ts.ts.min()\n",
    "    max_ts = ts.ts.max()\n",
    "    is_ts_sorted = np.all(ts.ts.values == ts.ts.sort_values().values)\n",
    "    return (x, is_sorted, min_ts, max_ts, is_ts_sorted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp = Parallel(14)(delayed(check_if_file_is_sorted)(x) for x in ts_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "660"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tmp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\"<class 'tuple'>\"}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(str(type(x)) for x in tmp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('./data/training_set/log_0_20180715_000000000000.csv.gz',\n",
       " True,\n",
       " Timestamp('2015-09-30 21:00:00'),\n",
       " Timestamp('2018-08-22 11:00:00'),\n",
       " False)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tmp[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame.from_records(tmp, columns=[\n",
    "    'filepath', 'is_sorted', 'min_ts', 'max_ts', 'is_ts_sorted'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.is_sorted.all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Timestamp('1969-12-31 16:00:00')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.min_ts.min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Timestamp('2032-05-26 00:00:00')"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.max_ts.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filepath</th>\n",
       "      <th>is_sorted</th>\n",
       "      <th>min_ts</th>\n",
       "      <th>max_ts</th>\n",
       "      <th>is_ts_sorted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>370</th>\n",
       "      <td>./data/training_set/log_5_20180824_00000000000...</td>\n",
       "      <td>True</td>\n",
       "      <td>2017-07-01 21:00:00</td>\n",
       "      <td>2018-10-08 06:00:00</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>420</th>\n",
       "      <td>./data/training_set/log_6_20180808_00000000000...</td>\n",
       "      <td>True</td>\n",
       "      <td>1970-01-01 20:00:00</td>\n",
       "      <td>2018-08-09 12:00:00</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              filepath  is_sorted  \\\n",
       "370  ./data/training_set/log_5_20180824_00000000000...       True   \n",
       "420  ./data/training_set/log_6_20180808_00000000000...       True   \n",
       "\n",
       "                 min_ts              max_ts  is_ts_sorted  \n",
       "370 2017-07-01 21:00:00 2018-10-08 06:00:00         False  \n",
       "420 1970-01-01 20:00:00 2018-08-09 12:00:00         False  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "del df\n",
    "del tmp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check which columns are (nearly) unique per session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts = pd.read_csv(ts_files[0], dtype='str')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>session_id</th>\n",
       "      <th>session_position</th>\n",
       "      <th>session_length</th>\n",
       "      <th>track_id_clean</th>\n",
       "      <th>skip_1</th>\n",
       "      <th>skip_2</th>\n",
       "      <th>skip_3</th>\n",
       "      <th>not_skipped</th>\n",
       "      <th>context_switch</th>\n",
       "      <th>no_pause_before_play</th>\n",
       "      <th>short_pause_before_play</th>\n",
       "      <th>long_pause_before_play</th>\n",
       "      <th>hist_user_behavior_n_seekfwd</th>\n",
       "      <th>hist_user_behavior_n_seekback</th>\n",
       "      <th>hist_user_behavior_is_shuffle</th>\n",
       "      <th>hour_of_day</th>\n",
       "      <th>date</th>\n",
       "      <th>premium</th>\n",
       "      <th>context_type</th>\n",
       "      <th>hist_user_behavior_reason_start</th>\n",
       "      <th>hist_user_behavior_reason_end</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0_00006f66-33e5-4de7-a324-2d18e439fc1e</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>t_0479f24c-27d2-46d6-a00c-7ec928f2b539</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>true</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>true</td>\n",
       "      <td>16</td>\n",
       "      <td>2018-07-15</td>\n",
       "      <td>true</td>\n",
       "      <td>editorial_playlist</td>\n",
       "      <td>trackdone</td>\n",
       "      <td>trackdone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0_00006f66-33e5-4de7-a324-2d18e439fc1e</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>t_9099cd7b-c238-47b7-9381-f23f2c1d1043</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>true</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>true</td>\n",
       "      <td>16</td>\n",
       "      <td>2018-07-15</td>\n",
       "      <td>true</td>\n",
       "      <td>editorial_playlist</td>\n",
       "      <td>trackdone</td>\n",
       "      <td>trackdone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0_00006f66-33e5-4de7-a324-2d18e439fc1e</td>\n",
       "      <td>3</td>\n",
       "      <td>20</td>\n",
       "      <td>t_fc5df5ba-5396-49a7-8b29-35d0d28249e0</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>true</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>true</td>\n",
       "      <td>16</td>\n",
       "      <td>2018-07-15</td>\n",
       "      <td>true</td>\n",
       "      <td>editorial_playlist</td>\n",
       "      <td>trackdone</td>\n",
       "      <td>trackdone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0_00006f66-33e5-4de7-a324-2d18e439fc1e</td>\n",
       "      <td>4</td>\n",
       "      <td>20</td>\n",
       "      <td>t_23cff8d6-d874-4b20-83dc-94e450e8aa20</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>true</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>true</td>\n",
       "      <td>16</td>\n",
       "      <td>2018-07-15</td>\n",
       "      <td>true</td>\n",
       "      <td>editorial_playlist</td>\n",
       "      <td>trackdone</td>\n",
       "      <td>trackdone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0_00006f66-33e5-4de7-a324-2d18e439fc1e</td>\n",
       "      <td>5</td>\n",
       "      <td>20</td>\n",
       "      <td>t_64f3743c-f624-46bb-a579-0f3f9a07a123</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>true</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>true</td>\n",
       "      <td>16</td>\n",
       "      <td>2018-07-15</td>\n",
       "      <td>true</td>\n",
       "      <td>editorial_playlist</td>\n",
       "      <td>trackdone</td>\n",
       "      <td>trackdone</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               session_id session_position session_length  \\\n",
       "0  0_00006f66-33e5-4de7-a324-2d18e439fc1e                1             20   \n",
       "1  0_00006f66-33e5-4de7-a324-2d18e439fc1e                2             20   \n",
       "2  0_00006f66-33e5-4de7-a324-2d18e439fc1e                3             20   \n",
       "3  0_00006f66-33e5-4de7-a324-2d18e439fc1e                4             20   \n",
       "4  0_00006f66-33e5-4de7-a324-2d18e439fc1e                5             20   \n",
       "\n",
       "                           track_id_clean skip_1 skip_2 skip_3 not_skipped  \\\n",
       "0  t_0479f24c-27d2-46d6-a00c-7ec928f2b539  false  false  false        true   \n",
       "1  t_9099cd7b-c238-47b7-9381-f23f2c1d1043  false  false  false        true   \n",
       "2  t_fc5df5ba-5396-49a7-8b29-35d0d28249e0  false  false  false        true   \n",
       "3  t_23cff8d6-d874-4b20-83dc-94e450e8aa20  false  false  false        true   \n",
       "4  t_64f3743c-f624-46bb-a579-0f3f9a07a123  false  false  false        true   \n",
       "\n",
       "  context_switch no_pause_before_play short_pause_before_play  \\\n",
       "0              0                    0                       0   \n",
       "1              0                    1                       0   \n",
       "2              0                    1                       0   \n",
       "3              0                    1                       0   \n",
       "4              0                    1                       0   \n",
       "\n",
       "  long_pause_before_play hist_user_behavior_n_seekfwd  \\\n",
       "0                      0                            0   \n",
       "1                      0                            0   \n",
       "2                      0                            0   \n",
       "3                      0                            0   \n",
       "4                      0                            0   \n",
       "\n",
       "  hist_user_behavior_n_seekback hist_user_behavior_is_shuffle hour_of_day  \\\n",
       "0                             0                          true          16   \n",
       "1                             0                          true          16   \n",
       "2                             0                          true          16   \n",
       "3                             0                          true          16   \n",
       "4                             0                          true          16   \n",
       "\n",
       "         date premium        context_type hist_user_behavior_reason_start  \\\n",
       "0  2018-07-15    true  editorial_playlist                       trackdone   \n",
       "1  2018-07-15    true  editorial_playlist                       trackdone   \n",
       "2  2018-07-15    true  editorial_playlist                       trackdone   \n",
       "3  2018-07-15    true  editorial_playlist                       trackdone   \n",
       "4  2018-07-15    true  editorial_playlist                       trackdone   \n",
       "\n",
       "  hist_user_behavior_reason_end  \n",
       "0                     trackdone  \n",
       "1                     trackdone  \n",
       "2                     trackdone  \n",
       "3                     trackdone  \n",
       "4                     trackdone  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "true     1533436\n",
       "false    1457173\n",
       "Name: skip_2, dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts.skip_2.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "false    1985901\n",
       "true     1004708\n",
       "Name: not_skipped, dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts.not_skipped.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "((ts.skip_2 == 'true') == (ts.not_skipped == 'false')).all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "178342"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ts.session_id.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(521578, 2)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts.loc[:, ['session_id', 'hist_user_behavior_reason_end']].drop_duplicates().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(554160, 2)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts.loc[:, ['session_id', 'hist_user_behavior_reason_start']].drop_duplicates().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(230742, 2)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts.loc[:, ['session_id', 'context_type']].drop_duplicates().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(178366, 2)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts.loc[:, ['session_id', 'premium']].drop_duplicates().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(181173, 2)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts.loc[:, ['session_id', 'date']].drop_duplicates().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(200598, 2)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts.loc[:, ['session_id', 'hist_user_behavior_is_shuffle']].drop_duplicates().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(240121, 2)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts.loc[:, ['session_id', 'hist_user_behavior_n_seekback']].drop_duplicates().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(232455, 2)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts.loc[:, ['session_id', 'hist_user_behavior_n_seekfwd']].drop_duplicates().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(336010, 2)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts.loc[:, ['session_id', 'long_pause_before_play']].drop_duplicates().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(325376, 2)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts.loc[:, ['session_id', 'short_pause_before_play']].drop_duplicates().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(356655, 2)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts.loc[:, ['session_id', 'no_pause_before_play']].drop_duplicates().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(241712, 2)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts.loc[:, ['session_id', 'context_switch']].drop_duplicates().shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract session related information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1161"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def uniq_session_ids(x):\n",
    "    ts = pd.read_csv(x, dtype='str')\n",
    "    ts = ts.groupby('session_id')['session_length', 'date', 'premium'].first().reset_index().assign(filepath = x)\n",
    "    return ts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp = Parallel(14)(delayed(uniq_session_ids)(x) for x in ts_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(124950714, 5)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.concat(tmp)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_pickle('./data/all_session_ids.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Distribution of session_length, year and premium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(124950714, 5)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_pickle('./data/all_session_ids.pkl.gz')\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "124950714"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df.session_id.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>session_id</th>\n",
       "      <th>session_length</th>\n",
       "      <th>date</th>\n",
       "      <th>premium</th>\n",
       "      <th>filepath</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0_00006f66-33e5-4de7-a324-2d18e439fc1e</td>\n",
       "      <td>20</td>\n",
       "      <td>2018-07-15</td>\n",
       "      <td>true</td>\n",
       "      <td>./data/training_set/log_0_20180715_00000000000...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0_0000a72b-09ac-412f-b452-9b9e79bded8f</td>\n",
       "      <td>20</td>\n",
       "      <td>2018-07-15</td>\n",
       "      <td>true</td>\n",
       "      <td>./data/training_set/log_0_20180715_00000000000...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0_00010fc5-b79e-4cdf-bc4c-f140d0f99a3a</td>\n",
       "      <td>20</td>\n",
       "      <td>2018-07-15</td>\n",
       "      <td>true</td>\n",
       "      <td>./data/training_set/log_0_20180715_00000000000...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0_00016a3d-9076-4f67-918f-f29e3ce160dc</td>\n",
       "      <td>20</td>\n",
       "      <td>2018-07-15</td>\n",
       "      <td>true</td>\n",
       "      <td>./data/training_set/log_0_20180715_00000000000...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0_00018b58-deb8-4f98-ac5e-d7e01b346130</td>\n",
       "      <td>11</td>\n",
       "      <td>2018-07-15</td>\n",
       "      <td>true</td>\n",
       "      <td>./data/training_set/log_0_20180715_00000000000...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               session_id session_length        date premium  \\\n",
       "0  0_00006f66-33e5-4de7-a324-2d18e439fc1e             20  2018-07-15    true   \n",
       "1  0_0000a72b-09ac-412f-b452-9b9e79bded8f             20  2018-07-15    true   \n",
       "2  0_00010fc5-b79e-4cdf-bc4c-f140d0f99a3a             20  2018-07-15    true   \n",
       "3  0_00016a3d-9076-4f67-918f-f29e3ce160dc             20  2018-07-15    true   \n",
       "4  0_00018b58-deb8-4f98-ac5e-d7e01b346130             11  2018-07-15    true   \n",
       "\n",
       "                                            filepath  \n",
       "0  ./data/training_set/log_0_20180715_00000000000...  \n",
       "1  ./data/training_set/log_0_20180715_00000000000...  \n",
       "2  ./data/training_set/log_0_20180715_00000000000...  \n",
       "3  ./data/training_set/log_0_20180715_00000000000...  \n",
       "4  ./data/training_set/log_0_20180715_00000000000...  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.session_length = df.session_length.astype('int64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10    11025730\n",
       "11     9611839\n",
       "12     8407692\n",
       "13     7344162\n",
       "14     6460309\n",
       "15     5664259\n",
       "16     4994670\n",
       "17     4400713\n",
       "18     3917839\n",
       "19     3477536\n",
       "20    59645965\n",
       "Name: session_length, dtype: int64"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.session_length.value_counts().sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10     8.824063\n",
       "11     7.692504\n",
       "12     6.728807\n",
       "13     5.877647\n",
       "14     5.170286\n",
       "15     4.533195\n",
       "16     3.997312\n",
       "17     3.521959\n",
       "18     3.135507\n",
       "19     2.783126\n",
       "20    47.735594\n",
       "Name: session_length, dtype: float64"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.session_length.value_counts(normalize=True, sort=False).sort_index()*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "true     0.816271\n",
       "false    0.183729\n",
       "Name: premium, dtype: float64"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.premium.value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1969     0.0001104\n",
       "1970     0.0001409\n",
       "1972     0.0000008\n",
       "1999     0.0000048\n",
       "2000     0.0000104\n",
       "2008     0.0000016\n",
       "2009     0.0000200\n",
       "2010     0.0000512\n",
       "2011     0.0000160\n",
       "2012     0.0000112\n",
       "2013     0.0000216\n",
       "2014     0.0000448\n",
       "2015     0.0001785\n",
       "2016     0.0002105\n",
       "2017     0.0011100\n",
       "2018    99.9980480\n",
       "2019     0.0000104\n",
       "2031     0.0000008\n",
       "2032     0.0000080\n",
       "Name: date, dtype: object"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(df.date.str[:4].value_counts(normalize=True, sort=False).sort_index() * 100)\\\n",
    "    .apply(lambda x: '%.7f' % x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}