tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def parse_amazon_data(file_path):\n",
    "# \t'''\n",
    "# \tThis function parses amazon data. It retrieves the reviews and good/bad label.\n",
    "\n",
    "# \t:param file_path: path to traing/testing files\n",
    "# \t\t\t-./data/test.ft.txt\n",
    "# \t\t\t-./data/train.ft.txt\n",
    "# \t:return: a tuple consisting of\n",
    "# \t\t\t1. a list of reviews (a list of strings)\n",
    "# \t\t\t2. a list of good/bad labels (stored as ints. 0 = bad, 1 = good)\n",
    "# \t'''\n",
    "# \tstopwords_set = set(stopwords.words('english'))\n",
    "# \tlemmatizer = WordNetLemmatizer().lemmatize\n",
    "\n",
    "# \treview_str_list = []\n",
    "# \tlabels_list = []\n",
    "\n",
    "# \twith open(file_path, 'r') as f:\n",
    "# \t\twords = []\n",
    "# \t\tfor line in f:\n",
    "# \t\t\tlabel_str, review_str = line.split(' ', 1)\n",
    "# \t\t\tlabel = int(label_str[-1]) - 1\t\t# original data is labeled with 1 and 2. change to labels of 0 and 1\n",
    "# \t\t\t# words = [lemmatizer(w) for w in review_str.lower().split() if w not in stopwords_set]\t# get the words out\n",
    "# \t\t\treview_str_list.append(review_str_list)\n",
    "\t\t\n",
    "# \t\t\t# Not too sure what the best DS is here but gonna store everything in list for now\n",
    "# \t\t\t# review_str_list.append(words)\n",
    "# \t\t\tlabels_list.append(label)\n",
    "\n",
    "# \treturn [review_str_list, labels_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_amazon = \"./data/train.ft.txt\"\n",
    "testing_amazon = \"./data/test.ft.txt\"\n",
    "training_imdb = \"./data/aclImdb/train\"\n",
    "testing_imdb = \"./data/aclImdb/test\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['.DS_Store', 'aclImdb', 'train.ft.txt', 'test.ft.txt.bz2', 'test.ft.txt', 'train.ft.txt.bz2', 'trainingandtestdata']\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
    "import bz2\n",
    "import os\n",
    "print(os.listdir(\"./data\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainfile = bz2.BZ2File('./data/test.ft.txt.bz2','r')\n",
    "lines = trainfile.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b\"__label__2 works fine, but Maha Energy is better: Check out Maha Energy's website. Their Powerex MH-C204F charger works in 100 minutes for rapid charge, with option for slower charge (better for batteries). And they have 2200 mAh batteries.\\n\""
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lines[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_amazon_data(lines):\n",
    "    review_and_ratings_list=[]\n",
    "    for i in range(len(lines)):\n",
    "        line_text = str(lines[i])\n",
    "        label, review_str = line_text.split(\" \", 1)\n",
    "        label = int(label[-1]) - 1         # label with 1 and 0\n",
    "        review_and_ratings_list.append([review_str, label])\n",
    "    return review_and_ratings_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "review_and_ratings_list=parse_amazon_data(lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Great CD: My lovely Pat has one of the GREAT voices of her generation. I have listened to this CD for YEARS and I still LOVE IT. When I\\\\\\'m in a good mood it makes me feel better. A bad mood just evaporates like sugar in the rain. This CD just oozes LIFE. Vocals are jusat STUUNNING and lyrics just kill. One of life\\\\\\'s hidden gems. This is a desert isle CD in my book. Why she never made it big is just beyond me. Everytime I play this, no matter black, white, young, old, male, female EVERYBODY says one thing \"Who was that singing ?\"\\\\n\\'',\n",
       " 1]"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "review_and_ratings_list[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Great CD: My lovely Pat has one of the GREAT v...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>One of the best game music soundtracks - for a...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Batteries died within a year ...: I bought thi...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>works fine, but Maha Energy is better: Check o...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Great for the non-audiophile: Reviewed quite a...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              review  rating\n",
       "0  Great CD: My lovely Pat has one of the GREAT v...       1\n",
       "1  One of the best game music soundtracks - for a...       1\n",
       "2  Batteries died within a year ...: I bought thi...       0\n",
       "3  works fine, but Maha Energy is better: Check o...       1\n",
       "4  Great for the non-audiophile: Reviewed quite a...       1"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = pd.DataFrame(review_and_ratings_list,columns=['review','rating'])\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    200000\n",
       "0    200000\n",
       "Name: rating, dtype: int64"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df['rating'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>rating</th>\n",
       "      <th>word_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Great CD: My lovely Pat has one of the GREAT v...</td>\n",
       "      <td>1</td>\n",
       "      <td>106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>One of the best game music soundtracks - for a...</td>\n",
       "      <td>1</td>\n",
       "      <td>148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Batteries died within a year ...: I bought thi...</td>\n",
       "      <td>0</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>works fine, but Maha Energy is better: Check o...</td>\n",
       "      <td>1</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Great for the non-audiophile: Reviewed quite a...</td>\n",
       "      <td>1</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              review  rating  word_count\n",
       "0  Great CD: My lovely Pat has one of the GREAT v...       1         106\n",
       "1  One of the best game music soundtracks - for a...       1         148\n",
       "2  Batteries died within a year ...: I bought thi...       0          60\n",
       "3  works fine, but Maha Energy is better: Check o...       1          37\n",
       "4  Great for the non-audiophile: Reviewed quite a...       1          69"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df['word_count'] = train_df['review'].str.lower().str.split().apply(len)\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "import string \n",
    "def remove_punc(s):\n",
    "    table = str.maketrans({key: None for key in string.punctuation})\n",
    "    return s.translate(table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(400000, 3)"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df['review'] = train_df['review'].apply(remove_punc)\n",
    "train_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>rating</th>\n",
       "      <th>word_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Great CD My lovely Pat has one of the GREAT vo...</td>\n",
       "      <td>1</td>\n",
       "      <td>106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>One of the best game music soundtracks  for a ...</td>\n",
       "      <td>1</td>\n",
       "      <td>148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Batteries died within a year  I bought this ch...</td>\n",
       "      <td>0</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>works fine but Maha Energy is better Check out...</td>\n",
       "      <td>1</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Great for the nonaudiophile Reviewed quite a b...</td>\n",
       "      <td>1</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              review  rating  word_count\n",
       "0  Great CD My lovely Pat has one of the GREAT vo...       1         106\n",
       "1  One of the best game music soundtracks  for a ...       1         148\n",
       "2  Batteries died within a year  I bought this ch...       0          60\n",
       "3  works fine but Maha Energy is better Check out...       1          37\n",
       "4  Great for the nonaudiophile Reviewed quite a b...       1          69"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25768"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_df['word_count'][train_df['word_count']<=25])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>rating</th>\n",
       "      <th>word_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Long and boring Ive read this book with much e...</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>Amazing love itvery touchits to bad that in th...</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>No another grill like this This grill is one o...</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>Creepy The creepiest book Ive ever read Its a ...</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>Possibly the funniest movie ever made It start...</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               review  rating  word_count\n",
       "20  Long and boring Ive read this book with much e...       0          19\n",
       "44  Amazing love itvery touchits to bad that in th...       1          15\n",
       "55  No another grill like this This grill is one o...       1          24\n",
       "88  Creepy The creepiest book Ive ever read Its a ...       1          22\n",
       "99  Possibly the funniest movie ever made It start...       1          22"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df1 = train_df[:][train_df['word_count']<=25]\n",
    "train_df1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    15790\n",
       "0     9978\n",
       "Name: rating, dtype: int64"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df1['rating'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction import text\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "st_wd = text.ENGLISH_STOP_WORDS\n",
    "c_vector = CountVectorizer(stop_words = st_wd,min_df=.0001,lowercase=1)\n",
    "X_counts = c_vector.fit_transform(train_df1['review'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<25768x9678 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 243841 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "y = train_df1['rating'].values\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_counts, y, test_size=0.1, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X_train.todense()\n",
    "X_test = X_test.todense()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(23191, 9678)"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "23191/23191 [==============================] - 36s 2ms/step - loss: 0.3239 - acc: 0.8577: 8s - loss: 0.3370 - acc: 0. \n",
      "Epoch 2/6\n",
      "23191/23191 [==============================] - 39s 2ms/step - loss: 0.1575 - acc: 0.9398\n",
      "Epoch 3/6\n",
      "23191/23191 [==============================] - 34s 1ms/step - loss: 0.0847 - acc: 0.9729\n",
      "Epoch 4/6\n",
      "23191/23191 [==============================] - 32s 1ms/step - loss: 0.0384 - acc: 0.9901\n",
      "Epoch 5/6\n",
      "23191/23191 [==============================] - 34s 1ms/step - loss: 0.0154 - acc: 0.9974\n",
      "Epoch 6/6\n",
      "23191/23191 [==============================] - 33s 1ms/step - loss: 0.0066 - acc: 0.9996\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation\n",
    "\n",
    "model1=  Sequential()\n",
    "model1.add(Dense(1000,input_shape=(9678,),activation='relu'))\n",
    "model1.add(Dense(1,activation='sigmoid'))\n",
    "\n",
    "model1.compile(optimizer='adam',\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "hist = model1.fit(X_train,y_train,epochs=6,batch_size=128,verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2577/2577 [==============================] - 2s 641us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.5244230652646497, 0.8820333721612746]"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model1.evaluate(X_test, y_test, batch_size=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23191/23191 [==============================] - 9s 400us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.0032562388347812084, 0.999870639472209]"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model1.evaluate(X_train, y_train, batch_size=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}