Part 6/Section-5 .ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "source": [
    "# Section- 5\n",
    "# NLP (Natural language processing)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section 5.1: Key Concepts, text data cleaning\n",
    "## Section 5.2: Count Vectorizer, TFIDF \n",
    "## Section 5.3: Example with Spam data \n",
    "## Section 5.4: Tweak model with Spam data\n",
    "## Section 5.5: Pipeline with Spam data "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section 5.1: Key Concepts, text data cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from collections import Counter\n",
    "import pandas as pd\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem.wordnet import WordNetLemmatizer \n",
    "from nltk.stem import SnowballStemmer\n",
    "import string\n",
    "from scipy.spatial.distance import pdist, squareform\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer \n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, VotingClassifier\n",
    "from sklearn.svm import SVC \n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.pipeline  import Pipeline, FeatureUnion, make_pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "stops = set(nltk.corpus.stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#stops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "corpus = [\"Jeff stole my octopus sandwich.\", \n",
    "    \"'Help!' I sobbed, sandwichlessly.\", \n",
    "    \"'Drop the sandwiches!' said the sandwich police.\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# How do I turn a corpus of documents into a feature matrix? \n",
    "# Words --> numbers?????"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Corpus: list of documents\n",
    "\n",
    "```\n",
    " [\n",
    " \"Jeff stole my octopus sandwich.\", \n",
    " \"'Help!' I sobbed, sandwichlessly.\", \n",
    " \"'Drop the sandwiches!' said the sandwich police.\"\n",
    " ]```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "def our_tokenizer(doc, stops=None, stemmer=None):\n",
    "    doc = word_tokenize(doc.lower())\n",
    "    tokens = [''.join([char for char in tok if char not in string.punctuation]) for tok in doc]\n",
    "    tokens = [tok for tok in tokens if tok]\n",
    "    if stops:\n",
    "        tokens = [tok for tok in tokens if (tok not in stops)]\n",
    "    if stemmer:\n",
    "        tokens = [stemmer.stem(tok) for tok in tokens]\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['jeff', 'stole', 'my', 'octopus', 'sandwich'],\n",
       " ['help', 'i', 'sobbed', 'sandwichlessly'],\n",
       " ['drop', 'the', 'sandwiches', 'said', 'the', 'sandwich', 'police']]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_docs = [our_tokenizer(doc) for doc in corpus]\n",
    "tokenized_docs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Step 1: lowercase, lose punction, split into tokens\n",
    "```\n",
    "[\n",
    " ['jeff', 'stole', 'my', 'octopus', 'sandwich'],\n",
    " ['help', 'i', 'sobbed', 'sandwichlessly'],\n",
    " ['drop', 'the', 'sandwiches', 'said', 'the', 'sandwich', 'police']\n",
    "]\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "stopwords = set(nltk.corpus.stopwords.words('english')) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'i' in stopwords "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['jeff', 'stole', 'octopus', 'sandwich'],\n",
       " ['help', 'sobbed', 'sandwichlessly'],\n",
       " ['drop', 'sandwiches', 'said', 'sandwich', 'police']]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_docs = [our_tokenizer(doc, stops=stopwords) for doc in corpus]\n",
    "tokenized_docs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Step 2: remove stop words\n",
    "```\n",
    "[\n",
    " ['jeff', 'stole', 'octopus', 'sandwich'],\n",
    " ['help', 'sobbed', 'sandwichlessly'],\n",
    " ['drop', 'sandwiches', 'said', 'sandwich', 'police']\n",
    "]\n",
    "```\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[u'jeff', u'stole', u'octopus', u'sandwich'],\n",
       " [u'help', u'sob', u'sandwichless'],\n",
       " [u'drop', u'sandwich', u'said', u'sandwich', u'polic']]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_docs = [our_tokenizer(doc, stops=stopwords, stemmer=SnowballStemmer('english')) for doc in corpus]\n",
    "tokenized_docs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Step 3: Stemming/Lemmatization\n",
    "```\n",
    "[\n",
    " ['jeff', 'stole', 'octopus', 'sandwich'],\n",
    " ['help', 'sobbed', 'sandwichlessly'],\n",
    " ['drop', u'sandwich', 'said', 'sandwich', 'police']\n",
    "]\n",
    "```\n",
    "### OK now what?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "Vocabulary:\n",
    "```\n",
    "['drop', 'help', 'jeff', 'octopus', 'police', 'said', 'sandwich', 'sandwichlessly', 'sobbed', 'stole']\n",
    "\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "vocab_set = set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "for doc in tokenized_docs:\n",
    "    vocab_set.update(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['drop', 'help', 'jeff', 'octopus', 'police', 'said', 'sandwich', 'sandwichlessly', 'sobbed', 'stole']\n"
     ]
    }
   ],
   "source": [
    "vocab = sorted(list(vocab_set))\n",
    "print vocab"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section 5.2: Count Vectorizer, TFIDF "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Count vectorization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "Vocabulary:\n",
    "```\n",
    "['drop', 'help', 'jeff', 'octopus', 'police', 'said', 'sandwich', 'sandwichlessly', 'sobbed', 'stole']\n",
    "\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "```\n",
    "['jeff', 'stole', 'octopus', 'sandwich']\n",
    "[0, 0, 1, 1, 0, 0, 1, 0, 0, 1]\n",
    "\n",
    "['help', 'sobbed', 'sandwichlessly']\n",
    "[0, 1, 0, 0, 0, 0, 0, 1, 1, 0]\n",
    "\n",
    "['drop', u'sandwich', 'said', 'sandwich', 'police']\n",
    "[1, 0, 0, 0, 1, 1, 2, 0, 0, 0]\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Term frequency\n",
    "$$TF_{word,document} = \\frac{\\#\\_of\\_times\\_word\\_appears\\_in\\_document}{total\\_\\#\\_of\\_words\\_in\\_document}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "```\n",
    "['jeff', 'stole', 'octopus', 'sandwich']\n",
    "[0, 0, 1/4, 1/4, 0, 0, 1/4, 0, 0, 1/4]\n",
    "\n",
    "['help', 'sobbed', 'sandwichlessly']\n",
    "[0, 1/3, 0, 0, 0, 0, 0, 1/3, 1/3, 0]\n",
    "\n",
    "['drop', u'sandwich', 'said', 'sandwich', 'police']\n",
    "[1/5, 0, 0, 0, 1/5, 1/5, 2/5, 0, 0, 0]\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Document frequency\n",
    "$$ DF_{word} = \\frac{\\#\\_of\\_documents\\_containing\\_word}{total\\_\\#\\_of\\_documents} $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "Vocabulary:\n",
    "```\n",
    "['drop', 'help', 'jeff', 'octopus', 'police', 'said', 'sandwich', 'sandwichlessly', 'sobbed', 'stole']\n",
    "```\n",
    "\n",
    "Document frequency for each word:\n",
    "```\n",
    "[1/3, 1/3, 1/3, 1/3, 1/3, 1/3, 2/3, 1/3, 1/3, 1/3]\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Inverse document frequency\n",
    "$$ IDF_{word} = \\log\\left(\\frac{total\\_\\#\\_of\\_documents}{\\#\\_of\\_documents\\_containing\\_word}\\right) $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "Vocabulary:\n",
    "```\n",
    "['drop', 'help', 'jeff', 'octopus', 'police', 'said', 'sandwich', 'sandwichlessly', 'sobbed', 'stole']\n",
    "```\n",
    "\n",
    "IDF for each word:\n",
    "```\n",
    "[1.099, 1.099, 1.099, 1.099, 1.099, 1.099, 0.405, 1.099, 1.099, 1.099]\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# TFIDF\n",
    "\n",
    "Vocabulary:\n",
    "```\n",
    "['drop', 'help', 'jeff', 'octopus', 'police', 'said', 'sandwich', 'sandwichlessly', 'sobbed', 'stole']\n",
    "```\n",
    "TF * IDF:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "```\n",
    "['jeff', 'stole', 'octopus', 'sandwich']\n",
    "[0, 0, 0.275, 0.275, 0, 0, 0.101, 0, 0, 0.275]\n",
    "\n",
    "['help', 'sobbed', 'sandwichlessly']\n",
    "[0, 0.366, 0, 0, 0, 0, 0, 0.366, 0.366, 0]\n",
    "\n",
    "['drop', u'sandwich', 'said', 'sandwich', 'police']\n",
    "[0.22, 0, 0, 0, 0.22, 0.22, 0.162, 0, 0, 0]\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Now that we have turned our DOCUMENTS into VECTORS, we can put them into whatever machine learning algorithm we want! We can use whatever kind of similarity measure we please!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Wow!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.        ,  0.08115802],\n",
       "       [ 0.08115802,  1.        ]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cosine_similarity([[0, 0, 0.275, 0.275, 0, 0, 0.101, 0, 0, 0.275],  [0.22, 0, 0, 0, 0.22, 0.22, 0.162, 0, 0, 0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.,  0.],\n",
       "       [ 0.,  1.]])"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cosine_similarity([[0, 0.366, 0, 0, 0, 0, 0, 0.366, 0.366, 0],  [0.22, 0, 0, 0, 0.22, 0.22, 0.162, 0, 0, 0]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section 5.3: Example with Spam data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#revisit spam ham example "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df= pd.read_table('data/SMSSpamCollection', header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      0                                                  1\n",
       "0   ham  Go until jurong point, crazy.. Available only ...\n",
       "1   ham                      Ok lar... Joking wif u oni...\n",
       "2  spam  Free entry in 2 a wkly comp to win FA Cup fina..."
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.columns=['spam', 'msg']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>spam</th>\n",
       "      <th>msg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  spam                                                msg\n",
       "0  ham  Go until jurong point, crazy.. Available only ...\n",
       "1  ham                      Ok lar... Joking wif u oni..."
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stopwords_set=set(stopwords)\n",
    "\n",
    "punctuation_set=set(string.punctuation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "179"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(stopwords_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(punctuation_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['msg_cleaned']= df.msg.apply(lambda x: ' '.join([word for word in x.split() if word not in stopwords_set \\\n",
    "                                                   and word not in punctuation_set]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Go until jurong point, crazy'"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str1='Go until jurong point, crazy'.split()\n",
    "' '.join(str1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>spam</th>\n",
       "      <th>msg</th>\n",
       "      <th>msg_cleaned</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "      <td>Go jurong point, crazy.. Available bugis n gre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  spam                                                msg  \\\n",
       "0  ham  Go until jurong point, crazy.. Available only ...   \n",
       "1  ham                      Ok lar... Joking wif u oni...   \n",
       "\n",
       "                                         msg_cleaned  \n",
       "0  Go jurong point, crazy.. Available bugis n gre...  \n",
       "1                      Ok lar... Joking wif u oni...  "
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['msg_cleaned']= df.msg_cleaned.str.lower()  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>spam</th>\n",
       "      <th>msg</th>\n",
       "      <th>msg_cleaned</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "      <td>go jurong point, crazy.. available bugis n gre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "      <td>ok lar... joking wif u oni...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  spam                                                msg  \\\n",
       "0  ham  Go until jurong point, crazy.. Available only ...   \n",
       "1  ham                      Ok lar... Joking wif u oni...   \n",
       "\n",
       "                                         msg_cleaned  \n",
       "0  go jurong point, crazy.. available bugis n gre...  \n",
       "1                      ok lar... joking wif u oni...  "
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "count_vect= CountVectorizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X= count_vect.fit_transform(df.msg_cleaned) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5572, 8703)"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y=df.spam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test= train_test_split(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.98277099784637478"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lg= LogisticRegression()\n",
    "\n",
    "lg.fit(X_train,y_train)\n",
    "y_pred=lg.predict(X_test)\n",
    "lg.score(X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1207,    1],\n",
       "       [  23,  162]])"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['ham', 'ham', 'ham', ..., 'ham', 'ham', 'ham'], dtype=object)"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Section 5.4: Tweak model with Spam data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## try tfidf  \n",
    "\n",
    "tfidf= TfidfVectorizer()  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>spam</th>\n",
       "      <th>msg</th>\n",
       "      <th>msg_cleaned</th>\n",
       "      <th>spam_num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "      <td>go jurong point, crazy.. available bugis n gre...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "      <td>ok lar... joking wif u oni...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  spam                                                msg  \\\n",
       "0  ham  Go until jurong point, crazy.. Available only ...   \n",
       "1  ham                      Ok lar... Joking wif u oni...   \n",
       "\n",
       "                                         msg_cleaned  spam_num  \n",
       "0  go jurong point, crazy.. available bugis n gre...         0  \n",
       "1                      ok lar... joking wif u oni...         0  "
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X= tfidf.fit_transform(df.msg_cleaned)\n",
    "y=df.spam \n",
    "X_train, X_test, y_train, y_test= train_test_split(X,y)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.97415649676956206"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## try random forest \n",
    "rf= RandomForestClassifier()\n",
    "rf.fit(X_train,y_train)\n",
    "y_pred=rf.predict(X_test)\n",
    "rf.score(X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1222,    2],\n",
       "       [  34,  135]])"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test, y_pred)   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.97343862167982775"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#try gradient boost \n",
    "gb= GradientBoostingClassifier()\n",
    "gb.fit(X_train,y_train)\n",
    "y_pred=gb.predict(X_test)\n",
    "gb.score(X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1216,    8],\n",
       "       [  29,  140]])"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Try tfidf with bigrams & trigrams \n",
    "tfidf=TfidfVectorizer(ngram_range=(1,3)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X= tfidf.fit_transform(df.msg_cleaned)\n",
    "y=df.spam\n",
    "X_train, X_test, y_train, y_test= train_test_split(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.96841349605168703"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#try gradient boost \n",
    "gb= GradientBoostingClassifier()\n",
    "gb.fit(X_train,y_train)\n",
    "y_pred=gb.predict(X_test)\n",
    "gb.score(X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1209,    0],\n",
       "       [  41,  143]])"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test, y_pred) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tfidf=TfidfVectorizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X=tfidf.fit_transform(df.msg_cleaned)\n",
    "y=df.spam\n",
    "X_train, X_test, y_train, y_test=  train_test_split(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.95764536970567127"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lg= LogisticRegression()\n",
    "lg.fit(X_train,y_train)\n",
    "y_pred=lg.predict(X_test)\n",
    "lg.score(X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1204,    1],\n",
       "       [  58,  130]])"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test, y_pred) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section 5.5: Pipeline with Spam data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pipeline= Pipeline([('countvect', CountVectorizer(stop_words=stopwords_set)),\\\n",
    "                    #('tfidf', TfidfVectorizer(stop_words=stopwords_set)),\\\n",
    "                    ('lg',  LogisticRegression())])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.97415649677\n",
      "[[1208    1]\n",
      " [  35  149]]\n"
     ]
    }
   ],
   "source": [
    "X=df.msg_cleaned #note we are passing the cleaned msg to the pipeline \n",
    "y=df.spam\n",
    "X_train, X_test, y_train, y_test= train_test_split(X,y) \n",
    "\n",
    "\n",
    "pipeline.fit(X_train, y_train) \n",
    "y_pred= pipeline.predict(X_test)\n",
    "print pipeline.score(X_test, y_test)\n",
    "print confusion_matrix(y_test, y_pred) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pipeline= Pipeline([#('countvect', CountVectorizer(stop_words=stopwords_set)),\\\n",
    "                    ('countvect', CountVectorizer(stop_words=stopwords_set)),\\\n",
    "                    ('rf',  RandomForestClassifier())])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.97272074659\n",
      "[[1188    7]\n",
      " [  31  167]]\n"
     ]
    }
   ],
   "source": [
    "X=df.msg_cleaned #note we are passing the cleaned msg to the pipeline \n",
    "y=df.spam\n",
    "X_train, X_test, y_train, y_test= train_test_split(X,y) \n",
    "\n",
    "\n",
    "pipeline.fit(X_train, y_train) \n",
    "y_pred= pipeline.predict(X_test)\n",
    "print pipeline.score(X_test, y_test)\n",
    "print confusion_matrix(y_test, y_pred)  \n",
    "\n",
    "# the best one so far!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pipeline= Pipeline([#('countvect', CountVectorizer(stop_words=stopwords_set)),\\\n",
    "                    ('countvect', CountVectorizer(stop_words=stopwords_set, ngram_range=(1,3))),\\\n",
    "                    ('rf',  RandomForestClassifier())])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.964824120603\n",
      "[[1221    0]\n",
      " [  49  123]]\n"
     ]
    }
   ],
   "source": [
    "X=df.msg_cleaned #note we are passing the cleaned msg to the pipeline \n",
    "y=df.spam\n",
    "X_train, X_test, y_train, y_test= train_test_split(X,y) \n",
    "\n",
    "\n",
    "pipeline.fit(X_train, y_train) \n",
    "y_pred= pipeline.predict(X_test)\n",
    "print pipeline.score(X_test, y_test)\n",
    "print confusion_matrix(y_test, y_pred) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  },
  "livereveal": {
   "transition": "none"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}