SMS_Spam.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "SMS_Spam.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "metadata": {
        "id": "3Cuqg1N_E4x3",
        "colab_type": "code",
        "outputId": "9bcb7b40-9588-4389-dda2-44b5ae77dbfa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "cell_type": "code",
      "source": [
        "!pip install -U -q PyDrive"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[?25l\r\u001b[K    1% |▎                               | 10kB 17.2MB/s eta 0:00:01\r\u001b[K    2% |▋                               | 20kB 1.6MB/s eta 0:00:01\r\u001b[K    3% |█                               | 30kB 2.4MB/s eta 0:00:01\r\u001b[K    4% |█▎                              | 40kB 1.7MB/s eta 0:00:01\r\u001b[K    5% |█▋                              | 51kB 2.0MB/s eta 0:00:01\r\u001b[K    6% |██                              | 61kB 2.4MB/s eta 0:00:01\r\u001b[K    7% |██▎                             | 71kB 2.8MB/s eta 0:00:01\r\u001b[K    8% |██▋                             | 81kB 3.2MB/s eta 0:00:01\r\u001b[K    9% |███                             | 92kB 2.5MB/s eta 0:00:01\r\u001b[K    10% |███▎                            | 102kB 2.7MB/s eta 0:00:01\r\u001b[K    11% |███▋                            | 112kB 2.8MB/s eta 0:00:01\r\u001b[K    12% |████                            | 122kB 4.0MB/s eta 0:00:01\r\u001b[K    13% |████▎                           | 133kB 4.0MB/s eta 0:00:01\r\u001b[K    14% |████▋                           | 143kB 7.4MB/s eta 0:00:01\r\u001b[K    15% |█████                           | 153kB 7.5MB/s eta 0:00:01\r\u001b[K    16% |█████▎                          | 163kB 7.5MB/s eta 0:00:01\r\u001b[K    17% |█████▋                          | 174kB 7.5MB/s eta 0:00:01\r\u001b[K    18% |██████                          | 184kB 7.5MB/s eta 0:00:01\r\u001b[K    19% |██████▎                         | 194kB 41.6MB/s eta 0:00:01\r\u001b[K    20% |██████▋                         | 204kB 37.6MB/s eta 0:00:01\r\u001b[K    21% |███████                         | 215kB 8.3MB/s eta 0:00:01\r\u001b[K    22% |███████▎                        | 225kB 8.3MB/s eta 0:00:01\r\u001b[K    23% |███████▋                        | 235kB 8.5MB/s eta 0:00:01\r\u001b[K    24% |████████                        | 245kB 8.5MB/s eta 0:00:01\r\u001b[K    25% |████████▎                       | 256kB 8.5MB/s eta 0:00:01\r\u001b[K    26% |████████▋                       | 266kB 8.2MB/s eta 0:00:01\r\u001b[K    27% |█████████                       | 276kB 8.3MB/s eta 0:00:01\r\u001b[K    29% |█████████▎                      | 286kB 8.3MB/s eta 0:00:01\r\u001b[K    30% |█████████▋                      | 296kB 8.3MB/s eta 0:00:01\r\u001b[K    31% |██████████                      | 307kB 8.5MB/s eta 0:00:01\r\u001b[K    32% |██████████▎                     | 317kB 43.4MB/s eta 0:00:01\r\u001b[K    33% |██████████▋                     | 327kB 44.0MB/s eta 0:00:01\r\u001b[K    34% |███████████                     | 337kB 44.9MB/s eta 0:00:01\r\u001b[K    35% |███████████▎                    | 348kB 38.7MB/s eta 0:00:01\r\u001b[K    36% |███████████▋                    | 358kB 38.5MB/s eta 0:00:01\r\u001b[K    37% |████████████                    | 368kB 44.6MB/s eta 0:00:01\r\u001b[K    38% |████████████▎                   | 378kB 44.3MB/s eta 0:00:01\r\u001b[K    39% |████████████▋                   | 389kB 44.7MB/s eta 0:00:01\r\u001b[K    40% |█████████████                   | 399kB 9.9MB/s eta 0:00:01\r\u001b[K    41% |█████████████▎                  | 409kB 9.8MB/s eta 0:00:01\r\u001b[K    42% |█████████████▋                  | 419kB 9.9MB/s eta 0:00:01\r\u001b[K    43% |██████████████                  | 430kB 9.8MB/s eta 0:00:01\r\u001b[K    44% |██████████████▎                 | 440kB 9.8MB/s eta 0:00:01\r\u001b[K    45% |██████████████▋                 | 450kB 10.0MB/s eta 0:00:01\r\u001b[K    46% |███████████████                 | 460kB 10.0MB/s eta 0:00:01\r\u001b[K    47% |███████████████▎                | 471kB 10.0MB/s eta 0:00:01\r\u001b[K    48% |███████████████▋                | 481kB 10.0MB/s eta 0:00:01\r\u001b[K    49% |████████████████                | 491kB 10.0MB/s eta 0:00:01\r\u001b[K    50% |████████████████▎               | 501kB 46.7MB/s eta 0:00:01\r\u001b[K    51% |████████████████▋               | 512kB 45.1MB/s eta 0:00:01\r\u001b[K    52% |█████████████████               | 522kB 44.5MB/s eta 0:00:01\r\u001b[K    53% |█████████████████▎              | 532kB 45.7MB/s eta 0:00:01\r\u001b[K    54% |█████████████████▋              | 542kB 9.0MB/s eta 0:00:01\r\u001b[K    55% |██████████████████              | 552kB 9.0MB/s eta 0:00:01\r\u001b[K    57% |██████████████████▎             | 563kB 9.0MB/s eta 0:00:01\r\u001b[K    58% |██████████████████▋             | 573kB 8.9MB/s eta 0:00:01\r\u001b[K    59% |███████████████████             | 583kB 8.9MB/s eta 0:00:01\r\u001b[K    60% |███████████████████▎            | 593kB 9.0MB/s eta 0:00:01\r\u001b[K    61% |███████████████████▋            | 604kB 8.9MB/s eta 0:00:01\r\u001b[K    62% |████████████████████            | 614kB 9.0MB/s eta 0:00:01\r\u001b[K    63% |████████████████████▎           | 624kB 9.1MB/s eta 0:00:01\r\u001b[K    64% |████████████████████▋           | 634kB 9.1MB/s eta 0:00:01\r\u001b[K    65% |█████████████████████           | 645kB 48.9MB/s eta 0:00:01\r\u001b[K    66% |█████████████████████▎          | 655kB 50.0MB/s eta 0:00:01\r\u001b[K    67% |█████████████████████▋          | 665kB 41.5MB/s eta 0:00:01\r\u001b[K    68% |██████████████████████          | 675kB 41.8MB/s eta 0:00:01\r\u001b[K    69% |██████████████████████▎         | 686kB 40.0MB/s eta 0:00:01\r\u001b[K    70% |██████████████████████▋         | 696kB 39.9MB/s eta 0:00:01\r\u001b[K    71% |███████████████████████         | 706kB 40.6MB/s eta 0:00:01\r\u001b[K    72% |███████████████████████▎        | 716kB 40.7MB/s eta 0:00:01\r\u001b[K    73% |███████████████████████▋        | 727kB 40.7MB/s eta 0:00:01\r\u001b[K    74% |████████████████████████        | 737kB 39.2MB/s eta 0:00:01\r\u001b[K    75% |████████████████████████▎       | 747kB 39.3MB/s eta 0:00:01\r\u001b[K    76% |████████████████████████▋       | 757kB 40.0MB/s eta 0:00:01\r\u001b[K    77% |████████████████████████▉       | 768kB 49.1MB/s eta 0:00:01\r\u001b[K    78% |█████████████████████████▏      | 778kB 50.7MB/s eta 0:00:01\r\u001b[K    79% |█████████████████████████▌      | 788kB 52.7MB/s eta 0:00:01\r\u001b[K    80% |█████████████████████████▉      | 798kB 51.6MB/s eta 0:00:01\r\u001b[K    81% |██████████████████████████▏     | 808kB 51.5MB/s eta 0:00:01\r\u001b[K    82% |██████████████████████████▌     | 819kB 51.5MB/s eta 0:00:01\r\u001b[K    83% |██████████████████████████▉     | 829kB 52.1MB/s eta 0:00:01\r\u001b[K    85% |███████████████████████████▏    | 839kB 54.0MB/s eta 0:00:01\r\u001b[K    86% |███████████████████████████▌    | 849kB 53.5MB/s eta 0:00:01\r\u001b[K    87% |███████████████████████████▉    | 860kB 48.5MB/s eta 0:00:01\r\u001b[K    88% |████████████████████████████▏   | 870kB 48.6MB/s eta 0:00:01\r\u001b[K    89% |████████████████████████████▌   | 880kB 48.0MB/s eta 0:00:01\r\u001b[K    90% |████████████████████████████▉   | 890kB 48.6MB/s eta 0:00:01\r\u001b[K    91% |█████████████████████████████▏  | 901kB 49.2MB/s eta 0:00:01\r\u001b[K    92% |█████████████████████████████▌  | 911kB 49.2MB/s eta 0:00:01\r\u001b[K    93% |█████████████████████████████▉  | 921kB 48.3MB/s eta 0:00:01\r\u001b[K    94% |██████████████████████████████▏ | 931kB 47.9MB/s eta 0:00:01\r\u001b[K    95% |██████████████████████████████▌ | 942kB 48.5MB/s eta 0:00:01\r\u001b[K    96% |██████████████████████████████▉ | 952kB 48.4MB/s eta 0:00:01\r\u001b[K    97% |███████████████████████████████▏| 962kB 53.9MB/s eta 0:00:01\r\u001b[K    98% |███████████████████████████████▌| 972kB 54.1MB/s eta 0:00:01\r\u001b[K    99% |███████████████████████████████▉| 983kB 21.8MB/s eta 0:00:01\r\u001b[K    100% |████████████████████████████████| 993kB 12.3MB/s \n",
            "\u001b[?25h  Building wheel for PyDrive (setup.py) ... \u001b[?25ldone\n",
            "\u001b[?25h"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "skiEuBmntSvV",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import os\n",
        "from pydrive.auth import GoogleAuth\n",
        "from pydrive.drive import GoogleDrive\n",
        "from google.colab import auth\n",
        "from oauth2client.client import GoogleCredentials\n",
        "\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense, CuDNNGRU, Embedding, Dropout, CuDNNLSTM, SimpleRNN, BatchNormalization\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "np.random.seed(1)\n",
        "tf.set_random_seed(1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Blvs22YVFo2F",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "auth.authenticate_user()\n",
        "gauth = GoogleAuth()\n",
        "gauth.credentials = GoogleCredentials.get_application_default()\n",
        "drive = GoogleDrive(gauth)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "_Acwl_4oF5HS",
        "colab_type": "code",
        "outputId": "8d461697-dea6-4107-e9ab-6e3a2c8fcd3e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "cell_type": "code",
      "source": [
        "download_path = os.path.expanduser('~/data')\n",
        "try:\n",
        "  os.makedirs(download_path)\n",
        "except: pass\n",
        "\n",
        "output_file = os.path.join(download_path)\n",
        "\n",
        "file_list = drive.ListFile(\n",
        "    {'q': \"'1B6wmxiZt8NHWnRe4gsZD5yYTrevTyZim' in parents\"}).GetList()\n",
        "\n",
        "for f in file_list:\n",
        "  print('title: %s, id: %s' % (f['title'], f['id']))\n",
        "  fname = os.path.join(download_path, f['title'])\n",
        "  print('downloading to {}'.format(fname))\n",
        "  f_ = drive.CreateFile({'id': f['id']})\n",
        "  f_.GetContentFile(fname)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "title: smsspamcollection.zip, id: 1kNI0uCWaJPqEeDqRkNlmh5jX7QzQHzLo\n",
            "downloading to /root/data/smsspamcollection.zip\n",
            "title: SMSSpamCollection.txt, id: 1p8nsjPoR5DKjMSKK_EobFYYpavSxU5sJ\n",
            "downloading to /root/data/SMSSpamCollection.txt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "oLKN0gTTtSvZ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "output_file = '/root/data/SMSSpamCollection.txt'\n",
        "\n",
        "with open(output_file) as f:\n",
        "    mylist = f.read().splitlines()\n",
        "    \n",
        "#mylist"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "zYQ2dFExtSve",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def text_to_df(list):\n",
        "    result = pd.DataFrame(columns=['text', 'spam'])\n",
        "    \n",
        "    for text in list:\n",
        "        if (text.find('ham\\t') == 0):\n",
        "            df = pd.DataFrame([[text[4:], 0]], columns=['text', 'spam'])\n",
        "            result = result.append(df)\n",
        "            \n",
        "        elif (text.find('spam\\t') == 0):\n",
        "            df = pd.DataFrame([[text[5:], 1]], columns=['text', 'spam'])\n",
        "            result = result.append(df)\n",
        "        \n",
        "    return result"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "YOGhS3T6tSvh",
        "colab_type": "code",
        "outputId": "4f9e7637-c751-4f73-d03e-8ac2a814bd1b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "cell_type": "code",
      "source": [
        "dataset = text_to_df(mylist)\n",
        "\n",
        "dataset.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>spam</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Ok lar... Joking wif u oni...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>U dun say so early hor... U c already then say...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text spam\n",
              "0  Go until jurong point, crazy.. Available only ...    0\n",
              "0                      Ok lar... Joking wif u oni...    0\n",
              "0  Free entry in 2 a wkly comp to win FA Cup fina...    1\n",
              "0  U dun say so early hor... U c already then say...    0\n",
              "0  Nah I don't think he goes to usf, he lives aro...    0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "metadata": {
        "id": "y0A2yyZ9tSvm",
        "colab_type": "code",
        "outputId": "bef8cae4-feb2-49b6-e342-cdce06619ac4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "cell_type": "code",
      "source": [
        "dataset.spam.value_counts()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    4827\n",
              "1     747\n",
              "Name: spam, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "metadata": {
        "id": "eNczKKM1tSvp",
        "colab_type": "code",
        "outputId": "5fedd511-bfa8-4bab-89a2-39280535d599",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "cell_type": "code",
      "source": [
        "x_data, y_data = dataset.text.values, dataset.spam.values\n",
        "\n",
        "print(len(x_data))\n",
        "print(len(y_data))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "5574\n",
            "5574\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "dS3gkV9ctSvt",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Tokenizing word\n",
        "num_words = 10000\n",
        "tokenizer = Tokenizer(num_words=num_words)\n",
        "tokenizer.fit_on_texts(x_data)\n",
        "\n",
        "if num_words is None:\n",
        "    num_words = len(tokenizer.word_index)\n",
        "    print(num_words)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "OpzwaLTbtSv3",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Print Token words\n",
        "#tokenizer.word_index\n",
        "#test_df = pd.tokenizer.word_index"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "kTunCWkMtSv9",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Get tokenize to data\n",
        "x_tokens = tokenizer.texts_to_sequences(x_data)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "4veTKVS1_WyV",
        "colab_type": "code",
        "outputId": "a3227a8e-c636-4699-f884-0356fa66cea8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "cell_type": "code",
      "source": [
        "# Print sample of data\n",
        "print(x_data[0])\n",
        "print(np.array(x_tokens[0]))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...\n",
            "[  49  471 4435  842  755  658   64    8 1327   88  123  351 1328  148\n",
            " 2996 1329   67   58 4436  144]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "5Lvw3bA7tSwG",
        "colab_type": "code",
        "outputId": "9ddc9e5d-8d88-4f72-fa9a-2c8624379472",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "cell_type": "code",
      "source": [
        "# Analysis for best shape in data\n",
        "num_tokens = [len(tokens) for tokens in x_data]\n",
        "num_tokens = np.array(num_tokens)\n",
        "\n",
        "print(np.mean(num_tokens))\n",
        "print(np.max(num_tokens))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "80.47829207032652\n",
            "910\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "XLwZe7-VtSwL",
        "colab_type": "code",
        "outputId": "da73d25c-71af-428d-b477-71a673427b8f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "max_tokens = np.mean(num_tokens) + 2 * np.std(num_tokens)\n",
        "max_tokens = int(max_tokens)\n",
        "max_tokens"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "200"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "metadata": {
        "id": "YkDLRZp3tSwP",
        "colab_type": "code",
        "outputId": "c674f0dd-00f4-4753-9827-e662f405ff5e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "np.sum(num_tokens < max_tokens) / len(num_tokens)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9795479009687836"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "metadata": {
        "id": "lo3QXspjtSwS",
        "colab_type": "code",
        "outputId": "5438dc9b-fb50-4467-ff4e-837c26d2a60a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "x_data_pad = pad_sequences(x_tokens, maxlen=max_tokens, padding='pre', truncating='post')\n",
        "#x_data_pad = x_data_pad / num_words\n",
        "\n",
        "x_data_pad.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(5574, 200)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "metadata": {
        "id": "Bz9sH0XFtSwX",
        "colab_type": "code",
        "outputId": "1bbb0d37-b6e7-4af0-fed9-c58abc0627b2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "cell_type": "code",
      "source": [
        "np.array(x_tokens[0])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([  49,  471, 4435,  842,  755,  658,   64,    8, 1327,   88,  123,\n",
              "        351, 1328,  148, 2996, 1329,   67,   58, 4436,  144])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "metadata": {
        "id": "dSDdFBQ9tSwe",
        "colab_type": "code",
        "outputId": "643b4ea3-692f-43ff-dc1d-8abf9c8f6999",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 340
        }
      },
      "cell_type": "code",
      "source": [
        "x_data_pad[0]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,   49,  471, 4435,  842,  755,  658,   64,\n",
              "          8, 1327,   88,  123,  351, 1328,  148, 2996, 1329,   67,   58,\n",
              "       4436,  144], dtype=int32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "metadata": {
        "id": "p52s5GZsjpMk",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Train Test Split by line\n",
        "x_train, y_train = x_data_pad[0:1674], y_data[0:1674]\n",
        "x_test, y_test = x_data_pad[1674:], y_data[1674:]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "UMRT82aJ-8vn",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def spam_caught_score(model):\n",
        "  y_pred = model.predict_classes(x_test)\n",
        "\n",
        "  all_spam = 0\n",
        "  spam_found = 0\n",
        "\n",
        "  for i in range(len(y_test)):\n",
        "    if y_test[i] == 1:\n",
        "      all_spam += 1\n",
        "      if y_pred[i][0] == y_test[i]:\n",
        "        spam_found += 1\n",
        "\n",
        "  print('Number of spam : ' + str(all_spam))\n",
        "  print('Number of spam caught : ' + str(spam_found))\n",
        "  print(spam_found / all_spam)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "uIICi-HjE3cU",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def block_ham_score(model):\n",
        "  y_pred = model.predict_classes(x_test)\n",
        "\n",
        "  all_ham = 0\n",
        "  block_ham_found = 0\n",
        "\n",
        "  for i in range(len(y_test)):\n",
        "    if y_test[i] == 0:\n",
        "      all_ham += 1\n",
        "      if y_pred[i][0] != y_test[i]:\n",
        "        block_ham_found += 1\n",
        "\n",
        "  print('Number of ham : ' + str(all_ham))\n",
        "  print('Number of blocked ham : ' + str(block_ham_found))\n",
        "  print(block_ham_found / all_ham)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "avoaxcH4k0ff",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Define parameter\n",
        "embedding_size = 32\n",
        "EPOCHS = 10\n",
        "batch_size = None"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "a7HyGZbBiR--",
        "colab_type": "code",
        "outputId": "78406934-2e13-4ef5-d972-8584e0c697db",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "x_train.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1674, 200)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "metadata": {
        "id": "1VCbZdrkiVDr",
        "colab_type": "code",
        "outputId": "9f9a8ea1-a5b2-47a3-ea7a-b784791da228",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "cell_type": "code",
      "source": [
        "print(num_words)\n",
        "print(embedding_size)\n",
        "print(max_tokens)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "10000\n",
            "32\n",
            "200\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "WV4AxQgBtSwh",
        "colab_type": "code",
        "outputId": "c43dace5-4182-4301-de10-e211953a90a1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 337
        }
      },
      "cell_type": "code",
      "source": [
        "# GRU\n",
        "model_gru = Sequential()\n",
        "\n",
        "#model_gru.add(Embedding(input_dim=num_words, output_dim=embedding_size, input_length=max_tokens))\n",
        "\n",
        "model_gru.add(CuDNNGRU(units=embedding_size, return_sequences=True))\n",
        "\n",
        "model_gru.add(CuDNNGRU(units=max_tokens, return_sequences=True))\n",
        "\n",
        "model_gru.add(CuDNNGRU(units=embedding_size))\n",
        "model_gru.add(Dropout(0.2))\n",
        "\n",
        "model_gru.add(Dense(embedding_size/2, activation='relu'))\n",
        "\n",
        "model_gru.add(Dense(1, activation='sigmoid'))\n",
        "\n",
        "optimizer = Adam(lr=0.01)\n",
        "\n",
        "model_gru.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])\n",
        "\n",
        "model_gru.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-27-3183dfc2aa37>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0mmodel_gru\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'binary_crossentropy'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m \u001b[0mmodel_gru\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msummary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/network.py\u001b[0m in \u001b[0;36msummary\u001b[0;34m(self, line_length, positions, print_fn)\u001b[0m\n\u001b[1;32m   1607\u001b[0m     \"\"\"\n\u001b[1;32m   1608\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuilt\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1609\u001b[0;31m       raise ValueError('This model has not yet been built. '\n\u001b[0m\u001b[1;32m   1610\u001b[0m                        \u001b[0;34m'Build the model first by calling `build()` or calling '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1611\u001b[0m                        \u001b[0;34m'`fit()` with some data, or specify '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mValueError\u001b[0m: This model has not yet been built. Build the model first by calling `build()` or calling `fit()` with some data, or specify an `input_shape` argument in the first layer(s) for automatic build."
          ]
        }
      ]
    },
    {
      "metadata": {
        "id": "HGi_-DlU08is",
        "colab_type": "code",
        "outputId": "3df9cdcb-daf1-43df-d810-91fc7871e535",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 442
        }
      },
      "cell_type": "code",
      "source": [
        "%%time\n",
        "model_gru.fit(x_train, y_train, epochs=EPOCHS, batch_size=batch_size)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/gradients_impl.py:108: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
            "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/10\n",
            "1674/1674 [==============================] - 5s 3ms/step - loss: 0.1563 - acc: 0.9456\n",
            "Epoch 2/10\n",
            "1674/1674 [==============================] - 3s 2ms/step - loss: 0.0095 - acc: 0.9982\n",
            "Epoch 3/10\n",
            "1674/1674 [==============================] - 3s 2ms/step - loss: 0.0035 - acc: 0.9988\n",
            "Epoch 4/10\n",
            "1674/1674 [==============================] - 3s 2ms/step - loss: 0.0030 - acc: 0.9994\n",
            "Epoch 5/10\n",
            "1674/1674 [==============================] - 3s 2ms/step - loss: 0.0071 - acc: 0.9970\n",
            "Epoch 6/10\n",
            "1674/1674 [==============================] - 3s 2ms/step - loss: 6.6598e-05 - acc: 1.0000\n",
            "Epoch 7/10\n",
            "1674/1674 [==============================] - 3s 2ms/step - loss: 3.1301e-05 - acc: 1.0000\n",
            "Epoch 8/10\n",
            "1674/1674 [==============================] - 3s 2ms/step - loss: 3.0467e-05 - acc: 1.0000\n",
            "Epoch 9/10\n",
            "1674/1674 [==============================] - 3s 2ms/step - loss: 2.1415e-05 - acc: 1.0000\n",
            "Epoch 10/10\n",
            "1674/1674 [==============================] - 3s 2ms/step - loss: 1.4350e-05 - acc: 1.0000\n",
            "CPU times: user 29.2 s, sys: 7.59 s, total: 36.7 s\n",
            "Wall time: 36.1 s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7f413105ee80>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 51
        }
      ]
    },
    {
      "metadata": {
        "id": "dqbOeyOv1DzE",
        "colab_type": "code",
        "outputId": "f1af74a4-abfa-43ab-a567-8513cfae8417",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "cell_type": "code",
      "source": [
        "model_gru.evaluate(x_test, y_test)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "3900/3900 [==============================] - 3s 801us/step\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.12697152848956952, 0.9802564102564103]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 52
        }
      ]
    },
    {
      "metadata": {
        "id": "Qk8R02gM1HjW",
        "colab_type": "code",
        "outputId": "51c8396d-db9c-4f8a-dd90-37b4ccbcc852",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        }
      },
      "cell_type": "code",
      "source": [
        "spam_caught_score(model_gru)\n",
        "block_ham_score(model_gru)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of spam : 509\n",
            "Number of spam caught : 439\n",
            "0.862475442043222\n",
            "Number of ham : 3391\n",
            "Number of blocked ham : 7\n",
            "0.0020642878207018578\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "jP6hLJZn47Zh",
        "colab_type": "code",
        "outputId": "17cd54fb-8fc9-4cca-8a77-dfb893001473",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 340
        }
      },
      "cell_type": "code",
      "source": [
        "# LSTM\n",
        "model_lstm = Sequential()\n",
        "\n",
        "model_lstm.add(Embedding(input_dim=num_words, output_dim=embedding_size, input_length=max_tokens))\n",
        "\n",
        "model_lstm.add(CuDNNLSTM(units=embedding_size, return_sequences=True))\n",
        "\n",
        "#model_lstm.add(CuDNNLSTM(units=max_tokens, return_sequences=True))\n",
        "\n",
        "model_lstm.add(CuDNNLSTM(units=embedding_size))\n",
        "model_lstm.add(Dropout(0.2))\n",
        "\n",
        "model_lstm.add(Dense(embedding_size/2, activation='relu'))\n",
        "model_lstm.add(Dense(1, activation='sigmoid'))\n",
        "\n",
        "optimizer = Adam(lr=0.01)\n",
        "\n",
        "model_lstm.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])\n",
        "\n",
        "model_lstm.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_2 (Embedding)      (None, 200, 32)           320000    \n",
            "_________________________________________________________________\n",
            "cu_dnnlstm (CuDNNLSTM)       (None, 200, 32)           8448      \n",
            "_________________________________________________________________\n",
            "cu_dnnlstm_1 (CuDNNLSTM)     (None, 32)                8448      \n",
            "_________________________________________________________________\n",
            "dropout_2 (Dropout)          (None, 32)                0         \n",
            "_________________________________________________________________\n",
            "dense_4 (Dense)              (None, 16)                528       \n",
            "_________________________________________________________________\n",
            "dense_5 (Dense)              (None, 1)                 17        \n",
            "=================================================================\n",
            "Total params: 337,441\n",
            "Trainable params: 337,441\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "p3Erkcz-tSwp",
        "colab_type": "code",
        "outputId": "98057440-3f30-43ff-a052-c4ea16f4db96",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 323
        }
      },
      "cell_type": "code",
      "source": [
        "%%time\n",
        "model_lstm.fit(x_train, y_train, epochs=EPOCHS, batch_size=batch_size)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/gradients_impl.py:108: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
            "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/10\n",
            "1674/1674 [==============================] - 4s 2ms/step - loss: 0.3682 - acc: 0.8596\n",
            "Epoch 2/10\n",
            "1674/1674 [==============================] - 3s 2ms/step - loss: 0.0490 - acc: 0.9881\n",
            "Epoch 3/10\n",
            "1674/1674 [==============================] - 3s 2ms/step - loss: 0.0093 - acc: 0.9982\n",
            "Epoch 4/10\n",
            "1674/1674 [==============================] - 3s 2ms/step - loss: 5.4148e-04 - acc: 1.0000\n",
            "Epoch 5/10\n",
            "1674/1674 [==============================] - 3s 2ms/step - loss: 1.9698e-04 - acc: 1.0000\n",
            "Epoch 6/10\n",
            "1674/1674 [==============================] - 3s 2ms/step - loss: 1.1745e-04 - acc: 1.0000\n",
            "Epoch 7/10\n",
            "1674/1674 [==============================] - 3s 2ms/step - loss: 1.0219e-04 - acc: 1.0000\n",
            "Epoch 8/10\n",
            " 736/1674 [============>.................] - ETA: 1s - loss: 6.9057e-05 - acc: 1.0000"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "_y29iqX6XnDM",
        "colab_type": "code",
        "outputId": "dfb366a0-0b6f-4133-faa9-4e61efc19a41",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "cell_type": "code",
      "source": [
        "model_lstm.evaluate(x_test, y_test)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "3900/3900 [==============================] - 5s 1ms/step\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.1887233146644865, 0.9815384615384616]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 205
        }
      ]
    },
    {
      "metadata": {
        "id": "uimBxP5v_bNo",
        "colab_type": "code",
        "outputId": "bcd402b1-797e-4322-b86e-7131287331e6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        }
      },
      "cell_type": "code",
      "source": [
        "spam_caught_score(model_lstm)\n",
        "block_ham_score(model_lstm)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of spam : 509\n",
            "Number of spam caught : 447\n",
            "0.8781925343811395\n",
            "Number of ham : 3391\n",
            "Number of blocked ham : 10\n",
            "0.002948982601002654\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}