Miniproject_on_Sequence_Modeling.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Miniproject on Sequence Modeling.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/leoliuuu/SequenceModeling/blob/main/Miniproject_on_Sequence_Modeling.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y83OebdlfMhz",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 487
        },
        "outputId": "424e9223-b34e-4c51-d55b-4bc2d0d28654"
      },
      "source": [
        "# Import the movie review data\n",
        "import gdown\n",
        "import pandas as pd\n",
        "\n",
        "gdown.download('https://drive.google.com/uc?authuser=0&id=1Z8bwIBa_0gFe9-C2W0goZ72lQfFMbxjS&export=download',\n",
        "               'labeledTrainData.tsv',\n",
        "               quiet=False)\n",
        "df = pd.read_csv('labeledTrainData.tsv', header=0, delimiter='\\t')\n",
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?authuser=0&id=1Z8bwIBa_0gFe9-C2W0goZ72lQfFMbxjS&export=download\n",
            "To: /content/labeledTrainData.tsv\n",
            "33.6MB [00:00, 75.1MB/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>sentiment</th>\n",
              "      <th>review</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>5814_8</td>\n",
              "      <td>1</td>\n",
              "      <td>With all this stuff going down at the moment w...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2381_9</td>\n",
              "      <td>1</td>\n",
              "      <td>\\The Classic War of the Worlds\\\" by Timothy Hi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>7759_3</td>\n",
              "      <td>0</td>\n",
              "      <td>The film starts with a manager (Nicholas Bell)...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3630_4</td>\n",
              "      <td>0</td>\n",
              "      <td>It must be assumed that those who praised this...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>9495_8</td>\n",
              "      <td>1</td>\n",
              "      <td>Superbly trashy and wondrously unpretentious 8...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24995</th>\n",
              "      <td>3453_3</td>\n",
              "      <td>0</td>\n",
              "      <td>It seems like more consideration has gone into...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24996</th>\n",
              "      <td>5064_1</td>\n",
              "      <td>0</td>\n",
              "      <td>I don't believe they made this film. Completel...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24997</th>\n",
              "      <td>10905_3</td>\n",
              "      <td>0</td>\n",
              "      <td>Guy is a loser. Can't get girls, needs to buil...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24998</th>\n",
              "      <td>10194_3</td>\n",
              "      <td>0</td>\n",
              "      <td>This 30 minute documentary Buñuel made in the ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24999</th>\n",
              "      <td>8478_8</td>\n",
              "      <td>1</td>\n",
              "      <td>I saw this movie as a child and it broke my he...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>25000 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "            id  sentiment                                             review\n",
              "0       5814_8          1  With all this stuff going down at the moment w...\n",
              "1       2381_9          1  \\The Classic War of the Worlds\\\" by Timothy Hi...\n",
              "2       7759_3          0  The film starts with a manager (Nicholas Bell)...\n",
              "3       3630_4          0  It must be assumed that those who praised this...\n",
              "4       9495_8          1  Superbly trashy and wondrously unpretentious 8...\n",
              "...        ...        ...                                                ...\n",
              "24995   3453_3          0  It seems like more consideration has gone into...\n",
              "24996   5064_1          0  I don't believe they made this film. Completel...\n",
              "24997  10905_3          0  Guy is a loser. Can't get girls, needs to buil...\n",
              "24998  10194_3          0  This 30 minute documentary Buñuel made in the ...\n",
              "24999   8478_8          1  I saw this movie as a child and it broke my he...\n",
              "\n",
              "[25000 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "33FIcFh-kBes",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        },
        "outputId": "f9f9181d-6488-4a38-fffe-697d0366aaa0"
      },
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "import numpy as np\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "y = np.array(df['sentiment'])\n",
        "\n",
        "dfX_train, dfX_test, y_train, y_test = train_test_split(df['review'], y)\n",
        "print(\"df_train.shape\",dfX_train.shape)\n",
        "print(\"y_train.shape\",y_train.shape)\n",
        "print(\"dfX_test.shape\",dfX_test.shape)\n",
        "print(\"y_test.shape\",y_test.shape)\n",
        "\n",
        "vectorizer = CountVectorizer(binary=True, min_df = 20) #convert a collection of text documents into a matrix of token counts\n",
        "vectorizer.fit(dfX_train) #learn a vocabulary dictionary of all tokens in the raw documents\n",
        "\n",
        "X_train = vectorizer.transform(dfX_train).todense() #transform to a document-term matrix\n",
        "X_test = vectorizer.transform(dfX_test).todense()\n",
        "print(\"X_test.shape\",X_test.shape)\n",
        "print(\"X_train.shape\", X_train.shape)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "df_train.shape (18750,)\n",
            "y_train.shape (18750,)\n",
            "dfX_test.shape (6250,)\n",
            "y_test.shape (6250,)\n",
            "X_test.shape (6250, 10055)\n",
            "X_train.shape (18750, 10055)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mrwh4PhbGw-d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 272
        },
        "outputId": "8b421119-8476-443b-a682-9068f500c1b2"
      },
      "source": [
        "# Looking at a review to make sure things work\n",
        "reviews_wrapped = dfX_train.str.wrap(80)\n",
        "terrible_index = vectorizer.get_feature_names().index('terrible')\n",
        "print(\"terrible occurs in\", X_train[y_train==1, terrible_index].mean(), \"for Y=1\")\n",
        "print(\"terrible occurs in\", X_train[y_train==0, terrible_index].mean(), \"for Y=0\")\n",
        "print(reviews_wrapped.iloc[1]) # Just in case you want to read a random review"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "terrible occurs in 0.016987179487179486 for Y=1\n",
            "terrible occurs in 0.08690095846645367 for Y=0\n",
            "Late one night on a desolate road, in an empty saloon Martin Sheen spins a yarn\n",
            "for Robert Carradine of Hopalong Cassidy and friends tracking a group of\n",
            "murderous cattle rustlers, who've killed a few men and kidnapped Cassidy's\n",
            "girl.<br /><br />Writer/director Christopher Coppola May have incurred the wrath\n",
            "of William Boyd purists by daring to make a modern low budget film featuring\n",
            "their beloved Hoppy, but I'm glad he did it! No character should be so tied to\n",
            "an an actor that no one else ever be allowed to play him or her again!<br /><br\n",
            "/>I thought it was good fun and an interesting updating of the classic\n",
            "programmers of the thirties and forties. Though guilty of some bad acting, this\n",
            "is earnest enough and unpretentious, making it hard for me to dislike.<br /><br\n",
            "/>The whole production is a bit odd though, but I really enjoyed the scenes\n",
            "between Sheen and Carridine. The fact that we're watching a story within a story\n",
            "makes the oddness and exaggerations more palatable.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "143B_QcCHG4g"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}