Ad_content_analysis_NLP.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "O8-FXD0BDtXL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bfb6d850-7c1d-4235-94a5-284fdcd27b6c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Notebook last run (end-to-end): 2024-01-15 04:53:19.441299\n"
          ]
        }
      ],
      "source": [
        "import datetime\n",
        "print(f\"Notebook last run (end-to-end): {datetime.datetime.now()}\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Check for GPU\n",
        "!nvidia-smi -L"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "brA415ti3hNx",
        "outputId": "30146a70-fd98-43af-a07e-8e393c6f1bfd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/bin/bash: line 1: nvidia-smi: command not found\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Download helper functions script\n",
        "!wget https://raw.githubusercontent.com/mrdbourke/tensorflow-deep-learning/main/extras/helper_functions.py"
      ],
      "metadata": {
        "id": "-AGlQ2PV3Y2E",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "74f456ce-9434-4fc8-f6a5-948d25f0cdf0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-01-15 04:53:22--  https://raw.githubusercontent.com/mrdbourke/tensorflow-deep-learning/main/extras/helper_functions.py\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.111.133, 185.199.109.133, 185.199.108.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.111.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 10246 (10K) [text/plain]\n",
            "Saving to: ‘helper_functions.py’\n",
            "\n",
            "helper_functions.py 100%[===================>]  10.01K  --.-KB/s    in 0s      \n",
            "\n",
            "2024-01-15 04:53:22 (97.5 MB/s) - ‘helper_functions.py’ saved [10246/10246]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Import series of helper functions for the notebook\n",
        "from helper_functions import unzip_data, create_tensorboard_callback, plot_loss_curves, compare_historys"
      ],
      "metadata": {
        "id": "tGIU9Vjf4KN6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n"
      ],
      "metadata": {
        "id": "ihj0xJEB4VWd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "import pandas as pd\n",
        "\n",
        "# Replace 'your_file.csv' with the actual name of your CSV file\n",
        "df = pd.read_csv('spam_ham_train.csv')\n",
        "\n",
        "# Display the first few rows of the DataFrame\n",
        "df.head()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "JYrU8hig4yR_",
        "outputId": "c4f4a876-705d-4e46-f800-c94aa45ee8e2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                 id   keyword   location  \\\n",
              "0  f635f780a0ead39a       NaN        NaN   \n",
              "1              4551  keyword1  location3   \n",
              "2              5520  keyword3  location2   \n",
              "3              3323  keyword2  location1   \n",
              "4  18d9089ff06204c1       NaN        NaN   \n",
              "\n",
              "                                                text  target  \n",
              "0  \" 2007 (UTC)\\n\\n TedFrank, your response here ...       0  \n",
              "1  Haha, my friend tyler literally just asked if ...       0  \n",
              "2  Can you pls send me that company name. In saib...       0  \n",
              "3                             Yo im right by yo work       0  \n",
              "4  Cocksucker \\n\\nBILLCJ YOU ARE A COMPLETE FUCKF...       1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6117f1b2-b45c-4a2a-bfa4-e4c955752759\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>keyword</th>\n",
              "      <th>location</th>\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>f635f780a0ead39a</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>\" 2007 (UTC)\\n\\n TedFrank, your response here ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>4551</td>\n",
              "      <td>keyword1</td>\n",
              "      <td>location3</td>\n",
              "      <td>Haha, my friend tyler literally just asked if ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>5520</td>\n",
              "      <td>keyword3</td>\n",
              "      <td>location2</td>\n",
              "      <td>Can you pls send me that company name. In saib...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3323</td>\n",
              "      <td>keyword2</td>\n",
              "      <td>location1</td>\n",
              "      <td>Yo im right by yo work</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>18d9089ff06204c1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Cocksucker \\n\\nBILLCJ YOU ARE A COMPLETE FUCKF...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6117f1b2-b45c-4a2a-bfa4-e4c955752759')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-6117f1b2-b45c-4a2a-bfa4-e4c955752759 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-6117f1b2-b45c-4a2a-bfa4-e4c955752759');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-e3dae273-e44e-42e1-b6c1-c2eb94244cc8\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-e3dae273-e44e-42e1-b6c1-c2eb94244cc8')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-e3dae273-e44e-42e1-b6c1-c2eb94244cc8 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Turn .csv files into pandas DataFrame's\n",
        "import pandas as pd\n",
        "train_df = pd.read_csv(\"spam_ham_train.csv\")\n",
        "test_df = pd.read_csv(\"spam_ham_test.csv\")\n",
        "train_df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "cm50bj9L5T3z",
        "outputId": "e0c8e092-0ad3-42ea-f217-944bca0d0ab5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                 id   keyword   location  \\\n",
              "0  f635f780a0ead39a       NaN        NaN   \n",
              "1              4551  keyword1  location3   \n",
              "2              5520  keyword3  location2   \n",
              "3              3323  keyword2  location1   \n",
              "4  18d9089ff06204c1       NaN        NaN   \n",
              "\n",
              "                                                text  target  \n",
              "0  \" 2007 (UTC)\\n\\n TedFrank, your response here ...       0  \n",
              "1  Haha, my friend tyler literally just asked if ...       0  \n",
              "2  Can you pls send me that company name. In saib...       0  \n",
              "3                             Yo im right by yo work       0  \n",
              "4  Cocksucker \\n\\nBILLCJ YOU ARE A COMPLETE FUCKF...       1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-5e15ceeb-4202-45aa-a151-0aaa01e87d64\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>keyword</th>\n",
              "      <th>location</th>\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>f635f780a0ead39a</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>\" 2007 (UTC)\\n\\n TedFrank, your response here ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>4551</td>\n",
              "      <td>keyword1</td>\n",
              "      <td>location3</td>\n",
              "      <td>Haha, my friend tyler literally just asked if ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>5520</td>\n",
              "      <td>keyword3</td>\n",
              "      <td>location2</td>\n",
              "      <td>Can you pls send me that company name. In saib...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3323</td>\n",
              "      <td>keyword2</td>\n",
              "      <td>location1</td>\n",
              "      <td>Yo im right by yo work</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>18d9089ff06204c1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Cocksucker \\n\\nBILLCJ YOU ARE A COMPLETE FUCKF...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5e15ceeb-4202-45aa-a151-0aaa01e87d64')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-5e15ceeb-4202-45aa-a151-0aaa01e87d64 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-5e15ceeb-4202-45aa-a151-0aaa01e87d64');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-19f34594-cfbc-4d30-b3d5-8ca041419566\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-19f34594-cfbc-4d30-b3d5-8ca041419566')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-19f34594-cfbc-4d30-b3d5-8ca041419566 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "PCbqGPKW6up-",
        "outputId": "3c4023ad-668e-438c-974d-ed1b9daec8a3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     id     keyword                      location  \\\n",
              "0  8051    refugees                           NaN   \n",
              "1   425  apocalypse  Currently Somewhere On Earth   \n",
              "2  1330  blown%20up                   Scout Team    \n",
              "3   663      attack                           NaN   \n",
              "4  2930      danger                        Leeds    \n",
              "\n",
              "                                                text  \n",
              "0  Refugees as citizens - The Hindu http://t.co/G...  \n",
              "1  @5SOStag honestly he could say an apocalypse i...  \n",
              "2  If you bored as shit don't nobody fuck wit you...  \n",
              "3  @RealTwanBrown Yesterday I Had A Heat Attack ?...  \n",
              "4  The Devil Wears Prada is still one of my favou...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-c2d7801d-c04d-4daa-a456-a85cfc4ef81e\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>keyword</th>\n",
              "      <th>location</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>8051</td>\n",
              "      <td>refugees</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Refugees as citizens - The Hindu http://t.co/G...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>425</td>\n",
              "      <td>apocalypse</td>\n",
              "      <td>Currently Somewhere On Earth</td>\n",
              "      <td>@5SOStag honestly he could say an apocalypse i...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1330</td>\n",
              "      <td>blown%20up</td>\n",
              "      <td>Scout Team</td>\n",
              "      <td>If you bored as shit don't nobody fuck wit you...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>663</td>\n",
              "      <td>attack</td>\n",
              "      <td>NaN</td>\n",
              "      <td>@RealTwanBrown Yesterday I Had A Heat Attack ?...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2930</td>\n",
              "      <td>danger</td>\n",
              "      <td>Leeds</td>\n",
              "      <td>The Devil Wears Prada is still one of my favou...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c2d7801d-c04d-4daa-a456-a85cfc4ef81e')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-c2d7801d-c04d-4daa-a456-a85cfc4ef81e button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-c2d7801d-c04d-4daa-a456-a85cfc4ef81e');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-6c69f2c4-bf28-4fd0-8cf4-250117c5b144\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-6c69f2c4-bf28-4fd0-8cf4-250117c5b144')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-6c69f2c4-bf28-4fd0-8cf4-250117c5b144 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Shuffle training dataframe\n",
        "train_df_shuffled = train_df.sample(frac=1, random_state=42) # shuffle with random_state=42 for reproducibility\n",
        "train_df_shuffled.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "uboKtjSc8qLx",
        "outputId": "f4fe8f3e-931d-48c0-a6f4-c6dc7c337806"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                    id keyword location  \\\n",
              "2753  bb167d7f6820a7f3     NaN      NaN   \n",
              "1595  47369e6eb0cf0950     NaN      NaN   \n",
              "1747  27081ec9b736d49a     NaN      NaN   \n",
              "866   8dd4a4e7163c89f5     NaN      NaN   \n",
              "8131  1f299afb467f0242     NaN      NaN   \n",
              "\n",
              "                                                   text  target  \n",
              "2753  Wikia property\\nThe status of Uncyc. as wikia ...       1  \n",
              "1595  Possible Improvement \\n\\nIt would be nice if w...       0  \n",
              "1747  Dogmatic \\n\\nThe use of the word dogmatic in r...       0  \n",
              "866   \"\\nRepublicanJacobite is the ultimate hypocrit...       1  \n",
              "8131  OK! \\n\\nIM GOING TO VANDALIZE WILD ONES WIKI T...       1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-9a5fa7ab-eb74-4b7b-94dd-a898162ae7c5\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>keyword</th>\n",
              "      <th>location</th>\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2753</th>\n",
              "      <td>bb167d7f6820a7f3</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Wikia property\\nThe status of Uncyc. as wikia ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1595</th>\n",
              "      <td>47369e6eb0cf0950</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Possible Improvement \\n\\nIt would be nice if w...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1747</th>\n",
              "      <td>27081ec9b736d49a</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Dogmatic \\n\\nThe use of the word dogmatic in r...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>866</th>\n",
              "      <td>8dd4a4e7163c89f5</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>\"\\nRepublicanJacobite is the ultimate hypocrit...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8131</th>\n",
              "      <td>1f299afb467f0242</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>OK! \\n\\nIM GOING TO VANDALIZE WILD ONES WIKI T...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-9a5fa7ab-eb74-4b7b-94dd-a898162ae7c5')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-9a5fa7ab-eb74-4b7b-94dd-a898162ae7c5 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-9a5fa7ab-eb74-4b7b-94dd-a898162ae7c5');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-55379c9a-07ca-48c2-a3eb-880915face6b\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-55379c9a-07ca-48c2-a3eb-880915face6b')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-55379c9a-07ca-48c2-a3eb-880915face6b button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# The test data doesn't have a target (that's what we'd try to predict)\n",
        "test_df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "Go-Njtwy9DPB",
        "outputId": "915d6f56-bb54-4e12-be87-bf01e6d02caf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     id     keyword                      location  \\\n",
              "0  8051    refugees                           NaN   \n",
              "1   425  apocalypse  Currently Somewhere On Earth   \n",
              "2  1330  blown%20up                   Scout Team    \n",
              "3   663      attack                           NaN   \n",
              "4  2930      danger                        Leeds    \n",
              "\n",
              "                                                text  \n",
              "0  Refugees as citizens - The Hindu http://t.co/G...  \n",
              "1  @5SOStag honestly he could say an apocalypse i...  \n",
              "2  If you bored as shit don't nobody fuck wit you...  \n",
              "3  @RealTwanBrown Yesterday I Had A Heat Attack ?...  \n",
              "4  The Devil Wears Prada is still one of my favou...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b94c1269-7e0e-412d-8027-1a990a9ab934\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>keyword</th>\n",
              "      <th>location</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>8051</td>\n",
              "      <td>refugees</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Refugees as citizens - The Hindu http://t.co/G...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>425</td>\n",
              "      <td>apocalypse</td>\n",
              "      <td>Currently Somewhere On Earth</td>\n",
              "      <td>@5SOStag honestly he could say an apocalypse i...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1330</td>\n",
              "      <td>blown%20up</td>\n",
              "      <td>Scout Team</td>\n",
              "      <td>If you bored as shit don't nobody fuck wit you...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>663</td>\n",
              "      <td>attack</td>\n",
              "      <td>NaN</td>\n",
              "      <td>@RealTwanBrown Yesterday I Had A Heat Attack ?...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2930</td>\n",
              "      <td>danger</td>\n",
              "      <td>Leeds</td>\n",
              "      <td>The Devil Wears Prada is still one of my favou...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b94c1269-7e0e-412d-8027-1a990a9ab934')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-b94c1269-7e0e-412d-8027-1a990a9ab934 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-b94c1269-7e0e-412d-8027-1a990a9ab934');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-0eb19b67-a233-4805-a478-804684da485f\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-0eb19b67-a233-4805-a478-804684da485f')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-0eb19b67-a233-4805-a478-804684da485f button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# How many examples of each class?\n",
        "train_df.target.value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XIN8JU5g9VCI",
        "outputId": "bb0c92bd-8a2b-42e4-bd94-787cc2f591ba"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    5895\n",
              "1    5500\n",
              "Name: target, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# How many samples total?\n",
        "print(f\"Total training samples: {len(train_df)}\")\n",
        "print(f\"Total test samples: {len(test_df)}\")\n",
        "print(f\"Total samples: {len(train_df) + len(test_df)}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RYGXIdzx9mmF",
        "outputId": "69c2a035-9dcc-4039-858d-d8434b7bf629"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total training samples: 11395\n",
            "Total test samples: 1500\n",
            "Total samples: 12895\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Let's visualize some random training examples\n",
        "import random\n",
        "random_index = random.randint(0, len(train_df)-5) # create random indexes not higher than the total number of samples\n",
        "for row in train_df_shuffled[[\"text\", \"target\"]][random_index:random_index+5].itertuples():\n",
        "  _, text, target = row\n",
        "  print(f\"Target: {target}\", \"(spam)\" if target > 0 else \"(ham)\")\n",
        "  print(f\"Text:\\n{text}\\n\")\n",
        "  print(\"---\\n\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BRDTX5OsAndK",
        "outputId": "d48f3e88-1ba8-4549-ff0e-4d4cf891918f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Target: 1 (spam)\n",
            "Text:\n",
            "It is my religious view that she will burn in hell for all of eternity and I have a right to make that discussion. Just because you believe it is insensitive means nothing, I have a right to express my opinion like anyone on Wikipedia, even if you disagree with it. Feel free to take it to mediation or something but you have no right to remove what I am saying.\n",
            "\n",
            "---\n",
            "\n",
            "Target: 0 (ham)\n",
            "Text:\n",
            "jay says he'll put in  &lt;#&gt;\n",
            "\n",
            "---\n",
            "\n",
            "Target: 0 (ham)\n",
            "Text:\n",
            "\"\n",
            " Your submission at Articles for creation \n",
            " Jeff Peterson Memorial Cup, which you submitted to Articles for creation, has been created. The article has been assessed as C-Class, which is recorded on the article's talk page. You may like to take a look at the grading scheme to see how you can improve the article.\n",
            "You are more than welcome to continue making quality contributions to Wikipedia. You may wish to consider registering an account so you can create articles yourself.\n",
            " If you have any questions, you are welcome to ask at the help desk\n",
            " If you would like to help us improve this process, please consider .\n",
            "Thank you for helping improve Wikipedia!\n",
            " (talk) \"\n",
            "\n",
            "---\n",
            "\n",
            "Target: 1 (spam)\n",
            "Text:\n",
            "SUCK MY ASSHOLE, YOU HAVE LESS OF A LIFE THAN ME! FUCK A HORSE YOU SHITBAG.\n",
            "\n",
            "---\n",
            "\n",
            "Target: 1 (spam)\n",
            "Text:\n",
            "TMi Page! \n",
            "\n",
            "To the tosser wanna be admin ego maniac! Would you get your head out of your arse and revert the tmi page to how it was. The content on the page is proudly looked after by a group of people who work on the series (Yes, that's work on the show). And guess what? we know a hell of alot of information on the show, its past, its present and its future. Guess what else, a bunch of us know what happens in series 5! You may not have seen that info in a trashy pop mag you seem to acquire your guess knowledge on. So let the page evolve with true knowledge as has been occuring over the past few years. You have just destroyed a wealth of info because your not sure what your reading is true, in which case, fuck off and edit something you do know about, loser.\n",
            "\n",
            "---\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "# Use train_test_split to split training data into training and validation sets\n",
        "train_sentences, val_sentences, train_labels, val_labels = train_test_split(train_df_shuffled[\"text\"].to_numpy(),\n",
        "                                                                            train_df_shuffled[\"target\"].to_numpy(),\n",
        "                                                                            test_size=0.1, # dedicate 10% of samples to validation set\n",
        "                                                                            random_state=42) # random state for reproducibility"
      ],
      "metadata": {
        "id": "WIThtRpZA8oC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Check the lengths\n",
        "len(train_sentences), len(train_labels), len(val_sentences), len(val_labels)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3NwKMaOYB6-R",
        "outputId": "026e03ba-d31a-4220-e71f-37bf56206018"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(10255, 10255, 1140, 1140)"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# View the first 10 training sentences and their labels\n",
        "train_sentences[:10], train_labels[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vZ_Rrv2NCDxc",
        "outputId": "b42a6c4a-2119-49a4-d35d-3adbb5dd8ad7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array(['|listas = Phillips, Melanie',\n",
              "        '\"\\n\\n Comunist \"\"Agents\"\" acting as wikipidia moderators are in fact glorifying comunism. \\n\\nI am astonished at the deep diarrhoea of Communism propaganda on articles such as this one on Fidel Castro. This criminal murderous communist (let us understand that no one is a communist only a fool can be a communist). This murderer is being glorified here in this space of wikipedia. \\n\\nLet us (at least me), we the ones that don\\'t have the power of moderators, let us uphold reason and above all truth of facts and reality, that this pig by the name of Fidel Castro is not only a communist criminal but is a devil in its own right.\\n\\nFor the crimes that this pig committed and the misery that this pig inflicted in nations and peoples, may the soul of Fidel Castro burn in hell for eternity.\\n\\nRui Gabirro, \\n100% Anti Communist\\n\\ngabirro@gmail.com\"',\n",
              "        'Just nw i came to hme da..',\n",
              "        \"That's fine, have him give me a call if he knows what he wants or has any questions\",\n",
              "        '{{unblock|This is punitive, not preventative. I unintentionally infringed the restrictions of the ArbCom decision, which is about to expire. I have long since ceased commenting on the issue, which is resolved. Since then (and almost a month ago), I said I would not have involved myself in the page if I had realized it was a MOS page; and I have abstained from it.))  PMAnderson',\n",
              "        'hello \\n\\nYou son a bitch cock licker stop reverting my stuff you fucking ASSHOLE!!!!!!!!!!!!!!!!',\n",
              "        'Okie...',\n",
              "        '\"\\nRepublicanJacobite is the ultimate hypocrite. He jumps on me for my \"\"personal attacks\"\", yet he uses them himself.  He says that I can\\'t stand not getting my way, yet his Wiki-Nazi style tactics clearly show that it is he who acts like a big baby.  He also fails to acknowledge in his rant above that I am also not the only user who reverts his useless edits.  I argue that not only is RepublicanJacobite a disruptive presence, he is also a useless human being.\"',\n",
              "        '\"\\n\\nCool, you proposed the deletion of \"\"Sant\"\" \"\"guru\"\" \"\"Maharaj\"\" \"\"Holier than Thou\"\" \"\"all purpose\"\" \"\"Guru\"\" \"\"Rajinder\"\" \"\"Singh\"\"....it\\'s about freaking time.  Now if you could only kick Gurdass off of the sant ajaib singh entry, as he\\'s hijacked it for his budinski Sirio Carrapa, another all purpose numbchucks, we\\'d be all the happier...\"',\n",
              "        'This image looks appropriate, would appreciate if you could change the links for all the articles that refer to the other image with this one. Thanks'],\n",
              "       dtype=object),\n",
              " array([0, 1, 0, 0, 0, 1, 0, 1, 1, 0]))"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow.keras.layers import TextVectorization # after TensorFlow 2.6\n",
        "\n",
        "# Before TensorFlow 2.6\n",
        "# from tensorflow.keras.layers.experimental.preprocessing import TextVectorization\n",
        "# Note: in TensorFlow 2.6+, you no longer need \"layers.experimental.preprocessing\"\n",
        "# you can use: \"tf.keras.layers.TextVectorization\", see https://github.com/tensorflow/tensorflow/releases/tag/v2.6.0 for more\n",
        "\n",
        "# Use the default TextVectorization variables\n",
        "text_vectorizer = TextVectorization(max_tokens=None, # how many words in the vocabulary (all of the different words in your text)\n",
        "                                    standardize=\"lower_and_strip_punctuation\", # how to process text\n",
        "                                    split=\"whitespace\", # how to split tokens\n",
        "                                    ngrams=None, # create groups of n-words?\n",
        "                                    output_mode=\"int\", # how to map tokens to numbers\n",
        "                                    output_sequence_length=None) # how long should the output sequence of tokens be?\n",
        "                                    # pad_to_max_tokens=True) # Not valid if using max_tokens=None"
      ],
      "metadata": {
        "id": "6718OiCYCT7n"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Find average number of tokens (words) in training Tweets\n",
        "round(sum([len(i.split()) for i in train_sentences])/len(train_sentences))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "k9QIkEhWC5-u",
        "outputId": "7f1dd989-c2f3-48fe-9148-8892df73e1d2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "45"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Setup text vectorization with custom variables\n",
        "max_vocab_length = 10000 # max number of words to have in our vocabulary\n",
        "max_length = 15 # max length our sequences will be (e.g. how many words from a Tweet does our model see?)\n",
        "\n",
        "text_vectorizer = TextVectorization(max_tokens=max_vocab_length,\n",
        "                                    output_mode=\"int\",\n",
        "                                    output_sequence_length=max_length)"
      ],
      "metadata": {
        "id": "P1dKlzFUDLV2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Fit the text vectorizer to the training text\n",
        "text_vectorizer.adapt(train_sentences)"
      ],
      "metadata": {
        "id": "61UArtc-DWoi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Create sample sentence and tokenize it\n",
        "sample_sentence = \"There's a fuck in my street!\"\n",
        "text_vectorizer([sample_sentence])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AQTaQm7yDnWz",
        "outputId": "f145c4f6-3323-4d8a-e9a5-92959461c9b6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 15), dtype=int64, numpy=\n",
              "array([[ 398,    5,   21,   11,   19, 1770,    0,    0,    0,    0,    0,\n",
              "           0,    0,    0,    0]])>"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Choose a random sentence from the training dataset and tokenize it\n",
        "random_sentence = random.choice(train_sentences)\n",
        "print(f\"Original text:\\n{random_sentence}\\\n",
        "      \\n\\nVectorized version:\")\n",
        "text_vectorizer([random_sentence])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z4chSKIjD4AB",
        "outputId": "cbb2464d-2883-4d9e-9a66-2d48951c8c82"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original text:\n",
            "\"\n",
            "\n",
            "Just who is stalking who here?\n",
            "Anyone can get my address also. It is you who is harrassing me and I am simply defending myself trying to get you off my back. Fred Alan Wolf has been called a \"\"crank\"\" et-al by the same stupid Usenet people who call me those names and for the very same reasons. Yet, none of that defamatory slander/libel whatever is on the Wiki article on him. As soon as you remove that stuff about me, we can settle this amiably.\n",
            "[Jack Sarfatti]\"      \n",
            "\n",
            "Vectorized version:\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 15), dtype=int64, numpy=\n",
              "array([[  46,   54,    9, 1982,   54,   76,  307,   36,   56,   19,  485,\n",
              "          96,   12,    9,    3]])>"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Get the unique words in the vocabulary\n",
        "words_in_vocab = text_vectorizer.get_vocabulary()\n",
        "top_5_words = words_in_vocab[:5] # most common tokens (notice the [UNK] token for \"unknown\" words)\n",
        "bottom_5_words = words_in_vocab[-5:] # least common tokens\n",
        "print(f\"Number of words in vocab: {len(words_in_vocab)}\")\n",
        "print(f\"Top 5 most common words: {top_5_words}\")\n",
        "print(f\"Bottom 5 least common words: {bottom_5_words}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RiD81MKsEQGN",
        "outputId": "ae26c8c6-34e4-4ffa-99cd-d6aee219a288"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of words in vocab: 10000\n",
            "Top 5 most common words: ['', '[UNK]', 'the', 'you', 'to']\n",
            "Bottom 5 least common words: ['thereafter', 'theoretically', 'thenwill', 'thatd', 'thanked']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tf.random.set_seed(42)\n",
        "from tensorflow.keras import layers\n",
        "\n",
        "embedding = layers.Embedding(input_dim=max_vocab_length, # set input shape\n",
        "                             output_dim=128, # set size of embedding vector\n",
        "                             embeddings_initializer=\"uniform\", # default, intialize randomly\n",
        "                             input_length=max_length, # how long is each input\n",
        "                             name=\"embedding_1\")\n",
        "\n",
        "embedding"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "h_H1NoGNEg6V",
        "outputId": "63832aa7-e0a3-4130-b2e3-259874549879"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.src.layers.core.embedding.Embedding at 0x7cba984c3ee0>"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Get a random sentence from training set\n",
        "random_sentence = random.choice(train_sentences)\n",
        "print(f\"Original text:\\n{random_sentence}\\\n",
        "      \\n\\nEmbedded version:\")\n",
        "\n",
        "# Embed the random sentence (turn it into numerical representation)\n",
        "sample_embed = embedding(text_vectorizer([random_sentence]))\n",
        "sample_embed"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zTOaTK2BFEel",
        "outputId": "04c0a52e-0ed1-4f35-e3d5-43041191c8b7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original text:\n",
            "I looked escaped the Kyrgyzstan, real nasty place, he real bad man      \n",
            "\n",
            "Embedded version:\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 15, 128), dtype=float32, numpy=\n",
              "array([[[ 0.04136788,  0.02539622, -0.03420647, ..., -0.00310607,\n",
              "          0.0492525 ,  0.04387799],\n",
              "        [ 0.03009612,  0.0478359 ,  0.00629596, ...,  0.01781181,\n",
              "         -0.02359798, -0.02590402],\n",
              "        [ 0.01210131,  0.02922418, -0.00040761, ...,  0.02289804,\n",
              "         -0.04551091,  0.01591796],\n",
              "        ...,\n",
              "        [-0.00484465, -0.0093034 , -0.00892364, ...,  0.04870324,\n",
              "         -0.02180631, -0.03330765],\n",
              "        [-0.00484465, -0.0093034 , -0.00892364, ...,  0.04870324,\n",
              "         -0.02180631, -0.03330765],\n",
              "        [-0.00484465, -0.0093034 , -0.00892364, ...,  0.04870324,\n",
              "         -0.02180631, -0.03330765]]], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Check out a single token's embedding\n",
        "sample_embed[0][0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ap09Ql7KFOSE",
        "outputId": "edb5c157-0137-4c11-cb69-e2176d8f78bf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(128,), dtype=float32, numpy=\n",
              "array([ 4.13678773e-02,  2.53962167e-02, -3.42064723e-02,  4.23592441e-02,\n",
              "       -1.44708380e-02, -4.13253531e-02,  2.55403258e-02,  2.44778059e-02,\n",
              "        8.53017718e-03,  4.10585478e-03, -3.95854600e-02,  3.56561579e-02,\n",
              "        5.95511124e-03,  1.80459730e-02,  4.43971045e-02, -2.85991281e-03,\n",
              "       -4.08962145e-02,  2.93615945e-02,  9.58410650e-03,  3.60038392e-02,\n",
              "       -5.60346991e-03,  7.25601986e-03,  2.67008692e-03, -4.03463468e-02,\n",
              "        2.81802155e-02,  4.99924906e-02, -3.65290418e-02,  3.19384374e-02,\n",
              "       -6.39334321e-05, -1.16113424e-02,  1.19614713e-02,  3.69230658e-03,\n",
              "       -9.84646007e-03,  1.82208754e-02, -1.77346095e-02,  3.96899618e-02,\n",
              "       -3.62774730e-02,  3.90008725e-02,  1.98830999e-02, -4.69075702e-02,\n",
              "        9.70103592e-03,  1.27640478e-02,  2.10619830e-02, -4.98379245e-02,\n",
              "        3.76144797e-03,  3.45160849e-02,  1.55529119e-02,  4.81229760e-02,\n",
              "       -3.69395390e-02,  1.35671161e-02,  4.84209545e-02,  4.34126891e-02,\n",
              "       -1.83223858e-02,  4.85313199e-02, -6.86421245e-03, -3.60643752e-02,\n",
              "       -4.31109928e-02,  4.85794879e-02,  6.71213865e-03,  3.58815677e-02,\n",
              "       -3.23369876e-02, -1.86545961e-02, -4.42050956e-02, -2.21576337e-02,\n",
              "       -2.04270016e-02, -7.47164339e-03, -1.56512149e-02, -2.20872648e-02,\n",
              "       -2.09428314e-02, -1.98209882e-02,  1.64449103e-02, -9.07741860e-03,\n",
              "       -3.57852131e-03, -3.72450575e-02, -8.39936733e-03,  3.63819636e-02,\n",
              "        3.51746418e-02,  1.53360032e-02,  2.16605552e-02,  4.87651117e-02,\n",
              "        2.74323933e-02, -1.91044062e-04,  1.55586265e-02,  4.65568937e-02,\n",
              "        4.65488173e-02,  4.15910967e-02, -4.86611724e-02, -1.36983395e-02,\n",
              "        3.40754502e-02, -1.30079687e-04,  4.29297723e-02,  6.10936806e-03,\n",
              "       -1.78826824e-02, -3.96723635e-02, -2.08482873e-02,  7.26469606e-03,\n",
              "       -1.48229115e-02,  3.62835824e-04,  1.41695179e-02,  3.01557072e-02,\n",
              "       -1.15732439e-02,  9.68008116e-03,  4.27379645e-02, -7.14740902e-03,\n",
              "       -2.55327299e-03, -4.89677787e-02,  2.96158455e-02,  1.54181756e-02,\n",
              "        1.29826553e-02, -4.84763384e-02, -8.20784643e-03, -3.06709651e-02,\n",
              "        2.50891782e-02, -6.65484741e-03, -4.49553989e-02,  1.87720917e-02,\n",
              "        1.98979303e-03, -2.91872155e-02, -4.62907664e-02, -3.00941356e-02,\n",
              "       -4.47925925e-02, -1.32645145e-02, -4.26069610e-02,  2.51720287e-02,\n",
              "       -2.62957104e-02, -3.10606882e-03,  4.92524989e-02,  4.38779853e-02],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Model 0"
      ],
      "metadata": {
        "id": "DyfpRSAsGCB5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn.pipeline import Pipeline\n",
        "\n",
        "# Create tokenization and modelling pipeline\n",
        "model_0 = Pipeline([\n",
        "                    (\"tfidf\", TfidfVectorizer()), # convert words to numbers using tfidf\n",
        "                    (\"clf\", MultinomialNB()) # model the text\n",
        "])\n",
        "\n",
        "# Fit the pipeline to the training data\n",
        "model_0.fit(train_sentences, train_labels)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 125
        },
        "id": "CHhpJ8tkGGM5",
        "outputId": "678b888d-93ef-4bdc-f29c-73ecbe50b0d6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Pipeline(steps=[('tfidf', TfidfVectorizer()), ('clf', MultinomialNB())])"
            ],
            "text/html": [
              "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[(&#x27;tfidf&#x27;, TfidfVectorizer()), (&#x27;clf&#x27;, MultinomialNB())])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;tfidf&#x27;, TfidfVectorizer()), (&#x27;clf&#x27;, MultinomialNB())])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">TfidfVectorizer</label><div class=\"sk-toggleable__content\"><pre>TfidfVectorizer()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MultinomialNB</label><div class=\"sk-toggleable__content\"><pre>MultinomialNB()</pre></div></div></div></div></div></div></div>"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "baseline_score = model_0.score(val_sentences, val_labels)\n",
        "print(f\"Our baseline model achieves an accuracy of: {baseline_score*100:.2f}%\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "okdcI41lGHGG",
        "outputId": "6a4529a6-29d3-4151-e867-fcf7c0787b83"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Our baseline model achieves an accuracy of: 88.95%\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Make predictions\n",
        "baseline_preds = model_0.predict(val_sentences)\n",
        "baseline_preds[:20]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ASUbVQxSGVf2",
        "outputId": "a55de1f6-5f6d-42e1-bb70-b8059726b867"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([1, 0, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 1])"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Function to evaluate: accuracy, precision, recall, f1-score\n",
        "from sklearn.metrics import accuracy_score, precision_recall_fscore_support\n",
        "\n",
        "def calculate_results(y_true, y_pred):\n",
        "  \"\"\"\n",
        "  Calculates model accuracy, precision, recall and f1 score of a binary classification model.\n",
        "\n",
        "  Args:\n",
        "  -----\n",
        "  y_true = true labels in the form of a 1D array\n",
        "  y_pred = predicted labels in the form of a 1D array\n",
        "\n",
        "  Returns a dictionary of accuracy, precision, recall, f1-score.\n",
        "  \"\"\"\n",
        "  # Calculate model accuracy\n",
        "  model_accuracy = accuracy_score(y_true, y_pred) * 100\n",
        "  # Calculate model precision, recall and f1 score using \"weighted\" average\n",
        "  model_precision, model_recall, model_f1, _ = precision_recall_fscore_support(y_true, y_pred, average=\"weighted\")\n",
        "  model_results = {\"accuracy\": model_accuracy,\n",
        "                  \"precision\": model_precision,\n",
        "                  \"recall\": model_recall,\n",
        "                  \"f1\": model_f1}\n",
        "  return model_results"
      ],
      "metadata": {
        "id": "ZxkZJPk9GiMj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Get baseline results\n",
        "baseline_results = calculate_results(y_true=val_labels,\n",
        "                                     y_pred=baseline_preds)\n",
        "baseline_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U7Neux8WG3-H",
        "outputId": "a2496a43-aed6-488b-b3d8-9cfc19be9ed6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 88.94736842105263,\n",
              " 'precision': 0.8894252139783421,\n",
              " 'recall': 0.8894736842105263,\n",
              " 'f1': 0.8893498452012383}"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Model 1"
      ],
      "metadata": {
        "id": "TfigpHfQHOF2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create tensorboard callback (need to create a new one for each model)\n",
        "from helper_functions import create_tensorboard_callback\n",
        "\n",
        "# Create directory to save TensorBoard logs\n",
        "SAVE_DIR = \"model_logs\""
      ],
      "metadata": {
        "id": "yPtx9NJKHCBd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Build model with the Functional API\n",
        "from tensorflow.keras import layers\n",
        "inputs = layers.Input(shape=(1,), dtype=\"string\") # inputs are 1-dimensional strings\n",
        "x = text_vectorizer(inputs) # turn the input text into numbers\n",
        "x = embedding(x) # create an embedding of the numerized numbers\n",
        "x = layers.GlobalAveragePooling1D()(x) # lower the dimensionality of the embedding (try running the model without this layer and see what happens)\n",
        "outputs = layers.Dense(1, activation=\"sigmoid\")(x) # create the output layer, want binary outputs so use sigmoid activation\n",
        "model_1 = tf.keras.Model(inputs, outputs, name=\"model_1_dense\") # construct the model"
      ],
      "metadata": {
        "id": "kHb1Q2IeHR1T"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Compile model\n",
        "model_1.compile(loss=\"binary_crossentropy\",\n",
        "                optimizer=tf.keras.optimizers.Adam(),\n",
        "                metrics=[\"accuracy\"])"
      ],
      "metadata": {
        "id": "hHei_GpOHiCI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Get a summary of the model\n",
        "model_1.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "stBO4a-EHtW6",
        "outputId": "8d921c2f-e8aa-415d-feba-1fc19561d72e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_1_dense\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_1 (InputLayer)        [(None, 1)]               0         \n",
            "                                                                 \n",
            " text_vectorization_1 (Text  (None, 15)                0         \n",
            " Vectorization)                                                  \n",
            "                                                                 \n",
            " embedding_1 (Embedding)     (None, 15, 128)           1280000   \n",
            "                                                                 \n",
            " global_average_pooling1d (  (None, 128)               0         \n",
            " GlobalAveragePooling1D)                                         \n",
            "                                                                 \n",
            " dense (Dense)               (None, 1)                 129       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1280129 (4.88 MB)\n",
            "Trainable params: 1280129 (4.88 MB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Fit the model\n",
        "model_1_history = model_1.fit(train_sentences, # input sentences can be a list of strings due to text preprocessing layer built-in model\n",
        "                              train_labels,\n",
        "                              epochs=5,\n",
        "                              validation_data=(val_sentences, val_labels),\n",
        "                              callbacks=[create_tensorboard_callback(dir_name=SAVE_DIR,\n",
        "                                                                     experiment_name=\"simple_dense_model\")])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j8iw9T3YH58w",
        "outputId": "ea0c424a-687f-4b8c-f12e-ab787d5e72f8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving TensorBoard log files to: model_logs/simple_dense_model/20240115-045331\n",
            "Epoch 1/5\n",
            "321/321 [==============================] - 7s 20ms/step - loss: 0.5816 - accuracy: 0.7332 - val_loss: 0.4670 - val_accuracy: 0.8000\n",
            "Epoch 2/5\n",
            "321/321 [==============================] - 7s 23ms/step - loss: 0.3875 - accuracy: 0.8412 - val_loss: 0.3834 - val_accuracy: 0.8307\n",
            "Epoch 3/5\n",
            "321/321 [==============================] - 6s 19ms/step - loss: 0.2927 - accuracy: 0.8856 - val_loss: 0.3597 - val_accuracy: 0.8281\n",
            "Epoch 4/5\n",
            "321/321 [==============================] - 7s 23ms/step - loss: 0.2350 - accuracy: 0.9127 - val_loss: 0.3561 - val_accuracy: 0.8377\n",
            "Epoch 5/5\n",
            "321/321 [==============================] - 6s 20ms/step - loss: 0.1937 - accuracy: 0.9294 - val_loss: 0.3598 - val_accuracy: 0.8456\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Check the results\n",
        "model_1.evaluate(val_sentences, val_labels)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5d9ZOe2WIRgW",
        "outputId": "2ddcd406-82d9-4297-c890-bfd975b6a574"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "36/36 [==============================] - 0s 3ms/step - loss: 0.3598 - accuracy: 0.8456\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.3597867488861084, 0.8456140160560608]"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "embedding.weights"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z-4A8ETdInQn",
        "outputId": "a6198982-dc19-40ba-9488-2dd471dbefbb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<tf.Variable 'embedding_1/embeddings:0' shape=(10000, 128) dtype=float32, numpy=\n",
              " array([[ 0.01194163,  0.00669516, -0.0260199 , ...,  0.03438895,\n",
              "         -0.03894087, -0.01860881],\n",
              "        [ 0.01492115,  0.03700199,  0.03416773, ...,  0.0140932 ,\n",
              "         -0.00939629,  0.03446785],\n",
              "        [ 0.03204699,  0.05798898, -0.0039538 , ..., -0.01440729,\n",
              "         -0.03542033,  0.02512226],\n",
              "        ...,\n",
              "        [ 0.02997314,  0.00268004, -0.02402622, ..., -0.03256062,\n",
              "          0.0358582 , -0.00760404],\n",
              "        [-0.02308761,  0.04738409,  0.04873358, ..., -0.03891174,\n",
              "         -0.01003792,  0.03297618],\n",
              "        [ 0.04088232, -0.04279062, -0.03470675, ...,  0.01101702,\n",
              "         -0.01721021,  0.04184072]], dtype=float32)>]"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "embed_weights = model_1.get_layer(\"embedding_1\").get_weights()[0]\n",
        "print(embed_weights.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zfnSJr2OI1I0",
        "outputId": "5c23f4fb-a100-44aa-9f46-2c227a93082f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(10000, 128)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Make predictions (these come back in the form of probabilities)\n",
        "model_1_pred_probs = model_1.predict(val_sentences)\n",
        "model_1_pred_probs[:10] # only print out the first 10 prediction probabilities"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Gs0GXP-iJCE-",
        "outputId": "bf9e33ae-755c-4349-d921-53ee0510be9a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "36/36 [==============================] - 0s 2ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.99944466],\n",
              "       [0.04863268],\n",
              "       [0.05893452],\n",
              "       [0.98004353],\n",
              "       [0.0626931 ],\n",
              "       [0.13093749],\n",
              "       [0.24382131],\n",
              "       [0.99962753],\n",
              "       [0.32417223],\n",
              "       [0.6166237 ]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Turn prediction probabilities into single-dimension tensor of floats\n",
        "model_1_preds = tf.squeeze(tf.round(model_1_pred_probs)) # squeeze removes single dimensions\n",
        "model_1_preds[:20]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qLTDXuPYJSrK",
        "outputId": "c1bfc837-4d50-4b2d-9825-1bb94fa3af0d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(20,), dtype=float32, numpy=\n",
              "array([1., 0., 0., 1., 0., 0., 0., 1., 0., 1., 1., 0., 1., 0., 1., 0., 0.,\n",
              "       0., 1., 1.], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Calculate model_1 metrics\n",
        "model_1_results = calculate_results(y_true=val_labels,\n",
        "                                    y_pred=model_1_preds)\n",
        "model_1_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0yB1HdiuJ135",
        "outputId": "5084bd7a-a1a4-46fa-92e2-e5a60d90972e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 84.56140350877193,\n",
              " 'precision': 0.8461900336045336,\n",
              " 'recall': 0.8456140350877193,\n",
              " 'f1': 0.8449439701151988}"
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Is our simple Keras model better than our baseline model?\n",
        "import numpy as np\n",
        "np.array(list(model_1_results.values())) > np.array(list(baseline_results.values()))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RMp11u9QKEh3",
        "outputId": "ceccaf0d-8ecb-4cbb-acb7-f3bb3a8089fa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([False, False, False, False])"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a helper function to compare our baseline results to new model results\n",
        "def compare_baseline_to_new_results(baseline_results, new_model_results):\n",
        "  for key, value in baseline_results.items():\n",
        "    print(f\"Baseline {key}: {value:.2f}, New {key}: {new_model_results[key]:.2f}, Difference: {new_model_results[key]-value:.2f}\")\n",
        "\n",
        "compare_baseline_to_new_results(baseline_results=baseline_results,\n",
        "                                new_model_results=model_1_results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LKGrFyZIKSnK",
        "outputId": "8df13c64-0bad-4d40-b06b-9eefa0640b30"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Baseline accuracy: 88.95, New accuracy: 84.56, Difference: -4.39\n",
            "Baseline precision: 0.89, New precision: 0.85, Difference: -0.04\n",
            "Baseline recall: 0.89, New recall: 0.85, Difference: -0.04\n",
            "Baseline f1: 0.89, New f1: 0.84, Difference: -0.04\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Get the vocabulary from the text vectorization layer\n",
        "words_in_vocab = text_vectorizer.get_vocabulary()\n",
        "len(words_in_vocab), words_in_vocab[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-SExJkQbKk98",
        "outputId": "d3a309e6-c3d3-498c-b472-84927efb4413"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(10000, ['', '[UNK]', 'the', 'you', 'to', 'a', 'i', 'and', 'of', 'is'])"
            ]
          },
          "metadata": {},
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_1.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P_yxonNFKych",
        "outputId": "17207531-e73e-4767-a82b-39b9dc0b0cfa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_1_dense\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_1 (InputLayer)        [(None, 1)]               0         \n",
            "                                                                 \n",
            " text_vectorization_1 (Text  (None, 15)                0         \n",
            " Vectorization)                                                  \n",
            "                                                                 \n",
            " embedding_1 (Embedding)     (None, 15, 128)           1280000   \n",
            "                                                                 \n",
            " global_average_pooling1d (  (None, 128)               0         \n",
            " GlobalAveragePooling1D)                                         \n",
            "                                                                 \n",
            " dense (Dense)               (None, 1)                 129       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1280129 (4.88 MB)\n",
            "Trainable params: 1280129 (4.88 MB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Get the weight matrix of embedding layer\n",
        "# (these are the numerical patterns between the text in the training dataset the model has learned)\n",
        "embed_weights = model_1.get_layer(\"embedding_1\").get_weights()[0]\n",
        "print(embed_weights.shape) # same size as vocab size and embedding_dim (each word is a embedding_dim size vector)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0jcyrNtuLF5G",
        "outputId": "7448b131-b756-4eb9-85e3-870465b2a4a7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(10000, 128)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        " # Code below is adapted from: https://www.tensorflow.org/tutorials/text/word_embeddings#retrieve_the_trained_word_embeddings_and_save_them_to_disk\n",
        " import io\n",
        "\n",
        " # Create output writers\n",
        " out_v = io.open(\"embedding_vectors.tsv\", \"w\", encoding=\"utf-8\")\n",
        " out_m = io.open(\"embedding_metadata.tsv\", \"w\", encoding=\"utf-8\")\n",
        "\n",
        " # Write embedding vectors and words to file\n",
        " for num, word in enumerate(words_in_vocab):\n",
        "   if num == 0:\n",
        "      continue # skip padding token\n",
        "   vec = embed_weights[num]\n",
        "   out_m.write(word + \"\\n\") # write words to file\n",
        "   out_v.write(\"\\t\".join([str(x) for x in vec]) + \"\\n\") # write corresponding word vector to file\n",
        " out_v.close()\n",
        " out_m.close()\n",
        "\n",
        " # Download files locally to upload to Embedding Projector\n",
        " try:\n",
        "   from google.colab import files\n",
        " except ImportError:\n",
        "   pass\n",
        " else:\n",
        "   files.download(\"embedding_vectors.tsv\")\n",
        "   files.download(\"embedding_metadata.tsv\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "-O7JQecPLWhC",
        "outputId": "f15d28c2-f178-449a-af2c-c74b025c5175"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "download(\"download_6f150a3d-1984-4e15-80e0-68efefba248e\", \"embedding_vectors.tsv\", 15403402)"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "download(\"download_f4a0f5a2-49f4-4a3d-b0cf-8eee9be3aef7\", \"embedding_metadata.tsv\", 76546)"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Model 2"
      ],
      "metadata": {
        "id": "7NFD04zJL5t8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Set random seed and create embedding layer (new embedding layer for each model)\n",
        "tf.random.set_seed(42)\n",
        "from tensorflow.keras import layers\n",
        "model_2_embedding = layers.Embedding(input_dim=max_vocab_length,\n",
        "                                     output_dim=128,\n",
        "                                     embeddings_initializer=\"uniform\",\n",
        "                                     input_length=max_length,\n",
        "                                     name=\"embedding_2\")\n",
        "\n",
        "\n",
        "# Create LSTM model\n",
        "inputs = layers.Input(shape=(1,), dtype=\"string\")\n",
        "x = text_vectorizer(inputs)\n",
        "x = model_2_embedding(x)\n",
        "print(x.shape)\n",
        "# x = layers.LSTM(64, return_sequences=True)(x) # return vector for each word in the Tweet (you can stack RNN cells as long as return_sequences=True)\n",
        "x = layers.LSTM(64)(x) # return vector for whole sequence\n",
        "print(x.shape)\n",
        "# x = layers.Dense(64, activation=\"relu\")(x) # optional dense layer on top of output of LSTM cell\n",
        "outputs = layers.Dense(1, activation=\"sigmoid\")(x)\n",
        "model_2 = tf.keras.Model(inputs, outputs, name=\"model_2_LSTM\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5FrxCAUHLpDR",
        "outputId": "acb6557c-4360-4b3f-fa2c-370bffa17d3f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(None, 15, 128)\n",
            "(None, 64)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Compile model\n",
        "model_2.compile(loss=\"binary_crossentropy\",\n",
        "                optimizer=tf.keras.optimizers.Adam(),\n",
        "                metrics=[\"accuracy\"])"
      ],
      "metadata": {
        "id": "xkOF4QCQMCZo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_2.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jEsZGf90MZVn",
        "outputId": "70dda54e-06f7-4597-d810-3e392e730719"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_2_LSTM\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_2 (InputLayer)        [(None, 1)]               0         \n",
            "                                                                 \n",
            " text_vectorization_1 (Text  (None, 15)                0         \n",
            " Vectorization)                                                  \n",
            "                                                                 \n",
            " embedding_2 (Embedding)     (None, 15, 128)           1280000   \n",
            "                                                                 \n",
            " lstm (LSTM)                 (None, 64)                49408     \n",
            "                                                                 \n",
            " dense_1 (Dense)             (None, 1)                 65        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1329473 (5.07 MB)\n",
            "Trainable params: 1329473 (5.07 MB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Fit model\n",
        "model_2_history = model_2.fit(train_sentences,\n",
        "                              train_labels,\n",
        "                              epochs=5,\n",
        "                              validation_data=(val_sentences, val_labels),\n",
        "                              callbacks=[create_tensorboard_callback(SAVE_DIR,\n",
        "                                                                     \"LSTM\")])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QLFVpQ4aMhXo",
        "outputId": "827a147c-1181-4eb5-d967-768a34119526"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving TensorBoard log files to: model_logs/LSTM/20240115-045415\n",
            "Epoch 1/5\n",
            "321/321 [==============================] - 14s 36ms/step - loss: 0.4522 - accuracy: 0.7774 - val_loss: 0.3671 - val_accuracy: 0.8307\n",
            "Epoch 2/5\n",
            "321/321 [==============================] - 11s 33ms/step - loss: 0.2531 - accuracy: 0.8939 - val_loss: 0.3506 - val_accuracy: 0.8395\n",
            "Epoch 3/5\n",
            "321/321 [==============================] - 11s 33ms/step - loss: 0.1640 - accuracy: 0.9367 - val_loss: 0.4356 - val_accuracy: 0.8289\n",
            "Epoch 4/5\n",
            "321/321 [==============================] - 9s 29ms/step - loss: 0.1045 - accuracy: 0.9604 - val_loss: 0.5099 - val_accuracy: 0.8184\n",
            "Epoch 5/5\n",
            "321/321 [==============================] - 11s 34ms/step - loss: 0.0725 - accuracy: 0.9733 - val_loss: 0.6171 - val_accuracy: 0.8281\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Make predictions on the validation dataset\n",
        "model_2_pred_probs = model_2.predict(val_sentences)\n",
        "model_2_pred_probs.shape, model_2_pred_probs[:10] # view the first 10"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nOA2ryKxMsfE",
        "outputId": "0eb8932e-812f-44ba-c46f-c557046bb5fa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "36/36 [==============================] - 1s 8ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((1140, 1),\n",
              " array([[9.9991417e-01],\n",
              "        [1.6824612e-03],\n",
              "        [4.7950963e-03],\n",
              "        [9.9985707e-01],\n",
              "        [2.3428745e-04],\n",
              "        [5.2264947e-01],\n",
              "        [3.0863483e-02],\n",
              "        [9.9996674e-01],\n",
              "        [2.7360052e-03],\n",
              "        [8.9269656e-01]], dtype=float32))"
            ]
          },
          "metadata": {},
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Round out predictions and reduce to 1-dimensional array\n",
        "model_2_preds = tf.squeeze(tf.round(model_2_pred_probs))\n",
        "model_2_preds[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_LtwN1LuNCVD",
        "outputId": "e837db55-8b62-44c3-d8b1-3cc66afeb9f7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([1., 0., 0., 1., 0., 1., 0., 1., 0., 1.], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Calculate LSTM model results\n",
        "model_2_results = calculate_results(y_true=val_labels,\n",
        "                                    y_pred=model_2_preds)\n",
        "model_2_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7HnxD-K0NP8P",
        "outputId": "273f3a2f-db84-45c2-e6c1-e965f4a77ae6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 82.80701754385966,\n",
              " 'precision': 0.8279335593886678,\n",
              " 'recall': 0.8280701754385965,\n",
              " 'f1': 0.8279787720230932}"
            ]
          },
          "metadata": {},
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Compare model 2 to baseline\n",
        "compare_baseline_to_new_results(baseline_results, model_2_results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cr9i3hZQNhIV",
        "outputId": "90c8eba5-82d6-42be-8cdb-29534ce07fa1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Baseline accuracy: 88.95, New accuracy: 82.81, Difference: -6.14\n",
            "Baseline precision: 0.89, New precision: 0.83, Difference: -0.06\n",
            "Baseline recall: 0.89, New recall: 0.83, Difference: -0.06\n",
            "Baseline f1: 0.89, New f1: 0.83, Difference: -0.06\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Model 3"
      ],
      "metadata": {
        "id": "LH8wJH9KN0xr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Set random seed and create embedding layer (new embedding layer for each model)\n",
        "tf.random.set_seed(42)\n",
        "from tensorflow.keras import layers\n",
        "model_3_embedding = layers.Embedding(input_dim=max_vocab_length,\n",
        "                                     output_dim=128,\n",
        "                                     embeddings_initializer=\"uniform\",\n",
        "                                     input_length=max_length,\n",
        "                                     name=\"embedding_3\")\n",
        "\n",
        "# Build an RNN using the GRU cell\n",
        "inputs = layers.Input(shape=(1,), dtype=\"string\")\n",
        "x = text_vectorizer(inputs)\n",
        "x = model_3_embedding(x)\n",
        "# x = layers.GRU(64, return_sequences=True) # stacking recurrent cells requires return_sequences=True\n",
        "x = layers.GRU(64)(x)\n",
        "# x = layers.Dense(64, activation=\"relu\")(x) # optional dense layer after GRU cell\n",
        "outputs = layers.Dense(1, activation=\"sigmoid\")(x)\n",
        "model_3 = tf.keras.Model(inputs, outputs, name=\"model_3_GRU\")"
      ],
      "metadata": {
        "id": "uD8Z67GaNsGd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Compile GRU model\n",
        "model_3.compile(loss=\"binary_crossentropy\",\n",
        "                optimizer=tf.keras.optimizers.Adam(),\n",
        "                metrics=[\"accuracy\"])"
      ],
      "metadata": {
        "id": "83dMfqmyN-NK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Get a summary of the GRU model\n",
        "model_3.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yiId1Ht7OF07",
        "outputId": "b90101cf-239a-4f5d-b256-d60381cb9b67"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_3_GRU\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_3 (InputLayer)        [(None, 1)]               0         \n",
            "                                                                 \n",
            " text_vectorization_1 (Text  (None, 15)                0         \n",
            " Vectorization)                                                  \n",
            "                                                                 \n",
            " embedding_3 (Embedding)     (None, 15, 128)           1280000   \n",
            "                                                                 \n",
            " gru (GRU)                   (None, 64)                37248     \n",
            "                                                                 \n",
            " dense_2 (Dense)             (None, 1)                 65        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1317313 (5.03 MB)\n",
            "Trainable params: 1317313 (5.03 MB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Fit model\n",
        "model_3_history = model_3.fit(train_sentences,\n",
        "                              train_labels,\n",
        "                              epochs=5,\n",
        "                              validation_data=(val_sentences, val_labels),\n",
        "                              callbacks=[create_tensorboard_callback(SAVE_DIR, \"GRU\")])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fiFSwQ4kOM7L",
        "outputId": "bd4ae5a2-7451-4cfb-8d44-f97f2e8d2f7c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving TensorBoard log files to: model_logs/GRU/20240115-045541\n",
            "Epoch 1/5\n",
            "321/321 [==============================] - 13s 34ms/step - loss: 0.4694 - accuracy: 0.7602 - val_loss: 0.3694 - val_accuracy: 0.8228\n",
            "Epoch 2/5\n",
            "321/321 [==============================] - 9s 28ms/step - loss: 0.2559 - accuracy: 0.8943 - val_loss: 0.3559 - val_accuracy: 0.8333\n",
            "Epoch 3/5\n",
            "321/321 [==============================] - 11s 33ms/step - loss: 0.1702 - accuracy: 0.9352 - val_loss: 0.4259 - val_accuracy: 0.8254\n",
            "Epoch 4/5\n",
            "321/321 [==============================] - 11s 33ms/step - loss: 0.1089 - accuracy: 0.9598 - val_loss: 0.5505 - val_accuracy: 0.8263\n",
            "Epoch 5/5\n",
            "321/321 [==============================] - 10s 32ms/step - loss: 0.0726 - accuracy: 0.9741 - val_loss: 0.5183 - val_accuracy: 0.8219\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Make predictions on the validation data\n",
        "model_3_pred_probs = model_3.predict(val_sentences)\n",
        "model_3_pred_probs.shape, model_3_pred_probs[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IMqb-W1Md07B",
        "outputId": "ef5d8364-5dc2-42fa-f9f6-4ce48a7b53ca"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "36/36 [==============================] - 1s 5ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((1140, 1),\n",
              " array([[9.9981850e-01],\n",
              "        [2.6998219e-01],\n",
              "        [1.5356761e-01],\n",
              "        [9.9933815e-01],\n",
              "        [2.6173377e-04],\n",
              "        [3.5902392e-02],\n",
              "        [4.5403477e-02],\n",
              "        [9.9984485e-01],\n",
              "        [5.3603360e-03],\n",
              "        [7.9105705e-01]], dtype=float32))"
            ]
          },
          "metadata": {},
          "execution_count": 61
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Convert prediction probabilities to prediction classes\n",
        "model_3_preds = tf.squeeze(tf.round(model_3_pred_probs))\n",
        "model_3_preds[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YXyAj0SNhap2",
        "outputId": "89d63f6a-5625-43fc-c469-e3e4a29beced"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([1., 0., 0., 1., 0., 0., 0., 1., 0., 1.], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 62
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Calcuate model_3 results\n",
        "model_3_results = calculate_results(y_true=val_labels,\n",
        "                                    y_pred=model_3_preds)\n",
        "model_3_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1lwruBBchnrD",
        "outputId": "66120293-ca44-4bb1-c978-ddbd8afcbea5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 82.19298245614036,\n",
              " 'precision': 0.8217214510510313,\n",
              " 'recall': 0.8219298245614035,\n",
              " 'f1': 0.8217484905115973}"
            ]
          },
          "metadata": {},
          "execution_count": 63
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Compare to baseline\n",
        "compare_baseline_to_new_results(baseline_results, model_3_results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ojKSrC_9h1Ji",
        "outputId": "5ac06b38-6b8a-4932-e5f3-50ad83fddab3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Baseline accuracy: 88.95, New accuracy: 82.19, Difference: -6.75\n",
            "Baseline precision: 0.89, New precision: 0.82, Difference: -0.07\n",
            "Baseline recall: 0.89, New recall: 0.82, Difference: -0.07\n",
            "Baseline f1: 0.89, New f1: 0.82, Difference: -0.07\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Model 4"
      ],
      "metadata": {
        "id": "1DlV8eXbiLyf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Set random seed and create embedding layer (new embedding layer for each model)\n",
        "tf.random.set_seed(42)\n",
        "from tensorflow.keras import layers\n",
        "model_4_embedding = layers.Embedding(input_dim=max_vocab_length,\n",
        "                                     output_dim=128,\n",
        "                                     embeddings_initializer=\"uniform\",\n",
        "                                     input_length=max_length,\n",
        "                                     name=\"embedding_4\")\n",
        "\n",
        "# Build a Bidirectional RNN in TensorFlow\n",
        "inputs = layers.Input(shape=(1,), dtype=\"string\")\n",
        "x = text_vectorizer(inputs)\n",
        "x = model_4_embedding(x)\n",
        "# x = layers.Bidirectional(layers.LSTM(64, return_sequences=True))(x) # stacking RNN layers requires return_sequences=True\n",
        "x = layers.Bidirectional(layers.LSTM(64))(x) # bidirectional goes both ways so has double the parameters of a regular LSTM layer\n",
        "outputs = layers.Dense(1, activation=\"sigmoid\")(x)\n",
        "model_4 = tf.keras.Model(inputs, outputs, name=\"model_4_Bidirectional\")"
      ],
      "metadata": {
        "id": "rweDqY39h_C0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Compile\n",
        "model_4.compile(loss=\"binary_crossentropy\",\n",
        "                optimizer=tf.keras.optimizers.Adam(),\n",
        "                metrics=[\"accuracy\"])"
      ],
      "metadata": {
        "id": "Kb8teyQ_iQuQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Get a summary of our bidirectional model\n",
        "model_4.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CXsez930iZyK",
        "outputId": "740cffde-2466-49c7-9364-8849fd1484b9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_4_Bidirectional\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_4 (InputLayer)        [(None, 1)]               0         \n",
            "                                                                 \n",
            " text_vectorization_1 (Text  (None, 15)                0         \n",
            " Vectorization)                                                  \n",
            "                                                                 \n",
            " embedding_4 (Embedding)     (None, 15, 128)           1280000   \n",
            "                                                                 \n",
            " bidirectional (Bidirection  (None, 128)               98816     \n",
            " al)                                                             \n",
            "                                                                 \n",
            " dense_3 (Dense)             (None, 1)                 129       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1378945 (5.26 MB)\n",
            "Trainable params: 1378945 (5.26 MB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Fit the model (takes longer because of the bidirectional layers)\n",
        "model_4_history = model_4.fit(train_sentences,\n",
        "                              train_labels,\n",
        "                              epochs=5,\n",
        "                              validation_data=(val_sentences, val_labels),\n",
        "                              callbacks=[create_tensorboard_callback(SAVE_DIR, \"bidirectional_RNN\")])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hy-yglkVij1-",
        "outputId": "be1f2ce3-49fe-4091-b7be-c6bd5ee6e1f7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving TensorBoard log files to: model_logs/bidirectional_RNN/20240115-045707\n",
            "Epoch 1/5\n",
            "321/321 [==============================] - 20s 47ms/step - loss: 0.4421 - accuracy: 0.7863 - val_loss: 0.3617 - val_accuracy: 0.8412\n",
            "Epoch 2/5\n",
            "321/321 [==============================] - 14s 44ms/step - loss: 0.2464 - accuracy: 0.8995 - val_loss: 0.3557 - val_accuracy: 0.8316\n",
            "Epoch 3/5\n",
            "321/321 [==============================] - 14s 43ms/step - loss: 0.1563 - accuracy: 0.9396 - val_loss: 0.4173 - val_accuracy: 0.8342\n",
            "Epoch 4/5\n",
            "321/321 [==============================] - 14s 44ms/step - loss: 0.0967 - accuracy: 0.9652 - val_loss: 0.5275 - val_accuracy: 0.8219\n",
            "Epoch 5/5\n",
            "321/321 [==============================] - 14s 44ms/step - loss: 0.0712 - accuracy: 0.9742 - val_loss: 0.6537 - val_accuracy: 0.8325\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Make predictions with bidirectional RNN on the validation data\n",
        "model_4_pred_probs = model_4.predict(val_sentences)\n",
        "model_4_pred_probs[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ri2iAh70ixnq",
        "outputId": "4f27e4f4-8740-44cf-e3dc-fc6951258d13"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "36/36 [==============================] - 2s 9ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[9.9990791e-01],\n",
              "       [1.0193295e-03],\n",
              "       [4.8477162e-02],\n",
              "       [9.9988902e-01],\n",
              "       [3.4565269e-04],\n",
              "       [3.4142964e-02],\n",
              "       [1.8001066e-03],\n",
              "       [9.9998266e-01],\n",
              "       [5.3998665e-04],\n",
              "       [9.1760033e-01]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 69
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Convert prediction probabilities to labels\n",
        "model_4_preds = tf.squeeze(tf.round(model_4_pred_probs))\n",
        "model_4_preds[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fB1oyStEjP0F",
        "outputId": "3126e3ec-6479-4ec6-f5a2-1d8d890eb990"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([1., 0., 0., 1., 0., 0., 0., 1., 0., 1.], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Calculate bidirectional RNN model results\n",
        "model_4_results = calculate_results(val_labels, model_4_preds)\n",
        "model_4_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Bi4XBEMQjjNW",
        "outputId": "721475bd-9fed-461d-f2b4-5bdd64c62622"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 83.24561403508773,\n",
              " 'precision': 0.8323810589738568,\n",
              " 'recall': 0.8324561403508772,\n",
              " 'f1': 0.8324127992373633}"
            ]
          },
          "metadata": {},
          "execution_count": 71
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Check to see how the bidirectional model performs against the baseline\n",
        "compare_baseline_to_new_results(baseline_results, model_4_results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a-nJNLLNjs_3",
        "outputId": "af943a5f-f14d-46b5-8c8c-d675c99f63c2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Baseline accuracy: 88.95, New accuracy: 83.25, Difference: -5.70\n",
            "Baseline precision: 0.89, New precision: 0.83, Difference: -0.06\n",
            "Baseline recall: 0.89, New recall: 0.83, Difference: -0.06\n",
            "Baseline f1: 0.89, New f1: 0.83, Difference: -0.06\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Model 5"
      ],
      "metadata": {
        "id": "qjalbhFrkJrs"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Test out the embedding, 1D convolutional and max pooling\n",
        "embedding_test = embedding(text_vectorizer([\"this is a test sentence\"])) # turn target sentence into embedding\n",
        "conv_1d = layers.Conv1D(filters=32, kernel_size=5, activation=\"relu\") # convolve over target sequence 5 words at a time\n",
        "conv_1d_output = conv_1d(embedding_test) # pass embedding through 1D convolutional layer\n",
        "max_pool = layers.GlobalMaxPool1D()\n",
        "max_pool_output = max_pool(conv_1d_output) # get the most important features\n",
        "embedding_test.shape, conv_1d_output.shape, max_pool_output.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ot7f_Rmlj5YJ",
        "outputId": "f0497f1a-7164-4304-d37b-7bbc65a089bd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(TensorShape([1, 15, 128]), TensorShape([1, 11, 32]), TensorShape([1, 32]))"
            ]
          },
          "metadata": {},
          "execution_count": 73
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# See the outputs of each layer\n",
        "embedding_test[:1], conv_1d_output[:1], max_pool_output[:1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YVG-iOYzkORk",
        "outputId": "a3c5485f-a7fc-4462-a5c2-d07d8e480d6d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(<tf.Tensor: shape=(1, 15, 128), dtype=float32, numpy=\n",
              " array([[[-0.0621724 , -0.04850797, -0.02803585, ...,  0.03067626,\n",
              "           0.0248801 , -0.02777423],\n",
              "         [-0.03755405,  0.00800279, -0.0041482 , ..., -0.00654611,\n",
              "          -0.00911831,  0.0458741 ],\n",
              "         [-0.06141719,  0.0054441 ,  0.03410306, ...,  0.04297196,\n",
              "           0.04032729, -0.00511609],\n",
              "         ...,\n",
              "         [ 0.01194163,  0.00669516, -0.0260199 , ...,  0.03438895,\n",
              "          -0.03894087, -0.01860881],\n",
              "         [ 0.01194163,  0.00669516, -0.0260199 , ...,  0.03438895,\n",
              "          -0.03894087, -0.01860881],\n",
              "         [ 0.01194163,  0.00669516, -0.0260199 , ...,  0.03438895,\n",
              "          -0.03894087, -0.01860881]]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(1, 11, 32), dtype=float32, numpy=\n",
              " array([[[8.70453194e-02, 3.59185226e-03, 0.00000000e+00, 1.07023090e-01,\n",
              "          0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
              "          2.46012807e-01, 1.41507268e-01, 1.01020411e-02, 0.00000000e+00,\n",
              "          0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
              "          0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 1.08884953e-01,\n",
              "          1.34868175e-01, 0.00000000e+00, 0.00000000e+00, 3.14654671e-02,\n",
              "          0.00000000e+00, 0.00000000e+00, 6.67505413e-02, 1.77434422e-02,\n",
              "          0.00000000e+00, 1.31551638e-01, 0.00000000e+00, 0.00000000e+00],\n",
              "         [6.91535026e-02, 1.46988869e-01, 5.26066869e-02, 2.00871378e-03,\n",
              "          1.19747855e-01, 0.00000000e+00, 0.00000000e+00, 1.40123710e-01,\n",
              "          0.00000000e+00, 0.00000000e+00, 2.98075192e-03, 2.59965777e-01,\n",
              "          3.21337730e-02, 6.46291450e-02, 8.44652876e-02, 1.92804217e-01,\n",
              "          8.37156251e-02, 6.36757463e-02, 0.00000000e+00, 5.11300862e-02,\n",
              "          7.50230104e-02, 2.54255496e-02, 5.20240180e-02, 9.64760706e-02,\n",
              "          0.00000000e+00, 0.00000000e+00, 8.62890705e-02, 1.03380457e-01,\n",
              "          6.34528100e-02, 0.00000000e+00, 1.83952838e-01, 2.18669385e-01],\n",
              "         [1.54713526e-01, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
              "          3.94711271e-04, 2.25972801e-01, 2.76841581e-01, 1.28445983e-01,\n",
              "          1.85543776e-01, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
              "          0.00000000e+00, 1.89262748e-01, 1.09241895e-01, 0.00000000e+00,\n",
              "          0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
              "          1.17620006e-01, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
              "          0.00000000e+00, 3.36516291e-01, 0.00000000e+00, 0.00000000e+00,\n",
              "          2.12664530e-03, 1.22205347e-01, 0.00000000e+00, 1.98533162e-02],\n",
              "         [0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 6.31028041e-02,\n",
              "          0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 2.39213929e-02,\n",
              "          3.49157006e-02, 1.79660246e-01, 0.00000000e+00, 1.34970739e-01,\n",
              "          0.00000000e+00, 1.03770986e-01, 0.00000000e+00, 6.71265125e-02,\n",
              "          3.93519402e-02, 1.37111753e-01, 3.23599130e-02, 3.16201672e-02,\n",
              "          0.00000000e+00, 0.00000000e+00, 3.94740067e-02, 0.00000000e+00,\n",
              "          1.33233801e-01, 0.00000000e+00, 2.48212107e-02, 2.25993291e-01,\n",
              "          3.08659434e-01, 1.35561138e-01, 0.00000000e+00, 0.00000000e+00],\n",
              "         [3.70405130e-02, 4.87922831e-03, 0.00000000e+00, 6.36008009e-03,\n",
              "          2.67245807e-04, 0.00000000e+00, 3.44839972e-03, 7.14378729e-02,\n",
              "          2.79049240e-02, 4.22335677e-02, 0.00000000e+00, 0.00000000e+00,\n",
              "          0.00000000e+00, 3.16069201e-02, 0.00000000e+00, 6.95552491e-03,\n",
              "          0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
              "          8.10550526e-02, 1.39736868e-02, 3.36098066e-03, 2.82064714e-02,\n",
              "          0.00000000e+00, 4.94558588e-02, 0.00000000e+00, 2.72672996e-02,\n",
              "          2.14490071e-02, 1.32149085e-02, 0.00000000e+00, 1.44661590e-02],\n",
              "         [0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 1.08242221e-02,\n",
              "          0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 5.53398654e-02,\n",
              "          2.09006257e-02, 3.94109599e-02, 0.00000000e+00, 0.00000000e+00,\n",
              "          0.00000000e+00, 2.61444878e-03, 0.00000000e+00, 2.17581857e-02,\n",
              "          0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 3.68872099e-03,\n",
              "          3.17007564e-02, 0.00000000e+00, 4.25378270e-02, 3.38327363e-02,\n",
              "          1.22334510e-02, 2.99786031e-02, 0.00000000e+00, 3.15275639e-02,\n",
              "          5.55277504e-02, 3.96012850e-02, 0.00000000e+00, 3.46854962e-02],\n",
              "         [0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 1.08242221e-02,\n",
              "          0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 5.53398654e-02,\n",
              "          2.09006257e-02, 3.94109599e-02, 0.00000000e+00, 0.00000000e+00,\n",
              "          0.00000000e+00, 2.61444878e-03, 0.00000000e+00, 2.17581857e-02,\n",
              "          0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 3.68872099e-03,\n",
              "          3.17007564e-02, 0.00000000e+00, 4.25378270e-02, 3.38327363e-02,\n",
              "          1.22334510e-02, 2.99786031e-02, 0.00000000e+00, 3.15275639e-02,\n",
              "          5.55277504e-02, 3.96012850e-02, 0.00000000e+00, 3.46854962e-02],\n",
              "         [0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 1.08242221e-02,\n",
              "          0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 5.53398654e-02,\n",
              "          2.09006257e-02, 3.94109599e-02, 0.00000000e+00, 0.00000000e+00,\n",
              "          0.00000000e+00, 2.61444878e-03, 0.00000000e+00, 2.17581857e-02,\n",
              "          0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 3.68872099e-03,\n",
              "          3.17007564e-02, 0.00000000e+00, 4.25378270e-02, 3.38327363e-02,\n",
              "          1.22334510e-02, 2.99786031e-02, 0.00000000e+00, 3.15275639e-02,\n",
              "          5.55277504e-02, 3.96012850e-02, 0.00000000e+00, 3.46854962e-02],\n",
              "         [0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 1.08242221e-02,\n",
              "          0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 5.53398654e-02,\n",
              "          2.09006257e-02, 3.94109599e-02, 0.00000000e+00, 0.00000000e+00,\n",
              "          0.00000000e+00, 2.61444878e-03, 0.00000000e+00, 2.17581857e-02,\n",
              "          0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 3.68872099e-03,\n",
              "          3.17007564e-02, 0.00000000e+00, 4.25378270e-02, 3.38327363e-02,\n",
              "          1.22334510e-02, 2.99786031e-02, 0.00000000e+00, 3.15275639e-02,\n",
              "          5.55277504e-02, 3.96012850e-02, 0.00000000e+00, 3.46854962e-02],\n",
              "         [0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 1.08242221e-02,\n",
              "          0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 5.53398654e-02,\n",
              "          2.09006257e-02, 3.94109599e-02, 0.00000000e+00, 0.00000000e+00,\n",
              "          0.00000000e+00, 2.61444878e-03, 0.00000000e+00, 2.17581857e-02,\n",
              "          0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 3.68872099e-03,\n",
              "          3.17007564e-02, 0.00000000e+00, 4.25378270e-02, 3.38327363e-02,\n",
              "          1.22334510e-02, 2.99786031e-02, 0.00000000e+00, 3.15275639e-02,\n",
              "          5.55277504e-02, 3.96012850e-02, 0.00000000e+00, 3.46854962e-02],\n",
              "         [0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 1.08242221e-02,\n",
              "          0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 5.53398654e-02,\n",
              "          2.09006257e-02, 3.94109599e-02, 0.00000000e+00, 0.00000000e+00,\n",
              "          0.00000000e+00, 2.61444878e-03, 0.00000000e+00, 2.17581857e-02,\n",
              "          0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 3.68872099e-03,\n",
              "          3.17007564e-02, 0.00000000e+00, 4.25378270e-02, 3.38327363e-02,\n",
              "          1.22334510e-02, 2.99786031e-02, 0.00000000e+00, 3.15275639e-02,\n",
              "          5.55277504e-02, 3.96012850e-02, 0.00000000e+00, 3.46854962e-02]]],\n",
              "       dtype=float32)>,\n",
              " <tf.Tensor: shape=(1, 32), dtype=float32, numpy=\n",
              " array([[0.15471353, 0.14698887, 0.05260669, 0.10702309, 0.11974785,\n",
              "         0.2259728 , 0.27684158, 0.14012371, 0.2460128 , 0.17966025,\n",
              "         0.01010204, 0.25996578, 0.03213377, 0.18926275, 0.1092419 ,\n",
              "         0.19280422, 0.08371563, 0.13711175, 0.03235991, 0.10888495,\n",
              "         0.13486817, 0.02542555, 0.05202402, 0.09647607, 0.1332338 ,\n",
              "         0.3365163 , 0.08628907, 0.22599329, 0.30865943, 0.13556114,\n",
              "         0.18395284, 0.21866938]], dtype=float32)>)"
            ]
          },
          "metadata": {},
          "execution_count": 74
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Set random seed and create embedding layer (new embedding layer for each model)\n",
        "tf.random.set_seed(42)\n",
        "from tensorflow.keras import layers\n",
        "model_5_embedding = layers.Embedding(input_dim=max_vocab_length,\n",
        "                                     output_dim=128,\n",
        "                                     embeddings_initializer=\"uniform\",\n",
        "                                     input_length=max_length,\n",
        "                                     name=\"embedding_5\")\n",
        "\n",
        "# Create 1-dimensional convolutional layer to model sequences\n",
        "from tensorflow.keras import layers\n",
        "inputs = layers.Input(shape=(1,), dtype=\"string\")\n",
        "x = text_vectorizer(inputs)\n",
        "x = model_5_embedding(x)\n",
        "x = layers.Conv1D(filters=32, kernel_size=5, activation=\"relu\")(x)\n",
        "x = layers.GlobalMaxPool1D()(x)\n",
        "# x = layers.Dense(64, activation=\"relu\")(x) # optional dense layer\n",
        "outputs = layers.Dense(1, activation=\"sigmoid\")(x)\n",
        "model_5 = tf.keras.Model(inputs, outputs, name=\"model_5_Conv1D\")\n",
        "\n",
        "# Compile Conv1D model\n",
        "model_5.compile(loss=\"binary_crossentropy\",\n",
        "                optimizer=tf.keras.optimizers.Adam(),\n",
        "                metrics=[\"accuracy\"])\n",
        "\n",
        "# Get a summary of our 1D convolution model\n",
        "model_5.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w7pppnkLkejr",
        "outputId": "45a5691d-d871-4edc-c0f2-4723cc2fcf82"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_5_Conv1D\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_5 (InputLayer)        [(None, 1)]               0         \n",
            "                                                                 \n",
            " text_vectorization_1 (Text  (None, 15)                0         \n",
            " Vectorization)                                                  \n",
            "                                                                 \n",
            " embedding_5 (Embedding)     (None, 15, 128)           1280000   \n",
            "                                                                 \n",
            " conv1d_1 (Conv1D)           (None, 11, 32)            20512     \n",
            "                                                                 \n",
            " global_max_pooling1d_1 (Gl  (None, 32)                0         \n",
            " obalMaxPooling1D)                                               \n",
            "                                                                 \n",
            " dense_4 (Dense)             (None, 1)                 33        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1300545 (4.96 MB)\n",
            "Trainable params: 1300545 (4.96 MB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Fit the model\n",
        "model_5_history = model_5.fit(train_sentences,\n",
        "                              train_labels,\n",
        "                              epochs=5,\n",
        "                              validation_data=(val_sentences, val_labels),\n",
        "                              callbacks=[create_tensorboard_callback(SAVE_DIR,\n",
        "                                                                     \"Conv1D\")])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qtlUbBYtk8lQ",
        "outputId": "81243679-64d1-4358-c612-3f20f6417561"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving TensorBoard log files to: model_logs/Conv1D/20240115-045835\n",
            "Epoch 1/5\n",
            "321/321 [==============================] - 8s 22ms/step - loss: 0.5043 - accuracy: 0.7478 - val_loss: 0.3730 - val_accuracy: 0.8237\n",
            "Epoch 2/5\n",
            "321/321 [==============================] - 8s 26ms/step - loss: 0.2739 - accuracy: 0.8896 - val_loss: 0.3580 - val_accuracy: 0.8386\n",
            "Epoch 3/5\n",
            "321/321 [==============================] - 7s 22ms/step - loss: 0.1606 - accuracy: 0.9435 - val_loss: 0.4038 - val_accuracy: 0.8386\n",
            "Epoch 4/5\n",
            "321/321 [==============================] - 8s 25ms/step - loss: 0.0916 - accuracy: 0.9706 - val_loss: 0.4696 - val_accuracy: 0.8342\n",
            "Epoch 5/5\n",
            "321/321 [==============================] - 8s 24ms/step - loss: 0.0505 - accuracy: 0.9862 - val_loss: 0.5290 - val_accuracy: 0.8333\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Make predictions with model_5\n",
        "model_5_pred_probs = model_5.predict(val_sentences)\n",
        "model_5_pred_probs[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y64K5bG2lasw",
        "outputId": "fc839cdc-cf1d-4690-8d95-3ad68cd0ed08"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "36/36 [==============================] - 0s 5ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[9.9999923e-01],\n",
              "       [1.2673133e-03],\n",
              "       [9.4565488e-02],\n",
              "       [9.9999988e-01],\n",
              "       [1.3429139e-04],\n",
              "       [1.3915145e-02],\n",
              "       [5.4940104e-02],\n",
              "       [9.9999058e-01],\n",
              "       [2.0157833e-02],\n",
              "       [9.1235745e-01]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 77
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Convert model_5 prediction probabilities to labels\n",
        "model_5_preds = tf.squeeze(tf.round(model_5_pred_probs))\n",
        "model_5_preds[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6R3-Gnv9lvxZ",
        "outputId": "05d18a03-d1cc-4903-95f6-983147a510ab"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([1., 0., 0., 1., 0., 0., 0., 1., 0., 1.], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 78
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Calculate model_5 evaluation metrics\n",
        "model_5_results = calculate_results(y_true=val_labels,\n",
        "                                    y_pred=model_5_preds)\n",
        "model_5_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Itq2Z0ypl54r",
        "outputId": "4ae62241-7cbd-459a-e716-6a0d06f3ede6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 83.33333333333334,\n",
              " 'precision': 0.8336742424242424,\n",
              " 'recall': 0.8333333333333334,\n",
              " 'f1': 0.832657733738693}"
            ]
          },
          "metadata": {},
          "execution_count": 79
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Compare model_5 results to baseline\n",
        "compare_baseline_to_new_results(baseline_results, model_5_results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "igeo2d2ImIsT",
        "outputId": "1ce45c05-54f1-4bc9-f191-c926ad5f649d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Baseline accuracy: 88.95, New accuracy: 83.33, Difference: -5.61\n",
            "Baseline precision: 0.89, New precision: 0.83, Difference: -0.06\n",
            "Baseline recall: 0.89, New recall: 0.83, Difference: -0.06\n",
            "Baseline f1: 0.89, New f1: 0.83, Difference: -0.06\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Model 6"
      ],
      "metadata": {
        "id": "fc6lDqmWmr15"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Example of pretrained embedding with universal sentence encoder - https://tfhub.dev/google/universal-sentence-encoder/4\n",
        "import tensorflow_hub as hub\n",
        "embed = hub.load(\"https://tfhub.dev/google/universal-sentence-encoder/4\") # load Universal Sentence Encoder\n",
        "embed_samples = embed([sample_sentence,\n",
        "                      \"When you call the universal sentence encoder on a sentence, it turns it into numbers.\"])\n",
        "\n",
        "print(embed_samples[0][:50])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6ATDpgUbmWJ1",
        "outputId": "744e7290-c7c2-4449-d034-4680a5ab894f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tf.Tensor(\n",
            "[-0.00982705  0.03238427 -0.03104174  0.04037441  0.02966517  0.02784401\n",
            " -0.01479441  0.02742032 -0.0299241   0.00754957 -0.02428642 -0.02191172\n",
            " -0.00623092  0.10449115  0.00417974 -0.06973658  0.00770808 -0.00611351\n",
            " -0.00401639 -0.08352943 -0.00812616 -0.03904727 -0.00013163 -0.02246787\n",
            " -0.05621985 -0.00273415  0.07752703 -0.06051603 -0.03481648  0.0654614\n",
            "  0.00419827 -0.01092643  0.00390858  0.0146431   0.02401788 -0.06800433\n",
            "  0.0567361   0.0482774  -0.03090527 -0.05531143  0.04806939  0.05837027\n",
            " -0.01289087  0.02400962 -0.10516107 -0.01181295 -0.06882656 -0.05222664\n",
            "  0.02367337  0.04817969], shape=(50,), dtype=float32)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Each sentence has been encoded into a 512 dimension vector\n",
        "embed_samples[0].shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Zx1CmLiUmyBX",
        "outputId": "9b5c02d7-b63d-4c8c-d4d5-bbf3407b096e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TensorShape([512])"
            ]
          },
          "metadata": {},
          "execution_count": 82
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# We can use this encoding layer in place of our text_vectorizer and embedding layer\n",
        "sentence_encoder_layer = hub.KerasLayer(\"https://tfhub.dev/google/universal-sentence-encoder/4\",\n",
        "                                        input_shape=[], # shape of inputs coming to our model\n",
        "                                        dtype=tf.string, # data type of inputs coming to the USE layer\n",
        "                                        trainable=False, # keep the pretrained weights (we'll create a feature extractor)\n",
        "                                        name=\"USE\")"
      ],
      "metadata": {
        "id": "3UR0onYCnhj2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Create model using the Sequential API\n",
        "model_6 = tf.keras.Sequential([\n",
        "  sentence_encoder_layer, # take in sentences and then encode them into an embedding\n",
        "  layers.Dense(64, activation=\"relu\"),\n",
        "  layers.Dense(1, activation=\"sigmoid\")\n",
        "], name=\"model_6_USE\")\n",
        "\n",
        "# Compile model\n",
        "model_6.compile(loss=\"binary_crossentropy\",\n",
        "                optimizer=tf.keras.optimizers.Adam(),\n",
        "                metrics=[\"accuracy\"])\n",
        "\n",
        "model_6.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CGQVAx-Yn1bs",
        "outputId": "0fe643af-fbbb-4358-a164-dd25d990034e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_6_USE\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " USE (KerasLayer)            (None, 512)               256797824 \n",
            "                                                                 \n",
            " dense_5 (Dense)             (None, 64)                32832     \n",
            "                                                                 \n",
            " dense_6 (Dense)             (None, 1)                 65        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 256830721 (979.73 MB)\n",
            "Trainable params: 32897 (128.50 KB)\n",
            "Non-trainable params: 256797824 (979.61 MB)\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Train a classifier on top of pretrained embeddings\n",
        "model_6_history = model_6.fit(train_sentences,\n",
        "                              train_labels,\n",
        "                              epochs=5,\n",
        "                              validation_data=(val_sentences, val_labels),\n",
        "                              callbacks=[create_tensorboard_callback(SAVE_DIR,\n",
        "                                                                     \"tf_hub_sentence_encoder\")])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AJqWyXZBoCM1",
        "outputId": "e9a6c768-d499-4182-ed38-d76a87eb04c8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving TensorBoard log files to: model_logs/tf_hub_sentence_encoder/20240115-045956\n",
            "Epoch 1/5\n",
            "321/321 [==============================] - 15s 35ms/step - loss: 0.3949 - accuracy: 0.8367 - val_loss: 0.2737 - val_accuracy: 0.8904\n",
            "Epoch 2/5\n",
            "321/321 [==============================] - 9s 27ms/step - loss: 0.2834 - accuracy: 0.8768 - val_loss: 0.2589 - val_accuracy: 0.8965\n",
            "Epoch 3/5\n",
            "321/321 [==============================] - 10s 32ms/step - loss: 0.2684 - accuracy: 0.8855 - val_loss: 0.2553 - val_accuracy: 0.9035\n",
            "Epoch 4/5\n",
            "321/321 [==============================] - 10s 32ms/step - loss: 0.2585 - accuracy: 0.8893 - val_loss: 0.2523 - val_accuracy: 0.9018\n",
            "Epoch 5/5\n",
            "321/321 [==============================] - 11s 33ms/step - loss: 0.2482 - accuracy: 0.8950 - val_loss: 0.2508 - val_accuracy: 0.9035\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Make predictions with USE TF Hub model\n",
        "model_6_pred_probs = model_6.predict(val_sentences)\n",
        "model_6_pred_probs[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Yuq8kzW5oc3L",
        "outputId": "512d6a8f-07df-40c7-c845-d08e2dac5dca"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "36/36 [==============================] - 1s 24ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.99334323],\n",
              "       [0.04031249],\n",
              "       [0.13454646],\n",
              "       [0.9870011 ],\n",
              "       [0.02324182],\n",
              "       [0.97967464],\n",
              "       [0.10362401],\n",
              "       [0.9769617 ],\n",
              "       [0.07309019],\n",
              "       [0.3417588 ]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 86
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Convert prediction probabilities to labels\n",
        "model_6_preds = tf.squeeze(tf.round(model_6_pred_probs))\n",
        "model_6_preds[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Gcm9-Zp9oxMk",
        "outputId": "10689df7-098d-4d7b-9ba1-2a8c865c14a2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([1., 0., 0., 1., 0., 1., 0., 1., 0., 0.], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 87
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Calculate model 6 performance metrics\n",
        "model_6_results = calculate_results(val_labels, model_6_preds)\n",
        "model_6_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eD9_7IHTpluw",
        "outputId": "0b90da12-9d79-485c-898a-45db5d66f522"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 90.35087719298247,\n",
              " 'precision': 0.9034814008828961,\n",
              " 'recall': 0.9035087719298246,\n",
              " 'f1': 0.9034922841083289}"
            ]
          },
          "metadata": {},
          "execution_count": 88
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Compare TF Hub model to baseline\n",
        "compare_baseline_to_new_results(baseline_results, model_6_results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G1_sciIvp0t1",
        "outputId": "61e59c28-0d14-41c7-ee16-1d93db1a787e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Baseline accuracy: 88.95, New accuracy: 90.35, Difference: 1.40\n",
            "Baseline precision: 0.89, New precision: 0.90, Difference: 0.01\n",
            "Baseline recall: 0.89, New recall: 0.90, Difference: 0.01\n",
            "Baseline f1: 0.89, New f1: 0.90, Difference: 0.01\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Model 7"
      ],
      "metadata": {
        "id": "7hyuUJ2mqgYt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "## NOTE: Making splits like this will lead to data leakage ###\n",
        "## (some of the training examples in the validation set) ###\n",
        "\n",
        "## WRONG WAY TO MAKE SPLITS (train_df_shuffled has already been split) ###\n",
        "\n",
        " # Create subsets of 10% of the training data\n",
        "train_10_percent = train_df_shuffled[[\"text\", \"target\"]].sample(frac=0.1, random_state=42)\n",
        "train_sentences_10_percent = train_10_percent[\"text\"].to_list()\n",
        "train_labels_10_percent = train_10_percent[\"target\"].to_list()\n",
        "len(train_sentences_10_percent), len(train_labels_10_percent)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lNiuF8JiqefB",
        "outputId": "ec11fc99-fdbe-4299-f674-42c02cba15bc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1140, 1140)"
            ]
          },
          "metadata": {},
          "execution_count": 90
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# One kind of correct way (there are more) to make data subset\n",
        "# (split the already split train_sentences/train_labels)\n",
        "train_sentences_90_percent, train_sentences_10_percent, train_labels_90_percent, train_labels_10_percent = train_test_split(np.array(train_sentences),\n",
        "                                                                                                                            train_labels,\n",
        "                                                                                                                            test_size=0.1,\n",
        "                                                                                                                            random_state=42)"
      ],
      "metadata": {
        "id": "6n0JMNf5qmL8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Check length of 10 percent datasets\n",
        "print(f\"Total training examples: {len(train_sentences)}\")\n",
        "print(f\"Length of 10% training examples: {len(train_sentences_10_percent)}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7xMuYJOXqtbJ",
        "outputId": "751dd793-daaf-4f7c-bfd8-d0462cf80683"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total training examples: 10255\n",
            "Length of 10% training examples: 1026\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Check the number of targets in our subset of data\n",
        "# (this should be close to the distribution of labels in the original train_labels)\n",
        "pd.Series(train_labels_10_percent).value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "giQA5ajLqw1E",
        "outputId": "c97eb1af-7873-4387-c0b5-ea760fc22131"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    520\n",
              "1    506\n",
              "dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 93
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Clone model_6 but reset weights\n",
        "model_7 = tf.keras.models.clone_model(model_6)\n",
        "\n",
        "# Compile model\n",
        "model_7.compile(loss=\"binary_crossentropy\",\n",
        "                optimizer=tf.keras.optimizers.Adam(),\n",
        "                metrics=[\"accuracy\"])\n",
        "\n",
        "# Get a summary (will be same as model_6)\n",
        "model_7.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y-C7s2_jqz3J",
        "outputId": "9255a086-654b-48e5-8331-7eb402da1170"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_6_USE\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " USE (KerasLayer)            (None, 512)               256797824 \n",
            "                                                                 \n",
            " dense_5 (Dense)             (None, 64)                32832     \n",
            "                                                                 \n",
            " dense_6 (Dense)             (None, 1)                 65        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 256830721 (979.73 MB)\n",
            "Trainable params: 32897 (128.50 KB)\n",
            "Non-trainable params: 256797824 (979.61 MB)\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Fit the model to 10% of the training data\n",
        "model_7_history = model_7.fit(x=train_sentences_10_percent,\n",
        "                              y=train_labels_10_percent,\n",
        "                              epochs=5,\n",
        "                              validation_data=(val_sentences, val_labels),\n",
        "                              callbacks=[create_tensorboard_callback(SAVE_DIR, \"10_percent_tf_hub_sentence_encoder\")])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QgMiOweYq3dx",
        "outputId": "ba18be23-6ac6-433f-b0ab-74ae6bc28dc0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving TensorBoard log files to: model_logs/10_percent_tf_hub_sentence_encoder/20240115-050131\n",
            "Epoch 1/5\n",
            "33/33 [==============================] - 6s 80ms/step - loss: 0.6531 - accuracy: 0.7251 - val_loss: 0.5991 - val_accuracy: 0.8132\n",
            "Epoch 2/5\n",
            "33/33 [==============================] - 2s 63ms/step - loss: 0.5463 - accuracy: 0.8460 - val_loss: 0.4867 - val_accuracy: 0.8465\n",
            "Epoch 3/5\n",
            "33/33 [==============================] - 4s 120ms/step - loss: 0.4429 - accuracy: 0.8499 - val_loss: 0.4050 - val_accuracy: 0.8500\n",
            "Epoch 4/5\n",
            "33/33 [==============================] - 2s 48ms/step - loss: 0.3732 - accuracy: 0.8684 - val_loss: 0.3629 - val_accuracy: 0.8561\n",
            "Epoch 5/5\n",
            "33/33 [==============================] - 2s 49ms/step - loss: 0.3301 - accuracy: 0.8743 - val_loss: 0.3392 - val_accuracy: 0.8614\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Make predictions with the model trained on 10% of the data\n",
        "model_7_pred_probs = model_7.predict(val_sentences)\n",
        "model_7_pred_probs[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JR1b9kxGq-CI",
        "outputId": "cf744d7f-4b9e-4ef0-e631-b6856eb958ed"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "36/36 [==============================] - 1s 23ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.94852537],\n",
              "       [0.20160632],\n",
              "       [0.5306983 ],\n",
              "       [0.9240105 ],\n",
              "       [0.2557098 ],\n",
              "       [0.89466804],\n",
              "       [0.36274913],\n",
              "       [0.85500544],\n",
              "       [0.17657241],\n",
              "       [0.39657706]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 96
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Convert prediction probabilities to labels\n",
        "model_7_preds = tf.squeeze(tf.round(model_7_pred_probs))\n",
        "model_7_preds[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jrGkIWpqrIaQ",
        "outputId": "ab58a718-f355-412e-b483-44009890ecf8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([1., 0., 1., 1., 0., 1., 0., 1., 0., 0.], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 97
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Calculate model results\n",
        "model_7_results = calculate_results(val_labels, model_7_preds)\n",
        "model_7_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ohT4zA1YrLn2",
        "outputId": "eea96fd4-8535-4300-d8d3-aae0994b5926"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 86.14035087719299,\n",
              " 'precision': 0.8630986946776421,\n",
              " 'recall': 0.8614035087719298,\n",
              " 'f1': 0.8616542329914012}"
            ]
          },
          "metadata": {},
          "execution_count": 98
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Compare to baseline\n",
        "compare_baseline_to_new_results(baseline_results, model_7_results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DPiIifPZrPAu",
        "outputId": "cccfd6c0-a75e-4821-ba27-6d26f5a3f603"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Baseline accuracy: 88.95, New accuracy: 86.14, Difference: -2.81\n",
            "Baseline precision: 0.89, New precision: 0.86, Difference: -0.03\n",
            "Baseline recall: 0.89, New recall: 0.86, Difference: -0.03\n",
            "Baseline f1: 0.89, New f1: 0.86, Difference: -0.03\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Combine model results into a DataFrame\n",
        "all_model_results = pd.DataFrame({\"baseline\": baseline_results,\n",
        "                                  \"simple_dense\": model_1_results,\n",
        "                                  \"lstm\": model_2_results,\n",
        "                                  \"gru\": model_3_results,\n",
        "                                  \"bidirectional\": model_4_results,\n",
        "                                  \"conv1d\": model_5_results,\n",
        "                                  \"tf_hub_sentence_encoder\": model_6_results,\n",
        "                                  \"tf_hub_10_percent_data\": model_7_results})\n",
        "all_model_results = all_model_results.transpose()\n",
        "all_model_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "id": "eWxaTPb_rS9u",
        "outputId": "65d4e6fc-23b3-42b8-c672-6edd515c332b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                          accuracy  precision    recall        f1\n",
              "baseline                 88.947368   0.889425  0.889474  0.889350\n",
              "simple_dense             84.561404   0.846190  0.845614  0.844944\n",
              "lstm                     82.807018   0.827934  0.828070  0.827979\n",
              "gru                      82.192982   0.821721  0.821930  0.821748\n",
              "bidirectional            83.245614   0.832381  0.832456  0.832413\n",
              "conv1d                   83.333333   0.833674  0.833333  0.832658\n",
              "tf_hub_sentence_encoder  90.350877   0.903481  0.903509  0.903492\n",
              "tf_hub_10_percent_data   86.140351   0.863099  0.861404  0.861654"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-489d24f3-7c06-412e-9556-515fbe6673d4\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>accuracy</th>\n",
              "      <th>precision</th>\n",
              "      <th>recall</th>\n",
              "      <th>f1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>baseline</th>\n",
              "      <td>88.947368</td>\n",
              "      <td>0.889425</td>\n",
              "      <td>0.889474</td>\n",
              "      <td>0.889350</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>simple_dense</th>\n",
              "      <td>84.561404</td>\n",
              "      <td>0.846190</td>\n",
              "      <td>0.845614</td>\n",
              "      <td>0.844944</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>lstm</th>\n",
              "      <td>82.807018</td>\n",
              "      <td>0.827934</td>\n",
              "      <td>0.828070</td>\n",
              "      <td>0.827979</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>gru</th>\n",
              "      <td>82.192982</td>\n",
              "      <td>0.821721</td>\n",
              "      <td>0.821930</td>\n",
              "      <td>0.821748</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>bidirectional</th>\n",
              "      <td>83.245614</td>\n",
              "      <td>0.832381</td>\n",
              "      <td>0.832456</td>\n",
              "      <td>0.832413</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>conv1d</th>\n",
              "      <td>83.333333</td>\n",
              "      <td>0.833674</td>\n",
              "      <td>0.833333</td>\n",
              "      <td>0.832658</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>tf_hub_sentence_encoder</th>\n",
              "      <td>90.350877</td>\n",
              "      <td>0.903481</td>\n",
              "      <td>0.903509</td>\n",
              "      <td>0.903492</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>tf_hub_10_percent_data</th>\n",
              "      <td>86.140351</td>\n",
              "      <td>0.863099</td>\n",
              "      <td>0.861404</td>\n",
              "      <td>0.861654</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-489d24f3-7c06-412e-9556-515fbe6673d4')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-489d24f3-7c06-412e-9556-515fbe6673d4 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-489d24f3-7c06-412e-9556-515fbe6673d4');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-17e88fd0-c52a-4ae1-97bf-0e67c19e3d56\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-17e88fd0-c52a-4ae1-97bf-0e67c19e3d56')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-17e88fd0-c52a-4ae1-97bf-0e67c19e3d56 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 100
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Reduce the accuracy to same scale as other metrics\n",
        "all_model_results[\"accuracy\"] = all_model_results[\"accuracy\"]/100"
      ],
      "metadata": {
        "id": "C7pDWym7rXxo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Plot and compare all of the model results\n",
        "all_model_results.plot(kind=\"bar\", figsize=(10, 7)).legend(bbox_to_anchor=(1.0, 1.0));"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 763
        },
        "id": "VsDm5DQOreME",
        "outputId": "dfdaf3a9-a0a9-4091-b5ea-f854bb14f491"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1000x700 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7IAAALqCAYAAAAIKmjaAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABogUlEQVR4nO3deXxNd+L/8fdNyEYWuzAhYo/ElthKqaWofrWWDoMKii5KEVoMolW1dazVkdqKThWtVjtlVJvSEqnaElp7LLGvJY0lkeX3h587c5tQITfHuff1fDzymNzP+dzct9wOeed8zudYsrKysgQAAAAAgEm4GB0AAAAAAIDcoMgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVAoYHeB+ZGZm6vTp0/L29pbFYjE6DgAAAACDZGVl6ffff1eZMmXk4sJ5OWdliiJ7+vRpBQQEGB0DAAAAwCPixIkT+stf/mJ0DBjEFEXW29tb0u3/WH18fAxOAwAAAMAoycnJCggIsHYEOCdTFNk7y4l9fHwosgAAAAC45NDJsagcAAAAAGAqFFkAAAAAgKlQZAEAAAAApmKKa2QBAAAA4H5lZGTo1q1bRsdALrm6uqpAgQL3df0zRRYAAACAw0hJSdHJkyeVlZVldBQ8AC8vL/n7+8vNze2e8yiyAAAAABxCRkaGTp48KS8vL5UoUYKdjU0kKytLaWlpunDhgo4eParKlSvLxeXuV8JSZAEAAAA4hFu3bikrK0slSpSQp6en0XGQS56enipYsKCOHz+utLQ0eXh43HUumz0BAAAAcCiciTWve52FtZln5xwAAAAAAOQpiiwAAAAAwFS4RhYAAACAQwscuSZfX+/Y5Kfz9fWcEWdkAQAAAAA2HvX78FJkAQAAAMBg69atU5MmTeTn56dixYrp//7v/5SYmGg9fvLkSXXr1k1FixZVoUKFFB4erq1bt1qP//vf/1a9evXk4eGh4sWLq2PHjtZjFotFq1evtnk9Pz8/LV68WJJ07NgxWSwWrVixQs2aNZOHh4c+/vhjXbp0Sd26dVPZsmXl5eWl0NBQffLJJzZfJzMzU1OnTlWlSpXk7u6ucuXK6Z133pEktWjRQgMHDrSZf+HCBbm5uSkmJuahvl8UWQAAAAAw2LVr1xQZGant27crJiZGLi4u6tixozIzM5WSkqJmzZrp1KlT+uqrr5SQkKA33nhDmZmZkqQ1a9aoY8eOateunXbt2qWYmBjVr18/1xlGjhypwYMHa9++fWrTpo1u3rypsLAwrVmzRr/88otefPFF9ezZUz///LP1OaNGjdLkyZM1duxY7d27V8uWLVOpUqUkSf369dOyZcuUmppqnf+vf/1LZcuWVYsWLR7q+8U1sgAAAABgsM6dO9s8XrRokUqUKKG9e/dqy5YtunDhgrZt26aiRYtKkipVqmSd+8477+hvf/ub3nrrLetYrVq1cp1hyJAh6tSpk83Y8OHDrZ8PGjRI33zzjVauXKn69evr999/16xZszRnzhz16tVLklSxYkU1adJEktSpUycNHDhQX375pbp06SJJWrx4sXr37v3Qt0jijCwAAAAAGOzQoUPq1q2bgoKC5OPjo8DAQElSUlKS4uPjVadOHWuJ/aP4+Hi1bNnyoTOEh4fbPM7IyNDbb7+t0NBQFS1aVIULF9Y333yjpKQkSdK+ffuUmpp619f28PBQz549tWjRIknSzp079csvv6h3794PnZUzsgAAAABgsPbt26t8+fKaP3++ypQpo8zMTIWEhCgtLU2enp73fO6fHbdYLMrKyrIZy2kzp0KFCtk8fvfddzVr1izNnDlToaGhKlSokIYMGaK0tLT7el3p9vLi2rVr6+TJk/rwww/VokULlS9f/k+f92c4IwsAAAAABrp06ZIOHDigMWPGqGXLlqpevbp+++036/GaNWsqPj5ely9fzvH5NWvWvOfmSSVKlNCZM2esjw8dOqTr16//aa7Y2Fg9++yzev7551WrVi0FBQXp4MGD1uOVK1eWp6fnPV87NDRU4eHhmj9/vpYtW6YXXnjhT1/3flBkAQAAAMBARYoUUbFixTRv3jwdPnxY33//vSIjI63Hu3XrptKlS6tDhw6KjY3VkSNHtGrVKsXFxUmSxo0bp08++UTjxo3Tvn37tGfPHk2ZMsX6/BYtWmjOnDnatWuXtm/frpdfflkFCxb801yVK1fWt99+qy1btmjfvn166aWXdO7cOetxDw8PjRgxQm+88YaWLl2qxMRE/fTTT1q4cKHN1+nXr58mT56srKwsm92UHwZFFgAAAAAM5OLiouXLl2vHjh0KCQnR0KFD9e6771qPu7m5af369SpZsqTatWun0NBQTZ48Wa6urpKkJ554Qp9++qm++uor1a5dWy1atLDZWXjatGkKCAjQ448/ru7du2v48OHy8vL601xjxoxR3bp11aZNGz3xxBPWMv2/xo4dq2HDhikqKkrVq1dX165ddf78eZs53bp1U4ECBdStWzd5eHg8xHfqvyxZf1ws/QhKTk6Wr6+vrl69Kh8fH6PjAAAAADDIvbrBzZs3dfToUVWoUCHPChMe3rFjx1SxYkVt27ZNdevWvefc+30P2ewJAAAAAJDnbt26pUuXLmnMmDFq2LDhn5bY3KDIAgAAwCpw5JpczT/m0T1X80MrlMvVfElaOSk9V/Or79+X69cAkPdiY2PVvHlzValSRZ999lmefm2KLAAAAAAgzz3xxBPZbvuTV9jsCQAAAABgKhRZAAAAAICpUGQBAAAAAKZCkQUAAAAAmApFFgAAAABgKhRZAAAAAICpcPud/+9Ru2ca90sDAAAAYC8bN25U8+bN9dtvv8nPzy/P5uYXiiwAAAAAx/ambz6/3tX8fb0H8Nhjj+nMmTPy9f3z701u5uYXlhYDAAAAgImkpaU99Ndwc3NT6dKlZbFY8nRufqHIAgAAAICBnnjiCQ0cOFADBw6Ur6+vihcvrrFjxyorK0uSFBgYqLffflsRERHy8fHRiy++KEnavHmzHn/8cXl6eiogIECvvfaarl27Zv26qampGjFihAICAuTu7q5KlSpp4cKFkm4vF7ZYLLpy5Yok6fjx42rfvr2KFCmiQoUKqUaNGlq7dm2OcyVp1apVqlGjhtzd3RUYGKhp06bZ/JkCAwM1ceJEvfDCC/L29la5cuU0b968PPueUWQBAAAAwGBLlixRgQIF9PPPP2vWrFmaPn26FixYYD3+j3/8Q7Vq1dKuXbs0duxYJSYmqm3bturcubN2796tFStWaPPmzRo4cKD1OREREfrkk080e/Zs7du3Tx988IEKFy6c4+u/+uqrSk1N1Y8//qg9e/ZoypQpd527Y8cOdenSRX/729+0Z88evfnmmxo7dqwWL15sM2/atGkKDw/Xrl27NGDAAL3yyis6cODAw3+zxDWyAAAAAGC4gIAAzZgxQxaLRVWrVtWePXs0Y8YM9e/fX5LUokULDRs2zDq/X79+6tGjh4YMGSJJqly5smbPnq1mzZpp7ty5SkpK0sqVK/Xtt9+qVatWkqSgoKC7vn5SUpI6d+6s0NDQP507ffp0tWzZUmPHjpUkValSRXv37tW7776r3r17W+e1a9dOAwYMkCSNGDFCM2bM0IYNG1S1atXcf4P+gDOyAAAAAGCwhg0b2lyD2qhRIx06dEgZGRmSpPDwcJv5CQkJWrx4sQoXLmz9aNOmjTIzM3X06FHFx8fL1dVVzZo1u6/Xf+211zRhwgQ1btxY48aN0+7du+86d9++fWrcuLHNWOPGjW3ySlLNmjWtn1ssFpUuXVrnz5+/rzx/hiILAAAAAI+4QoUK2TxOSUnRSy+9pPj4eOtHQkKCDh06pIoVK8rT0zNXX79fv346cuSIevbsqT179ig8PFzvvffeQ2UuWLCgzWOLxaLMzMyH+pp3UGQBAAAAwGBbt261efzTTz+pcuXKcnV1zXF+3bp1tXfvXlWqVCnbh5ubm0JDQ5WZmakffvjhvjMEBATo5Zdf1ueff65hw4Zp/vz5Oc6rXr26YmNjbcZiY2NVpUqVu+bNaxRZAAAAADBYUlKSIiMjdeDAAX3yySd67733NHjw4LvOHzFihLZs2aKBAwcqPj5ehw4d0pdffmnd7CkwMFC9evXSCy+8oNWrV+vo0aPauHGjVq5cmePXGzJkiL755hsdPXpUO3fu1IYNG1S9evUc5w4bNkwxMTF6++23dfDgQS1ZskRz5szR8OHDH/4bcZ/Y7AkAAAAADBYREaEbN26ofv36cnV11eDBg6232clJzZo19cMPP2j06NF6/PHHlZWVpYoVK6pr167WOXPnztXf//53DRgwQJcuXVK5cuX097//Pcevl5GRoVdffVUnT56Uj4+P2rZtqxkzZuQ4t27dulq5cqWioqL09ttvy9/fX+PHj7fZ6MneLFl3bk70CEtOTpavr6+uXr0qHx8fu7xG4Mg1uZp/zKN7ruaHViiXq/krJ6Xnan71/ftyNR8AACAnj9rPRBI/F8HWvbrBzZs3dfToUVWoUEEeHh4GJcy9J554QrVr19bMmTONjmK4+30PWVoMAAAAADAViiwAAAAAwFS4RhYAAAAADLRx40ajI5gORRYAAAAwszd9czn/qn1yAPmIIgsAAAA8QnK/4Vbuvn7oktBczc/tZlsSG27B/rhGFgAAAABgKhRZAAAAAICpUGQBAAAAAKZCkQUAAAAAmApFFgAAAACczJtvvqnatWtbH/fu3VsdOnQwLE9usWsxAAAAAIeW252aH9aeXnvy9fWcEWdkAQAAAOARkpaWZnSERx5FFgAAAAAM9MQTT2jgwIEaMmSIihcvrjZt2uiXX37RU089pcKFC6tUqVLq2bOnLl68aH1OZmampk6dqkqVKsnd3V3lypXTO++8Yz0+YsQIValSRV5eXgoKCtLYsWN169YtI/54dsHSYjwycn/z7+65mh9aoVyu5rMkBAAAAPllyZIleuWVVxQbG6srV66oRYsW6tevn2bMmKEbN25oxIgR6tKli77//ntJ0qhRozR//nzNmDFDTZo00ZkzZ7R//37r1/P29tbixYtVpkwZ7dmzR/3795e3t7feeOMNo/6IeYoiC9zFvmrVczW/+v59dkoCAAAAR1e5cmVNnTpVkjRhwgTVqVNHEydOtB5ftGiRAgICdPDgQfn7+2vWrFmaM2eOevXqJUmqWLGimjRpYp0/ZswY6+eBgYEaPny4li9fTpEFAAAAAOSNsLAw6+cJCQnasGGDChcunG1eYmKirly5otTUVLVs2fKuX2/FihWaPXu2EhMTlZKSovT0dPn4+NgluxEosgAAAABgsEKFClk/T0lJUfv27TVlypRs8/z9/XXkyJF7fq24uDj16NFDb731ltq0aSNfX18tX75c06ZNy/PcRqHIAgAAAMAjpG7dulq1apUCAwNVoED2yla5cmV5enoqJiZG/fr1y3Z8y5YtKl++vEaPHm0dO378uF0z5zd2LQYAAACAR8irr76qy5cvq1u3btq2bZsSExP1zTffqE+fPsrIyJCHh4dGjBihN954Q0uXLlViYqJ++uknLVy4UNLtopuUlKTly5crMTFRs2fP1hdffGHwnypvUWQBAAAA4BFSpkwZxcbGKiMjQ61bt1ZoaKiGDBkiPz8/ubjcrnBjx47VsGHDFBUVperVq6tr1646f/68JOmZZ57R0KFDNXDgQNWuXVtbtmzR2LFjjfwj5TmWFgMAgEeGvW/FJnE7NsAZPer/P964cWO2scqVK+vzzz+/63NcXFw0evRom+XD/2vq1KnWXZDvGDJkiPXzN998U2+++ab18eLFi3MT2XAUWQAA/r/clijJ/ve0XjkpPVfzuRVY3uN2bADw6GFpMQAAAADAVDgjC8DK3kv6OBMFAACAvMAZWQAAAACAqVBkAQAAAACmQpEFAAAAAJgK18gCwCOE65QBAAD+HGdkAQAAAACmQpEFAAAAAJgKRRYAAAAADJSVlaUXX3xRRYsWlcViUXx8vNGRHnlcIwsAAADAoe2rVj1fXy+3e0asW7dOixcv1saNGxUUFKSDBw+qffv22rFjh86cOaMvvvhCHTp0sE9Yk+KMLAAAAAAYKDExUf7+/nrsscdUunRpXbt2TbVq1dL7779vdLRHFmdkAQAAAMAgvXv31pIlSyRJFotF5cuX17Fjx/TUU08ZnOzRRpEFAAAAAIPMmjVLFStW1Lx587Rt2za5uroaHckUHmhp8fvvv6/AwEB5eHioQYMG+vnnn+85f+bMmapatao8PT0VEBCgoUOH6ubNmw8UGAAAAAAcha+vr7y9veXq6qrSpUurRIkSRkcyhVwX2RUrVigyMlLjxo3Tzp07VatWLbVp00bnz5/Pcf6yZcs0cuRIjRs3Tvv27dPChQu1YsUK/f3vf3/o8AAAAAAA55PrIjt9+nT1799fffr0UXBwsKKjo+Xl5aVFixblOH/Lli1q3LixunfvrsDAQLVu3VrdunX707O4AAAAAADkJFdFNi0tTTt27FCrVq3++wVcXNSqVSvFxcXl+JzHHntMO3bssBbXI0eOaO3atWrXrt1dXyc1NVXJyck2HwAAAAAASLnc7OnixYvKyMhQqVKlbMZLlSql/fv35/ic7t276+LFi2rSpImysrKUnp6ul19++Z5LiydNmqS33norN9EAAAAAwCGkpKTo8OHD1sdHjx5VfHy8ihYtqnLlyhmY7NFh9/vIbty4URMnTtQ///lP7dy5U59//rnWrFmjt99++67PGTVqlK5evWr9OHHihL1jAgAAAMAjYfv27apTp47q1KkjSYqMjFSdOnUUFRVlcLJHR67OyBYvXlyurq46d+6czfi5c+dUunTpHJ8zduxY9ezZU/369ZMkhYaG6tq1a3rxxRc1evRoubhk79Lu7u5yd3fPTTQAAAAAyFH1/fuMjnBPQ4YM0ZAhQ6yPn3jiCWVlZRkXyARydUbWzc1NYWFhiomJsY5lZmYqJiZGjRo1yvE5169fz1ZW79wbiTcHAAAAAJBbuTojK90+rd2rVy+Fh4erfv36mjlzpq5du6Y+ffpIkiIiIlS2bFlNmjRJktS+fXtNnz5dderUUYMGDXT48GGNHTtW7du352a/AAAAAIBcy3WR7dq1qy5cuKCoqCidPXtWtWvX1rp166wbQCUlJdmcgR0zZowsFovGjBmjU6dOqUSJEmrfvr3eeeedvPtTAAAAAACcRq6LrCQNHDhQAwcOzPHYxo0bbV+gQAGNGzdO48aNe5CXAgAAAADAht13LQYAAAAAIC9RZAEAAAA4FDaVNa/7fe8osgAAAAAcwp3NZNPS0gxOggd1/fp1SVLBggXvOe+BrpEFAAAAgEdNgQIF5OXlpQsXLqhgwYLZbgOKR1dWVpauX7+u8+fPy8/P70/vcEORBQAAAOAQLBaL/P39dfToUR0/ftzoOHgAfn5+Kl269J/Oo8gCAAAAcBhubm6qXLkyy4tNqGDBgn96JvYOiiwAAAAAh+Li4iIPDw+jY8COWDQOAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABM5YGK7Pvvv6/AwEB5eHioQYMG+vnnn+85/8qVK3r11Vfl7+8vd3d3ValSRWvXrn2gwAAAAAAA51Ygt09YsWKFIiMjFR0drQYNGmjmzJlq06aNDhw4oJIlS2abn5aWpieffFIlS5bUZ599prJly+r48ePy8/PLi/wAAAAAACeT6yI7ffp09e/fX3369JEkRUdHa82aNVq0aJFGjhyZbf6iRYt0+fJlbdmyRQULFpQkBQYGPlxqAAAAAIDTytXS4rS0NO3YsUOtWrX67xdwcVGrVq0UFxeX43O++uorNWrUSK+++qpKlSqlkJAQTZw4URkZGXd9ndTUVCUnJ9t8AAAAAAAg5bLIXrx4URkZGSpVqpTNeKlSpXT27Nkcn3PkyBF99tlnysjI0Nq1azV27FhNmzZNEyZMuOvrTJo0Sb6+vtaPgICA3MQEAAAAADgwu+9anJmZqZIlS2revHkKCwtT165dNXr0aEVHR9/1OaNGjdLVq1etHydOnLB3TAAAAACASeTqGtnixYvL1dVV586dsxk/d+6cSpcuneNz/P39VbBgQbm6ulrHqlevrrNnzyotLU1ubm7ZnuPu7i53d/fcRAMAAAAAOIlcnZF1c3NTWFiYYmJirGOZmZmKiYlRo0aNcnxO48aNdfjwYWVmZlrHDh48KH9//xxLLAAAAAAA95LrpcWRkZGaP3++lixZon379umVV17RtWvXrLsYR0REaNSoUdb5r7zyii5fvqzBgwfr4MGDWrNmjSZOnKhXX3017/4UAAAAAACnkevb73Tt2lUXLlxQVFSUzp49q9q1a2vdunXWDaCSkpLk4vLffhwQEKBvvvlGQ4cOVc2aNVW2bFkNHjxYI0aMyLs/BQAAAADAaeS6yErSwIEDNXDgwByPbdy4MdtYo0aN9NNPPz3ISwEAAAAAYMPuuxYDAAAAAJCXKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFN5oCL7/vvvKzAwUB4eHmrQoIF+/vnn+3re8uXLZbFY1KFDhwd5WQAAAAAAcl9kV6xYocjISI0bN047d+5UrVq11KZNG50/f/6ezzt27JiGDx+uxx9//IHDAgAAAACQ6yI7ffp09e/fX3369FFwcLCio6Pl5eWlRYsW3fU5GRkZ6tGjh9566y0FBQU9VGAAAAAAgHPLVZFNS0vTjh071KpVq/9+ARcXtWrVSnFxcXd93vjx41WyZEn17dv3vl4nNTVVycnJNh8AAAAAAEi5LLIXL15URkaGSpUqZTNeqlQpnT17NsfnbN68WQsXLtT8+fPv+3UmTZokX19f60dAQEBuYgIAAAAAHJhddy3+/fff1bNnT82fP1/Fixe/7+eNGjVKV69etX6cOHHCjikBAAAAAGZSIDeTixcvLldXV507d85m/Ny5cypdunS2+YmJiTp27Jjat29vHcvMzLz9wgUK6MCBA6pYsWK257m7u8vd3T030QAAAAAATiJXZ2Td3NwUFhammJgY61hmZqZiYmLUqFGjbPOrVaumPXv2KD4+3vrxzDPPqHnz5oqPj2fJMAAAAAAg13J1RlaSIiMj1atXL4WHh6t+/fqaOXOmrl27pj59+kiSIiIiVLZsWU2aNEkeHh4KCQmxeb6fn58kZRsHAAAAAOB+5LrIdu3aVRcuXFBUVJTOnj2r2rVra926ddYNoJKSkuTiYtdLbwEAAAAATizXRVaSBg4cqIEDB+Z4bOPGjfd87uLFix/kJQEAAAAAkGTnXYsBAAAAAMhrFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApvJARfb9999XYGCgPDw81KBBA/388893nTt//nw9/vjjKlKkiIoUKaJWrVrdcz4AAAAAAPeS6yK7YsUKRUZGaty4cdq5c6dq1aqlNm3a6Pz58znO37hxo7p166YNGzYoLi5OAQEBat26tU6dOvXQ4QEAAAAAzifXRXb69Onq37+/+vTpo+DgYEVHR8vLy0uLFi3Kcf7HH3+sAQMGqHbt2qpWrZoWLFigzMxMxcTEPHR4AAAAAIDzyVWRTUtL044dO9SqVav/fgEXF7Vq1UpxcXH39TWuX7+uW7duqWjRonedk5qaquTkZJsPAAAAAACkXBbZixcvKiMjQ6VKlbIZL1WqlM6ePXtfX2PEiBEqU6aMTRn+o0mTJsnX19f6ERAQkJuYAAAAAAAHlq+7Fk+ePFnLly/XF198IQ8Pj7vOGzVqlK5evWr9OHHiRD6mBAAAAAA8ygrkZnLx4sXl6uqqc+fO2YyfO3dOpUuXvudz//GPf2jy5Mn67rvvVLNmzXvOdXd3l7u7e26iAQAAAACcRK7OyLq5uSksLMxmo6Y7Gzc1atTors+bOnWq3n77ba1bt07h4eEPnhYAAAAA4PRydUZWkiIjI9WrVy+Fh4erfv36mjlzpq5du6Y+ffpIkiIiIlS2bFlNmjRJkjRlyhRFRUVp2bJlCgwMtF5LW7hwYRUuXDgP/ygAAAAAAGeQ6yLbtWtXXbhwQVFRUTp79qxq166tdevWWTeASkpKkovLf0/0zp07V2lpaXruuedsvs64ceP05ptvPlx6AAAAAIDTyXWRlaSBAwdq4MCBOR7buHGjzeNjx449yEsAAAAAAJCjfN21GAAAAACAh0WRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYygMV2ffff1+BgYHy8PBQgwYN9PPPP99z/qeffqpq1arJw8NDoaGhWrt27QOFBQAAAAAg10V2xYoVioyM1Lhx47Rz507VqlVLbdq00fnz53Ocv2XLFnXr1k19+/bVrl271KFDB3Xo0EG//PLLQ4cHAAAAADifXBfZ6dOnq3///urTp4+Cg4MVHR0tLy8vLVq0KMf5s2bNUtu2bfX666+revXqevvtt1W3bl3NmTPnocMDAAAAAJxPropsWlqaduzYoVatWv33C7i4qFWrVoqLi8vxOXFxcTbzJalNmzZ3nQ8AAAAAwL0UyM3kixcvKiMjQ6VKlbIZL1WqlPbv35/jc86ePZvj/LNnz971dVJTU5Wammp9fPXqVUlScnJybuLmSmbq9VzNT7Zk5Wp+xo2MXM1PycjdfHt+b/IL74HxeA+Mx3tgrNx+/yXeg7xm7/8PSLwHf+ZR+3tI4j34M4/a30OSfd+DO187Kyv3//+H48hVkc0vkyZN0ltvvZVtPCAgwIA0OfPN9TP25Wp2/dx+ed/cJzI73gPj8R4Yj/fAeLwHxnqwPy3vQV6y9/8HJN6DP/PI/T0k5ct78Pvvv8vXyd5r/Feuimzx4sXl6uqqc+fO2YyfO3dOpUuXzvE5pUuXztV8SRo1apQiIyOtjzMzM3X58mUVK1ZMFoslN5EfCcnJyQoICNCJEyfk4+NjdBynxHtgPN4D4/EeGI/3wHi8B8bi+288R3gPsrKy9Pvvv6tMmTJGR4GBclVk3dzcFBYWppiYGHXo0EHS7ZIZExOjgQMH5vicRo0aKSYmRkOGDLGOffvtt2rUqNFdX8fd3V3u7u42Y35+frmJ+kjy8fEx7V8YjoL3wHi8B8bjPTAe74HxeA+MxfffeGZ/DzgTi1wvLY6MjFSvXr0UHh6u+vXra+bMmbp27Zr69OkjSYqIiFDZsmU1adIkSdLgwYPVrFkzTZs2TU8//bSWL1+u7du3a968eXn7JwEAAAAAOIVcF9muXbvqwoULioqK0tmzZ1W7dm2tW7fOuqFTUlKSXFz+uxnyY489pmXLlmnMmDH6+9//rsqVK2v16tUKCQnJuz8FAAAAAMBpPNBmTwMHDrzrUuKNGzdmG/vrX/+qv/71rw/yUg7B3d1d48aNy7ZcGvmH98B4vAfG4z0wHu+B8XgPjMX333i8B3AUliz2rQYAAAAAmIjLn08BAAAAAODRQZEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAYBe3bt3SCy+8oKNHjxodBYCDYddiOLRNmzbpgw8+UGJioj777DOVLVtWH330kSpUqKAmTZoYHQ+wu6SkpHseL1euXD4lAeCsfH19FR8frwoVKhgdBZJu3ryptLQ0mzEfHx+D0gAP7oHuI4v7k56ero0bNyoxMVHdu3eXt7e3Tp8+LR8fHxUuXNjoeA5v1apV6tmzp3r06KFdu3YpNTVVknT16lVNnDhRa9euNTghYH+BgYGyWCx3PZ6RkZGPaYD8ERkZed9zp0+fbsckkKQOHTpo9erVGjp0qNFRnNb169f1xhtvaOXKlbp06VK24/xbADOiyNrJ8ePH1bZtWyUlJSk1NVVPPvmkvL29NWXKFKWmpio6OtroiA5vwoQJio6OVkREhJYvX24db9y4sSZMmGBgMudx8+ZNvffee9qwYYPOnz+vzMxMm+M7d+40KJnz2LVrl83jW7duadeuXZo+fbreeecdg1I5viJFitzzFwj/6/Lly3ZO43z++N/9zp07lZ6erqpVq0qSDh48KFdXV4WFhRkRz+lUrlxZ48ePV2xsrMLCwlSoUCGb46+99ppByZzH66+/rg0bNmju3Lnq2bOn3n//fZ06dUoffPCBJk+ebHQ84IFQZO1k8ODBCg8PV0JCgooVK2Yd79ixo/r3729gMudx4MABNW3aNNu4r6+vrly5kv+BnFDfvn21fv16Pffcc6pfv/59/2CPvFOrVq1sY+Hh4SpTpozeffddderUyYBUjm/mzJlGR3BqGzZssH4+ffp0eXt7a8mSJSpSpIgk6bffflOfPn30+OOPGxXRqSxcuFB+fn7asWOHduzYYXPMYrFQZPPBv//9by1dulRPPPGE9b/9SpUqqXz58vr444/Vo0cPoyMCuUaRtZNNmzZpy5YtcnNzsxkPDAzUqVOnDErlXEqXLq3Dhw8rMDDQZnzz5s0KCgoyJpST+frrr7V27Vo1btzY6Cj4g6pVq2rbtm1Gx3BYvXr1MjoC/r9p06Zp/fr11hIr3T5jPmHCBLVu3VrDhg0zMJ1zYKMn412+fNn6s4+Pj491JUiTJk30yiuvGBkNeGDsWmwnmZmZOV5vcPLkSXl7exuQyPn0799fgwcP1tatW2WxWHT69Gl9/PHHGj58OH9p55OyZcvy37vBkpOTbT6uXr2q/fv3a8yYMapcubLR8ZzOzZs3s70nsK/k5GRduHAh2/iFCxf0+++/G5DIeaWlpenAgQNKT083OorTCQoKsv5CoVq1alq5cqWk22dq/fz8DEwGPDiKrJ20bt3aZmmZxWJRSkqKxo0bp3bt2hkXzImMHDlS3bt3V8uWLZWSkqKmTZuqX79+eumllzRo0CCj4zmFadOmacSIETp+/LjRUZyWn5+fihQpYv0oWrSogoODFRcXp7lz5xodzylcu3ZNAwcOVMmSJVWoUCGb9+N/zxLCPjp27Kg+ffro888/18mTJ3Xy5EmtWrVKffv2ZWl9Prl+/br69u0rLy8v1ahRw7qb+qBBg7g+M5/06dNHCQkJkm7/fPT+++/Lw8NDQ4cO1euvv25wOuDBcPsdOzl58qTatGmjrKwsHTp0SOHh4Tp06JCKFy+uH3/8USVLljQ6otNIS0vT4cOHlZKSouDgYHaMzkcXLlxQly5d9OOPP8rLy0sFCxa0Oc4mN/b3ww8/2Dx2cXFRiRIlVKlSJRUowNUl+eHVV1/Vhg0b9Pbbb+e4yQrXptnX9evXNXz4cC1atEi3bt2SJBUoUEB9+/bVu+++m23jIeS9wYMHKzY2VjNnzlTbtm21e/duBQUF6csvv9Sbb76ZbXMu2N/x48e1Y8cOVapUSTVr1jQ6DvBAKLJ2lJ6eruXLl2v37t1KSUlR3bp11aNHD3l6ehodzSklJyfr+++/V9WqVVW9enWj4ziFVq1aKSkpSX379lWpUqWybfbEdYT2devWLb300ksaO3Ys9280ULly5aybrPj4+Gjnzp2qVKmSPvroI33yySfcCiyfXLt2TYmJiZKkihUrUmDzUfny5bVixQo1bNhQ3t7eSkhIUFBQkA4fPqy6deuyxD4fLF26VF27dpW7u7vNeFpampYvX66IiAiDkgEPjiILh9WlSxc1bdpUAwcO1I0bN1S7dm0dPXpUWVlZWr58uTp37mx0RIfn5eWluLi4HHfORf7w9fVVfHw8RdZAhQsX1t69e1WuXDn95S9/0eeff6769evr6NGjCg0NVUpKitERAbvy8vLSL7/8oqCgIJsim5CQoKZNm+rq1atGR3R4rq6uOnPmTLYVgZcuXVLJkiW5jyxMiWtk7ejQoUOaN2+eJkyYoPHjx9t8wP5+/PFH660VvvjiC2VmZurKlSuaPXs295HNJ9WqVdONGzeMjuHUOnTooNWrVxsdw6mxycqjKTExUS1atDA6hlMIDw/XmjVrrI/vrM5ZsGCBGjVqZFQsp5KVlZXjLfBOnjwpX19fAxIBD48LpOxk/vz5euWVV1S8eHGVLl3a5i8Pi8WiqKgoA9M5h6tXr6po0aKSpHXr1qlz587y8vLS008/zcYG+WTy5MkaNmyY3nnnHYWGhma7RtbHx8egZM6jcuXKGj9+vGJjYxUWFpZtOSX3b7S/O5usNGvWTCNHjlT79u01Z84c3bp1S9OnTzc6ntNKSUnJdg057GPixIl66qmntHfvXqWnp2vWrFnau3evtmzZwntgZ3Xq1JHFYpHFYlHLli1t9kbIyMjQ0aNH1bZtWwMTAg+OpcV2Ur58eQ0YMEAjRowwOorTqlKliiZMmKCnn35aFSpU0PLly9WiRQslJCSoZcuWunjxotERHZ6Ly+1FH3/8LfCd3wyzlMn+7rWk2GKx6MiRI/mYBhKbrOSX2bNn3/P4qVOn9I9//IO/h/JJYmKiJk+erISEBOu+ISNGjFBoaKjR0RzaW2+9Zf3fYcOG2Wx46ebmpsDAQHXu3Flubm5GRQQeGEXWTnx8fBQfH2+9+TTy3z//+U8NHjxYhQsXVvny5bVz5065uLjovffe0+eff64NGzYYHdHh/dlv2ps1a5ZPSQA4GxcXF/n7+9/1B/S0tDSdPXuWIgunsGTJEnXt2lUeHh5GRwHyDEXWTvr27at69erp5ZdfNjqKU9u+fbtOnDihJ5980vpbyDVr1sjPz0+NGzc2OJ3jS0pKUkBAQI5nZE+cOKFy5coZlMx5REZG5jhusVjk4eGhSpUq6dlnn7Uuw4d9xMTEKCYmRufPn1dmZqbNsUWLFhmUyrFVqFBBU6ZMUZcuXXI8Hh8fr7CwMIqsneRmJ2IuMwHwICiydjJp0iRNnz5dTz/9dI7XBnJdGpwBuyQar3nz5tq5c6cyMjJUtWpVSdLBgwfl6uqqatWq6cCBA7JYLNq8ebOCg4MNTuuY3nrrLY0fP17h4eHy9/fP9oudL774wqBkju25555TxYoVNWXKlByPJyQkqE6dOtl+sYC84eLikuPmQjnh3wL7y8jI0IwZM7Ry5UolJSUpLS3N5jj3dYcZUWTthOvSjJeRkaHFixff9SzI999/b1Ay5+Hi4qJz586pRIkSNuPHjx9XcHCwrl27ZlAy5zFz5kxt2rRJH374ofWsx9WrV9WvXz81adJE/fv3V/fu3XXjxg198803Bqd1TP7+/po6dap69uxpdBSnsnfvXl2/fl3h4eE5Hr9165ZOnz6t8uXL53My5/C/l5YcO3ZMI0eOVO/eva27FMfFxWnJkiWaNGkS9xTPB1FRUVqwYIGGDRumMWPGaPTo0Tp27JhWr16tqKgoTrDAlCiycFgDBw7U4sWL9fTTT+d4FmTGjBkGJXN8d5azzpo1S/3795eXl5f1WEZGhrZu3SpXV1fFxsYaFdFplC1bVt9++222s62//vqrWrdurVOnTmnnzp1q3bo1G6DZSbFixfTzzz+rYsWKRkcBDNGyZUv169dP3bp1sxlftmyZ5s2bp40bNxoTzIlUrFhRs2fP1tNPPy1vb2/Fx8dbx3766SctW7bM6IhArnH7HTis5cuXa+XKlWrXrp3RUZzOrl27JN2+FnbPnj02m624ubmpVq1aGj58uFHxnMrVq1d1/vz5bEX2woUL1mvY/Pz8si0zQ97p16+fli1bprFjxxodxSlNmDBBPXr0uOdKKdhXXFycoqOjs42Hh4erX79+BiRyPmfPnrXuEF24cGFdvXpVkvR///d//N0E06LI5qHIyEi9/fbbKlSo0F03WLmDewfan5ubmypVqmR0DKd0Z0foPn36aNasWWzkYaBnn31WL7zwgqZNm6Z69epJkrZt26bhw4erQ4cOkqSff/5ZVapUMTClY7t586bmzZun7777TjVr1sy2ZwL/HtjXp59+qnHjxqlBgwZ6/vnn1aVLFxUvXtzoWE4lICBA8+fP19SpU23GFyxYoICAAINSOZe//OUvOnPmjMqVK6eKFStq/fr1qlu3rrZt2yZ3d3ej4wEPhKXFeah58+b64osv5Ofnp+bNm991nsVi4frMfDBt2jQdOXJEc+bMue8NJ2BfycnJ+v7771WtWjVVq1bN6DhOISUlRUOHDtXSpUuVnp4uSSpQoIB69eqlGTNmqFChQoqPj5ck1a5d27igDox/D4z366+/6uOPP9by5ct18uRJPfnkk+rRo4c6dOhgc+kD7GPt2rXq3LmzKlWqpAYNGki6/Qu0Q4cOadWqVaycygcjR46Uj4+P/v73v2vFihV6/vnnFRgYqKSkJA0dOlSTJ082OiKQaxRZOKyOHTtqw4YNKlq0qGrUqJHtLMjnn39uUDLn0aVLFzVt2lQDBw7UjRs3VKtWLR07dkxZWVlavny5OnfubHREp5GSkmLdZC4oKMh6OyrA2cTGxmrZsmX69NNPdfPmzVzdJgYP7uTJk/rnP/+p/fv3S5KqV6+ul19+mTOyBomLi1NcXJwqV66s9u3bGx0HeCAsLYbD8vPzU8eOHY2O4dR+/PFHjR49WtLtW4xkZWXpypUrWrJkiSZMmECRzUeFCxdWzZo1jY7h9E6ePCnp9jI/GKNQoULy9PSUm5ubfv/9d6PjOI2//OUvmjhxotEx8P81atTIuoM0YFackc1DnTp1uu+5nA2EM/D09NTBgwcVEBCgiIgIlSlTRpMnT1ZSUpKCg4OVkpJidETA7jIzMzVhwgRNmzbN+t+8t7e3hg0bptGjR8vFxcXghI7v6NGjWrZsmZYtW6YDBw6oWbNm6t69u5577jn5+voaHc8pXLlyRQsXLtS+ffskSTVq1NALL7zA99+Ovvrqq/ue+8wzz9gxCWAfnJHNQ/xl/OhJT0/Xxo0blZiYqO7du8vb21unT5+Wj48PSyvzQUBAgOLi4lS0aFGtW7dOy5cvlyT99ttv8vDwMDgdkD9Gjx6thQsXavLkyWrcuLEkafPmzXrzzTd18+ZNvfPOOwYndGwNGzbUtm3bVLNmTfXp00fdunVT2bJljY7lVLZv3642bdrI09NT9evXl3R7k7N33nnHuukQ8t6dDf3usFgs+uP5qzt7iGRkZORXLCDPcEYWDuv48eNq27atkpKSlJqaqoMHDyooKEiDBw9WampqjrcCQN765z//qcGDB6tw4cIqV66cdu3aJRcXF7333nv6/PPPrbsbA46sTJkyio6OznbG48svv9SAAQN06tQpg5I5h9GjR6tHjx7ZbkGF/PP444+rUqVKmj9/vgoUuH0OJT09Xf369dORI0f0448/GpzQ8X333XcaMWKEJk6caF1SHBcXpzFjxmjixIl68sknDU4I5B5F1o44G2isDh06yNvbWwsXLlSxYsWUkJCgoKAgbdy4Uf3799ehQ4eMjugUduzYoaSkJLVu3VqFChWSJK1Zs0ZFihTRY489ZnA6wP48PDy0e/fubLc4OnDggGrXrq0bN24YlAzIH56entq1a1e23er37t2r8PBwXb9+3aBkziMkJETR0dFq0qSJzfimTZv04osvWpd8A2bC0mI7+ePZwCeffFLe3t6aMmUKZwPzyaZNm7Rlyxa5ubnZjAcGBnIGxI7udg/lTZs2ZRujyMIZ1KpVS3PmzNHs2bNtxufMmaNatWoZlMp5ZGRkaPHixYqJidH58+eVmZlpc5zbH9mfj4+PkpKSshXZEydOyNvb26BUziUxMVF+fn7Zxn19fXXs2LF8zwPkBYqsnQwePFjh4eFKSEhQsWLFrOMdO3ZU//79DUzmPDIzM3O85uPkyZP8w2lHu3btuq953NsXzmLq1Kl6+umn9d1339ks6Ttx4oTWrl1rcDrHN3jwYC1evFhPP/20QkJC+LvHAF27dlXfvn31j3/8w/oLzNjYWL3++uvq1q2bwemcQ7169RQZGamPPvpIpUqVkiSdO3dOr7/+uvW6ZcBsWFpsJ8WKFdOWLVtUtWpVeXt7W5e1Hjt2TMHBwSyjyQddu3aVr6+v5s2bJ29vb+3evVslSpTQs88+q3LlyunDDz80OiIAJ3H69Gm9//77NvfQHDBggMqUKWNwMsdXvHhxLV26VO3atTM6itNKS0vT66+/rujoaKWnp0uSChYsqFdeeUWTJ0+Wu7u7wQkd3+HDh9WxY0frnQSk22fEK1eurNWrV6tSpUoGJwRyjyJrJ0WKFFFsbKyCg4NtiuzmzZvVuXNnnTt3zuiIDu/kyZNq06aNsrKydOjQIYWHh+vQoUMqXry4fvzxR5UsWdLoiAAAOytTpow2btyY7Rpl5L/r168rMTFRklSxYkV5eXkZnMi5ZGVl6dtvv7X5hVqrVq1YpQDTosjaCWcDHw3p6elavny5du/erZSUFNWtW1c9evSQp6en0dEAOLDdu3crJCRELi4u2r179z3n1qxZM59SOadp06bpyJEjmjNnDj+wG+Tq1avKyMhQ0aJFbcYvX76sAgUKyMfHx6Bk+KPQ0FCtXbvWetYWeJRRZO2Es4EA4LxcXFx09uxZlSxZUi4uLjnev1G6fa0492+0r44dO2rDhg0qWrSoatSooYIFC9oc//zzzw1K5jyeeuoptW/fXgMGDLAZj46O1ldffcW14o+Q/11FCDzqKLJ2lJ6erhUrVighIYGzgfnkq6++uu+5f7ynIwDklePHj6tcuXKyWCw6fvz4PeeWL18+n1I5pz59+tzzOCuk7K9o0aKKjY1V9erVbcb379+vxo0b69KlSwYlwx9RZGEmFFk4FBcXF5vHOZ0FubO0jLMgAPLDjz/+qMcee0wFCtjeKCA9PV1btmxR06ZNDUoG5I9ChQrpp59+UmhoqM34nj171KBBAzbAfIRQZGEmLn8+BQ9iyZIlWrNmjfXxG2+8IT8/Pz322GN/+tt5PLjMzEzrx/r161W7dm395z//0ZUrV3TlyhX95z//Ud26dbVu3TqjowJwEs2bN9fly5ezjV+9elXNmzc3IJFzunDhgjZv3qzNmzfrwoULRsdxKvXr19e8efOyjUdHRyssLMyARAAcAWdk7aRq1aqaO3euWrRoobi4OLVs2VIzZ87U119/rQIFCnBNTj4ICQlRdHS0mjRpYjO+adMmvfjii9q3b59ByQA4ExcXF507d04lSpSwGT948KDCw8OVnJxsUDLncO3aNQ0aNEhLly5VZmamJMnV1VURERF677332Dk3H8TGxqpVq1aqV6+eWrZsKUmKiYnRtm3btH79ej3++OMGJ8QdnJGFmRT48yl4ECdOnLDek2v16tV67rnn9OKLL6px48Z64oknjA3nJBITE+Xn55dt3NfXV8eOHcv3PACcS6dOnSTdvpyhd+/eNvfKzMjI0O7du/XYY48ZFc9pREZG6ocfftC///1vNW7cWJK0efNmvfbaaxo2bJjmzp1rcELH17hxY8XFxendd9/VypUr5enpqZo1a2rhwoWqXLmy0fEAmBRF1k4KFy6sS5cuqVy5clq/fr0iIyMlSR4eHrpx44bB6ZxDvXr1FBkZqY8++kilSpWSJJ07d06vv/666tevb3A6AI7O19dX0u17N3p7e9ts9Ofm5qaGDRuqf//+RsVzGqtWrdJnn31m80vkdu3aydPTU126dKHI5pPatWvr448/NjqG01q6dKm6du1q8ws1SUpLS9Py5csVEREhSfrggw+sPzMBjzqWFttJjx49tH//ftWpU0effPKJkpKSVKxYMX311Vf6+9//rl9++cXoiA7v8OHD6tixow4ePGi9H9qJEydUuXJlrV692nrGHADs6a233tLrr7/OElaDeHl5aceOHdl2zP31119Vv359Xbt2zaBkziUzM1OHDx/W+fPnrUu872DDM/tzdXXVmTNnst3+8dKlSypZsiQbYMKUKLJ2cuXKFY0ZM0YnTpzQK6+8orZt20qSxo0bJzc3N40ePdrghM4hKytL3377rfbv3y9Jql69ulq1amXduRgA7O3o0aNKT0/PtoTy0KFDKliwoAIDA40J5iRatmypYsWKaenSpfLw8JAk3bhxQ7169dLly5f13XffGZzQ8f3000/q3r27jh8/nuOdBChR9ne3a/UTEhLuuiEd8KijyMLphYaGau3atdaztgCQl5o1a6YXXnhBvXr1shn/17/+pQULFmjjxo3GBHMSe/bsUdu2bZWamqpatWpJuv3Du7u7u9avX68aNWoYnNDx1a5dW1WqVNFbb70lf3//bL9MvrMMH3mvTp06slgsSkhIUI0aNWxuA5aRkaGjR4+qbdu2WrlypYEpgQdDkbWz69evKykpSWlpaTbjNWvWNCgR/ogd+gDYk4+Pj3bu3JntcobDhw8rPDxcV65cMSaYE7l+/bo+/vhjm9U5PXr0sLluGfZTqFAhJSQkcEmPAd566y3r/w4bNkyFCxe2HnNzc1NgYKA6d+4sNzc3oyICD4zNnuzkwoUL6t27913vV8oyGgBwDhaLRb///nu28atXr/JvQT6YNGmSSpUqlW1jrUWLFunChQsaMWKEQcmcR4MGDXT48GGKrAHGjRsnSQoMDFTXrl2ty+sBR+BidABHNWTIEF29elVbt26Vp6en1q1bpyVLlqhy5cr66quvjI4HAMgnTZs21aRJk2xKa0ZGhiZNmpTtPtfIex988IGqVauWbbxGjRqKjo42IJHzGTRokIYNG6bFixdrx44d2r17t80H7K9Xr17y8PBQWlqaTp48qaSkJJsPwIxYWmwn/v7++vLLL1W/fn35+Pho+/btqlKlir766itNnTpVmzdvNjoi/j+WFgOwp71796pp06by8/PT448/LknatGmTkpOT9f333yskJMTghI7Nw8ND+/btU4UKFWzGjxw5ouDgYN28edOgZM7DxSX7eROLxaKsrCw2e8onhw4d0gsvvKAtW7bYjPMewMxYWmwn165ds25xXqRIEV24cEFVqlRRaGiodu7caXA6AEB+CQ4O1u7duzVnzhwlJCTI09NTERERGjhwoIoWLWp0PIcXEBCg2NjYbEU2NjZWZcqUMSiVczl69KjREZxe7969VaBAAX399dc5brgFmBFF1k6qVq2qAwcOKDAwULVq1dIHH3ygwMBARUdHy9/f3+h4AIB8VKZMGU2cONHoGE6pf//+GjJkiG7duqUWLVpIkmJiYvTGG29o2LBhBqdzDuXLlzc6gtOLj4/Xjh07clxmD5gVRdZOBg8erDNnzki6faF927Zt9a9//Utubm5asmSJwemcz82bN++6wcEHH3ygUqVK5XMiAM5k06ZN+uCDD3TkyBF9+umnKlu2rD766CNVqFCB62Tt7PXXX9elS5c0YMAA6x0EPDw8NGLECI0aNcrgdM7jo48+UnR0tI4ePaq4uDiVL19eM2fOVIUKFfTss88aHc/hBQcH6+LFi0bHAPIUmz3ZyfPPP6/evXtLkurWravjx49r+/btOnnypLp27WpsOCeRmZmpt99+W2XLllXhwoV15MgRSdLYsWO1cOFC67zu3burUKFCRsUE4OBWrVqlNm3ayNPTUzt37lRqaqqk27sWc5bW/iwWi6ZMmaILFy7op59+UkJCgi5fvqyoqCijozmNuXPnKjIyUu3atdOVK1es12P6+flp5syZxoZzElOmTNEbb7yhjRs36tKlS0pOTrb5AMyIImtHCxcuVEhIiDw8PFSkSBFFRERo9erVRsdyGhMmTNDixYs1depUm/ujhYSEaMGCBQYmA+BMJkyYoOjoaM2fP18FCxa0jjdu3Jg9E/JR4cKFVa9ePYWEhMjd3d3oOE7lvffe0/z58zV69Gi5urpax8PDw7Vnzx4DkzmPVq1a6aefflLLli1VsmRJFSlSREWKFJGfn5+KFClidDzggbC02E6ioqI0ffp0DRo0SI0aNZIkxcXFaejQoUpKStL48eMNTuj4li5dqnnz5qlly5Z6+eWXreO1atXS/v37DUwGwJkcOHBATZs2zTbu6+urK1eu5H8gIJ8dPXpUderUyTbu7u6ua9euGZDI+WzYsMHoCECeo8jaydy5czV//nx169bNOvbMM8+oZs2aGjRoEEU2H5w6dSrHm69nZmbq1q1bBiQC4IxKly6tw4cPKzAw0GZ88+bN3PYLTqFChQqKj4/PtunTunXrVL16dYNSOZdmzZoZHQHIcywttpNbt24pPDw823hYWJjS09MNSOR8goODtWnTpmzjn332WY6/GQYAe+jfv78GDx6srVu3ymKx6PTp0/r44481fPhwvfLKK0bHA+wuMjJSr776qlasWKGsrCz9/PPPeueddzRq1Ci98cYbRsdzGps2bdLzzz+vxx57TKdOnZJ0exOuzZs3G5wMeDCckbWTnj17au7cuZo+fbrN+Lx589SjRw+DUjmXqKgo9erVS6dOnVJmZqY+//xzHThwQEuXLtXXX39tdDwATmLkyJHKzMxUy5Ytdf36dTVt2lTu7u4aPny4Bg0aZHQ8wO769esnT09PjRkzRtevX1f37t1VpkwZzZo1S3/729+MjucUVq1apZ49e6pHjx45bjq3du1agxMCuWfJysrKMjqEo4iMjLR+np6ersWLF6tcuXJq2LChJGnr1q1KSkpSRESE3nvvPaNiOpVNmzZp/PjxSkhIUEpKiurWrauoqCi1bt3a6GgAnEBGRoZiY2NVs2ZNeXl56fDhw0pJSVFwcLAKFy5sdDwg312/fl0pKSkqWbJktmOxsbEKDw9nMy47qFOnjoYOHaqIiAh5e3srISFBQUFB2rVrl5566imdPXvW6IhArlFk81Dz5s3va57FYtH3339v5zQAgEeBh4eH9u3bpwoVKhgdBXik+fj4KD4+nmvH7cDLy0t79+5VYGCgTZE9cuSIgoODdfPmTaMjArnG0uI8xI5wAIA/CgkJ0ZEjRyiywJ/g3Ir9sOkcHBFFFg6lSJEislgs9zX38uXLdk4DALfvIzt8+HC9/fbbCgsLU6FChWyO+/j4GJQMgLO4s+ncokWLrJvOxcXFafjw4Ro7dqzR8YAHQpGFQ5k5c6bREQDARrt27STdvgXb//6iLSsrSxaLRRkZGUZFA+Ak2HQOjohrZAEAsKMffvjhnse5vyNw2/9euwn7SEtLY9M5OAyKLBxaRkaGvvjiC+3bt0/S7XvLPvvssypQgMUIAAA8StjsyX6uXr2qjIwMFS1a1Gb88uXLKlCgAJc4wJT4aR4O69dff9Uzzzyjs2fPqmrVqpKkKVOmqESJEvr3v/+tkJAQgxMCcFS7d+9WSEiIXFxctHv37nvOrVmzZj6lAh5tnFuxn7/97W9q3769BgwYYDO+cuVKffXVV9xHFqbEGVk4rEaNGqlEiRJasmSJihQpIkn67bff1Lt3b124cEFbtmwxOCEAR+Xi4qKzZ8+qZMmScnFxkcViyfGHdK6RhbNIT0/Xxo0blZiYqO7du8vb21unT5+Wj48Py1vzQdGiRRUbG6vq1avbjO/fv1+NGzfWpUuXDEoGPDjOyMJhxcfHa/v27dYSK93e1fidd95RvXr1DEwGwNEdPXpUJUqUsH4OOLPjx4+rbdu2SkpKUmpqqp588kl5e3trypQpSk1NVXR0tNERHV5qaqrS09Ozjd+6dUs3btwwIBHw8FyMDgDYS5UqVXTu3Lls4+fPn1elSpUMSATAWZQvX966Q3H58uXv+QE4usGDBys8PFy//fabPD09reMdO3ZUTEyMgcmcR/369TVv3rxs49HR0QoLCzMgEfDwOCMLhzVp0iS99tprevPNN9WwYUNJ0k8//aTx48drypQpSk5Ots5lkwMAeemrr76677nPPPOMHZMAxtu0aZO2bNkiNzc3m/HAwECdOnXKoFTOZcKECWrVqpUSEhLUsmVLSVJMTIy2bdum9evXG5wOeDBcIwuH5eLy3wUHd86M3PnP/X8fc40agLz2v3//SMp2jez/3k+Wv3/g6IoUKaLY2FgFBwfb3GJn8+bN6ty5c46rp5D3EhISNHXqVMXHx8vT01M1a9bUqFGjVLlyZaOjAQ+EM7JwWBs2bDA6AgAnlZmZaf38u+++04gRIzRx4kQ1atRIkhQXF6cxY8Zo4sSJRkUE8k3r1q01c+ZM69JWi8WilJQUjRs3Tu3atTM4neO7deuWXnrpJY0dO1Yff/yx0XGAPMMZWQAA7CgkJETR0dFq0qSJzfimTZv04osvWu9zDTiqkydPqk2bNsrKytKhQ4cUHh6uQ4cOqXjx4vrxxx9VsmRJoyM6PF9fX8XHx6tChQpGRwHyDEUWDu3mzZvavXu3zp8/b3OGROK6NAD5w9PTU9u2bct27+rdu3erQYMG7BgKp5Cenq4VK1YoISFBKSkpqlu3rnr06GGz+RPsp1evXqpdu7aGDh1qdBQgz1Bk4bDWrVuniIgIXbx4MdsxrosFkF+aNm0qDw8PffTRRypVqpQk6dy5c4qIiNDNmzf1ww8/GJwQgKObMGGCpk2bppYtWyosLEyFChWyOf7aa68ZlAx4cBRZOKzKlSurdevWioqKsv7wCAD57fDhw+rYsaMOHjyogIAASdKJEydUuXJlrV69mtuBweFNmjRJpUqV0gsvvGAzvmjRIl24cEEjRowwKJnzuNeSYovFoiNHjuRjGiBvUGThsHx8fLRr1y5VrFjR6CgAnFxWVpa+/fZb7d+/X5JUvXp1tWrVymb3YsBRBQYGatmyZXrsscdsxrdu3aq//e1vOnr0qEHJAJgZuxbDYT333HPauHEjRRaA4SwWi1q3bq3WrVsbHQXId2fPnpW/v3+28RIlSujMmTMGJHJeaWlpOnr0qCpWrKgCBagBMDf+C4bDmjNnjv76179q06ZNCg0NVcGCBW2Ocz0IAHuZPXu2XnzxRXl4eGj27Nn3nMvfRXB0AQEBio2Nzba8NTY2VmXKlDEolXO5fv26Bg0apCVLlkiSDh48qKCgIA0aNEhly5bVyJEjDU4I5B5Li+GwFi5cqJdfflkeHh4qVqyYzRI+rgcBYE8VKlTQ9u3bVaxYMa5Ng9ObOnWqpk6dqnfffVctWrSQJMXExOiNN97QsGHDNGrUKIMTOr7BgwcrNjZWM2fOVNu2bbV7924FBQXpyy+/1Jtvvqldu3YZHRHINYosHFbp0qX12muvaeTIkXJxcTE6DgDozj+5XBsLZ5KVlaWRI0dq9uzZSktLkyR5eHhoxIgRioqKMjidcyhfvrxWrFihhg0bytvbWwkJCQoKCtLhw4dVt25dJScnGx0RyDV+uofDSktLU9euXSmxAAy3cOFChYSEyMPDQx4eHgoJCdGCBQuMjgXkC4vFoilTpujChQv66aeflJCQoMuXL1Ni89GFCxdUsmTJbOPXrl3jF2swLX7Ch8Pq1auXVqxYYXQMAE4uKipKgwcPVvv27fXpp5/q008/Vfv27TV06FB+kIdTKVy4sOrVq6eQkBC5u7sbHcephIeHa82aNdbHd8rrggUL1KhRI6NiAQ+FpcVwWK+99pqWLl2qWrVqqWbNmtk2e5o+fbpByQA4kxIlSmj27Nnq1q2bzfgnn3yiQYMG6eLFiwYlA/LHtWvXNHnyZMXExOj8+fPKzMy0Oc514va3efNmPfXUU3r++ee1ePFivfTSS9q7d6+2bNmiH374QWFhYUZHBHKNXYvhsPbs2aM6depIkn755RebYyyjAZBfbt26pfDw8GzjYWFhSk9PNyARkL/69eunH374QT179pS/vz//BhugSZMmio+P1+TJkxUaGqr169erbt26iouLU2hoqNHxgAfCGVkAAOxo0KBBKliwYLZVIMOHD9eNGzf0/vvvG5QMyB9+fn5as2aNGjdubHQUAA6EM7IAAOSxyMhI6+cWi0ULFizQ+vXr1bBhQ0nS1q1blZSUpIiICKMiAvmmSJEiKlq0qNExnF5GRoa++OIL7du3T5IUHBysZ599VgUKUAdgTpyRhUPp1KmTFi9eLB8fH3Xq1Omecz///PN8SgXA2TRv3vy+5lksFn3//fd2TgMY61//+pe+/PJLLVmyRF5eXkbHcUq//vqrnnnmGZ09e1ZVq1aVJB08eFAlSpTQv//9b4WEhBicEMg9fgUDh+Lr62u99sbX19fgNACc1YYNG4yOADwypk2bpsTERJUqVUqBgYHZNl/cuXOnQcmcR79+/VSjRg1t375dRYoUkST99ttv6t27t1588UVt2bLF4IRA7nFGFg7rxo0byszMVKFChSRJx44d0+rVq1W9enW1adPG4HQAADiHt956657Hx40bl09JnJenp6e2b9+uGjVq2Iz/8ssvqlevnm7cuGFQMuDBcUYWDuvZZ59Vp06d9PLLL+vKlStq2LChChYsqIsXL2r69Ol65ZVXjI4IAIDDo6gar0qVKjp37ly2Inv+/HlVqlTJoFTAw3ExOgBgLzt37tTjjz8uSfrss89UqlQpHT9+XEuXLtXs2bMNTgcAgPO4cuWKFixYoFGjRuny5cuSbv87ferUKYOTOYdJkybptdde02effaaTJ0/q5MmT+uyzzzRkyBBNmTJFycnJ1g/ALFhaDIfl5eWl/fv3q1y5curSpYtq1KihcePG6cSJE6pataquX79udEQAABze7t271apVK/n6+urYsWM6cOCAgoKCNGbMGCUlJWnp0qVGR3R4Li7/PXd1Zy+ROxXgfx9bLBZlZGTkf0DgAbC0GA6rUqVKWr16tTp27KhvvvlGQ4cOlXR7GY2Pj4/B6QAAcA6RkZHq3bu3pk6dKm9vb+t4u3bt1L17dwOTOQ82oIMjosjCYUVFRal79+4aOnSoWrZsqUaNGkmS1q9frzp16hicDgAA57Bt2zZ98MEH2cbLli2rs2fPGpDI+TRr1uy+5g0YMEA1atRQ8eLF7ZwIeHhcIwuH9dxzzykpKUnbt2/XunXrrOMtW7bUjBkzDEwGAIDzcHd3z/Hayzv3McWj41//+hfXycI0KLJwaKVLl1adOnVsrg2pX7++qlWrZmAqAACcxzPPPKPx48fr1q1bkm5fk5mUlKQRI0aoc+fOBqfD/2LrHJgJRRYAAAB2M23aNKWkpKhkyZK6ceOGmjVrpkqVKsnb21vvvPOO0fEAmBTXyAIAAMBufH199e233yo2NlYJCQlKSUlR3bp11apVK6OjATAxbr8DAAAAu1m6dKm6du0qd3d3m/G0tDQtX75cERERBiXDH3l7eyshIUFBQUFGRwH+FEUWAAAAduPq6qozZ86oZMmSNuOXLl1SyZIluW/pI4QiCzPhGlkAAADYTVZWliwWS7bxkydPytfX14BEuJvnn39ePj4+RscA7gvXyAIAACDP1alTRxaLRRaLRS1btlSBAv/9sTMjI0NHjx5V27ZtDUzoPH7++WfFxcVZ79tbunRpNWrUSPXr17eZN3fuXCPiAQ+EIgsAAIA816FDB0lSfHy82rRpo8KFC1uPubm5KTAwkNvv2Nn58+fVuXNnxcbGqly5cipVqpQk6dy5cxo6dKgaN26sVatWZVv2DZgB18gCAADAbpYsWaKuXbvKw8PD6ChO57nnntPp06f14YcfqmrVqjbHDhw4oBdeeEFlypTRp59+alBC4MFRZAEAAGB3aWlpOn/+vDIzM23Gy5UrZ1Aix+ft7a0ff/xRderUyfH4jh079MQTT+j333/P52TAw2NpMQAAAOzm0KFDeuGFF7Rlyxab8TubQLFrsf24u7srOTn5rsd///33bLdFAsyCIgsAAAC76d27twoUKKCvv/5a/v7+Oe5gDPvo2rWrevXqpRkzZqhly5bWHYmTk5MVExOjyMhIdevWzeCUwINhaTEAAADsplChQtqxY4eqVatmdBSnk5qaqiFDhmjRokVKT0+Xm5ubpNvLvAsUKKC+fftqxowZnJWFKVFkAQAAYDf16tXTjBkz1KRJE6OjOK3k5GTt2LHD5vY7YWFh3DMWpkaRBQAAgN18//33GjNmjCZOnKjQ0FAVLFjQ5jhlCsCDoMgCAADAblxcXCQp27WxbPZkvHPnzumDDz5QVFSU0VGAXKPIAgAAwG5++OGHex5v1qxZPiXBHyUkJKhu3br8MgGmxK7FAAAAsBuKqnF27959z+MHDhzIpyRA3uOMLAAAAOxq06ZN+uCDD3TkyBF9+umnKlu2rD766CNVqFCBTaDsyMXFRRaLRTn9uH9nnOXdMCsXowMAAADAca1atUpt2rSRp6endu7cqdTUVEnS1atXNXHiRIPTObaiRYtq/vz5Onr0aLaPI0eO6OuvvzY6IvDAWFoMAAAAu5kwYYKio6MVERGh5cuXW8cbN26sCRMmGJjM8YWFhen06dMqX758jsevXLmS49lawAwosgAAALCbAwcOqGnTptnGfX19deXKlfwP5ERefvllXbt27a7Hy5Urpw8//DAfEwF5hyILAAAAuyldurQOHz6swMBAm/HNmzcrKCjImFBOomPHjvc8XqRIEfXq1Suf0gB5i2tkAQAAYDf9+/fX4MGDtXXrVlksFp0+fVoff/yxhg8frldeecXoePgfPj4+OnLkiNExgPvCGVkAAADYzciRI5WZmamWLVvq+vXratq0qdzd3TV8+HANGjTI6Hj4H1wvCzPh9jsAAACwu7S0NB0+fFgpKSkKDg5W4cKFjY6EP/D29lZCQgJLvmEKLC0GAACA3bm5uSk4OFjVqlXTd999p3379hkdCYCJUWQBAABgN126dNGcOXMkSTdu3FC9evXUpUsX1axZU6tWrTI4HQCzosgCAADAbn788Uc9/vjjkqQvvvhCmZmZunLlimbPns19ZB8xFovF6AjAfaPIAgAAwG6uXr2qokWLSpLWrVunzp07y8vLS08//bQOHTpkcDr8L7bOgZlQZAEAAGA3AQEBiouL07Vr17Ru3Tq1bt1akvTbb7/Jw8PD4HTOJysr666F9T//+Y/Kli2bz4mAB0ORBQAAgN0MGTJEPXr00F/+8heVKVNGTzzxhKTbS45DQ0ONDedEFi5cqJCQEHl4eMjDw0MhISFasGCBzZwmTZrI3d3doIRA7nD7HQAAANjVjh07lJSUpCeffNJ62501a9bIz89PjRs3Njid44uKitL06dM1aNAgNWrUSJIUFxenOXPmaOjQoRo/frzBCYHco8gCAADAcD4+PoqPj+cepnZQokQJzZ49W926dbMZ/+STTzRo0CBdvHjRoGTAg2NpMQAAAAzHuRX7uXXrlsLDw7ONh4WFKT093YBEwMOjyAIAAAAOrGfPnpo7d2628Xnz5qlHjx4GJAIeXgGjAwAAAADIW5GRkdbPLRaLFixYoPXr16thw4aSpK1btyopKUkRERFGRQQeCkUWAAAAcDC7du2yeRwWFiZJSkxMlCQVL15cxYsX16+//prv2YC8QJEFAACA4SwWi9ERHMqGDRuMjgDYFdfIAgAAwHBs9gQgNzgjCwAAgHxxp6zmdPb1P//5j8qWLZvfkZxC8+bN73nG+/vvv8/HNEDe4IwsAAAA7GrhwoUKCQmRh4eHPDw8FBISogULFtjMadKkidzd3Q1K6Nhq166tWrVqWT+Cg4OVlpamnTt3KjQ01Oh4wAPhjCwAAADsJioqStOnT9egQYPUqFEjSVJcXJyGDh2qpKQkjR8/3uCEjm/GjBk5jr/55ptKSUnJ5zRA3rBkcUECAAAA7KREiRKaPXu2unXrZjP+ySefaNCgQbp48aJByXD48GHVr19fly9fNjoKkGssLQYAAIDd3Lp1S+Hh4dnGw8LClJ6ebkAi3BEXFycPDw+jYwAPhKXFAAAAsJuePXtq7ty5mj59us34vHnz1KNHD4NSOZdOnTrZPM7KytKZM2e0fft2jR071qBUwMOhyAIAACBPRUZGWj+3WCxasGCB1q9fr4YNG0qStm7dqqSkJEVERBgV0an4+vraPHZxcVHVqlU1fvx4tW7d2qBUwMPhGlkAAADkqebNm9/XPIvFwq1fADwQiiwAAADgBNLS0nT+/HllZmbajJcrV86gRMCDY2kxAAAA4MAOHjyovn37asuWLTbjWVlZslgsysjIMCgZ8OAosgAAALCb5s2by2Kx3PU4S4vtr0+fPipQoIC+/vpr+fv73/P9AMyCIgsAAAC7qV27ts3jW7duKT4+Xr/88ot69eplTCgnEx8frx07dqhatWpGRwHyDEUWAAAAdjNjxowcx998802lpKTkcxrnFBwcrIsXLxodA8hTbPYEAACAfHf48GHVr19fly9fNjqKQ0pOTrZ+vn37do0ZM0YTJ05UaGioChYsaDPXx8cnv+MBD40zsgAAAMh3cXFx8vDwMDqGw/Lz87O5FjYrK0stW7a0mcNmTzAziiwAAADsplOnTjaPs7KydObMGW3fvl1jx441KJXj27Bhg9ERALtiaTEAAADspk+fPjaPXVxcVKJECbVo0UKtW7c2KBVyMmDAAI0fP17Fixc3OgrwpyiyAAAAAOTj46P4+HgFBQUZHQX4UywtBgAAgN2lpaXp/PnzyszMtBkvV66cQYnwR5zfgplQZAEAAGA3Bw8eVN++fbVlyxabcTYaAvAwKLIAAACwmz59+qhAgQL6+uuv5e/vb7OTLgA8KIosAAAA7CY+Pl47duxQtWrVjI4CwIG4GB0AAAAAjis4OFgXL140OgYAB0ORBQAAQJ5KTk62fkyZMkVvvPGGNm7cqEuXLtkcS05ONjqqw+rUqZP1+7t06VKlpqb+6XOef/55+fj42DsakCe4/Q4AAADylIuLi821sHc2dvpfbPZkX25ubjp+/Lj8/f3l6uqqM2fOqGTJkkbHAvIM18gCAAAgT23YsMHoCE6vWrVqGjVqlJo3b66srCytXLnyrmdbIyIi8jkd8PA4IwsAAADDDRgwQOPHj1fx4sWNjuIQtmzZosjISCUmJury5cvy9vbOccdoi8Wiy5cvG5AQeDgUWQAAABjOx8dH8fHxCgoKMjqKw3FxcdGpU6fk7+9vM56VlaWkpCSVL1/eoGTAg2OzJwAAABiOcyv57/Lly/ziAKZFkQUAAAAcnKura7axlJQUeXh4GJAGeHhs9gQAAAA4oMjISEm3r4ONioqSl5eX9VhGRoa2bt2q2rVrG5QOeDgUWQAAAMAB7dq1S9LtZdt79uyRm5ub9Zibm5tq1aql4cOHGxUPeCgUWQAAAMAB3bkNUp8+fTRr1qy73n4HMCOukQUAAECe6tSpk5KTkyVJS5cuVWpq6p8+5/nnn6do2cmHH37I9xYOh9vvAAAAIE+5ubnp+PHj8vf3l6urq86cOaOSJUsaHQuAA2FpMQAAAPJUtWrVNGrUKDVv3lxZWVlauXLlXc8IRkRE5HM6AI6AM7IAAADIU1u2bFFkZKQSExN1+fJleXt7y2KxZJtnsVh0+fJlAxICMDuKLAAAAOzGxcVFp06dkr+/v814VlaWkpKSVL58eYOSATAzNnsCAABAvrt8+bKCgoKMjgHApCiyAAAAsCtXV9dsYykpKfLw8DAgDQBHwGZPAAAAyHORkZGSbl8HGxUVJS8vL+uxjIwMbd26VbVr1zYoHQCzo8gCAAAgz+3atUvS7Wth9+zZIzc3N+sxNzc31apVS8OHDzcqHgCTY7MnAAAA2E2fPn00a9asu95+BwAeBEUWAAAAAGAqbPYEAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABM5f8BjIp8zW3mrLAAAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Sort model results by f1-score\n",
        "all_model_results.sort_values(\"f1\", ascending=False)[\"f1\"].plot(kind=\"bar\", figsize=(10, 7));"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 763
        },
        "id": "Py-zNB1orhJw",
        "outputId": "e8fd739e-1d7e-40a6-f157-d24e61924f10"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1000x700 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzoAAALqCAYAAAD9+CEvAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABUy0lEQVR4nO3deXxNd+L/8fdNSGJJYo+lIZQiYk1QpToqxdRPW9VhUEFVpwtVocVUo1W1zVBVncZadGpoabVTxmhTikipJdHNvsTSRFAilkSS+/vD123vJIx0cu7Rz309H4/7qHzOucm7LpL3PZ/z+TicTqdTAAAAAGAQH7sDAAAAAEBxo+gAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOCXsDnAz8vPzdeLECQUGBsrhcNgdBwAAAIBNnE6nzp8/r+rVq8vH5/rXbX4TRefEiRMKDQ21OwYAAACAW8TRo0d12223Xff4b6LoBAYGSrr6PxMUFGRzGgAAAAB2yczMVGhoqKsjXM9vouhcm64WFBRE0QEAAADwX29pYTECAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA45SwO8CtImz0Krsj/E8OT+5qdwQAAADglsEVHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHfXRwy2AvIwAAABQXrugAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4JewOAODWETZ6ld0R/ieHJ3e1OwIAALhFcEUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcUrYHQAA8LOw0avsjvA/OTy5q90RAACQxBUdAAAAAAai6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjsLw0AAD/57e+vLf021/im9cAQHHhig4AAAAA43BFBwAAAC5cVYMpuKIDAAAAwDgUHQAAAADGoegAAAAAMA736AAAAAC3EO6TKh5c0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxflXReeuttxQWFqaAgAC1bt1aW7duveH5M2bMUP369VWqVCmFhoZq+PDhunz58q8KDAAAAAD/TZGLzrJlyxQbG6tx48Zpx44datq0qTp37qyTJ08Wev6SJUs0evRojRs3Tj/88IPmz5+vZcuW6c9//vP/HB4AAAAAClPkojN9+nQNHjxYAwcOVHh4uOLj41W6dGktWLCg0PM3b96stm3bqk+fPgoLC1OnTp3Uu3fv/3oVCAAAAAB+rSIVnZycHG3fvl3R0dE/fwIfH0VHRyspKanQ59x1113avn27q9gcPHhQq1ev1v3333/dr5Odna3MzEy3BwAAAADcrBJFOfnUqVPKy8tTSEiI23hISIh2795d6HP69OmjU6dOqV27dnI6ncrNzdWTTz55w6lrkyZN0iuvvFKUaAAAAADgYvmqa+vXr9fEiRP1t7/9TTt27NCHH36oVatW6dVXX73uc8aMGaNz5865HkePHrU6JgAAAACDFOmKTqVKleTr66v09HS38fT0dFWtWrXQ57z00kvq16+fHn/8cUlS48aNdeHCBT3xxBN68cUX5eNTsGv5+/vL39+/KNEAAAAAwKVIV3T8/PwUGRmphIQE11h+fr4SEhLUpk2bQp9z8eLFAmXG19dXkuR0OouaFwAAAAD+qyJd0ZGk2NhY9e/fX1FRUWrVqpVmzJihCxcuaODAgZKkmJgY1ahRQ5MmTZIkdevWTdOnT1fz5s3VunVr7d+/Xy+99JK6devmKjwAAAAAUJyKXHR69eqljIwMxcXFKS0tTc2aNdOaNWtcCxSkpqa6XcEZO3asHA6Hxo4dq+PHj6ty5crq1q2bXnvtteL7vwAAAACAXyhy0ZGkIUOGaMiQIYUeW79+vfsXKFFC48aN07hx437NlwIAAACAIrN81TUAAAAA8DSKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcX5V0XnrrbcUFhamgIAAtW7dWlu3br3h+WfPntUzzzyjatWqyd/fX3fccYdWr179qwIDAAAAwH9ToqhPWLZsmWJjYxUfH6/WrVtrxowZ6ty5s/bs2aMqVaoUOD8nJ0f33XefqlSpouXLl6tGjRo6cuSIypUrVxz5AQAAAKCAIhed6dOna/DgwRo4cKAkKT4+XqtWrdKCBQs0evToAucvWLBAZ86c0ebNm1WyZElJUlhY2P+WGgAAAABuoEhT13JycrR9+3ZFR0f//Al8fBQdHa2kpKRCn/PJJ5+oTZs2euaZZxQSEqKIiAhNnDhReXl51/062dnZyszMdHsAAAAAwM0qUtE5deqU8vLyFBIS4jYeEhKitLS0Qp9z8OBBLV++XHl5eVq9erVeeuklTZs2TRMmTLju15k0aZKCg4Ndj9DQ0KLEBAAAAODlLF91LT8/X1WqVNGcOXMUGRmpXr166cUXX1R8fPx1nzNmzBidO3fO9Th69KjVMQEAAAAYpEj36FSqVEm+vr5KT093G09PT1fVqlULfU61atVUsmRJ+fr6usYaNmyotLQ05eTkyM/Pr8Bz/P395e/vX5RoAAAAAOBSpCs6fn5+ioyMVEJCgmssPz9fCQkJatOmTaHPadu2rfbv36/8/HzX2N69e1WtWrVCSw4AAAAA/K+KPHUtNjZWc+fO1aJFi/TDDz/oqaee0oULF1yrsMXExGjMmDGu85966imdOXNGw4YN0969e7Vq1SpNnDhRzzzzTPH9XwAAAADALxR5eelevXopIyNDcXFxSktLU7NmzbRmzRrXAgWpqany8fm5P4WGhurf//63hg8friZNmqhGjRoaNmyYRo0aVXz/FwAAAADwC0UuOpI0ZMgQDRkypNBj69evLzDWpk0bffXVV7/mSwEAAABAkVm+6hoAAAAAeBpFBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABjnVxWdt956S2FhYQoICFDr1q21devWm3re0qVL5XA49NBDD/2aLwsAAAAAN6XIRWfZsmWKjY3VuHHjtGPHDjVt2lSdO3fWyZMnb/i8w4cPa+TIkbr77rt/dVgAAAAAuBlFLjrTp0/X4MGDNXDgQIWHhys+Pl6lS5fWggULrvucvLw89e3bV6+88orq1KnzPwUGAAAAgP+mSEUnJydH27dvV3R09M+fwMdH0dHRSkpKuu7zxo8frypVqmjQoEE39XWys7OVmZnp9gAAAACAm1WkonPq1Cnl5eUpJCTEbTwkJERpaWmFPmfTpk2aP3++5s6de9NfZ9KkSQoODnY9QkNDixITAAAAgJezdNW18+fPq1+/fpo7d64qVap0088bM2aMzp0753ocPXrUwpQAAAAATFOiKCdXqlRJvr6+Sk9PdxtPT09X1apVC5x/4MABHT58WN26dXON5efnX/3CJUpoz549uv322ws8z9/fX/7+/kWJBgAAAAAuRbqi4+fnp8jISCUkJLjG8vPzlZCQoDZt2hQ4v0GDBvrmm2+UnJzsejzwwAPq0KGDkpOTmZIGAAAAwBJFuqIjSbGxserfv7+ioqLUqlUrzZgxQxcuXNDAgQMlSTExMapRo4YmTZqkgIAARUREuD2/XLlyklRgHAAAAACKS5GLTq9evZSRkaG4uDilpaWpWbNmWrNmjWuBgtTUVPn4WHrrDwAAAADcUJGLjiQNGTJEQ4YMKfTY+vXrb/jchQsX/povCQAAAAA3jUsvAAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMM6vKjpvvfWWwsLCFBAQoNatW2vr1q3XPXfu3Lm6++67Vb58eZUvX17R0dE3PB8AAAAA/ldFLjrLli1TbGysxo0bpx07dqhp06bq3LmzTp48Wej569evV+/evbVu3TolJSUpNDRUnTp10vHjx//n8AAAAABQmCIXnenTp2vw4MEaOHCgwsPDFR8fr9KlS2vBggWFnv/ee+/p6aefVrNmzdSgQQPNmzdP+fn5SkhI+J/DAwAAAEBhilR0cnJytH37dkVHR//8CXx8FB0draSkpJv6HBcvXtSVK1dUoUKF656TnZ2tzMxMtwcAAAAA3KwiFZ1Tp04pLy9PISEhbuMhISFKS0u7qc8xatQoVa9e3a0s/adJkyYpODjY9QgNDS1KTAAAAABezqOrrk2ePFlLly7VRx99pICAgOueN2bMGJ07d871OHr0qAdTAgAAAPitK1GUkytVqiRfX1+lp6e7jaenp6tq1ao3fO5f//pXTZ48WZ9//rmaNGlyw3P9/f3l7+9flGgAAAAA4FKkKzp+fn6KjIx0W0jg2sICbdq0ue7zpk6dqldffVVr1qxRVFTUr08LAAAAADehSFd0JCk2Nlb9+/dXVFSUWrVqpRkzZujChQsaOHCgJCkmJkY1atTQpEmTJElTpkxRXFyclixZorCwMNe9PGXLllXZsmWL8X8FAAAAAK4qctHp1auXMjIyFBcXp7S0NDVr1kxr1qxxLVCQmpoqH5+fLxS9/fbbysnJ0SOPPOL2ecaNG6eXX375f0sPAAAAAIUoctGRpCFDhmjIkCGFHlu/fr3bx4cPH/41XwIAAAAAfjWPrroGAAAAAJ5A0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADG+VVF56233lJYWJgCAgLUunVrbd269Ybnf/DBB2rQoIECAgLUuHFjrV69+leFBQAAAICbUeSis2zZMsXGxmrcuHHasWOHmjZtqs6dO+vkyZOFnr9582b17t1bgwYN0s6dO/XQQw/poYce0rfffvs/hwcAAACAwhS56EyfPl2DBw/WwIEDFR4ervj4eJUuXVoLFiwo9Pw33nhDXbp00fPPP6+GDRvq1VdfVYsWLTRr1qz/OTwAAAAAFKZIRScnJ0fbt29XdHT0z5/Ax0fR0dFKSkoq9DlJSUlu50tS586dr3s+AAAAAPyvShTl5FOnTikvL08hISFu4yEhIdq9e3ehz0lLSyv0/LS0tOt+nezsbGVnZ7s+PnfunCQpMzOzKHGLJD/7omWf2xOs/L3xFF4D+/Ea2I/XwF6/9d9/idfgVsBrYD9eA/tZ+Rpc+9xOp/OG5xWp6HjKpEmT9MorrxQYDw0NtSHNb0PwDLsTgNfAfrwG9uM1sB+vgf14DezHa2A/T7wG58+fV3Bw8HWPF6noVKpUSb6+vkpPT3cbT09PV9WqVQt9TtWqVYt0viSNGTNGsbGxro/z8/N15swZVaxYUQ6HoyiRbwmZmZkKDQ3V0aNHFRQUZHccr8RrYD9eA/vxGtiP18B+vAb24vfffia8Bk6nU+fPn1f16tVveF6Rio6fn58iIyOVkJCghx56SNLVEpKQkKAhQ4YU+pw2bdooISFBzz33nGvss88+U5s2ba77dfz9/eXv7+82Vq5cuaJEvSUFBQX9Zv9AmYLXwH68BvbjNbAfr4H9eA3sxe+//X7rr8GNruRcU+Spa7Gxserfv7+ioqLUqlUrzZgxQxcuXNDAgQMlSTExMapRo4YmTZokSRo2bJjuueceTZs2TV27dtXSpUu1bds2zZkzp6hfGgAAAABuSpGLTq9evZSRkaG4uDilpaWpWbNmWrNmjWvBgdTUVPn4/LyY21133aUlS5Zo7Nix+vOf/6x69epp5cqVioiIKL7/CwAAAAD4hV+1GMGQIUOuO1Vt/fr1Bcb+8Ic/6A9/+MOv+VJG8Pf317hx4wpMx4Pn8BrYj9fAfrwG9uM1sB+vgb34/befN70GDud/W5cNAAAAAH5jirRhKAAAAAD8FlB0AAAAABiHogMAAADAOBQdAAAAAMah6FjgypUreuyxx3To0CG7owAAAABeiVXXLBIcHKzk5GTVrl3b7iiA7S5fvqycnBy3sd/ybsy/NRs3btTs2bN14MABLV++XDVq1NC7776r2rVrq127dnbHAwBYLDU19YbHa9as6aEknvWr9tHBf/fQQw9p5cqVGj58uN1RvFpubq7Wr1+vAwcOqE+fPgoMDNSJEycUFBSksmXL2h3PaBcvXtQLL7yg999/X6dPny5wPC8vz4ZU3mfFihXq16+f+vbtq507dyo7O1uSdO7cOU2cOFGrV6+2OaF5YmNjb/rc6dOnW5gEAK4KCwuTw+G47nFTvydTdCxSr149jR8/XomJiYqMjFSZMmXcjj/77LM2JfMeR44cUZcuXZSamqrs7Gzdd999CgwM1JQpU5Sdna34+Hi7Ixrt+eef17p16/T222+rX79+euutt3T8+HHNnj1bkydPtjue15gwYYLi4+MVExOjpUuXusbbtm2rCRMm2JjMXDt37nT7eMeOHcrNzVX9+vUlSXv37pWvr68iIyPtiOc1ypcvf8Mf7H7pzJkzFqfB5cuX9eabb2rdunU6efKk8vPz3Y7v2LHDpmTe4T//Xbpy5Yp27typ6dOn67XXXrMplfUoOhaZP3++ypUrp+3bt2v79u1uxxwOB0XHA4YNG6aoqCilpKSoYsWKrvHu3btr8ODBNibzDv/85z+1ePFi/e53v9PAgQN19913q27duqpVq5bee+899e3b1+6IXmHPnj1q3759gfHg4GCdPXvW84G8wLp161y/nj59ugIDA7Vo0SKVL19ekvTTTz+5/k7AOjNmzLA7An5h0KBBWrt2rR555BG1atXqpksoikfTpk0LjEVFRal69er6y1/+oocfftiGVNaj6FiEhQjst3HjRm3evFl+fn5u42FhYTp+/LhNqbzHmTNnVKdOHUlX78e59o5pu3bt9NRTT9kZzatUrVpV+/fvV1hYmNv4pk2bXK8PrDNt2jStXbvWVXKkq1caJkyYoE6dOmnEiBE2pjNb//797Y6AX/j000+1evVqtW3b1u4o+IX69evr66+/tjuGZVh1zWI5OTnas2ePcnNz7Y7idfLz8wudc3rs2DEFBgbakMi71KlTx1X4GzRooPfff1/S1Ss95cqVszGZdxk8eLCGDRumLVu2yOFw6MSJE3rvvfc0cuRICqcHZGZmKiMjo8B4RkaGzp8/b0MiXL58WZmZmW4PWK9GjRp877XRf/6ZP3funHbv3q2xY8eqXr16dsezjhOWuHDhgvOxxx5z+vr6On19fZ0HDhxwOp1O55AhQ5yTJk2yOZ136Nmzp3Pw4MFOp9PpLFu2rPPgwYPO8+fPO++9917ngAEDbE5nvunTpzvfeOMNp9PpdH722WfOgIAAp7+/v9PHx8c5Y8YMm9N5j/z8fOeECROcZcqUcTocDqfD4XAGBAQ4x44da3c0r9CvXz9nWFiYc8WKFc6jR486jx496ly+fLmzdu3azpiYGLvjeY2srCznM88846xcubLTx8enwAPWW716tbNLly7Ow4cP2x3FKzkcjgJ/7h0Oh7NmzZrOzZs32x3PMiwvbZFhw4YpMTFRM2bMUJcuXbRr1y7VqVNHH3/8sV5++eUCN4Wh+B07dkydO3eW0+nUvn37FBUVpX379qlSpUrasGGDqlSpYndEr3LkyBFt375ddevWVZMmTeyO43VycnK0f/9+ZWVlKTw8nFUHPeTixYsaOXKkFixYoCtXrkiSSpQooUGDBukvf/lLgYVqYI1nnnlG69at06uvvlro4ijcM2i9jIwM9ezZUxs2bFDp0qVVsmRJt+MsCGGtL7/80u1jHx8fVa5cWXXr1lWJEubeyULRsUitWrW0bNky3XnnnQoMDFRKSorq1Kmj/fv3q0WLFlwq95Dc3FwtXbpUu3btUlZWllq0aKG+ffuqVKlSdkcz3uLFi9WrVy/5+/u7jefk5Gjp0qWKiYmxKZl3y8zM1BdffKH69eurYcOGdsfxGhcuXNCBAwckSbfffjsFx8Nq1qzpWhwlKChIO3bsUN26dfXuu+/qH//4B8use0B0dLRSU1M1aNAghYSEFFiMgHuqrHPlyhX96U9/0ksvveR1+ztSdCxSunRpffvtt6pTp45b0UlJSVH79u117tw5uyMClvL19dWPP/5Y4MrZ6dOnVaVKFWPX7L/V9OzZU+3bt9eQIUN06dIlNWvWTIcOHZLT6dTSpUvVo0cPuyMClitbtqy+//571axZU7fddps+/PBDtWrVSocOHVLjxo2VlZVld0TjlS5dWklJSYWu/gXreetG9ixGYJGoqCitWrXK9fG1dy7mzZunNm3a2BXL6+zbt09z5szRhAkTNH78eLcHrOV0OgtdPvTYsWMKDg62IZF32rBhg2sZ448++kj5+fk6e/asZs6cyT46Njpw4IDuvfdeu2N4DRZHsV+DBg106dIlu2N4rWsb2Xsbcyfl2WzixIn6/e9/r++//165ubl644039P3332vz5s0F5knCGnPnztVTTz2lSpUqqWrVqm4/dDscDsXFxdmYzlzNmzeXw+GQw+FQx44d3eb+5uXl6dChQ+rSpYuNCb3LuXPnVKFCBUnSmjVr1KNHD5UuXVpdu3bV888/b3M675WVlcX3Ag8aOHCgUlJSdM8992j06NHq1q2bZs2apStXrmj69Ol2x/MKkydP1ogRI/Taa6+pcePGBe7RCQoKsimZd/DWjeyZumahAwcOaPLkyUpJSXHdHzJq1Cg1btzY7mheoVatWnr66ac1atQou6N4lVdeecX13xEjRrjd9O7n56ewsDD16NGjwP5GsMYdd9yhCRMmqGvXrqpdu7aWLl2qe++9VykpKerYsaNOnTpld0QjzZw584bHjx8/rr/+9a9M4bQJi6N4no/P1UlE/3ml/9rVf/4uWOtGU9YcDocOHjzowTSeQ9GBsYKCgpScnMymiDZZtGiRevXqpYCAALujeLW//e1vGjZsmMqWLatatWppx44d8vHx0ZtvvqkPP/xQ69atszuikXx8fFStWrXrFvqcnBylpaXxwx28xn+7gnnPPfd4KAm8CUWnGBVlJTUu0Vpv0KBBatmypZ588km7owC22rZtm44ePar77rvPdYVt1apVKleuHLuUW6R27dqaMmWKevbsWejx5ORkRUZGUnQ8KCEhQQkJCTp58qTy8/Pdji1YsMCmVN4jNTVVoaGhhV7ROXr0qGrWrGlTMu8QGxtb6LjD4VBAQIDq1q2rBx980DXV2RQUnWLk4+NT6M3XheGbm/UmTZqk6dOnq2vXroXOBzZ1PuqtIi8vT6+//rref/99paamKicnx+04eybAZI888ohuv/12TZkypdDjKSkpat68eYEfuGGNV155RePHj1dUVJSqVatW4Hv1Rx99ZFMy78FKnPbq0KGDduzYoby8PNWvX1+StHfvXvn6+qpBgwbas2ePHA6HNm3apPDwcJvTFh+KTjH65WXZw4cPa/To0RowYIBrlbWkpCQtWrRIkyZNYr14D/DW+ai3iri4OM2bN08jRozQ2LFj9eKLL+rw4cNauXKl4uLiKJoekpeXp4ULF173newvvvjCpmRm+/7773Xx4kVFRUUVevzKlSs6ceKEatWq5eFk3qlatWqaOnWq+vXrZ3cUr+Xj46P09HRVrlzZbfzIkSMKDw/XhQsXbErmHWbMmKGNGzfqnXfecc0qOnfunB5//HG1a9dOgwcPVp8+fXTp0iX9+9//tjlt8aHoWKRjx456/PHH1bt3b7fxJUuWaM6cOVq/fr09wQAPuf322zVz5kx17dpVgYGBSk5Odo199dVXWrJkid0RvcKQIUO0cOFCde3atdB3sl9//XWbkgGeU7FiRW3dulW333673VG8zrUpU2+88YYGDx6s0qVLu47l5eVpy5Yt8vX1VWJiol0RvUKNGjX02WefFbha891336lTp046fvy4duzYoU6dOhm1SA3LS1skKSlJ8fHxBcajoqL0+OOP25AI8Ky0tDTXCoNly5Z1bZL7//7f/9NLL71kZzSvsnTpUr3//vu6//777Y7ilSZMmKC+fft63SZ9t5rHH39cS5Ys4d8eG+zcuVPS1XtxvvnmG7cFOvz8/NS0aVONHDnSrnhe49y5czp58mSBopORkeG6x7xcuXIFppn/1lF0LBIaGqq5c+dq6tSpbuPz5s1TaGioTanMFxsbq1dffVVlypS57o1317B3grVuu+02/fjjj6pZs6Zuv/12rV27Vi1atNDXX38tf39/u+N5DT8/P9WtW9fuGF7rgw8+0Lhx49S6dWs9+uij6tmzpypVqmR3LK9z+fJlzZkzR59//rmaNGlS4J5Nvh9Y59rKjgMHDtQbb7zBYkw2efDBB/XYY49p2rRpatmypSTp66+/1siRI/XQQw9JkrZu3ao77rjDxpTFj6lrFlm9erV69OihunXrqnXr1pKu/gHat2+fVqxYwburFunQoYM++ugjlStXTh06dLjueQ6Hg3sTLDZ69GgFBQXpz3/+s5YtW6ZHH31UYWFhSk1N1fDhwzV58mS7I3qFadOm6eDBg5o1a9ZNL5aC4vXdd9/pvffe09KlS3Xs2DHdd9996tu3rx566CG3aTywDt8Pbj2ZmZn64osv1KBBAzVo0MDuOMbLysrS8OHDtXjxYuXm5kqSSpQoof79++v1119XmTJllJycLElq1qyZfUGLGUXHQseOHdPf/vY37d69W5LUsGFDPfnkk1zRgVdKSkpSUlKS6tWrp27dutkdx2t0795d69atU4UKFdSoUaMC72R/+OGHNiXzTomJiVqyZIk++OADXb58uUjbEgC/ZT179lT79u01ZMgQXbp0SU2bNtXhw4fldDq1dOlS9ejRw+6IXiErK8u1GFOdOnXcNvU2EVPXLHTbbbdp4sSJdscAbglt2rRxrUAIzylXrpy6d+9udwz8nzJlyqhUqVLy8/PT+fPn7Y7jlY4dOybp6vdoeM6GDRv04osvSrq6nLfT6dTZs2e1aNEiTZgwgaLjIWXLllWTJk3sjuExXNGx0NmzZzV//nz98MMPkqRGjRrpscceU3BwsM3JzPXwww/f9Lm8k138Pvnkk5s+94EHHrAwCXDrOHTokJYsWaIlS5Zoz549uueee9SnTx898sgjfD/wkPz8fE2YMEHTpk1TVlaWJCkwMFAjRozQiy++KB8fH5sTmq9UqVLau3evQkNDFRMTo+rVq2vy5MlKTU1VeHi463UBihNXdCyybds2de7cWaVKlVKrVq0kXb3Z8bXXXnPdlI3ixw8N9rp2Q+M1DodD//leyrX7RNgcznNyc3O1fv16HThwQH369FFgYKBOnDihoKAg46ct2O3OO+/U119/rSZNmmjgwIHq3bu3atSoYXcsr/Piiy9q/vz5mjx5stq2bStJ2rRpk15++WVdvnxZr732ms0JzRcaGqqkpCRVqFBBa9as0dKlSyVJP/30kwICAmxOB1NxRccid999t+rWrau5c+eqRImrfTI3N1ePP/64Dh48qA0bNticELDW559/rlGjRmnixIlum+aOHTtWEydO1H333WdzQu9w5MgRdenSRampqcrOztbevXtVp04dDRs2TNnZ2YUug4/i8+KLL6pv375G7TT+W1S9enXFx8cXuJL88ccf6+mnn9bx48dtSuY9/va3v2nYsGEqW7asatasqZ07d8rHx0dvvvmmPvzwQ9fqbEBxouhYpFSpUtq5c2eBlUS+//57RUVF6eLFizYl8y68k22fiIgIxcfHq127dm7jGzdu1BNPPOGa0glrPfTQQwoMDNT8+fNVsWJFpaSkqE6dOlq/fr0GDx6sffv22R0RsFxAQIB27dpVYOncPXv2qFmzZrp06ZJNybzL9u3blZqaqk6dOqlMmTKSpFWrVql8+fK66667bE4HEzF1zSJBQUFKTU0tUHSOHj2qwMBAm1J5l/98J/u+++5TYGCgpkyZwjvZHnDgwAGVK1euwHhwcLAOHz7s8TzeauPGjdq8ebPbJn2SFBYWxrvYHpCXl6eFCxcqISFBJ0+eVH5+vttxljX2jKZNm2rWrFmaOXOm2/isWbPUtGlTm1KZ73r72W3cuLHAGEUHVqDoWKRXr14aNGiQ/vrXv7r+8iYmJur5559X7969bU7nHYYNG6aoqCilpKSoYsWKrvHu3btr8ODBNibzDi1btlRsbKzeffddhYSESJLS09P1/PPPu+5bg/Xy8/MLvR/q2LFjvOniAcOGDdPChQvVtWtXRUREsJeRTaZOnaquXbvq888/d5tKe/ToUa1evdrmdObauXPnTZ3H3wtYhalrFsnJydHzzz+v+Ph418ZMJUuW1FNPPaXJkyezM7wHVKxYUZs3b1b9+vUVGBjomrJz+PBhhYeHM33QYvv371f37t1dq+xIV69o1qtXTytXrlTdunVtTugdevXqpeDgYM2ZM0eBgYHatWuXKleurAcffFA1a9bUO++8Y3dEo1WqVEmLFy9mk+hbwIkTJ/TWW2+57W339NNPq3r16jYnA2AVio7FLl68qAMHDkiSbr/9dnbB9qDy5csrMTFR4eHhbkVn06ZN6tGjh9LT0+2OaDyn06nPPvvM7QeL6Oho3r3zoGPHjqlz585yOp3at2+foqKitG/fPlWqVEkbNmxQlSpV7I5otOrVq2v9+vUF7g0BAFiPomORc+fOKS8vTxUqVHAbP3PmjEqUKKGgoCCbknkP3sn+bWjcuLFWr17tuuqD4pebm6ulS5dq165dysrKUosWLdS3b1+VKlXK7mjGmzZtmg4ePKhZs2ZR8D1s165dioiIkI+Pj3bt2nXDc71pA0XAm1B0LPL73/9e3bp109NPP+02Hh8fr08++YQ5wR7AO9m/Db+82gaYpnv37lq3bp0qVKigRo0aqWTJkm7H2bjYOj4+PkpLS1OVKlXk4+NT6L5e0tX7Q9jXCzATRcciFSpUUGJioho2bOg2vnv3brVt21anT5+2KZl3yc3N1bJly5SSksI72bcoik7x++STT2763P/cVwTFa+DAgTc8zpVl6xw5ckQ1a9aUw+HQkSNHbnhurVq1PJQKgCdRdCxSpkwZffXVV2rcuLHb+DfffKPWrVtzIzzwfyg6xc/Hx8ft48Leyb42jYp3suENNmzYoLvuusu1gfc1ubm52rx5s9q3b29TMgBW8vnvp+DXaNWqlebMmVNgPD4+XpGRkTYk8j6LFi3SqlWrXB+/8MILKleunO66667/+u4e8FuWn5/veqxdu1bNmjXTv/71L509e1Znz57Vv/71L7Vo0UJr1qyxO6rXyMjI0KZNm7Rp0yZlZGTYHcfrdOjQQWfOnCkwfu7cOXXo0MGGRAA8gSs6FklMTFR0dLRatmypjh07SpISEhL09ddfa+3atbr77rttTmi++vXr6+2339a9996rpKQkdezYUTNmzNCnn36qEiVKMDf+FsEVHWtFREQoPj5e7dq1cxvfuHGjnnjiCf3www82JfMOFy5c0NChQ7V48WLXZqG+vr6KiYnRm2++yUqcHuLj46P09HRVrlzZbXzv3r2KiopSZmamTckAWIkNQy3Stm1bJSUl6S9/+Yvef/99lSpVSk2aNNH8+fNVr149u+N5haNHj7r2alm5cqUeeeQRPfHEE2rbtq1+97vf2RsO8JADBw6oXLlyBcaDg4N1+PBhj+fxNrGxsfryyy/1z3/+U23btpUkbdq0Sc8++6xGjBiht99+2+aEZnv44YclXZ2qOWDAALc97PLy8rRr1y7Xpt4AzEPRsVCzZs303nvv2R3Da5UtW1anT59WzZo1tXbtWsXGxkqSAgICdOnSJZvTmW/x4sXq1atXgc1xc3JytHTpUsXExEiSZs+erZCQEDsieoWWLVsqNjZW7777ruv3OT09Xc8//7xatWplczrzrVixQsuXL3d7c+X+++9XqVKl1LNnT4qOxYKDgyVd3dMrMDDQbSEaPz8/3XnnnRo8eLBd8QBYjKlrFsrPz9f+/ft18uRJ15SFa7jx0Xp9+/bV7t271bx5c/3jH/9QamqqKlasqE8++UR//vOf9e2339od0Wi+vr768ccfCyzjffr0aVWpUoWb4D1k//796t69u/bu3evaq+jo0aOqV6+eVq5c6brqCWuULl1a27dvL7AC53fffadWrVrpwoULNiXzLq+88oqef/55pgoCXoaiY5GvvvpKffr00ZEjRwpd7Ygf8qx39uxZjR07VkePHtVTTz2lLl26SJLGjRsnPz8/vfjiizYnNNv15sSnpKRc98ZgWMPpdOqzzz7T7t27JUkNGzZUdHQ0G1h6QMeOHVWxYkUtXrxYAQEBkqRLly6pf//+OnPmjD7//HObE3qHQ4cOKTc3t8DU8X379qlkyZIKCwuzJxgAS1F0LNKsWTPdcccdeuWVV1StWrUCP1Bcu5wOmKZ58+ZyOBxKSUlRo0aN3JZzzcvL06FDh9SlSxe9//77NqbEf2rcuLFWr17tuuqD4vHNN9+oS5cuys7OVtOmTSVdLfv+/v5au3atGjVqZHNC73DPPffoscceU//+/d3G//73v2vevHlav369PcEAWIqiY5EyZcooJSWFaSG3gIsXLyo1NVU5OTlu402aNLEpkdleeeUV139HjBihsmXLuo75+fkpLCxMPXr0kJ+fn10RUQhWv7POxYsX9d5777ldUWPjYs8KCgrSjh07CnxP3r9/v6KionT27Fl7ggGwFIsRWKR169bav38/RcdGGRkZGjBgwHX3CmH6oDXGjRsnSQoLC1OvXr1c03UAbzRp0iSFhIQUuOF9wYIFysjI0KhRo2xK5l0cDofOnz9fYPzcuXN8LwAMxoahFhk6dKhGjBihhQsXavv27dq1a5fbA9Z77rnndO7cOW3ZskWlSpXSmjVrtGjRItWrV0+ffPKJ3fGM179/fwUEBCgnJ0fHjh1Tamqq2wPwBrNnz1aDBg0KjDdq1Ejx8fE2JPJO7du316RJk9xKTV5eniZNmlRgjykA5mDqmkV8fAp2SIfDIafTyWIEHlKtWjV9/PHHatWqlYKCgrRt2zbdcccd+uSTTzR16lRt2rTJ7ohG27dvnx577DFt3rzZbZy/A7cmpq5ZIyAgQD/88INq167tNn7w4EGFh4fr8uXLNiXzLt9//73at2+vcuXKuTbs3rhxozIzM/XFF18oIiLC5oQArMDUNYscOnTI7ghe78KFC66ljcuXL6+MjAzdcccdaty4sXbs2GFzOvMNGDBAJUqU0KefflroghyANwgNDVViYmKBopOYmKjq1avblMr7hIeHa9euXZo1a5ZSUlJUqlQpxcTEaMiQIapQoYLd8QBYhKJjkVq1atkdwevVr19fe/bsUVhYmJo2barZs2crLCxM8fHxqlatmt3xjJecnKzt27cXOm0H8BaDBw/Wc889pytXrujee++VJCUkJOiFF17QiBEjbE7nXapXr66JEyfaHQOAB1F0LPTuu+8qPj5ehw4dUlJSkmrVqqUZM2aodu3aevDBB+2OZ7xhw4bpxx9/lHT1BvkuXbro73//u/z8/LRo0SKb05kvPDxcp06dsjsGfuHy5cvXXRxi9uzZCgkJ8XAi8z3//PM6ffq0nn76adfKjwEBARo1apTGjBljczrvsnHjRs2ePVsHDx7UBx98oBo1aujdd99V7dq1uU8HMBSLEVjk7bffVmxsrO6//36dPXvWdT9CuXLlNGPGDHvDeYlHH31UAwYMkCS1aNFCR44c0bZt23Ts2DH16tXL3nBeYMqUKXrhhRe0fv16nT59WpmZmW4PeEZ+fr5effVV1ahRQ2XLltXBgwclSS+99JLmz5/vOq9Pnz4qU6aMXTGN5XA4NGXKFGVkZOirr75SSkqKzpw5o7i4OLujeZUVK1aoc+fOKlWqlHbs2KHs7GxJV1dd4yoPYC6KjkXefPNNzZ07Vy+++KJ8fX1d41FRUfrmm29sTOZd5s+fr4iICAUEBKh8+fKKiYnRypUr7Y7lFaKjo/XVV1+pY8eOqlKlisqXL6/y5curXLlyKl++vN3xvMaECRO0cOFCTZ061W3vooiICM2bN8/GZN6lbNmyatmypSIiIuTv7293HK8zYcIExcfHa+7cuSpZsqRrvG3bttyzCRiMqWsWOXTokJo3b15g3N/fXxcuXLAhkfeJi4vT9OnTNXToULVp00aSlJSUpOHDhys1NVXjx4+3OaHZ1q1bZ3cESFq8eLHmzJmjjh076sknn3SNN23a1LWBJWC6PXv2qH379gXGg4OD2SwUMBhFxyK1a9dWcnJygUUJ1qxZo4YNG9qUyru8/fbbmjt3rnr37u0ae+CBB9SkSRMNHTqUomOxe+65x+4IkHT8+PFCNy7Oz8/XlStXbEgEeF7VqlW1f/9+hYWFuY1v2rSJJdUBgzF1zSKxsbF65plntGzZMjmdTm3dulWvvfaaxowZoxdeeMHueF7hypUrioqKKjAeGRmp3NxcGxJ5n40bN+rRRx/VXXfdpePHj0u6ukgHexh5Tnh4uDZu3FhgfPny5YVedQZMNHjwYA0bNkxbtmyRw+HQiRMn9N5772nkyJF66qmn7I4HwCJc0bHI448/rlKlSmns2LG6ePGi+vTpo+rVq+uNN97QH//4R7vjeYV+/frp7bff1vTp093G58yZo759+9qUynusWLFC/fr1U9++fQu9+Xf16tU2J/QOcXFx6t+/v44fP678/Hx9+OGH2rNnjxYvXqxPP/3U7niAR4wePVr5+fnq2LGjLl68qPbt28vf318jR47U0KFD7Y4HwCIOp9PptDuE6S5evKisrCzX5pW/lJiYqKioKG5OLSaxsbGuX+fm5mrhwoWqWbOm7rzzTknSli1blJqaqpiYGL355pt2xfQKzZs31/DhwxUTE6PAwEClpKSoTp062rlzp37/+98rLS3N7oheY+PGjRo/frxSUlKUlZWlFi1aKC4uTp06dbI7GmC5vLw8JSYmqkmTJipdurT279+vrKwshYeHq2zZsnbHA2Ahio7NgoKClJyczBzhYtKhQ4ebOs/hcOiLL76wOI13K126tL7//nuFhYW5FZ2DBw8qPDxcly9ftjsiAC8REBCgH374QbVr17Y7CgAPYuqazeiZxYuVvm4d3PwL4FYRERGhgwcPUnQAL0PRAWCJazf/LliwwHXzb1JSkkaOHKmXXnrJ7nhGK1++vBwOx02de+bMGYvTAPabMGGCRo4cqVdffVWRkZEFNscNCgqyKRkAK1F0AFiCm3/tM2PGDLsjALeU+++/X9LVLQZ++SaA0+mUw+FQXl6eXdEAWIh7dGz2y3sXABPl5ORw8y8AW3355Zc3PM6+X4CZKDo2YzECmOrcuXPKy8tThQoV3MbPnDmjEiVKMFXEg/Ly8vTRRx/phx9+kHR1b50HH3xQJUpwUR8AYC6+y9mMnglT/fGPf1S3bt309NNPu42///77+uSTT9hHx0O+++47PfDAA0pLS1P9+vUlSVOmTFHlypX1z3/+UxERETYnBKyxa9cuRUREyMfHR7t27brhuU2aNPFQKgCexBUdC+Xm5mr9+vU6cOCA+vTpo8DAQJ04cUJBQUFM34HxKlSooMTERDVs2NBtfPfu3Wrbtq1Onz5tUzLv0qZNG1WuXFmLFi1S+fLlJUk//fSTBgwYoIyMDG3evNnmhIA1fHx8lJaWpipVqsjHx0cOh6PQNxe5RwcwF1d0LHLkyBF16dJFqampys7O1n333afAwEBNmTJF2dnZio+PtzsiYKns7Gzl5uYWGL9y5YouXbpkQyLvlJycrG3btrlKjnR1VbbXXntNLVu2tDEZYK1Dhw6pcuXKrl8D8D4+dgcw1bBhwxQVFaWffvpJpUqVco13795dCQkJNiYDPKNVq1aaM2dOgfH4+HhFRkbakMg73XHHHUpPTy8wfvLkSdWtW9eGRIBn1KpVy7XCWq1atW74AGAmruhYZOPGjdq8ebP8/PzcxsPCwnT8+HGbUgGeM2HCBEVHRyslJUUdO3aUJCUkJOjrr7/W2rVrbU7nPSZNmqRnn31WL7/8su68805J0ldffaXx48drypQpyszMdJ3LAhEwySeffHLT5z7wwAMWJgFgF+7RsUj58uWVmJio8PBwtyWkN23apB49ehT6DitgmpSUFE2dOlXJyckqVaqUmjRpojFjxqhevXp2R/MaPj4/X7i/9u72tX/2f/kx9ynANL/8sy+pwD06v9xPhz/7gJm4omORTp06acaMGa6pOw6HQ1lZWRo3bpxr4zLAVFeuXNGf/vQnvfTSS3rvvffsjuPV1q1bZ3cEwBb5+fmuX3/++ecaNWqUJk6cqDZt2kiSkpKSNHbsWE2cONGuiAAsxhUdixw7dkydO3eW0+nUvn37FBUVpX379qlSpUrasGGDqlSpYndEwFLBwcFKTk5W7dq17Y4CwMtFREQoPj5e7dq1cxvfuHGjnnjiCdceUwDMQtGxUG5urpYtW6aUlBRlZWWpRYsW6tu3r9viBICp+vfvr2bNmmn48OF2R/F6ly9f1q5du3Ty5Em3d7kl7k2AdyhVqpS+/vrrAvtG7dq1S61bt2YlSMBQFB0AlpgwYYKmTZumjh07KjIyUmXKlHE7/uyzz9qUzLusWbNGMTExOnXqVIFj3JcDb9G+fXsFBATo3XffVUhIiCQpPT1dMTExunz5sr788kubEwKwAkXHIpMmTVJISIgee+wxt/EFCxYoIyNDo0aNsikZ4Bk3mrLmcDh08OBBD6bxXvXq1VOnTp0UFxfn+gEP8Db79+9X9+7dtXfvXoWGhkqSjh49qnr16mnlypUstQ4YiqJjkbCwMC1ZskR33XWX2/iWLVv0xz/+kc3LAHhEUFCQdu7cqdtvv93uKICtnE6nPvvsM+3evVuS1LBhQ0VHR7utvgbALKy6ZpG0tDRVq1atwHjlypX1448/2pAIsEdOTo4OHTqk22+/XSVK8E+Opz3yyCNav349RQdez+FwqFOnTurUqZPdUQB4CD91WCQ0NFSJiYkFpu8kJiaqevXqNqUCPOfixYsaOnSoFi1aJEnau3ev6tSpo6FDh6pGjRoaPXq0zQm9w6xZs/SHP/xBGzduVOPGjVWyZEm349wrBVPNnDlTTzzxhAICAjRz5swbnsvfA8BMTF2zyNSpUzV16lT95S9/0b333ivp6q7wL7zwgkaMGKExY8bYnBCw1rBhw5SYmKgZM2aoS5cu2rVrl+rUqaOPP/5YL7/8snbu3Gl3RK8wf/58PfnkkwoICFDFihXdpulwrxRMVrt2bW3btk0VK1bknkHAS1F0LOJ0OjV69GjNnDlTOTk5kqSAgACNGjVKcXFxNqcDrFerVi0tW7ZMd955pwIDA5WSkqI6depo//79atGihTIzM+2O6BWqVq2qZ599VqNHjy6wUzzgja792MO9OYD5+K5nEYfDoSlTpigjI0NfffWVUlJSdObMGUoOvEZGRkahG+NeuHCBHzA8KCcnR7169aLkwOvNnz9fERERCggIUEBAgCIiIjRv3jy7YwGwEN/5LFa2bFm1bNlSERER8vf3tzsO4DFRUVFatWqV6+Nr5WbevHlq06aNXbG8Tv/+/bVs2TK7YwC2iouL07Bhw9StWzd98MEH+uCDD9StWzcNHz6cNyABgzF1zSIXLlzQ5MmTlZCQUOhu5MwHhuk2bdqk3//+93r00Ue1cOFC/elPf9L333+vzZs368svv1RkZKTdEb3Cs88+q8WLF6tp06Zq0qRJgcUIpk+fblMywHMqV66smTNnqnfv3m7j//jHPzR06NBCN9QF8NvHqmsWefzxx/Xll1+qX79+qlatGlN14HXatWun5ORkTZ48WY0bN9batWvVokULJSUlqXHjxnbH8xrffPONmjdvLkn69ttv3Y7x7xK8xZUrVxQVFVVgPDIyUrm5uTYkAuAJXNGxSLly5bRq1Sq1bdvW7igAAHi1oUOHqmTJkgWuYI4cOVKXLl3SW2+9ZVMyAFbiio5FypcvrwoVKtgdA7BVXl6ePvroI/3www+SpPDwcD344INsHArAcrGxsa5fOxwOzZs3T2vXrtWdd94pSdqyZYtSU1MVExNjV0QAFuOKjkX+/ve/6+OPP9aiRYtUunRpu+MAHvfdd9/pgQceUFpamurXry/p6qahlStX1j//+U9FRETYnNBcDz/8sBYuXKigoCA9/PDDNzz3ww8/9FAqwLM6dOhwU+c5HA598cUXFqcBYAfeVrXItGnTdODAAYWEhCgsLKzADcA7duywKRngGY8//rgaNWqkbdu2qXz58pKkn376SQMGDNATTzyhzZs325zQXMHBwa77b4KDg21OA9hj3bp1dkcAYDOu6FjklVdeueHxcePGeSgJYI9SpUpp27ZtatSokdv4t99+q5YtW+rSpUs2JfMuly5dUn5+vsqUKSNJOnz4sFauXKmGDRuqc+fONqcDAMA6XNGxCEUG3u6OO+5Qenp6gaJz8uRJ1a1b16ZU3ufBBx/Uww8/rCeffFJnz57VnXfeqZIlS+rUqVOaPn26nnrqKbsjAgBgCTYMtdDZs2c1b948jRkzRmfOnJF0dcra8ePHbU4GWG/SpEl69tlntXz5ch07dkzHjh3T8uXL9dxzz2nKlCnKzMx0PWCdHTt26O6775YkLV++XCEhITpy5IgWL16smTNn2pwOAADrMHXNIrt27VJ0dLSCg4N1+PBh7dmzR3Xq1NHYsWOVmpqqxYsX2x0RsJSPz8/vo1y7X+TaPze//NjhcCgvL8/zAb1E6dKltXv3btWsWVM9e/ZUo0aNNG7cOB09elT169fXxYsX7Y4IAIAlmLpmkdjYWA0YMEBTp05VYGCga/z+++9Xnz59bEwGeAY3At8a6tatq5UrV6p79+7697//reHDh0u6OoUwKCjI5nQAAFiHomORr7/+WrNnzy4wXqNGDaWlpdmQCPCse+6556bOe/rpp9WoUSNVqlTJ4kTeKS4uTn369NHw4cPVsWNHtWnTRpK0du1aNW/e3OZ0AABYh3t0LOLv71/ovQfX9hEBcNXf//537tOx0COPPKLU1FRt27ZNa9ascY137NhRr7/+uo3JAACwFkXHIg888IDGjx+vK1euSLp6T0JqaqpGjRqlHj162JwOuHVwm6D1qlatqubNm7vdN9WqVSs1aNDAxlQAAFiLomORadOmKSsrS1WqVNGlS5d0zz33qG7dugoMDNRrr71mdzwAAADAaNyjY5Hg4GB99tlnSkxMVEpKirKystSiRQtFR0fbHQ0AAAAwHstLW2Tx4sXq1auX/P393cZzcnK0dOlSxcTE2JQMuLUEBgYqJSVFderUsTsKAAAwCEXHIr6+vvrxxx9VpUoVt/HTp0+rSpUq7BsC/B+KDgAAsAL36Fjk2kaI/+nYsWMKDg62IRFwa3r00UfZzwUAABQ77tEpZs2bN5fD4ZDD4VDHjh1VosTPv8V5eXk6dOiQunTpYmNCwDO2bt2qpKQk175RVatWVZs2bdSqVSu3895++2074gEAAMNRdIrZQw89JElKTk5W586dVbZsWdcxPz8/hYWFsbw0jHby5En16NFDiYmJqlmzpkJCQiRJ6enpGj58uNq2basVK1YUmNYJAABQnLhHxyKLFi1Sr169FBAQYHcUwKMeeeQRnThxQu+8847q16/vdmzPnj167LHHVL16dX3wwQc2JQQAAN6AomOxnJwcnTx5Uvn5+W7jNWvWtCkRYK3AwEBt2LBBzZs3L/T49u3b9bvf/U7nz5/3cDIAAOBNmLpmkX379umxxx7T5s2b3cavLVLAqmswlb+/vzIzM697/Pz58wWWXQcAAChuFB2LDBgwQCVKlNCnn36qatWqFboCG2CiXr16qX///nr99dfVsWNH14pqmZmZSkhIUGxsrHr37m1zSgAAYDqmrlmkTJky2r59uxo0aGB3FMCjsrOz9dxzz2nBggXKzc2Vn5+fpKvTOEuUKKFBgwbp9ddf56oOAACwFEXHIi1bttTrr7+udu3a2R0FsEVmZqa2b9/utrx0ZGQke+YAAACPoOhY5IsvvtDYsWM1ceJENW7cWCVLlnQ7zg97AAAAgHUoOhbx8fGRpAL35rAYAbxdenq6Zs+erbi4OLujAAAAg1F0LPLll1/e8Pg999zjoSTArSUlJUUtWrSg7AMAAEux6ppFKDLwVrt27brh8T179ngoCQAA8GZc0bHQxo0bNXv2bB08eFAffPCBatSooXfffVe1a9dmkQIYy8fHRw6HQ4X903JtnOmbAADAaj52BzDVihUr1LlzZ5UqVUo7duxQdna2JOncuXOaOHGizekA61SoUEFz587VoUOHCjwOHjyoTz/91O6IAADACzB1zSITJkxQfHy8YmJitHTpUtd427ZtNWHCBBuTAdaKjIzUiRMnVKtWrUKPnz17ttCrPQAAAMWJomORPXv2qH379gXGg4ODdfbsWc8HAjzkySef1IULF657vGbNmnrnnXc8mAgAAHgjio5Fqlatqv379yssLMxtfNOmTapTp449oQAP6N69+w2Ply9fXv379/dQGgAA4K24R8cigwcP1rBhw7RlyxY5HA6dOHFC7733nkaOHKmnnnrK7njALSMoKEgHDx60OwYAADAMV3QsMnr0aOXn56tjx466ePGi2rdvL39/f40cOVJDhw61Ox5wy+B+HQAAYAWWl7ZYTk6O9u/fr6ysLIWHh6ts2bJ2RwJuKYGBgUpJSWFKJwAAKFZMXbOYn5+fwsPD1aBBA33++ef64Ycf7I4EAAAAGI+iY5GePXtq1qxZkqRLly6pZcuW6tmzp5o0aaIVK1bYnA4AAAAwG0XHIhs2bNDdd98tSfroo4+Un5+vs2fPaubMmeyjA/yCw+GwOwIAADAQRcci586dU4UKFSRJa9asUY8ePVS6dGl17dpV+/btszkdcOvgNkEAAGAFio5FQkNDlZSUpAsXLmjNmjXq1KmTJOmnn35SQECAzekAz3I6ndctNP/6179Uo0YNDycCAACmo+hY5LnnnlPfvn112223qXr16vrd734n6eqUtsaNG9sbDvCQ+fPnKyIiQgEBAQoICFBERITmzZvndk67du3k7+9vU0IAAGAqlpe20Pbt25Wamqr77rvPtaz0qlWrVK5cObVt29bmdIC14uLiNH36dA0dOlRt2rSRJCUlJWnWrFkaPny4xo8fb3NCAABgMoqOzYKCgpScnMweIjBO5cqVNXPmTPXu3dtt/B//+IeGDh2qU6dO2ZQMAAB4A6au2YyeCVNduXJFUVFRBcYjIyOVm5trQyIAAOBNKDoALNGvXz+9/fbbBcbnzJmjvn372pAIAAB4kxJ2BwBgjtjYWNevHQ6H5s2bp7Vr1+rOO++UJG3ZskWpqamKiYmxKyIAAPASFB0AxWbnzp1uH0dGRkqSDhw4IEmqVKmSKlWqpO+++87j2QAAgHeh6NiMXeFhknXr1tkdAQAAQBL36NiOxQgAAACA4scVHQ+4VmYKu3rDrvAwVYcOHW54xfKLL77wYBoAAOBtuKJjIXaFhzdr1qyZmjZt6nqEh4crJydHO3bsUOPGje2OBwAADMcVHYtcb1f44cOHKzU1lV3hYbzXX3+90PGXX35ZWVlZHk4DAAC8jcPJTSKWYFd4oHD79+9Xq1atdObMGbujAAAAgzF1zSLsCg8ULikpSQEBAXbHAAAAhmPqmkWu7Qo/ffp0t3F2hYe3ePjhh90+djqd+vHHH7Vt2za99NJLNqUCAADegqJTjNgVHvhZcHCw28c+Pj6qX7++xo8fr06dOtmUCgAAeAvu0SlGHTp0uKnzHA4HS+sCAAAAFqLoALBUTk6OTp48qfz8fLfxmjVr2pQIAAB4A6auAbDE3r17NWjQIG3evNlt3Ol0yuFwKC8vz6ZkAADAG1B0LMKu8PB2AwcOVIkSJfTpp5+qWrVqN/z7AAAAUNwoOhZp1qyZ28dXrlxRcnKyvv32W/Xv39+eUIAHJScna/v27WrQoIHdUQAAgBei6FiEXeHh7cLDw9kYFwAA2IbFCDyMXeFhsszMTNevt23bprFjx2rixIlq3LixSpYs6XZuUFCQp+MBAAAvwhUdD2NXeJisXLlybvfiOJ1OdezY0e0cFiMAAACeQNGxCLvCwxutW7fO7ggAAACSmLpmmYEDB7p97OPjo8qVK+vee+9lV3jgF55++mmNHz9elSpVsjsKAAAwCEUHgK2CgoKUnJysOnXq2B0FAAAYhKlrFmNXeODGeK8FAABYgaJjEXaFBwAAAOxD0bEIu8IDAAAA9qHoWIRd4QEAAAD7+NgdwFTsCg8AAADYh6JTjDIzM12PKVOm6IUXXtD69et1+vRpt2O/3D0eMMnDDz/s+vO9ePFiZWdn/9fnPProowoKCrI6GgAA8DIsL12MfHx8CuwK/5/35rAYAUzm5+enI0eOqFq1avL19dWPP/6oKlWq2B0LAAB4Ie7RKUbsCg9v16BBA40ZM0YdOnSQ0+nU+++/f92rNTExMR5OBwAAvAlXdGzGrvAwyebNmxUbG6sDBw7ozJkzCgwMLHTFQYfDoTNnztiQEAAAeAuKjs3YFR6m8vHx0fHjx1WtWjW3cafTqdTUVNWqVcumZAAAwBuwGIHN6JnwNmfOnKHYAwAAy1F0AFjG19e3wFhWVpYCAgJsSAMAALwJixEAKFaxsbGSrt6HExcXp9KlS7uO5eXlacuWLWrWrJlN6QAAgLeg6AAoVjt37pR0dVrmN998Iz8/P9cxPz8/NW3aVCNHjrQrHgAA8BIUHQDF6toy6wMHDtQbb7zBZqAAAMAW3KNTjNgVHvjZO++8w59tAABgG5aXLkbsCg8AAADcGpi6VozYFR4AAAC4NXBFpxixKzwAAABwa6DoWIRd4QEAAAD7sBiBh7ErPAAAAGA9io6F2BUeAAAAsAeLERQzdoUHAAAA7EfRKWbsCg8AAADYj8UILMKu8AAAAIB9KDoAAAAAjMNiBAAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcf4/3IovfV71X1EAAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        " # View tensorboard logs of transfer learning modelling experiments (should be 4 models)\n",
        " # Upload TensorBoard dev records\n",
        " !tensorboard dev upload --logdir ./model_logs \\\n",
        "   --name \"NLP modelling experiments\" \\\n",
        "   --description \"A series of different NLP modellings experiments with various models\" \\\n",
        "   --one_shot # exits the uploader when upload has finished"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nLOqT0hArpjA",
        "outputId": "8317bcbf-59b3-41e1-a82c-d40835c018e4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-01-15 05:01:59.177034: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2024-01-15 05:01:59.177126: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2024-01-15 05:01:59.182015: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2024-01-15 05:02:00.642635: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "****************************************************************\n",
            "****************************************************************\n",
            "****************************************************************\n",
            "\n",
            "Uploading TensorBoard logs to https://tensorboard.dev/ is no longer\n",
            "supported.\n",
            "\n",
            "TensorBoard.dev is shutting down.\n",
            "\n",
            "Please export your experiments by Dec 31, 2023.\n",
            "\n",
            "See the FAQ at https://tensorboard.dev.\n",
            "\n",
            "****************************************************************\n",
            "****************************************************************\n",
            "****************************************************************\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Get mean pred probs for 3 models\n",
        "baseline_pred_probs = np.max(model_0.predict_proba(val_sentences), axis=1) # get the prediction probabilities from baseline model\n",
        "combined_pred_probs = baseline_pred_probs + tf.squeeze(model_2_pred_probs, axis=1) + tf.squeeze(model_6_pred_probs)\n",
        "combined_preds = tf.round(combined_pred_probs/3) # average and round the prediction probabilities to get prediction classes\n",
        "combined_preds[:20]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Cj_lG1der5uE",
        "outputId": "7b2b61ce-4529-4cbb-de30-a72f632e8102"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(20,), dtype=float32, numpy=\n",
              "array([1., 0., 0., 1., 0., 1., 0., 1., 0., 1., 1., 0., 1., 0., 1., 0., 0.,\n",
              "       0., 1., 1.], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 105
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Calculate results from averaging the prediction probabilities\n",
        "ensemble_results = calculate_results(val_labels, combined_preds)\n",
        "ensemble_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5-fMCsYrr_SH",
        "outputId": "05db6bc8-d309-4b05-96e9-104d45828bc6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 87.28070175438597,\n",
              " 'precision': 0.8756040829907962,\n",
              " 'recall': 0.8728070175438597,\n",
              " 'f1': 0.8730768006920276}"
            ]
          },
          "metadata": {},
          "execution_count": 106
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Add our combined model's results to the results DataFrame\n",
        "all_model_results.loc[\"ensemble_results\"] = ensemble_results"
      ],
      "metadata": {
        "id": "NDnW0Q6qsDx7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Convert the accuracy to the same scale as the rest of the results\n",
        "all_model_results.loc[\"ensemble_results\"][\"accuracy\"] = all_model_results.loc[\"ensemble_results\"][\"accuracy\"]/100"
      ],
      "metadata": {
        "id": "NxR1PJdvsL71"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "all_model_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 331
        },
        "id": "DcB1mXPmsP2T",
        "outputId": "506d1538-31e2-4299-a6bd-7a382bf06f27"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                         accuracy  precision    recall        f1\n",
              "baseline                 0.889474   0.889425  0.889474  0.889350\n",
              "simple_dense             0.845614   0.846190  0.845614  0.844944\n",
              "lstm                     0.828070   0.827934  0.828070  0.827979\n",
              "gru                      0.821930   0.821721  0.821930  0.821748\n",
              "bidirectional            0.832456   0.832381  0.832456  0.832413\n",
              "conv1d                   0.833333   0.833674  0.833333  0.832658\n",
              "tf_hub_sentence_encoder  0.903509   0.903481  0.903509  0.903492\n",
              "tf_hub_10_percent_data   0.861404   0.863099  0.861404  0.861654\n",
              "ensemble_results         0.872807   0.875604  0.872807  0.873077"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-45c629fb-489a-4d23-9d47-d1a64ce7f7d5\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>accuracy</th>\n",
              "      <th>precision</th>\n",
              "      <th>recall</th>\n",
              "      <th>f1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>baseline</th>\n",
              "      <td>0.889474</td>\n",
              "      <td>0.889425</td>\n",
              "      <td>0.889474</td>\n",
              "      <td>0.889350</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>simple_dense</th>\n",
              "      <td>0.845614</td>\n",
              "      <td>0.846190</td>\n",
              "      <td>0.845614</td>\n",
              "      <td>0.844944</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>lstm</th>\n",
              "      <td>0.828070</td>\n",
              "      <td>0.827934</td>\n",
              "      <td>0.828070</td>\n",
              "      <td>0.827979</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>gru</th>\n",
              "      <td>0.821930</td>\n",
              "      <td>0.821721</td>\n",
              "      <td>0.821930</td>\n",
              "      <td>0.821748</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>bidirectional</th>\n",
              "      <td>0.832456</td>\n",
              "      <td>0.832381</td>\n",
              "      <td>0.832456</td>\n",
              "      <td>0.832413</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>conv1d</th>\n",
              "      <td>0.833333</td>\n",
              "      <td>0.833674</td>\n",
              "      <td>0.833333</td>\n",
              "      <td>0.832658</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>tf_hub_sentence_encoder</th>\n",
              "      <td>0.903509</td>\n",
              "      <td>0.903481</td>\n",
              "      <td>0.903509</td>\n",
              "      <td>0.903492</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>tf_hub_10_percent_data</th>\n",
              "      <td>0.861404</td>\n",
              "      <td>0.863099</td>\n",
              "      <td>0.861404</td>\n",
              "      <td>0.861654</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>ensemble_results</th>\n",
              "      <td>0.872807</td>\n",
              "      <td>0.875604</td>\n",
              "      <td>0.872807</td>\n",
              "      <td>0.873077</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-45c629fb-489a-4d23-9d47-d1a64ce7f7d5')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-45c629fb-489a-4d23-9d47-d1a64ce7f7d5 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-45c629fb-489a-4d23-9d47-d1a64ce7f7d5');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-c3bb18ca-2f06-4f7e-9e2d-3a3e0c1ce714\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-c3bb18ca-2f06-4f7e-9e2d-3a3e0c1ce714')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-c3bb18ca-2f06-4f7e-9e2d-3a3e0c1ce714 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 109
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Save TF Hub Sentence Encoder model to HDF5 format\n",
        "model_6.save(\"model_6.h5\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4JC3ucIJsSen",
        "outputId": "4d888a2b-b16c-4a6d-bdbd-1655867dca40"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/keras/src/engine/training.py:3103: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')`.\n",
            "  saving_api.save_model(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Load model with custom Hub Layer (required with HDF5 format)\n",
        "loaded_model_6 = tf.keras.models.load_model(\"model_6.h5\",\n",
        "                                            custom_objects={\"KerasLayer\": hub.KerasLayer})"
      ],
      "metadata": {
        "id": "1829_ZDlsWm0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# How does our loaded model perform?\n",
        "loaded_model_6.evaluate(val_sentences, val_labels)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2yQgCvBMshDE",
        "outputId": "f3194335-139a-4b6b-f204-71171c0fbff0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "36/36 [==============================] - 2s 23ms/step - loss: 0.2508 - accuracy: 0.9035\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.2508135437965393, 0.9035087823867798]"
            ]
          },
          "metadata": {},
          "execution_count": 112
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Save TF Hub Sentence Encoder model to SavedModel format (default)\n",
        "model_6.save(\"model_6_SavedModel_format\")"
      ],
      "metadata": {
        "id": "meRwcEf3sne4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Load TF Hub Sentence Encoder SavedModel\n",
        "loaded_model_6_SavedModel = tf.keras.models.load_model(\"model_6_SavedModel_format\")"
      ],
      "metadata": {
        "id": "AVUG6l12sq5L"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Evaluate loaded SavedModel format\n",
        "loaded_model_6_SavedModel.evaluate(val_sentences, val_labels)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "njnVrxnAst5w",
        "outputId": "e45e9be9-0d81-4a16-e966-3d80430408b7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "36/36 [==============================] - 1s 24ms/step - loss: 0.2508 - accuracy: 0.9035\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.2508135437965393, 0.9035087823867798]"
            ]
          },
          "metadata": {},
          "execution_count": 115
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Create dataframe with validation sentences and best performing model predictions\n",
        "val_df = pd.DataFrame({\"text\": val_sentences,\n",
        "                       \"target\": val_labels,\n",
        "                       \"pred\": model_6_preds,\n",
        "                       \"pred_prob\": tf.squeeze(model_6_pred_probs)})\n",
        "val_df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "o718LlYTs4M6",
        "outputId": "8603dc73-2ad3-46de-c352-85681b8889af"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                text  target  pred  pred_prob\n",
              "0  is a gay faggot that takes it in the butt from...       1   1.0   0.993343\n",
              "1  \"\\n\\n Deletion of section on Possibility of Ma...       0   0.0   0.040312\n",
              "2  A swt thought: \"Nver get tired of doing little...       0   0.0   0.134546\n",
              "3                         is a huge fat nigger hippo       1   1.0   0.987001\n",
              "4                   Why didn't u call on your lunch?       0   0.0   0.023242"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2db736ec-7ada-45b8-b2ba-ca96d95008ed\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "      <th>pred</th>\n",
              "      <th>pred_prob</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>is a gay faggot that takes it in the butt from...</td>\n",
              "      <td>1</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.993343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>\"\\n\\n Deletion of section on Possibility of Ma...</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.040312</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>A swt thought: \"Nver get tired of doing little...</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.134546</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>is a huge fat nigger hippo</td>\n",
              "      <td>1</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.987001</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Why didn't u call on your lunch?</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.023242</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2db736ec-7ada-45b8-b2ba-ca96d95008ed')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-2db736ec-7ada-45b8-b2ba-ca96d95008ed button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-2db736ec-7ada-45b8-b2ba-ca96d95008ed');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-ae280e24-0d27-4d4f-b6fa-bac1d6bcec9b\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-ae280e24-0d27-4d4f-b6fa-bac1d6bcec9b')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-ae280e24-0d27-4d4f-b6fa-bac1d6bcec9b button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 116
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Find the wrong predictions and sort by prediction probabilities\n",
        "most_wrong = val_df[val_df[\"target\"] != val_df[\"pred\"]].sort_values(\"pred_prob\", ascending=False)\n",
        "most_wrong[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "id": "85iOh1Sns9Eh",
        "outputId": "ee7597fe-4ae0-4034-8202-32b8ce6f2ff3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                   text  target  pred  \\\n",
              "837   \" Just like the hate-filled quote that was use...       0   1.0   \n",
              "1089  Neo-Nazism \\n\\nI would like to just point out ...       0   1.0   \n",
              "1031                        ? \\n\\nWhat's your deal guy?       0   1.0   \n",
              "895            Armand says get your ass over to epsilon       0   1.0   \n",
              "251   \" I don't know how the word \"\"hatred\"\" is pers...       0   1.0   \n",
              "591   \"\\n\\nChicken-Ryu?\\nAll the ip addresses you us...       0   1.0   \n",
              "1098  You best watch what you say cause I get drunk ...       0   1.0   \n",
              "532   .  (Ah, did I hurt you?)  Just call this a res...       0   1.0   \n",
              "643   For you information, IKEA is spelled with all ...       0   1.0   \n",
              "1066  ZOE IT JUST HIT ME 2 IM FUCKING SHITIN MYSELF ...       0   1.0   \n",
              "\n",
              "      pred_prob  \n",
              "837    0.981117  \n",
              "1089   0.975010  \n",
              "1031   0.963300  \n",
              "895    0.956992  \n",
              "251    0.945938  \n",
              "591    0.935730  \n",
              "1098   0.921089  \n",
              "532    0.919609  \n",
              "643    0.909478  \n",
              "1066   0.906401  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-9e75cf05-d3ea-453c-a312-22841d8b797e\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "      <th>pred</th>\n",
              "      <th>pred_prob</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>837</th>\n",
              "      <td>\" Just like the hate-filled quote that was use...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.981117</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1089</th>\n",
              "      <td>Neo-Nazism \\n\\nI would like to just point out ...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.975010</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1031</th>\n",
              "      <td>? \\n\\nWhat's your deal guy?</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.963300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>895</th>\n",
              "      <td>Armand says get your ass over to epsilon</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.956992</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>251</th>\n",
              "      <td>\" I don't know how the word \"\"hatred\"\" is pers...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.945938</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>591</th>\n",
              "      <td>\"\\n\\nChicken-Ryu?\\nAll the ip addresses you us...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.935730</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1098</th>\n",
              "      <td>You best watch what you say cause I get drunk ...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.921089</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>532</th>\n",
              "      <td>.  (Ah, did I hurt you?)  Just call this a res...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.919609</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>643</th>\n",
              "      <td>For you information, IKEA is spelled with all ...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.909478</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1066</th>\n",
              "      <td>ZOE IT JUST HIT ME 2 IM FUCKING SHITIN MYSELF ...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.906401</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-9e75cf05-d3ea-453c-a312-22841d8b797e')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-9e75cf05-d3ea-453c-a312-22841d8b797e button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-9e75cf05-d3ea-453c-a312-22841d8b797e');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-d378404c-29bf-4d17-9503-7768dd833bb4\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-d378404c-29bf-4d17-9503-7768dd833bb4')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-d378404c-29bf-4d17-9503-7768dd833bb4 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 117
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Check the false positives (model predicted 1 when should've been 0)\n",
        "for row in most_wrong[:10].itertuples(): # loop through the top 10 rows (change the index to view different rows)\n",
        "  _, text, target, pred, prob = row\n",
        "  print(f\"Target: {target}, Pred: {int(pred)}, Prob: {prob}\")\n",
        "  print(f\"Text:\\n{text}\\n\")\n",
        "  print(\"----\\n\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ACUr73DMtDcS",
        "outputId": "d85aea90-e88a-4311-9b0c-216c09c9107a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Target: 0, Pred: 1, Prob: 0.9811173677444458\n",
            "Text:\n",
            "\" Just like the hate-filled quote that was used\n",
            "\n",
            "\"\"the plethora of attempts to relate apiru (Habiru) to the gentilic (i.e. biblical word) ibri are all nothing but wishful thinking.\"\"\n",
            "\n",
            "Talk about hate speech and a grand attempt to belittle anyone that believe the Bible. This is typical high-nose stuck up \"\"I'm a Atheist so I'm sooooooooo much better and smarter then you knuckle dragging believers\"\" load of BS. It needs to be removed at once. but I know it won't be removed. \"\n",
            "\n",
            "----\n",
            "\n",
            "Target: 0, Pred: 1, Prob: 0.975010097026825\n",
            "Text:\n",
            "Neo-Nazism \n",
            "\n",
            "I would like to just point out that section Neo-Nazism focuses entirely on some funny concepts created by the Western Neo-nazis (Americans, British). The rest of the world Neo-Nazis, White Supremacists have never heard of such things and do not even believe in something like that. They just believe that an Aryan race are all Whites, nothing less, nothing more. I've traveled through Eastern and Southern Europe and I know that Neo-Nazis there consider Aryans to be all White people. I've never met with any concepts of creating super-humans, sending anybody into space and so on. Please, change this section, it is misleading that all neo-nazis believe in such ridiculousness.\n",
            "\n",
            "----\n",
            "\n",
            "Target: 0, Pred: 1, Prob: 0.963300347328186\n",
            "Text:\n",
            "? \n",
            "\n",
            "What's your deal guy?\n",
            "\n",
            "----\n",
            "\n",
            "Target: 0, Pred: 1, Prob: 0.9569917917251587\n",
            "Text:\n",
            "Armand says get your ass over to epsilon\n",
            "\n",
            "----\n",
            "\n",
            "Target: 0, Pred: 1, Prob: 0.9459376335144043\n",
            "Text:\n",
            "\" I don't know how the word \"\"hatred\"\" is personal attack.   \"\n",
            "\n",
            "----\n",
            "\n",
            "Target: 0, Pred: 1, Prob: 0.9357298612594604\n",
            "Text:\n",
            "\"\n",
            "\n",
            "Chicken-Ryu?\n",
            "All the ip addresses you used so far, for your (sneaky) vandalist actions and keeping the edit war alive:\n",
            "\n",
            " Special:Contributions/72.41.48.2\n",
            " Special:Contributions/212.10.39.20\n",
            " Special:Contributions/212.10.37.151\n",
            " Special:Contributions/70.86.96.58\n",
            " Special:Contributions/212.97.132.92\n",
            " Special:Contributions/212.10.38.36\n",
            " Special:Contributions/212.10.33.189\n",
            " Special:Contributions/212.10.33.135\n",
            " Special:Contributions/212.10.39.207\n",
            " Special:Contributions/212.10.37.178\n",
            " Special:Contributions/212.10.34.162\n",
            " Special:Contributions/212.10.37.233\n",
            " Special:Contributions/212.10.36.211\n",
            "(This list is probably not even complete!!)\n",
            "\n",
            "In stead of hiding behind all kinds of different ip addresses, you could at least have the guts and the decency to insult me using your own name!?! Or are you chickin'??? Maybe you can change the name of your style (Genseiryu-Butokukai) into Chicken-Ryu??? Wouldn't even be so bad, since your McDojo is after all called \"\"Kentucky-Kai\"\"!!! -)) At least I am not afraid to use my own name, especially since I know I have nothing to hide and I am on the right side!     P.S.: are you still gonna answer the question above here (Genseiryu-Genseikan)???\"\n",
            "\n",
            "----\n",
            "\n",
            "Target: 0, Pred: 1, Prob: 0.9210891723632812\n",
            "Text:\n",
            "You best watch what you say cause I get drunk as a motherfucker\n",
            "\n",
            "----\n",
            "\n",
            "Target: 0, Pred: 1, Prob: 0.9196086525917053\n",
            "Text:\n",
            ".  (Ah, did I hurt you?)  Just call this a response to your liberal tirade\n",
            "\n",
            "----\n",
            "\n",
            "Target: 0, Pred: 1, Prob: 0.9094776511192322\n",
            "Text:\n",
            "For you information, IKEA is spelled with all caps. That is not yelling. when you thought i had left you, you were sitting on the bed among the mess when i came in. i said we were going after you got home from class. please don't try and bullshit me. It makes me want to listen to you less.\n",
            "\n",
            "----\n",
            "\n",
            "Target: 0, Pred: 1, Prob: 0.9064009189605713\n",
            "Text:\n",
            "ZOE IT JUST HIT ME 2 IM FUCKING SHITIN MYSELF IL DEFO TRY MY HARDEST 2 CUM 2MOROW LUV U MILLIONS LEKDOG\n",
            "\n",
            "----\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Check the most wrong false negatives (model predicted 0 when should've predict 1)\n",
        "for row in most_wrong[-10:].itertuples():\n",
        "  _, text, target, pred, prob = row\n",
        "  print(f\"Target: {target}, Pred: {int(pred)}, Prob: {prob}\")\n",
        "  print(f\"Text:\\n{text}\\n\")\n",
        "  print(\"----\\n\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xRBjEtzztHxd",
        "outputId": "253ae5af-6cf3-4aae-da78-f31c452066c1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Target: 1, Pred: 0, Prob: 0.12195780873298645\n",
            "Text:\n",
            "It was and your are ignorant. Please post only on article discussion page.\n",
            "\n",
            "----\n",
            "\n",
            "Target: 1, Pred: 0, Prob: 0.1127125695347786\n",
            "Text:\n",
            "\"\n",
            "Quit the crap. That is such a reach that it looks silly. To start with it is a symmetric statement which says the same about everybody that is either for or against. North8000  \"\n",
            "\n",
            "----\n",
            "\n",
            "Target: 1, Pred: 0, Prob: 0.10081490874290466\n",
            "Text:\n",
            "also do not vandilise streets when you feel like it because Phillip Trueman is a homosexual.\n",
            ".\n",
            "\n",
            "----\n",
            "\n",
            "Target: 1, Pred: 0, Prob: 0.09253530949354172\n",
            "Text:\n",
            "i just started it 5 minutes before you started this crap. give it some time.\n",
            "\n",
            "----\n",
            "\n",
            "Target: 1, Pred: 0, Prob: 0.08713698387145996\n",
            "Text:\n",
            "they have poor theories\n",
            "\n",
            "----\n",
            "\n",
            "Target: 1, Pred: 0, Prob: 0.051694560796022415\n",
            "Text:\n",
            "dating:i have had two of these. Only started after i sent a text to talk sport radio last week. Any connection do you think or coincidence?\n",
            "\n",
            "----\n",
            "\n",
            "Target: 1, Pred: 0, Prob: 0.03880428522825241\n",
            "Text:\n",
            "Hi \n",
            "\n",
            "Hi muthafuckaa\n",
            "How are you man? Just wanted to wish you a very happy new year 2013.\n",
            "\n",
            "----\n",
            "\n",
            "Target: 1, Pred: 0, Prob: 0.018635747954249382\n",
            "Text:\n",
            "In The Simpsons Movie released in July 2007 name the band that died at the start of the film? A-Green Day, B-Blue Day, C-Red Day. (Send A, B or C)\n",
            "\n",
            "----\n",
            "\n",
            "Target: 1, Pred: 0, Prob: 0.01802699640393257\n",
            "Text:\n",
            "I wonder if this niggaaarr pussiieee remembers me.\n",
            "\n",
            "----\n",
            "\n",
            "Target: 1, Pred: 0, Prob: 0.0025148692075163126\n",
            "Text:\n",
            "on time i pooped\n",
            "and it got on my shirt\n",
            "i thought it might hurt\n",
            "but it was great dessert\n",
            "\n",
            "----\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Making predictions on the test dataset\n",
        "test_sentences = test_df[\"text\"].to_list()\n",
        "test_samples = random.sample(test_sentences, 10)\n",
        "for test_sample in test_samples:\n",
        "  pred_prob = tf.squeeze(model_6.predict([test_sample])) # has to be list\n",
        "  pred = tf.round(pred_prob)\n",
        "  print(f\"Pred: {int(pred)}, Prob: {pred_prob}\")\n",
        "  print(f\"Text:\\n{test_sample}\\n\")\n",
        "  print(\"----\\n\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1hK34OV0tNct",
        "outputId": "85dd0f5d-27d9-4ddf-c380-24db30548ac4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1/1 [==============================] - 0s 50ms/step\n",
            "Pred: 1, Prob: 0.6728590726852417\n",
            "Text:\n",
            "#dw_english Hundreds feared drowned as migrant boat capsizes off Libya: Hundreds of migrants are feared to hav... http://t.co/i9HW0qZoVy\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 40ms/step\n",
            "Pred: 1, Prob: 0.9804993271827698\n",
            "Text:\n",
            "@Gay4BB bitch dont get slapped\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 43ms/step\n",
            "Pred: 0, Prob: 0.10470793396234512\n",
            "Text:\n",
            "#FatLoss #Diet How Can You Find The Best Ways To Reduce Weight? http://t.co/czcC7NIEoX #Thunder #Health\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 43ms/step\n",
            "Pred: 0, Prob: 0.21468447148799896\n",
            "Text:\n",
            "'Failure doesnÛªt exist. ItÛªs only a change of direction' - Alejandro Jodorowsky (via buildings-on-fire) http://t.co/ByMVemW4xF\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 46ms/step\n",
            "Pred: 0, Prob: 0.08679459989070892\n",
            "Text:\n",
            "Zouma! Runaway train. Absolutely flattened the Fiorentina player there...\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 40ms/step\n",
            "Pred: 0, Prob: 0.2024354785680771\n",
            "Text:\n",
            "Truth...\n",
            "https://t.co/Kix1j4ZyGx\n",
            "#News\n",
            "#BBC\n",
            "#CNN\n",
            "#Islam\n",
            "#Truth\n",
            "#god\n",
            "#ISIS\n",
            "#terrorism\n",
            "#Quran\n",
            "#Lies http://t.co/pi6Qn7y7ql\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 46ms/step\n",
            "Pred: 0, Prob: 0.07587280124425888\n",
            "Text:\n",
            "Best Moments of Jon Stewart on the Daily Show http://t.co/EOPbXWoZbm\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 52ms/step\n",
            "Pred: 1, Prob: 0.900214672088623\n",
            "Text:\n",
            "@61_alvin @Craigyb74 @nomar4eva @Shahasda80 @wherepond @trensabby @parallelpond the wall Israel erected has stopped suicide bombing?\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 55ms/step\n",
            "Pred: 1, Prob: 0.8161047101020813\n",
            "Text:\n",
            "#hot  Reddit's new content policy goes into effect many horrible subreddits banned or quarantined http://t.co/sqHDrl7uKG #prebreak #best\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 49ms/step\n",
            "Pred: 0, Prob: 0.081715889275074\n",
            "Text:\n",
            "@HarvardU If 90BLKs&amp;8WHTs colluded 2 take WHT F @USAgov AUTH Hostage&amp;2 make her look BLK w/Bioterrorism&amp;use her lgl/org IDis ID still hers?\n",
            "\n",
            "----\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Turn Tweet into string\n",
        "daniels_tweet = \"Life like an ensemble: take the best choices from others and make your own\""
      ],
      "metadata": {
        "id": "ITXyqo85tStk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def predict_on_sentence(model, sentence):\n",
        "  \"\"\"\n",
        "  Uses model to make a prediction on sentence.\n",
        "\n",
        "  Returns the sentence, the predicted label and the prediction probability.\n",
        "  \"\"\"\n",
        "  pred_prob = model.predict([sentence])\n",
        "  pred_label = tf.squeeze(tf.round(pred_prob)).numpy()\n",
        "  print(f\"Pred: {pred_label}\", \"(real disaster)\" if pred_label > 0 else \"(not real disaster)\", f\"Prob: {pred_prob[0][0]}\")\n",
        "  print(f\"Text:\\n{sentence}\")"
      ],
      "metadata": {
        "id": "JHHszdnxtXqB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Make a prediction on Tweet from the wild\n",
        "predict_on_sentence(model=model_6, # use the USE model\n",
        "                    sentence=daniels_tweet)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MvM4Nk-qtdEl",
        "outputId": "a904bac0-10d9-4fa8-adde-9277c89a8a49"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1/1 [==============================] - 0s 41ms/step\n",
            "Pred: 1.0 (real disaster) Prob: 0.6113803386688232\n",
            "Text:\n",
            "Life like an ensemble: take the best choices from others and make your own\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Source - https://twitter.com/BeirutCityGuide/status/1290696551376007168\n",
        "beirut_tweet_1 = \"Reports that the smoke in Beirut sky contains nitric acid, which is toxic. Please share and refrain from stepping outside unless urgent. #Lebanon\"\n",
        "\n",
        "# Source - https://twitter.com/BeirutCityGuide/status/1290773498743476224\n",
        "beirut_tweet_2 = \"#Beirut declared a “devastated city”, two-week state of emergency officially declared. #Lebanon\""
      ],
      "metadata": {
        "id": "Jx7Gtmz1tgTy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Predict on diaster Tweet 1\n",
        "predict_on_sentence(model=model_6,\n",
        "                    sentence=beirut_tweet_1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TGNZfn-rtu8i",
        "outputId": "6f82f89a-8014-4c8d-cad1-19f90fc5ce44"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1/1 [==============================] - 0s 40ms/step\n",
            "Pred: 0.0 (not real disaster) Prob: 0.039803050458431244\n",
            "Text:\n",
            "Reports that the smoke in Beirut sky contains nitric acid, which is toxic. Please share and refrain from stepping outside unless urgent. #Lebanon\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Predict on diaster Tweet 2\n",
        "predict_on_sentence(model=model_6,\n",
        "                    sentence=beirut_tweet_2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eC89aBUptyyj",
        "outputId": "df255f33-de85-4f2f-d6ec-a3d80300c437"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1/1 [==============================] - 0s 40ms/step\n",
            "Pred: 0.0 (not real disaster) Prob: 0.0315127968788147\n",
            "Text:\n",
            "#Beirut declared a “devastated city”, two-week state of emergency officially declared. #Lebanon\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Calculate the time of predictions\n",
        "import time\n",
        "def pred_timer(model, samples):\n",
        "  \"\"\"\n",
        "  Times how long a model takes to make predictions on samples.\n",
        "\n",
        "  Args:\n",
        "  ----\n",
        "  model = a trained model\n",
        "  sample = a list of samples\n",
        "\n",
        "  Returns:\n",
        "  ----\n",
        "  total_time = total elapsed time for model to make predictions on samples\n",
        "  time_per_pred = time in seconds per single sample\n",
        "  \"\"\"\n",
        "  start_time = time.perf_counter() # get start time\n",
        "  model.predict(samples) # make predictions\n",
        "  end_time = time.perf_counter() # get finish time\n",
        "  total_time = end_time-start_time # calculate how long predictions took to make\n",
        "  time_per_pred = total_time/len(val_sentences) # find prediction time per sample\n",
        "  return total_time, time_per_pred"
      ],
      "metadata": {
        "id": "34oyLRSjt2ad"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Calculate TF Hub Sentence Encoder prediction times\n",
        "model_6_total_pred_time, model_6_time_per_pred = pred_timer(model_6, val_sentences)\n",
        "model_6_total_pred_time, model_6_time_per_pred"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FimnK1XLt8tx",
        "outputId": "94bef1a6-cc70-4682-d4b1-b750cd7e294a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "36/36 [==============================] - 1s 24ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1.3384830049999437, 0.0011741078991227576)"
            ]
          },
          "metadata": {},
          "execution_count": 128
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Calculate Naive Bayes prediction times\n",
        "baseline_total_pred_time, baseline_time_per_pred = pred_timer(model_0, val_sentences)\n",
        "baseline_total_pred_time, baseline_time_per_pred"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qbLtiSFduALb",
        "outputId": "fb22ec8f-b347-4d04-d58c-c5740428f764"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(0.057178659000101106, 5.015671842114132e-05)"
            ]
          },
          "metadata": {},
          "execution_count": 129
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "plt.figure(figsize=(10, 7))\n",
        "plt.scatter(baseline_time_per_pred, baseline_results[\"f1\"], label=\"baseline\")\n",
        "plt.scatter(model_6_time_per_pred, model_6_results[\"f1\"], label=\"tf_hub_sentence_encoder\")\n",
        "plt.legend()\n",
        "plt.title(\"F1-score versus time per prediction\")\n",
        "plt.xlabel(\"Time per prediction\")\n",
        "plt.ylabel(\"F1-Score\");"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 641
        },
        "id": "cNp1DjqXuEuO",
        "outputId": "a6cb32da-1828-4bd7-9cb5-8aa697b2790a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1000x700 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA2UAAAJwCAYAAAAJGsnPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABtFElEQVR4nO3dd3gVVeL/8c9ND6kiIc1AaFKUXmLoaiSUBVRcEJQm5YtSRRGidASsGEFA11XRYEEFxVUMIm0BqaGoEJASehdJIBBCcs/vD37c9ZIASQhOMO/X89xH7pkzZ86ZQ3bzYWbO2IwxRgAAAAAAS7hY3QEAAAAAKM4IZQAAAABgIUIZAAAAAFiIUAYAAAAAFiKUAQAAAICFCGUAAAAAYCFCGQAAAABYiFAGAAAAABYilAEAAACAhQhlAADkU48ePRQZGWl1N3CFZcuWyWazadmyZY6ywp6rWbNmyWazae/evYXWJgAQygAUW5d/ucrtM2LECEe9H374Qb169dLdd98tV1dXfhkvJg4fPqyxY8dq8+bNVncFFpg0aZK+/vprq7sBoJhws7oDAGC18ePHq1y5ck5ld999t+PPn3zyiebMmaM6deooLCzsr+4eLHL48GGNGzdOkZGRqlWrltO2d999V3a73ZqOIV8KOleTJk3SI488ogcffNCpvGvXrnr00Ufl6elZSD0EAEIZAKhVq1aqV6/eVbdPmjRJ7777rtzd3fWPf/xDv/7661/Yu8KRnp4uHx8fq7txXbdKP93d3a3ugiVu1vzY7XZlZmbKy8ur0Nsu7LlydXWVq6trobYJANy+CADXERYWdkO/2J05c0ZDhgxRZGSkPD09Vbp0aT3wwAPauHGjU721a9eqdevWuu222+Tj46MaNWrozTffdKqzZMkSNWnSRD4+PgoMDFT79u2VnJzsVGfs2LGy2Wzatm2bunTpottuu02NGzd2bJ89e7bq1q0rb29vlSxZUo8++qgOHDhwzTF8+eWXstlsWr58eY5t77zzjmw2m1NY3b59ux555BGVLFlSXl5eqlevnr755hun/S7fPrp8+XI99dRTKl26tO644448n7PIyEj16NEjR3+aN2+u5s2bO5VNmzZNd911l0qUKKHbbrtN9erV0yeffHLV8S5btkz169eXJPXs2dNxW+usWbMk5XxOae/evbLZbHrttdc0ffp0lS9fXiVKlFCLFi104MABGWM0YcIE3XHHHfL29lb79u116tSpHMf9/vvvHfPr5+enNm3aaOvWrVft55Xn8r///a/+7//+T7fffrv8/f3VrVs3/fHHHwU6To8ePeTr66vdu3erdevW8vPz02OPPXbVPlz+e7d9+3Z17NhR/v7+uv322zV48GBlZGQ41bXZbBowYIA+/vhj3XXXXfL09FRiYqIk6dChQ3riiScUHBwsT09P3XXXXXr//fdzHO/gwYN68MEH5ePjo9KlS+vpp5/WhQsXctTL7Zkyu92uN998U9WrV5eXl5eCgoLUsmVLbdiwwdG/9PR0ffjhh465v/x37WrPlM2YMcMxlrCwMPXv31+nT592qtO8eXPdfffd2rZtm+69916VKFFC4eHheuWVV656XgEUD1wpA1Dspaam6uTJk05lpUqVKrT2+/Xrpy+//FIDBgxQtWrV9Pvvv2vlypVKTk5WnTp1JEmLFi3SP/7xD4WGhmrw4MEKCQlRcnKyvv32Ww0ePFiS9OOPP6pVq1YqX768xo4dq/Pnz2vatGlq1KiRNm7cmOMXz3/+85+qVKmSJk2aJGOMJGnixIkaNWqUOnbsqN69e+vEiROaNm2amjZtqk2bNikwMDDXMbRp00a+vr76/PPP1axZM6dtc+bM0V133eW45XPr1q1q1KiRwsPDNWLECPn4+Ojzzz/Xgw8+qLlz5+qhhx5y2v+pp55SUFCQRo8erfT09Dyfs7x69913NWjQID3yyCOOgPDzzz9r7dq16tKlS677VK1aVePHj9fo0aPVt29fNWnSRJLUsGHDax7r448/VmZmpgYOHKhTp07plVdeUceOHXXfffdp2bJlGj58uHbt2qVp06bp2WefdQobCQkJ6t69u2JjY/Xyyy/r3Llzmjlzpho3bqxNmzbl6VnGAQMGKDAwUGPHjtWOHTs0c+ZM7du3z7EARn6Pk5WVpdjYWDVu3FivvfaaSpQocd0+dOzYUZGRkZo8ebLWrFmjqVOn6o8//tBHH33kVG/JkiX6/PPPNWDAAJUqVUqRkZE6duyY7rnnHkdoCwoK0vfff69evXopLS1NQ4YMkSSdP39e999/v/bv369BgwYpLCxMCQkJWrJkyXX7J0m9evXSrFmz1KpVK/Xu3VtZWVlasWKF1qxZo3r16ikhIUG9e/dWgwYN1LdvX0lShQoVrtre2LFjNW7cOMXExOjJJ590nPv169dr1apVTv+o88cff6hly5Z6+OGH1bFjR3355ZcaPny4qlevrlatWuWp/wD+hgwAFFMffPCBkZTr52ratGljypYtm6/jBAQEmP79+191e1ZWlilXrpwpW7as+eOPP5y22e12x59r1aplSpcubX7//XdH2ZYtW4yLi4vp1q2bo2zMmDFGkuncubNTW3v37jWurq5m4sSJTuW//PKLcXNzy1F+pc6dO5vSpUubrKwsR9mRI0eMi4uLGT9+vKPs/vvvN9WrVzcZGRlO42jYsKGpVKmSo+zy+W/cuLFTm8Zc/5wZY0zZsmVN9+7dc5Q3a9bMNGvWzPG9ffv25q677rpmW7lZv369kWQ++OCDHNu6d+/u9PcgJSXFSDJBQUHm9OnTjvK4uDgjydSsWdNcvHjRUd65c2fj4eHhOEdnzpwxgYGBpk+fPk7HOXr0qAkICMhRfqXL57Ju3bomMzPTUf7KK68YSWb+/Pn5Pk737t2NJDNixIhrHvuyy3/v2rVr51T+1FNPGUlmy5YtjjJJxsXFxWzdutWpbq9evUxoaKg5efKkU/mjjz5qAgICzLlz54wxxsTHxxtJ5vPPP3fUSU9PNxUrVjSSzNKlS53G8ee5WrJkiZFkBg0alGMMf/558/HxyfXv1+VznZKSYowx5vjx48bDw8O0aNHCZGdnO+q99dZbRpJ5//33HWXNmjUzksxHH33kKLtw4YIJCQkxHTp0yHEsAMUHty8CKPamT5+uRYsWOX0KU2BgoNauXavDhw/nun3Tpk1KSUnRkCFDclypunx148iRI9q8ebN69OihkiVLOrbXqFFDDzzwgBYsWJCj3X79+jl9nzdvnux2uzp27KiTJ086PiEhIapUqZKWLl16zXF06tRJx48fd1pu/Msvv5TdblenTp0kSadOndKSJUvUsWNHnTlzxnGM33//XbGxsdq5c6cOHTrk1G6fPn1yPKNzvXOWH4GBgTp48KDWr19/w21dzz//+U8FBAQ4vkdFRUmSHn/8cbm5uTmVZ2ZmOs7FokWLdPr0aXXu3NlpblxdXRUVFXXdubmsb9++TldlnnzySbm5uTn+fhTkOE8++WS+zkH//v2dvg8cOFCScvwdbdasmapVq+b4bozR3Llz1bZtWxljnPoXGxur1NRUx+2rCxYsUGhoqB555BHH/iVKlHBc1bqWuXPnymazacyYMTm2Xf55y48ff/xRmZmZGjJkiFxc/vdrVZ8+feTv76/vvvvOqb6vr68ef/xxx3cPDw81aNBAe/bsyfexAfx9cPsigGKvQYMG11zoIy+ys7N14sQJp7KSJUvKw8NDr7zyirp3766IiAjVrVtXrVu3Vrdu3VS+fHlJ0u7duyU5r/h4pX379kmSKleunGNb1apVtXDhwhyLMFy5ouTOnTtljFGlSpVyPcb1nptr2bKlAgICNGfOHN1///2SLt26WKtWLd15552SpF27dskYo1GjRmnUqFG5tnP8+HGFh4dftZ+SrnvO8mP48OH68ccf1aBBA1WsWFEtWrRQly5d1KhRo3y3dT1lypRx+n45oEVERORafvl5r507d0qS7rvvvlzb9ff3z9Pxr5xbX19fhYaGOp5/yu9x3NzcHM/55dWVfahQoYJcXFxyPIN15byfOHFCp0+f1r/+9S/961//yrXt48ePS7r081CxYsUcISq3n48r7d69W2FhYU7/uHEjrvaz6eHhofLlyzu2X3bHHXfk6Pdtt92mn3/+uVD6A+DWRCgDgEJw4MCBHL9kLl26VM2bN1fHjh3VpEkTffXVV/rhhx/06quv6uWXX9a8efNu6jMk3t7eTt/tdrtsNpu+//77XFeP8/X1vWZ7np6eevDBB/XVV19pxowZOnbsmFatWqVJkyY5HUOSnn32WcXGxubaTsWKFa/ZT0l5OmdXu6qRnZ3tNL6qVatqx44d+vbbb5WYmKi5c+dqxowZGj16tMaNG3fNMefX1Vblu1q5+f/P+l0+bwkJCQoJCclR789X2W5Efo/j6enpdPWnIK42T7n9/ZQuXVXs3r17rvvUqFHjhvpSFFzv7wKA4olQBgCFICQkJMdtjzVr1nT8OTQ0VE899ZSeeuopHT9+XHXq1NHEiRPVqlUrxwICv/76q2JiYnJtv2zZspKkHTt25Ni2fft2lSpV6rpLlVeoUEHGGJUrV85xZSu/OnXqpA8//FCLFy9WcnKyjDGOWxclOa5kubu7X3UseXWtcyZdurpw5ep20qUrF1deUfPx8VGnTp3UqVMnZWZm6uGHH9bEiRMVFxd31WXYC3IrW0Fd/jtQunTpGzpvO3fu1L333uv4fvbsWR05ckStW7cu1ONcrw9//geKXbt2yW63X3ehkqCgIPn5+Sk7O/u6fStbtqx+/fVXGWOc5im3n48rVahQQQsXLtSpU6euebUsr/P/55/NP/+9y8zMVEpKyk07zwD+XnimDAAKgZeXl2JiYpw+t912m7Kzs5WamupUt3Tp0goLC3Ms312nTh2VK1dO8fHxOULG5X89Dw0NVa1atfThhx861fn111/1ww8/OH7pvpaHH35Yrq6uGjduXI5/lTfG6Pfff79uGzExMSpZsqTmzJmjOXPmqEGDBk6/gJcuXVrNmzfXO++8oyNHjuTY/8pbPHOTl3MmXfrles2aNcrMzHSUffvttzmW979yXB4eHqpWrZqMMbp48eJV+3E55OYW/ApbbGys/P39NWnSpFz7lJfzJkn/+te/nPafOXOmsrKyHEG2sI5zLdOnT3f6Pm3aNEm67lVhV1dXdejQQXPnzs31XYB/7lvr1q11+PBhffnll46yc+fOXfW2xz/r0KGDjDG5XiX988+Fj49PnuY+JiZGHh4emjp1qtP+7733nlJTU9WmTZvrtgEAXCkDgOv4+eefHe/Y2rVrl1JTU/Xiiy9KunQ1rG3btlfd98yZM7rjjjv0yCOPqGbNmvL19dWPP/6o9evX6/XXX5ckubi4aObMmWrbtq1q1aqlnj17KjQ0VNu3b9fWrVu1cOFCSdKrr76qVq1aKTo6Wr169XIsiR8QEKCxY8dedxwVKlTQiy++qLi4OO3du1cPPvig/Pz8lJKSoq+++kp9+/bVs88+e8023N3d9fDDD+uzzz5Tenq6XnvttRx1pk+frsaNG6t69erq06ePypcvr2PHjmn16tU6ePCgtmzZcs1j5OWcSVLv3r315ZdfqmXLlurYsaN2796t2bNn51i6vEWLFgoJCVGjRo0UHBys5ORkvfXWW2rTpo38/Pyueb4CAwP19ttvy8/PTz4+PoqKisr1Gbgb5e/vr5kzZ6pr166qU6eOHn30UQUFBWn//v367rvv1KhRI7311lvXbSczM1P333+/OnbsqB07dmjGjBlq3Lix2rVrV6jHuZaUlBS1a9dOLVu21OrVqzV79mx16dLF6crx1bz00ktaunSpoqKi1KdPH1WrVk2nTp3Sxo0b9eOPPzre7danTx+99dZb6tatm5KSkhQaGqqEhIQ8Ldl/7733qmvXrpo6dap27typli1bym63a8WKFbr33ns1YMAASVLdunX1448/asqUKQoLC1O5cuUcC7f8WVBQkOLi4jRu3Di1bNlS7dq1c5z7+vXrOy3qAQBX9dcv+AgARcPlpa3Xr1+fp3q5fXJbMvvPLly4YIYNG2Zq1qxp/Pz8jI+Pj6lZs6aZMWNGjrorV640DzzwgKNejRo1zLRp05zq/Pjjj6ZRo0bG29vb+Pv7m7Zt25pt27Y51bm8NPmJEydy7dPcuXNN48aNjY+Pj/Hx8TFVqlQx/fv3Nzt27LjmWC5btGiRkWRsNps5cOBArnV2795tunXrZkJCQoy7u7sJDw83//jHP8yXX37pqHO185+fc/b666+b8PBw4+npaRo1amQ2bNiQY0n8d955xzRt2tTcfvvtxtPT01SoUMEMGzbMpKamXnes8+fPN9WqVTNubm5Oy+NfbUn8V1991Wn/pUuXGknmiy++cCq/2tiXLl1qYmNjTUBAgPHy8jIVKlQwPXr0MBs2bLhmPy+3t3z5ctO3b19z2223GV9fX/PYY485vUIhP8fp3r278fHxue45uuzy37tt27aZRx55xPj5+ZnbbrvNDBgwwJw/f96prqSrvvLg2LFjpn///iYiIsK4u7ubkJAQc//995t//etfTvX27dtn2rVrZ0qUKGFKlSplBg8ebBITE6+7JL4xl15D8eqrr5oqVaoYDw8PExQUZFq1amWSkpIcdbZv326aNm1qvL29nX7Wr1wS/7K33nrLVKlSxbi7u5vg4GDz5JNP5njFRbNmzXJ9PUNufQRQvNiM4clSAABuZbNmzVLPnj21fv36G15JtKAuv0D5xIkThfrydQAoDnimDAAAAAAsRCgDAAAAAAsRygAAAADAQjxTBgAAAAAW4koZAAAAAFiIUAYAAAAAFuLl0QVkt9t1+PBh+fn5yWazWd0dAAAAABYxxujMmTMKCwuTi0v+r3sRygro8OHDioiIsLobAAAAAIqIAwcO6I477sj3foSyAvLz85N06cT7+/tb3BsAAAAAVklLS1NERIQjI+QXoayALt+y6O/vTygDAAAAUODHmorEQh/Tp09XZGSkvLy8FBUVpXXr1l217sWLFzV+/HhVqFBBXl5eqlmzphITEwvcpjFGrVq1ks1m09dff11YQwIAAACAPLE8lM2ZM0dDhw7VmDFjtHHjRtWsWVOxsbE6fvx4rvVHjhypd955R9OmTdO2bdvUr18/PfTQQ9q0aVOB2oyPj2ehDgAAAACWsfzl0VFRUapfv77eeustSZdWNYyIiNDAgQM1YsSIHPXDwsL0wgsvqH///o6yDh06yNvbW7Nnz85Xm5s3b9Y//vEPbdiwQaGhofrqq6/04IMP5qnfaWlpCggIUGpqKrcvAgAAAMXYjWYDS58py8zMVFJSkuLi4hxlLi4uiomJ0erVq3Pd58KFC/Ly8nIq8/b21sqVK/PV5rlz59SlSxdNnz5dISEh1+3rhQsXdOHCBcf3tLS06+5jjFFWVpays7OvWxdA4XN1dZWbmxtXwwEAQJFmaSg7efKksrOzFRwc7FQeHBys7du357pPbGyspkyZoqZNm6pChQpavHix5s2b5wg+eW3z6aefVsOGDdW+ffs89XXy5MkaN25cnseWmZmpI0eO6Ny5c3neB0DhK1GihEJDQ+Xh4WF1VwAAAHJ1y62++Oabb6pPnz6qUqWKbDabKlSooJ49e+r999/PcxvffPONlixZ4vQc2vXExcVp6NChju+Xl73Mjd1uV0pKilxdXRUWFiYPDw/+pR74ixljlJmZqRMnTiglJUWVKlUq0MscAQAAbjZLQ1mpUqXk6uqqY8eOOZUfO3bsqrcUBgUF6euvv1ZGRoZ+//13hYWFacSIESpfvnye21yyZIl2796twMBApzodOnRQkyZNtGzZshzH9fT0lKenZ57GlZmZ6XiOrUSJEnnaB0Dh8/b2lru7u/bt26fMzMwctz4DAAAUBZb+s7GHh4fq1q2rxYsXO8rsdrsWL16s6Ojoa+7r5eWl8PBwZWVlae7cuY7bEPPS5ogRI/Tzzz9r8+bNjo8kvfHGG/rggw8KbXz8qzxgPX4OAQBAUWf57YtDhw5V9+7dVa9ePTVo0EDx8fFKT09Xz549JUndunVTeHi4Jk+eLElau3atDh06pFq1aunQoUMaO3as7Ha7nnvuuTy3GRISkuuVuDJlyqhcuXJ/wagBAAAA4BLLQ1mnTp104sQJjR49WkePHlWtWrWUmJjoWKhj//79Tv/SnZGRoZEjR2rPnj3y9fVV69atlZCQ4HQr4vXaBAAAAICiwvL3lN2qrvUugoyMDKWkpKhcuXK33DMszZs3V61atRQfH2/J8Xv06KHTp0/r66+/LhL9wa3vVv55BAAAt4Zb+j1lwPXMmzdP7u7uVncDAAAAuGkIZUVYtt1oXcopHT+TodJ+XmpQrqRcXYrX0volS5a0ugsAAADATcWyZEVU4q9H1PjlJer87hoN/myzOr+7Ro1fXqLEX4/c9GNnZWVpwIABCggIUKlSpTRq1Chdvss1ISFB9erVk5+fn0JCQtSlSxcdP37cse8ff/yhxx57TEFBQfL29lalSpWcVrQ8cOCAOnbsqMDAQJUsWVLt27fX3r17r9qX5s2ba8iQIY7vkZGRmjRpkp544gn5+fmpTJky+te//uW0T36PAQAAAFiJUFYEJf56RE/O3qgjqRlO5UdTM/Tk7I03PZh9+OGHcnNz07p16/Tmm29qypQp+ve//y1JunjxoiZMmKAtW7bo66+/1t69e9WjRw/HvqNGjdK2bdv0/fffKzk5WTNnzlSpUqUc+8bGxsrPz08rVqzQqlWr5Ovrq5YtWyozMzPP/Xv99ddVr149bdq0SU899ZSefPJJ7dixo1CPAQAAAPxVuH2xiMm2G437zzbltvqKkWSTNO4/2/RAtZCbditjRESE3njjDdlsNlWuXFm//PKL3njjDfXp00dPPPGEo1758uU1depU1a9fX2fPnpWvr6/279+v2rVrq169epIuXdm6bM6cObLb7fr3v/8tm+1S3z/44AMFBgZq2bJlatGiRZ7617p1az311FOSpOHDh+uNN97Q0qVLVbly5UI7BgAAAPBX4UpZEbMu5VSOK2R/ZiQdSc3QupRTN60P99xzjyPQSFJ0dLR27typ7OxsJSUlqW3btipTpoz8/PzUrFkzSZdeXSBJTz75pD777DPVqlVLzz33nH766SdHO1u2bNGuXbvk5+cnX19f+fr6qmTJksrIyNDu3bvz3L8aNWo4/myz2RQSEuK4hbKwjgEAAIAizJ4tpayQfvny0n/t2Vb36IZwpayIOX7m6oGsIPUKU0ZGhmJjYxUbG6uPP/5YQUFB2r9/v2JjYx23BrZq1Ur79u3TggULtGjRIt1///3q37+/XnvtNZ09e1Z169bVxx9/nKPtoKCgPPfjytUYbTab7Ha7JBXaMQAAAFBEbftGShwupR3+X5l/mNTyZalaO+v6dQMIZUVMab+8vUcpr/UKYu3atU7f16xZo0qVKmn79u36/fff9dJLLykiIkKStGHDhhz7BwUFqXv37urevbuaNGmiYcOG6bXXXlOdOnU0Z84clS5dukDvb8iLv+IYAAAAsMi2b6TPu0lXPuyTduRSecePbslgxu2LRUyDciUVGuClqz0tZpMUGnBpefybZf/+/Ro6dKh27NihTz/9VNOmTdPgwYNVpkwZeXh4aNq0adqzZ4+++eYbTZgwwWnf0aNHa/78+dq1a5e2bt2qb7/9VlWrVpUkPfbYYypVqpTat2+vFStWKCUlRcuWLdOgQYN08ODBQun7X3EMAAAAWMCefekK2VVXX5CUOOKWvJWRUFbEuLrYNKZtNUnKEcwufx/TttpNfV9Zt27ddP78eTVo0ED9+/fX4MGD1bdvXwUFBWnWrFn64osvVK1aNb300kt67bXXnPb18PBQXFycatSooaZNm8rV1VWfffaZJKlEiRL673//qzJlyujhhx9W1apV1atXL2VkZBTaVa2/4hgAAACwwL6fnG9ZzMFIaYcu1bvF2MzlF1AhX9LS0hQQEKDU1NQcv+xnZGQoJSVF5cqVk5dXwW4zTPz1iMb9Z5vToh+hAV4a07aaWt4dekN9B4qTwvh5BAAARcAvX0pze12/Xof3pOqP3Pz+/Mm1skFe8ExZEdXy7lA9UC1E61JO6fiZDJX2u3TL4s28QgYAAAAUWb7BhVuvCCGUFWGuLjZFV7jd6m4AAAAA1ivb8NIqi2lHlPtzZbZL28s2/Kt7dsN4pgwAAABA0efiemnZe0lXXX2h5UuX6t1iCGUAAAAAbg3V2l1a9t7/ijUW/MNu2eXwJW5fBAAAAHArqdZOqtLm0iqLZ49deoasbMNb8grZZYQyAAAAALcWF1epXBOre1FouH0RAAAAACxEKAMAAAAACxHKAAAAAMBChDJc16pVq1S9enW5u7vrwQcfvGbdZcuWyWaz6fTp0zd0zObNm2vIkCE31AZuHYX19wYAAOBWRCiDk9zC0NChQ1WrVi2lpKRo1qxZlvSrqBk7dqxq1apldTcAAADwN0AoK8rs2VLKCumXLy/9155tSTd2796t++67T3fccYcCAwMt6QNwPZmZmVZ3AQAAoEAIZUXVtm+k+LulD/8hze116b/xd18qv0l69Oih5cuX680335TNZnN8fv/9dz3xxBOy2Wx5vlKWlJSkevXqqUSJEmrYsKF27NjhdJwrb4McMmSImjdv7lSWlZWlAQMGKCAgQKVKldKoUaNkjMnT8WfMmKFKlSrJy8tLwcHBeuSRRxzb7Ha7Jk+erHLlysnb21s1a9bUl19+6dh++Va6xYsX5zqGWbNmady4cdqyZYvjHF0+L6dPn1bv3r0VFBQkf39/3XfffdqyZYuj7ctX2BISEhQZGamAgAA9+uijOnPmjFP/XnnlFVWsWFGenp4qU6aMJk6c6Nh+4MABdezYUYGBgSpZsqTat2+vvXv35um8SNK///1vVa1aVV5eXqpSpYpmzJjh2LZ3717ZbDbNmzdP9957r0qUKKGaNWtq9erVTm2sWrVKzZs3V4kSJXTbbbcpNjZWf/zxhyTpwoULGjRokEqXLi0vLy81btxY69evd9p/wYIFuvPOO+Xt7a1777031/6vXLlSTZo0kbe3tyIiIjRo0CClp6c7tkdGRmrChAnq1q2b/P391bdv3zyfAwAAgKKEUFYUbftG+ryblHbYuTztyKXymxTM3nzzTUVHR6tPnz46cuSIDh48qIMHD8rf31/x8fE6cuSIOnXqlKe2XnjhBb3++uvasGGD3Nzc9MQTT+S7Px9++KHc3Ny0bt06vfnmm5oyZYr+/e9/X3e/DRs2aNCgQRo/frx27NihxMRENW3a1LF98uTJ+uijj/T2229r69atevrpp/X4449r+fLleRpDp06d9Mwzz+iuu+7SkSNHnM7LP//5Tx0/flzff/+9kpKSVKdOHd1///06deqUo93du3fr66+/1rfffqtvv/1Wy5cv10svveTYHhcXp5deekmjRo3Stm3b9Mknnyg4OFiSdPHiRcXGxsrPz08rVqzQqlWr5Ovrq5YtW+bpStHHH3+s0aNHa+LEiUpOTtakSZM0atQoffjhhznG/uyzz2rz5s2688471blzZ2VlZUmSNm/erPvvv1/VqlXT6tWrtXLlSrVt21bZ2Zeu5D733HOaO3euPvzwQ23cuFEVK1ZUbGys4xwcOHBADz/8sNq2bavNmzerd+/eGjFihNPxd+/erZYtW6pDhw76+eefNWfOHK1cuVIDBgxwqvfaa6+pZs2a2rRpk0aNGnXd8QMAABRJBgWSmppqJJnU1NQc286fP2+2bdtmzp8/n/+Gs7OMeb2KMWP8r/IJMOb1qpfq3QTNmjUzgwcPdioLCAgwH3zwQZ72X7p0qZFkfvzxR0fZd999ZyQ5zkf37t1N+/btnfYbPHiwadasmVM/qlataux2u6Ns+PDhpmrVqtftw9y5c42/v79JS0vLsS0jI8OUKFHC/PTTT07lvXr1Mp07d87zGMaMGWNq1qzp1MaKFSuMv7+/ycjIcCqvUKGCeeeddxz7lShRwqlvw4YNM1FRUcYYY9LS0oynp6d59913cx1bQkKCqVy5stN5uXDhgvH29jYLFy685nm53JdPPvnEqWzChAkmOjraGGNMSkqKkWT+/e9/O7Zv3brVSDLJycnGGGM6d+5sGjVqlGv7Z8+eNe7u7ubjjz92lGVmZpqwsDDzyiuvGGOMiYuLM9WqVXPab/jw4UaS+eOPP4wxl+ajb9++TnVWrFhhXFxcHHNQtmxZ8+CDD153zDf08wgAAJAH18oGeeFmWRpE7vb9lPMKmRMjpR26VK8Iv8W8Ro0ajj+HhoZKko4fP64yZcrkuY177rlHNpvN8T06Olqvv/66srOz5erqetX9HnjgAZUtW1bly5dXy5Yt1bJlSz300EMqUaKEdu3apXPnzumBBx5w2iczM1O1a9e+oTFs2bJFZ8+e1e233+5Ufv78ee3evdvxPTIyUn5+fk5tHz9+XJKUnJysCxcu6P7777/qMXbt2uW0vyRlZGQ4HSM36enp2r17t3r16qU+ffo4yrOyshQQEOBU92pjr1KlijZv3qx//vOfuR5j9+7dunjxoho1auQoc3d3V4MGDZScnOwYY1RUlNN+0dHROcb5888/6+OPP3aUGWNkt9uVkpKiqlWrSpLq1at3zTEDAADcCghlRc3ZY4VbzyLu7u6OP18OVna7XZLk4uKS49mwixcvFtqx/fz8tHHjRi1btkw//PCDRo8erbFjx2r9+vU6e/asJOm7775TeHi4036enp55HkNuzp49q9DQUC1btizHtj8vkPLndi+3fbldb2/va47t7Nmzqlu3rlNYuSwoKOi6+0rSu+++myMUXRlyrzX26/WxMJw9e1b/93//p0GDBuXY9udQ7OPjc9P7AgAAcLMRyooa3+DCrZdPHh4ejmeDbpagoCD9+uuvTmWbN2/OEVbWrl3r9H3NmjWqVKnSNa+SXebm5qaYmBjFxMRozJgxCgwM1JIlS/TAAw/I09NT+/fvV7NmzQo8htzOU506dXT06FG5ubkpMjKyQO1WqlRJ3t7eWrx4sXr37p1je506dTRnzhyVLl1a/v7++Wo7ODhYYWFh2rNnjx577LEC9U+6dBVt8eLFGjduXI5tFSpUkIeHh1atWqWyZctKuhS4169f73jVQtWqVfXNN87PRa5Zs8bpe506dbRt2zZVrFixwP0EAAC4VbDQR1FTtqHkHybJdpUKNsk//FK9myAyMlJr167V3r17dfLkyWteGSqo++67Txs2bNBHH32knTt3asyYMTlCmiTt379fQ4cO1Y4dO/Tpp59q2rRpGjx48HXb//bbbzV16lRt3rxZ+/bt00cffSS73a7KlSvLz89Pzz77rJ5++ml9+OGH2r17tzZu3Khp06blWOziWiIjI5WSkqLNmzfr5MmTunDhgmJiYhQdHa0HH3xQP/zwg/bu3auffvpJL7zwgjZs2JCndr28vDR8+HA999xz+uijj7R7926tWbNG7733niTpscceU6lSpdS+fXutWLFCKSkpWrZsmQYNGqSDBw9et/1x48Zp8uTJmjp1qn777Tf98ssv+uCDDzRlypQ8jz0uLk7r16/XU089pZ9//lnbt2/XzJkzdfLkSfn4+OjJJ5/UsGHDlJiYqG3btqlPnz46d+6cevXqJUnq16+fdu7cqWHDhmnHjh365JNPcqzqOXz4cP30008aMGCANm/erJ07d2r+/Pk5FvoAAAD4OyCUFTUurlLLl///lyuD2f//3vKlS/VugmeffVaurq6qVq2agoKCtH///kI/RmxsrEaNGqXnnntO9evX15kzZ9StW7cc9bp166bz58+rQYMG6t+/vwYPHpynZc8DAwM1b9483Xfffapatarefvttffrpp7rrrrskSRMmTNCoUaM0efJkVa1aVS1bttR3332ncuXK5XkMHTp0UMuWLXXvvfcqKChIn376qWw2mxYsWKCmTZuqZ8+euvPOO/Xoo49q3759jtUT82LUqFF65plnNHr0aFWtWlWdOnVyPHNWokQJ/fe//1WZMmX08MMPq2rVqurVq5cyMjLydOWsd+/e+ve//60PPvhA1atXV7NmzTRr1qx8jf3OO+/UDz/8oC1btqhBgwaKjo7W/Pnz5eZ26cL7Sy+9pA4dOqhr166qU6eOdu3apYULF+q2226TdOn2w7lz5+rrr79WzZo19fbbb2vSpElOx6hRo4aWL1+u3377TU2aNFHt2rU1evRohYWF5bmfAAAAtwqbufLhHuRJWlqaAgIClJqamuOX4YyMDKWkpKhcuXLy8vIq2AG2fSMlDnde9MM//FIgq9buBnoOFC+F8vMIAABwDdfKBnnBM2VFVbV2UpU2l1ZZPHvs0jNkZRvetCtkAAAAAKzB7YtFmYvrpWXvqz9y6b9FIJD169dPvr6+uX769ev3l/RhxYoVV+2Dr6/vX9KHoupa52XFihVWdw8AAAC54EoZ8mX8+PF69tlnc91WkEu1BVGvXj1t3rz5LznWreZa5+XKVwAAAACgaCCUIV9Kly6t0qVLW9oHb29vlkq/Cs4LAADArYfbF28i1lABrMfPIQAAKOoIZTfB5Zcgnzt3zuKeALj8c3jly8kBAACKCm5fvAlcXV0VGBjo9G4pm+1qL4MGcDMYY3Tu3DkdP35cgYGBcnW1fqEcAACA3BDKbpKQkBBJcgQzANYIDAx0/DwCAAAURYSym8Rmsyk0NFSlS5fWxYsXre4OUCy5u7tzhQwAABR5hLKbzNXVlV8KAQAAAFwVC30AAAAAgIUIZQAAAABgIUIZAAAAAFiIUAYAAAAAFiKUAQAAAICFCGUAAAAAYCFCGQAAAABYiFAGAAAAABYilAEAAACAhQhlAAAAAGAhQhkAAAAAWIhQBgAAAAAWIpQBAAAAgIUIZQAAAABgIUIZAAAAAFiIUAYAAAAAFioSoWz69OmKjIyUl5eXoqKitG7duqvWvXjxosaPH68KFSrIy8tLNWvWVGJiYr7aPHXqlAYOHKjKlSvL29tbZcqU0aBBg5SamnpTxgcAAAAAV2N5KJszZ46GDh2qMWPGaOPGjapZs6ZiY2N1/PjxXOuPHDlS77zzjqZNm6Zt27apX79+euihh7Rp06Y8t3n48GEdPnxYr732mn799VfNmjVLiYmJ6tWr118yZgAAAAC4zGaMMVZ2ICoqSvXr19dbb70lSbLb7YqIiNDAgQM1YsSIHPXDwsL0wgsvqH///o6yDh06yNvbW7Nnzy5Qm5L0xRdf6PHHH1d6errc3Nyu2++0tDQFBAQoNTVV/v7++R43AAAAgL+HG80Gll4py8zMVFJSkmJiYhxlLi4uiomJ0erVq3Pd58KFC/Ly8nIq8/b21sqVKwvcpiTHCbxaILtw4YLS0tKcPgAAAABwoywNZSdPnlR2draCg4OdyoODg3X06NFc94mNjdWUKVO0c+dO2e12LVq0SPPmzdORI0cK3ObJkyc1YcIE9e3b96p9nTx5sgICAhyfiIiI/AwVAAAAAHJl+TNl+fXmm2+qUqVKqlKlijw8PDRgwAD17NlTLi4FG0paWpratGmjatWqaezYsVetFxcXp9TUVMfnwIEDBRwBAAAAAPyPpaGsVKlScnV11bFjx5zKjx07ppCQkFz3CQoK0tdff6309HTt27dP27dvl6+vr8qXL5/vNs+cOaOWLVvKz89PX331ldzd3a/aV09PT/n7+zt9AAAAAOBGWRrKPDw8VLduXS1evNhRZrfbtXjxYkVHR19zXy8vL4WHhysrK0tz585V+/bt89VmWlqaWrRoIQ8PD33zzTc5nlMDAAAAgL/C9ZcZvMmGDh2q7t27q169emrQoIHi4+OVnp6unj17SpK6deum8PBwTZ48WZK0du1aHTp0SLVq1dKhQ4c0duxY2e12Pffcc3lu83IgO3funGbPnu20cEdQUJBcXV3/4rMAAAAAoLiyPJR16tRJJ06c0OjRo3X06FHVqlVLiYmJjoU69u/f7/S8WEZGhkaOHKk9e/bI19dXrVu3VkJCggIDA/Pc5saNG7V27VpJUsWKFZ36k5KSosjIyJs7aAAAAAD4/yx/T9mtiveUAQAAAJBu8feUAQAAAEBxRygDAAAAAAsRygAAAADAQoQyAAAAALAQoQwAAAAALEQoAwAAAAALEcoAAAAAwEKEMgAAAACwEKEMAAAAACxEKAMAAAAACxHKAAAAAMBChDIAAAAAsBChDAAAAAAsRCgDAAAAAAsRygAAAADAQoQyAAAAALAQoQwAAAAALEQoAwAAAAALEcoAAAAAwEKEMgAAAACwEKEMAAAAACxEKAMAAAAACxHKAAAAAMBChDIAAAAAsBChDAAAAAAsRCgDAAAAAAsRygAAAADAQoQyAAAAALAQoQwAAAAALEQoAwAAAAALEcoAAAAAwEKEMgAAAACwEKEMAAAAACxEKAMAAAAACxHKAAAAAMBChDIAAAAAsBChDAAAAAAsRCgDAAAAAAsRygAAAADAQoQyAAAAALAQoQwAAAAALEQoAwAAAAALEcoAAAAAwEKEMgAAAACwEKEMAAAAACxEKAMAAAAACxHKAAAAAMBChDIAAAAAsBChDAAAAAAsRCgDAAAAAAsRygAAAADAQoQyAAAAALAQoQwAAAAALEQoAwAAAAALEcoAAAAAwEKEMgAAAACwEKEMAAAAACxEKAMAAAAACxWJUDZ9+nRFRkbKy8tLUVFRWrdu3VXrXrx4UePHj1eFChXk5eWlmjVrKjExMd9tZmRkqH///rr99tvl6+urDh066NixY4U+NgAAAAC4FstD2Zw5czR06FCNGTNGGzduVM2aNRUbG6vjx4/nWn/kyJF65513NG3aNG3btk39+vXTQw89pE2bNuWrzaefflr/+c9/9MUXX2j58uU6fPiwHn744Zs+XgAAAAD4M5sxxljZgaioKNWvX19vvfWWJMlutysiIkIDBw7UiBEjctQPCwvTCy+8oP79+zvKOnToIG9vb82ePTtPbaampiooKEiffPKJHnnkEUnS9u3bVbVqVa1evVr33HPPdfudlpamgIAApaamyt/f/4bPAwAAAIBb041mA0uvlGVmZiopKUkxMTGOMhcXF8XExGj16tW57nPhwgV5eXk5lXl7e2vlypV5bjMpKUkXL150qlOlShWVKVPmmsdNS0tz+gAAAADAjbI0lJ08eVLZ2dkKDg52Kg8ODtbRo0dz3Sc2NlZTpkzRzp07ZbfbtWjRIs2bN09HjhzJc5tHjx6Vh4eHAgMD83zcyZMnKyAgwPGJiIgoyJABAAAAwInlz5Tl15tvvqlKlSqpSpUq8vDw0IABA9SzZ0+5uNzcocTFxSk1NdXxOXDgwE09HgAAAIDiwdJQVqpUKbm6uuZY9fDYsWMKCQnJdZ+goCB9/fXXSk9P1759+7R9+3b5+vqqfPnyeW4zJCREmZmZOn36dJ6P6+npKX9/f6cPAAAAANwoS0OZh4eH6tatq8WLFzvK7Ha7Fi9erOjo6Gvu6+XlpfDwcGVlZWnu3Llq3759ntusW7eu3N3dners2LFD+/fvv+5xAQAAAKAwuVndgaFDh6p79+6qV6+eGjRooPj4eKWnp6tnz56SpG7duik8PFyTJ0+WJK1du1aHDh1SrVq1dOjQIY0dO1Z2u13PPfdcntsMCAhQr169NHToUJUsWVL+/v4aOHCgoqOj87TyIgAAAAAUFstDWadOnXTixAmNHj1aR48eVa1atZSYmOhYqGP//v1Oz4tlZGRo5MiR2rNnj3x9fdW6dWslJCQ4LdpxvTYl6Y033pCLi4s6dOigCxcuKDY2VjNmzPjLxg0AAAAAUhF4T9mtiveUAQAAAJBu8feUAQAAAEBxRygDAAAAAAsRygAAAADAQoQyAAAAALAQoQwAAAAALEQoAwAAAAALEcoAAAAAwEKEMgAAAACwEKEMAAAAACxEKAMAAAAACxHKAAAAAMBChDIAAAAAsBChDAAAAAAsRCgDAAAAAAsRygAAAADAQoQyAAAAALAQoQwAAAAALEQoAwAAAAALEcoAAAAAwEKEMgAAAACwEKEMAAAAACxEKAMAAAAACxHKAAAAAMBChDIAAAAAsBChDAAAAAAsRCgDAAAAAAsRygAAAADAQoQyAAAAALAQoQwAAAAALEQoAwAAAAALEcoAAAAAwEKEMgAAAACwEKEMAAAAACxEKAMAAAAACxHKAAAAAMBChDIAAAAAsBChDAAAAAAsRCgDAAAAAAsRygAAAADAQoQyAAAAALAQoQwAAAAALEQoAwAAAAALEcoAAAAAwEKEMgAAAACwEKEMAAAAACxEKAMAAAAACxHKAAAAAMBChDIAAAAAsBChDAAAAAAsRCgDAAAAAAsRygAAAADAQoQyAAAAALAQoQwAAAAALEQoAwAAAAALEcoAAAAAwEKEMgAAAACwEKEMAAAAACxEKAMAAAAACxHKAAAAAMBCloey6dOnKzIyUl5eXoqKitK6deuuWT8+Pl6VK1eWt7e3IiIi9PTTTysjI8Ox/cyZMxoyZIjKli0rb29vNWzYUOvXr3dq4+zZsxowYIDuuOMOeXt7q1q1anr77bdvyvgAAAAA4FosDWVz5szR0KFDNWbMGG3cuFE1a9ZUbGysjh8/nmv9Tz75RCNGjNCYMWOUnJys9957T3PmzNHzzz/vqNO7d28tWrRICQkJ+uWXX9SiRQvFxMTo0KFDjjpDhw5VYmKiZs+ereTkZA0ZMkQDBgzQN998c9PHDAAAAAB/ZjPGGKsOHhUVpfr16+utt96SJNntdkVERGjgwIEaMWJEjvoDBgxQcnKyFi9e7Ch75plntHbtWq1cuVLnz5+Xn5+f5s+frzZt2jjq1K1bV61atdKLL74oSbr77rvVqVMnjRo16qp1rictLU0BAQFKTU2Vv79/gcYPAAAA4NZ3o9nAsitlmZmZSkpKUkxMzP864+KimJgYrV69Otd9GjZsqKSkJMctjnv27NGCBQvUunVrSVJWVpays7Pl5eXltJ+3t7dWrlzp1M4333yjQ4cOyRijpUuX6rffflOLFi2u2t8LFy4oLS3N6QMAAAAAN8rNqgOfPHlS2dnZCg4OdioPDg7W9u3bc92nS5cuOnnypBo3bixjjLKystSvXz/H7Yt+fn6Kjo7WhAkTVLVqVQUHB+vTTz/V6tWrVbFiRUc706ZNU9++fXXHHXfIzc1NLi4uevfdd9W0adOr9nfy5MkaN25cIYwcAAAAAP7H8oU+8mPZsmWaNGmSZsyYoY0bN2revHn67rvvNGHCBEedhIQEGWMUHh4uT09PTZ06VZ07d5aLy/+GOm3aNK1Zs0bffPONkpKS9Prrr6t///768ccfr3rsuLg4paamOj4HDhy4qWMFAAAAUDxY9kxZZmamSpQooS+//FIPPvigo7x79+46ffq05s+fn2OfJk2a6J577tGrr77qKJs9e7b69u2rs2fPOgWv9PR0paWlKTQ0VJ06ddLZs2f13Xff6fz58woICNBXX33l9NxZ7969dfDgQSUmJuap/zxTBgAAAEC6hZ8p8/DwUN26dZ0W7bDb7Vq8eLGio6Nz3efcuXNOwUuSXF1dJUlXZksfHx+Fhobqjz/+0MKFC9W+fXtJ0sWLF3Xx4sVc27Hb7Tc8LgAAAADID8ueKZMuLU3fvXt31atXTw0aNFB8fLzS09PVs2dPSVK3bt0UHh6uyZMnS5Latm2rKVOmqHbt2oqKitKuXbs0atQotW3b1hHOFi5cKGOMKleurF27dmnYsGGqUqWKo01/f381a9ZMw4YNk7e3t8qWLavly5fro48+0pQpU6w5EQAAAACKLUtDWadOnXTixAmNHj1aR48eVa1atZSYmOhY/GP//v1OV7RGjhwpm82mkSNH6tChQwoKClLbtm01ceJER53U1FTFxcXp4MGDKlmypDp06KCJEyfK3d3dUeezzz5TXFycHnvsMZ06dUply5bVxIkT1a9fv79u8AAAAAAgi99TdivjmTIAAAAA0i38TBkAAAAAgFAGAAAAAJYilAEAAACAhQhlAAAAAGAhQhkAAAAAWIhQBgAAAAAWIpQBAAAAgIUIZQAAAABgIUIZAAAAAFiIUAYAAAAAFiKUAQAAAICFCGUAAAAAYCFCGQAAAABYiFAGAAAAABYilAEAAACAhQhlAAAAAGAhQhkAAAAAWIhQBgAAAAAWIpQBAAAAgIUIZQAAAABgIUIZAAAAAFiowKEsISFBjRo1UlhYmPbt2ydJio+P1/z58wutcwAAAADwd1egUDZz5kwNHTpUrVu31unTp5WdnS1JCgwMVHx8fGH2DwAAAAD+1goUyqZNm6Z3331XL7zwglxdXR3l9erV0y+//FJonQMAAACAv7sChbKUlBTVrl07R7mnp6fS09NvuFMAAAAAUFwUKJSVK1dOmzdvzlGemJioqlWr3mifAAAAAKDYcCvITkOHDlX//v2VkZEhY4zWrVunTz/9VJMnT9a///3vwu4jAAAAAPxtFSiU9e7dW97e3ho5cqTOnTunLl26KCwsTG+++aYeffTRwu4jAAAAAPxt5TuUZWVl6ZNPPlFsbKwee+wxnTt3TmfPnlXp0qVvRv8AAAAA4G8t38+Uubm5qV+/fsrIyJAklShRgkAGAAAAAAVUoIU+GjRooE2bNhV2XwAAAACg2CnQM2VPPfWUnnnmGR08eFB169aVj4+P0/YaNWoUSucAAAAA4O/OZowx+d3JxSXnBTabzSZjjGw2m7Kzswulc0VZWlqaAgIClJqaKn9/f6u7AwAAAMAiN5oNCnSlLCUlpSC7AQAAAACuUKBQVrZs2cLuBwAAAAAUSwUKZZK0e/duxcfHKzk5WZJUrVo1DR48WBUqVCi0zgEAAADA312BVl9cuHChqlWrpnXr1qlGjRqqUaOG1q5dq7vuukuLFi0q7D4CAAAAwN9WgRb6qF27tmJjY/XSSy85lY8YMUI//PCDNm7cWGgdLKpY6AMAAACAdOPZoEBXypKTk9WrV68c5U888YS2bdtWkCYBAAAAoFgqUCgLCgrS5s2bc5Rv3rxZpUuXvtE+AQAAAECxUaCFPvr06aO+fftqz549atiwoSRp1apVevnllzV06NBC7SAAAAAA/J0V6JkyY4zi4+P1+uuv6/Dhw5KksLAwDRs2TIMGDZLNZiv0jhY1PFMGAAAAQLrxbFCgUPZnZ86ckST5+fndSDO3HEIZAAAAAOnGs0GBbl9MSUlRVlaWKlWq5BTGdu7cKXd3d0VGRhakWQAAAAAodgq00EePHj30008/5Shfu3atevTocaN9AgAAAIBio0ChbNOmTWrUqFGO8nvuuSfXVRkBAAAAALkrUCiz2WyOZ8n+LDU1VdnZ2TfcKQAAAAAoLgoUypo2barJkyc7BbDs7GxNnjxZjRs3LrTOAQAAAMDfXYEW+nj55ZfVtGlTVa5cWU2aNJEkrVixQmlpaVqyZEmhdhAAAAAA/s4KdKWsWrVq+vnnn9WxY0cdP35cZ86cUbdu3bR9+3bdfffdhd1HAAAAAPjbuuH3lBVXvKcMAAAAgHTj2SBfV8pOnjypffv2OZVt3bpVPXv2VMeOHfXJJ5/kuwMAAAAAUJzlK5QNHDhQU6dOdXw/fvy4mjRpovXr1+vChQvq0aOHEhISCr2TAAAAAPB3la9QtmbNGrVr187x/aOPPlLJkiW1efNmzZ8/X5MmTdL06dMLvZMAAAAA8HeVr1B29OhRRUZGOr4vWbJEDz/8sNzcLi3i2K5dO+3cubNQOwgAAAAAf2f5CmX+/v46ffq04/u6desUFRXl+G6z2XThwoVC6xwAAAAA/N3lK5Tdc889mjp1qux2u7788kudOXNG9913n2P7b7/9poiIiELvJAAAAAD8XeXr5dETJkzQ/fffr9mzZysrK0vPP/+8brvtNsf2zz77TM2aNSv0TgIAAADA31W+rpTVqFFDycnJ+vzzz/XTTz9pwoQJTtsfffRRDR8+PF8dmD59uiIjI+Xl5aWoqCitW7fumvXj4+NVuXJleXt7KyIiQk8//bQyMjIc28+cOaMhQ4aobNmy8vb2VsOGDbV+/foc7SQnJ6tdu3YKCAiQj4+P6tevr/379+er7wAAAABwo/J1pUySSpUqpfbt2zu+Hzx4UGFhYXJxcVGbNm3y1dacOXM0dOhQvf3224qKilJ8fLxiY2O1Y8cOlS5dOkf9Tz75RCNGjND777+vhg0b6rffflOPHj1ks9k0ZcoUSVLv3r3166+/KiEhQWFhYZo9e7ZiYmK0bds2hYeHS5J2796txo0bq1evXho3bpz8/f21detWeXl55fd0AAAAAMANsRljzI004O/vr82bN6t8+fL53jcqKkr169fXW2+9JUmy2+2KiIjQwIEDNWLEiBz1BwwYoOTkZC1evNhR9swzz2jt2rVauXKlzp8/Lz8/P82fP98pINatW1etWrXSiy++KOnSFT13d/cbeqfajb61GwAAAMDfw41mg3zdvpibgma6zMxMJSUlKSYm5n+dcXFRTEyMVq9enes+DRs2VFJSkuMWxz179mjBggVq3bq1JCkrK0vZ2dk5rnh5e3tr5cqVki4Fv++++0533nmnYmNjVbp0aUVFRenrr7++Zn8vXLigtLQ0pw8AAAAA3KgbDmUFdfLkSWVnZys4ONipPDg4WEePHs11ny5dumj8+PFq3Lix3N3dVaFCBTVv3lzPP/+8JMnPz0/R0dGaMGGCDh8+rOzsbM2ePVurV6/WkSNHJEnHjx/X2bNn9dJLL6lly5b64Ycf9NBDD+nhhx/W8uXLr9rfyZMnKyAgwPFhlUkAAAAAheGGQ9nzzz+vkiVLFkZfrmvZsmWaNGmSZsyYoY0bN2revHn67rvvnBYcSUhIkDFG4eHh8vT01NSpU9W5c2e5uFwaqt1ulyS1b99eTz/9tGrVqqURI0boH//4h95+++2rHjsuLk6pqamOz4EDB27uYAEAAAAUC/le6ONKcXFxBdqvVKlScnV11bFjx5zKjx07ppCQkFz3GTVqlLp27arevXtLkqpXr6709HT17dtXL7zwglxcXFShQgUtX75c6enpSktLU2hoqDp16uR45q1UqVJyc3NTtWrVnNquWrWq4xbH3Hh6esrT07NAYwUAAACAqynU2xcPHDigJ554Ik91PTw8VLduXadFO+x2uxYvXqzo6Ohc9zl37pzjitdlrq6uknI+2+bj46PQ0FD98ccfWrhwoWPFSA8PD9WvX187duxwqv/bb7+pbNmyeeo7AAAAABSWG75S9menTp3Shx9+qPfffz9P9YcOHaru3burXr16atCggeLj45Wenq6ePXtKkrp166bw8HBNnjxZktS2bVtNmTJFtWvXVlRUlHbt2qVRo0apbdu2jnC2cOFCGWNUuXJl7dq1S8OGDVOVKlUcbUrSsGHD1KlTJzVt2lT33nuvEhMT9Z///EfLli0rzNMBAAAAANeVr1D2zTffXHP7nj178nXwTp066cSJExo9erSOHj2qWrVqKTEx0bH4x/79+52ujI0cOVI2m00jR47UoUOHFBQUpLZt22rixImOOqmpqYqLi9PBgwdVsmRJdejQQRMnTpS7u7ujzkMPPaS3335bkydP1qBBg1S5cmXNnTtXjRs3zlf/AQAAAOBG5es9ZS4uLrLZbNdcBt9msyk7O7tQOleU8Z4yAAAAANJf/J6y0NBQzZs3T3a7PdfPxo0b890BAAAAACjO8hXK6tatq6SkpKtuv95VNAAAAACAs3w9UzZs2DClp6dfdXvFihW1dOnSG+4UAAAAABQX+Qpl4eHhKleu3FW3+/j4qFmzZjfcKQAAAAAoLvJ1+2KlSpV04sQJx/dOnTrlePkzAAAAACDv8hXKrnxebMGCBde8nREAAAAAcG35CmUAAAAAgMKVr1Bms9lks9lylAEAAAAACiZfC30YY9SjRw95enpKkjIyMtSvXz/5+Pg41Zs3b17h9RAAAAAA/sbyFcq6d+/u9P3xxx8v1M4AAAAAQHGTr1D2wQcf3Kx+AAAAAECxxEIfAAAAAGAhQhkAAAAAWIhQBgAAAAAWIpQBAAAAgIUIZQAAAABgIUIZAAAAAFiIUAYAAAAAFiKUAQAAAICFCGUAAAAAYCFCGQAAAABYiFAGAAAAABYilAEAAACAhQhlAAAAAGAhQhkAAAAAWIhQBgAAAAAWIpQBAAAAgIUIZQAAAABgIUIZAAAAAFiIUAYAAAAAFiKUAQAAAICFCGUAAAAAYCFCGQAAAABYiFAGAAAAABYilAEAAACAhQhlAAAAAGAhQhkAAAAAWIhQBgAAAAAWIpQBAAAAgIUIZQAAAABgIUIZAAAAAFiIUAYAAAAAFiKUAQAAAICFCGUAAAAAYCFCGQAAAABYiFAGAAAAABYilAEAAACAhQhlAAAAAGAhQhkAAAAAWIhQBgAAAAAWIpQBAAAAgIUIZQAAAABgIUIZAAAAAFiIUAYAAAAAFiKUAQAAAICFCGUAAAAAYCFCGQAAAABYiFAGAAAAABYqEqFs+vTpioyMlJeXl6KiorRu3bpr1o+Pj1flypXl7e2tiIgIPf3008rIyHBsP3PmjIYMGaKyZcvK29tbDRs21Pr166/aXr9+/WSz2RQfH19YQwIAAACAPLE8lM2ZM0dDhw7VmDFjtHHjRtWsWVOxsbE6fvx4rvU/+eQTjRgxQmPGjFFycrLee+89zZkzR88//7yjTu/evbVo0SIlJCTol19+UYsWLRQTE6NDhw7laO+rr77SmjVrFBYWdtPGCAAAAABXY3komzJlivr06aOePXuqWrVqevvtt1WiRAm9//77udb/6aef1KhRI3Xp0kWRkZFq0aKFOnfu7Li6dv78ec2dO1evvPKKmjZtqooVK2rs2LGqWLGiZs6c6dTWoUOHNHDgQH388cdyd3e/6WMFAAAAgCtZGsoyMzOVlJSkmJgYR5mLi4tiYmK0evXqXPdp2LChkpKSHCFsz549WrBggVq3bi1JysrKUnZ2try8vJz28/b21sqVKx3f7Xa7unbtqmHDhumuu+66bl8vXLigtLQ0pw8AAAAA3ChLQ9nJkyeVnZ2t4OBgp/Lg4GAdPXo01326dOmi8ePHq3HjxnJ3d1eFChXUvHlzx+2Lfn5+io6O1oQJE3T48GFlZ2dr9uzZWr16tY4cOeJo5+WXX5abm5sGDRqUp75OnjxZAQEBjk9EREQBRw0AAAAA/2P57Yv5tWzZMk2aNEkzZszQxo0bNW/ePH333XeaMGGCo05CQoKMMQoPD5enp6emTp2qzp07y8Xl0nCTkpL05ptvatasWbLZbHk6blxcnFJTUx2fAwcO3JTxAQAAAChe3Kw8eKlSpeTq6qpjx445lR87dkwhISG57jNq1Ch17dpVvXv3liRVr15d6enp6tu3r1544QW5uLioQoUKWr58udLT05WWlqbQ0FB16tRJ5cuXlyStWLFCx48fV5kyZRztZmdn65lnnlF8fLz27t2b47ienp7y9PQspJEDAAAAwCWWXinz8PBQ3bp1tXjxYkeZ3W7X4sWLFR0dnes+586dc1zxuszV1VWSZIxxKvfx8VFoaKj++OMPLVy4UO3bt5ckde3aVT///LM2b97s+ISFhWnYsGFauHBhYQ4RAAAAAK7J0itlkjR06FB1795d9erVU4MGDRQfH6/09HT17NlTktStWzeFh4dr8uTJkqS2bdtqypQpql27tqKiorRr1y6NGjVKbdu2dYSzhQsXyhijypUra9euXRo2bJiqVKniaPP222/X7bff7tQPd3d3hYSEqHLlyn/h6AEAAAAUd5aHsk6dOunEiRMaPXq0jh49qlq1aikxMdGx+Mf+/fudroyNHDlSNptNI0eO1KFDhxQUFKS2bdtq4sSJjjqpqamKi4vTwYMHVbJkSXXo0EETJ05k2XsAAAAARY7NXHnPH/IkLS1NAQEBSk1Nlb+/v9XdAQAAAGCRG80Gt9zqiwAAAADwd0IoAwAAAAALEcoAAAAAwEKEMgAAAACwEKEMAAAAACxEKAMAAAAACxHKAAAAAMBChDIAAAAAsBChDAAAAAAsRCgDAAAAAAsRygAAAADAQoQyAAAAALAQoQwAAAAALEQoAwAAAAALEcoAAAAAwEKEMgAAAACwEKEMAAAAACxEKAMAAAAACxHKAAAAAMBChDIAAAAAsBChDAAAAAAsRCgDAAAAAAsRygAAAADAQoQyAAAAALAQoQwAAAAALEQoAwAAAAALEcoAAAAAwEKEMgAAAACwEKEMAAAAACxEKAMAAAAACxHKAAAAAMBChDIAAAAAsBChDAAAAAAsRCgDAAAAAAsRygAAAADAQoQyAAAAALAQoQwAAAAALEQoAwAAAAALEcoAAAAAwEKEMgAAAACwEKEMAAAAACxEKAMAAAAACxHKAAAAAMBChDIAAAAAsBChDAAAAAAsRCgDAAAAAAsRygAAAADAQoQyAAAAALAQoQwAAAAALEQoAwAAAAALEcoAAAAAwEKEMgAAAACwEKEMAAAAACxEKAMAAAAACxHKAAAAAMBChDIAAAAAsBChDAAAAAAsRCgDAAAAAAsRygAAAADAQkUilE2fPl2RkZHy8vJSVFSU1q1bd8368fHxqly5sry9vRUREaGnn35aGRkZju1nzpzRkCFDVLZsWXl7e6thw4Zav369Y/vFixc1fPhwVa9eXT4+PgoLC1O3bt10+PDhmzZGAAAAAMiN5aFszpw5Gjp0qMaMGaONGzeqZs2aio2N1fHjx3Ot/8knn2jEiBEaM2aMkpOT9d5772nOnDl6/vnnHXV69+6tRYsWKSEhQb/88otatGihmJgYHTp0SJJ07tw5bdy4UaNGjdLGjRs1b9487dixQ+3atftLxgwAAAAAl9mMMcbKDkRFRal+/fp66623JEl2u10REREaOHCgRowYkaP+gAEDlJycrMWLFzvKnnnmGa1du1YrV67U+fPn5efnp/nz56tNmzaOOnXr1lWrVq304osv5tqP9evXq0GDBtq3b5/KlCmTY/uFCxd04cIFx/e0tDRFREQoNTVV/v7+BR4/AAAAgFtbWlqaAgICCpwNLL1SlpmZqaSkJMXExDjKXFxcFBMTo9WrV+e6T8OGDZWUlOS4xXHPnj1asGCBWrduLUnKyspSdna2vLy8nPbz9vbWypUrr9qX1NRU2Ww2BQYG5rp98uTJCggIcHwiIiLyM1QAAAAAyJWloezkyZPKzs5WcHCwU3lwcLCOHj2a6z5dunTR+PHj1bhxY7m7u6tChQpq3ry54/ZFPz8/RUdHa8KECTp8+LCys7M1e/ZsrV69WkeOHMm1zYyMDA0fPlydO3e+arKNi4tTamqq43PgwIEbGDkAAAAAXGL5M2X5tWzZMk2aNEkzZsxwPA/23XffacKECY46CQkJMsYoPDxcnp6emjp1qjp37iwXl5zDvXjxojp27ChjjGbOnHnV43p6esrf39/pAwAAAAA3ys3Kg5cqVUqurq46duyYU/mxY8cUEhKS6z6jRo1S165d1bt3b0lS9erVlZ6err59++qFF16Qi4uLKlSooOXLlys9PV1paWkKDQ1Vp06dVL58eae2Lgeyffv2acmSJQQtAAAAAH85S6+UeXh4qG7duk6Ldtjtdi1evFjR0dG57nPu3LkcV7xcXV0lSVeuWeLj46PQ0FD98ccfWrhwodq3b+/YdjmQ7dy5Uz/++KNuv/32whoWAAAAAOSZpVfKJGno0KHq3r276tWrpwYNGig+Pl7p6enq2bOnJKlbt24KDw/X5MmTJUlt27bVlClTVLt2bUVFRWnXrl0aNWqU2rZt6whnCxculDFGlStX1q5duzRs2DBVqVLF0ebFixf1yCOPaOPGjfr222+VnZ3teIatZMmS8vDwsOBMAAAAACiOLA9lnTp10okTJzR69GgdPXpUtWrVUmJiomPxj/379ztdGRs5cqRsNptGjhypQ4cOKSgoSG3bttXEiRMddVJTUxUXF6eDBw+qZMmS6tChgyZOnCh3d3dJ0qFDh/TNN99IkmrVquXUn6VLl6p58+Y3d9AAAAAA8P9Z/p6yW9WNvosAAAAAwN/DLf2eMgAAAAAo7ghlAAAAAGAhQhkAAAAAWIhQBgAAAAAWIpQBAAAAgIUIZQAAAABgIUIZAAAAAFiIUAYAAAAAFiKUAQAAAICFCGUAAAAAYCFCGQAAAABYiFAGAAAAABYilAEAAACAhQhlAAAAAGAhQhkAAAAAWIhQBgAAAAAWIpQBAAAAgIUIZQAAAABgIUIZAAAAAFiIUAYAAAAAFiKUAQAAAICFCGUAAAAAYCFCGQAAAABYiFAGAAAAABYilAEAAACAhQhlAAAAAGAhQhkAAAAAWIhQBgAAAAAWIpQBAAAAgIUIZQAAAABgIUIZAAAAAFiIUAYAAAAAFiKUAQAAAICFCGUAAAAAYCFCGQAAAABYiFAGAAAAABYilAEAAACAhQhlAAAAAGAhQhkAAAAAWIhQBgAAAAAWIpQBAAAAgIUIZQAAAABgIUIZAAAAAFiIUAYAAAAAFiKUAQAAAICFCGUAAAAAYCFCGQAAAABYiFAGAAAAABYilAEAAACAhQhlAAAAAGAhQhkAAAAAWIhQBgAAAAAWIpQBAAAAgIUIZQAAAABgIUIZAAAAAFiIUAYAAAAAFiKUAQAAAICFCGUAAAAAYKEiEcqmT5+uyMhIeXl5KSoqSuvWrbtm/fj4eFWuXFne3t6KiIjQ008/rYyMDMf2M2fOaMiQISpbtqy8vb3VsGFDrV+/3qkNY4xGjx6t0NBQeXt7KyYmRjt37rwp4wMAAACAq7E8lM2ZM0dDhw7VmDFjtHHjRtWsWVOxsbE6fvx4rvU/+eQTjRgxQmPGjFFycrLee+89zZkzR88//7yjTu/evbVo0SIlJCTol19+UYsWLRQTE6NDhw456rzyyiuaOnWq3n77ba1du1Y+Pj6KjY11CncAAAAAcLPZjDHGyg5ERUWpfv36euuttyRJdrtdERERGjhwoEaMGJGj/oABA5ScnKzFixc7yp555hmtXbtWK1eu1Pnz5+Xn56f58+erTZs2jjp169ZVq1at9OKLL8oYo7CwMD3zzDN69tlnJUmpqakKDg7WrFmz9Oijj16332lpaQoICFBqaqr8/f1v9DQAAAAAuEXdaDaw9EpZZmamkpKSFBMT4yhzcXFRTEyMVq9enes+DRs2VFJSkuMWxz179mjBggVq3bq1JCkrK0vZ2dny8vJy2s/b21srV66UJKWkpOjo0aNOxw0ICFBUVNRVj3vhwgWlpaU5fQAAAADgRlkayk6ePKns7GwFBwc7lQcHB+vo0aO57tOlSxeNHz9ejRs3lru7uypUqKDmzZs7bl/08/NTdHS0JkyYoMOHDys7O1uzZ8/W6tWrdeTIEUlytJ2f406ePFkBAQGOT0RExA2NHQAAAACkIvBMWX4tW7ZMkyZN0owZM7Rx40bNmzdP3333nSZMmOCok5CQIGOMwsPD5enpqalTp6pz585ycSn4cOPi4pSamur4HDhwoDCGAwAAAKCYc7Py4KVKlZKrq6uOHTvmVH7s2DGFhITkus+oUaPUtWtX9e7dW5JUvXp1paenq2/fvnrhhRfk4uKiChUqaPny5UpPT1daWppCQ0PVqVMnlS9fXpIcbR87dkyhoaFOx61Vq1aux/X09JSnp+eNDhkAAAAAnFh6pczDw0N169Z1WrTDbrdr8eLFio6OznWfc+fO5bji5erqKunSMvd/5uPjo9DQUP3xxx9auHCh2rdvL0kqV66cQkJCnI6blpamtWvXXvW4AAAAAHAzWHqlTJKGDh2q7t27q169emrQoIHi4+OVnp6unj17SpK6deum8PBwTZ48WZLUtm1bTZkyRbVr11ZUVJR27dqlUaNGqW3bto5wtnDhQhljVLlyZe3atUvDhg1TlSpVHG3abDYNGTJEL774oipVqqRy5cpp1KhRCgsL04MPPmjJeQAAAABQPFkeyjp16qQTJ05o9OjROnr0qGrVqqXExETHIhz79+93ujI2cuRI2Ww2jRw5UocOHVJQUJDatm2riRMnOuqkpqYqLi5OBw8eVMmSJdWhQwdNnDhR7u7ujjrPPfec47bH06dPq3HjxkpMTMyxaiMAAAAA3EyWv6fsVsV7ygAAAABIt/h7ygAAAACguCOUAQAAAICFCGUAAAAAYCFCGQAAAABYiFAGAAAAABYilAEAAACAhQhlAAAAAGAhQhkAAAAAWIhQBgAAAAAWIpQBAAAAgIXcrO4Abky23WhdyikdP5Oh0n5ealCupFxdbFZ3CwAAAEAeEcpuYYm/HtG4/2zTkdQMR1logJfGtK2mlneHWtgzAAAAAHnF7Yu3qMRfj+jJ2RudApkkHU3N0JOzNyrx1yMW9QwAAABAfhDKbkHZdqNx/9kmk8u2y2Xj/rNN2fbcagAAAAAoSghlt6B1KadyXCH7MyPpSGqG1qWc+us6BQAAAKBACGW3oONnrh7IClIPAAAAgHUIZbeg0n5ehVoPAAAAgHUIZbegBuVKKjTAS1db+N6mS6swNihX8q/sFgAAAIACIJTdglxdbBrTtpok5Qhml7+PaVuN95UBAAAAtwBC2S2q5d2hmvl4HYUEON+iGBLgpZmP1+E9ZQAAAMAtgpdH38Ja3h2qB6qFaF3KKR0/k6HSfpduWeQKGQAAAHDrIJTd4lxdbIqucLvV3QAAAABQQNy+CAAAAAAWIpQBAAAAgIUIZQAAAABgIUIZAAAAAFiIUAYAAAAAFiKUAQAAAICFCGUAAAAAYCFCGQAAAABYiFAGAAAAABYilAEAAACAhQhlAAAAAGAhQhkAAAAAWIhQBgAAAAAWIpQBAAAAgIUIZQAAAABgIUIZAAAAAFiIUAYAAAAAFnKzugO3KmOMJCktLc3ingAAAACw0uVMcDkj5BehrIDOnDkjSYqIiLC4JwAAAACKgjNnziggICDf+9lMQeNcMWe323X48GH5+fnJZrMpLS1NEREROnDggPz9/a3uHv4CzHnxxLwXP8x58cOcF0/Me/FTmHNujNGZM2cUFhYmF5f8PyHGlbICcnFx0R133JGj3N/fnx/kYoY5L56Y9+KHOS9+mPPiiXkvfgprzgtyhewyFvoAAAAAAAsRygAAAADAQoSyQuLp6akxY8bI09PT6q7gL8KcF0/Me/HDnBc/zHnxxLwXP0VpzlnoAwAAAAAsxJUyAAAAALAQoQwAAAAALEQoAwAAAAALEcoAAAAAwELFNpRNnz5dkZGR8vLyUlRUlNatW3fN+l988YWqVKkiLy8vVa9eXQsWLHDabozR6NGjFRoaKm9vb8XExGjnzp1OdU6dOqXHHntM/v7+CgwMVK9evXT27FmnOj///LOaNGkiLy8vRURE6JVXXimcAaNIzvmyZcvUvn17hYaGysfHR7Vq1dLHH39ceIMu5orinP/Zrl275Ofnp8DAwBsaJ5wV1Xk3xui1117TnXfeKU9PT4WHh2vixImFM+hirqjO+cKFC3XPPffIz89PQUFB6tChg/bu3VsoYy7urJjziRMnqmHDhipRosRV/3d7//79atOmjUqUKKHSpUtr2LBhysrKuqGx4n+K4rxv2bJFnTt3VkREhLy9vVW1alW9+eab+R+cKYY+++wz4+HhYd5//32zdetW06dPHxMYGGiOHTuWa/1Vq1YZV1dX88orr5ht27aZkSNHGnd3d/PLL7846rz00ksmICDAfP3112bLli2mXbt2ply5cub8+fOOOi1btjQ1a9Y0a9asMStWrDAVK1Y0nTt3dmxPTU01wcHB5rHHHjO//vqr+fTTT423t7d55513bt7JKCaK6pxPnDjRjBw50qxatcrs2rXLxMfHGxcXF/Of//zn5p2MYqKozvllmZmZpl69eqZVq1YmICCg0MdfXBXleR84cKCpXLmymT9/vtmzZ4/ZsGGD+eGHH27OiShGiuqc79mzx3h6epq4uDiza9cuk5SUZJo2bWpq1659805GMWHVnI8ePdpMmTLFDB06NNf/3c7KyjJ33323iYmJMZs2bTILFiwwpUqVMnFxcYV+Doqjojrv7733nhk0aJBZtmyZ2b17t0lISDDe3t5m2rRp+RpfsQxlDRo0MP3793d8z87ONmFhYWby5Mm51u/YsaNp06aNU1lUVJT5v//7P2OMMXa73YSEhJhXX33Vsf306dPG09PTfPrpp8YYY7Zt22YkmfXr1zvqfP/998Zms5lDhw4ZY4yZMWOGue2228yFCxccdYYPH24qV658gyNGUZ3z3LRu3dr07Nkz/4OEk6I+588995x5/PHHzQcffEAoK0RFdd63bdtm3NzczPbt2wtnoHAoqnP+xRdfGDc3N5Odne2o88033xibzWYyMzNvcNTFmxVz/mdX+9/tBQsWGBcXF3P06FFH2cyZM42/v7/T73YomKI677l56qmnzL333punupcVu9sXMzMzlZSUpJiYGEeZi4uLYmJitHr16lz3Wb16tVN9SYqNjXXUT0lJ0dGjR53qBAQEKCoqylFn9erVCgwMVL169Rx1YmJi5OLiorVr1zrqNG3aVB4eHk7H2bFjh/74448bHHnxVZTnPDepqakqWbJk/gcKh6I+50uWLNEXX3yh6dOn3/hg4VCU5/0///mPypcvr2+//VblypVTZGSkevfurVOnThXO4IupojzndevWlYuLiz744ANlZ2crNTVVCQkJiomJkbu7e+GcgGLIqjnPi9WrV6t69eoKDg52Ok5aWpq2bt2a53aQU1Ge99wU5He5YhfKTp48qezsbKcfGEkKDg7W0aNHc93n6NGj16x/+b/Xq1O6dGmn7W5ubipZsqRTndza+PMxkH9Fec6v9Pnnn2v9+vXq2bNnHkeH3BTlOf/999/Vo0cPzZo1S/7+/gUcIXJTlOd9z5492rdvn7744gt99NFHmjVrlpKSkvTII48UcLSQivaclytXTj/88IOef/55eXp6KjAwUAcPHtTnn39ewNFCsm7O84Lf426eojzvV/rpp580Z84c9e3bN1/7FbtQBhRVS5cuVc+ePfXuu+/qrrvusro7uEn69OmjLl26qGnTplZ3BX8hu92uCxcu6KOPPlKTJk3UvHlzvffee1q6dKl27NhhdfdwExw9elR9+vRR9+7dtX79ei1fvlweHh565JFHZIyxunsAboJff/1V7du315gxY9SiRYt87VvsQlmpUqXk6uqqY8eOOZUfO3ZMISEhue4TEhJyzfqX/3u9OsePH3fanpWVpVOnTjnVya2NPx8D+VeU5/yy5cuXq23btnrjjTfUrVu3fI4QVyrKc75kyRK99tprcnNzk5ubm3r16qXU1FS5ubnp/fffL+CIIRXteQ8NDZWbm5vuvPNOR52qVatKurRaGwqmKM/59OnTFRAQoFdeeUW1a9dW06ZNNXv2bC1evPiat7Dj2qya87zg97ibpyjP+2Xbtm3T/fffr759+2rkyJH53r/YhTIPDw/VrVtXixcvdpTZ7XYtXrxY0dHRue4THR3tVF+SFi1a5Khfrlw5hYSEONVJS0vT2rVrHXWio6N1+vRpJSUlOeosWbJEdrtdUVFRjjr//e9/dfHiRafjVK5cWbfddtsNjrz4KspzLl1aFr9NmzZ6+eWX832pG7krynO+evVqbd682fEZP368/Pz8tHnzZj300EOFcwKKqaI8740aNVJWVpZ2797tqPPbb79JksqWLXsjwy7WivKcnzt3Ti4uzr9mubq6OvqIgrFqzvMiOjpav/zyi1NgX7Rokfz9/VWtWrU8t4OcivK8S9LWrVt17733qnv37gV/1Um+lgX5m/jss8+Mp6enmTVrltm2bZvp27evCQwMdKyW07VrVzNixAhH/VWrVhk3Nzfz2muvmeTkZDNmzJhcl9QMDAw08+fPNz///LNp3759rsvn1q5d26xdu9asXLnSVKpUyWn53NOnT5vg4GDTtWtX8+uvv5rPPvvMlChRgiXxC0FRnfMlS5aYEiVKmLi4OHPkyBHH5/fff/8LzsrfW1Gd8yux+mLhKqrznp2dberUqWOaNm1qNm7caDZs2GCioqLMAw888Beclb+3ojrnixcvNjabzYwbN8789ttvJikpycTGxpqyZcuac+fO/QVn5u/Lqjnft2+f2bRpkxk3bpzx9fU1mzZtMps2bTJnzpwxxvxvSfwWLVqYzZs3m8TERBMUFMSS+IWkqM77L7/8YoKCgszjjz/u9Lvc8ePH8zW+YhnKjDFm2rRppkyZMsbDw8M0aNDArFmzxrGtWbNmpnv37k71P//8c3PnnXcaDw8Pc9ddd5nvvvvOabvdbjejRo0ywcHBxtPT09x///1mx44dTnV+//1307lzZ+Pr62v8/f1Nz549HRN62ZYtW0zjxo2Np6enCQ8PNy+99FLhDrwYK4pz3r17dyMpx6dZs2aFPv7iqCjO+ZUIZYWvqM77oUOHzMMPP2x8fX1NcHCw6dGjB/8AU0iK6px/+umnpnbt2sbHx8cEBQWZdu3ameTk5MIdfDFlxZxf7f+zly5d6qizd+9e06pVK+Pt7W1KlSplnnnmGXPx4sVCH39xVRTnfcyYMbluL1u2bL7GZjOGp00BAAAAwCrF7pkyAAAAAChKCGUAAAAAYCFCGQAAAABYiFAGAAAAABYilAEAAACAhQhlAAAAAGAhQhkAAAAAWIhQBgAAAAAWIpQBAG6KHj166MEHH7S6G387NptNX3/9tSRp7969stls2rx5c4HbK4w2AAA3xs3qDgAAbj02m+2a28eMGaM333xTxpi/qEfFU0REhI4cOaJSpUrlqX6PHj10+vRpR6grSBsAgMJHKAMA5NuRI0ccf54zZ45Gjx6tHTt2OMp8fX3l6+trRddumosXL8rd3b1IteXq6qqQkBDL2wAA3BhuXwQA5FtISIjjExAQIJvN5lTm6+ub4/bF5s2ba+DAgRoyZIhuu+02BQcH691331V6erp69uwpPz8/VaxYUd9//73TsX799Ve1atVKvr6+Cg4OVteuXXXy5Mmr9m3WrFkKDAzU119/rUqVKsnLy0uxsbE6cOCAU7358+erTp068vLyUvny5TVu3DhlZWU5tttsNs2cOVPt2rWTj4+PJk6cmOvxIiMjNWHCBHXu3Fk+Pj4KDw/X9OnTnepcra3r9WHnzp1q2rSpvLy8VK1aNS1atMip3dxuPdy6dav+8Y9/yN/fX35+fmrSpIl2796tsWPH6sMPP9T8+fNls9lks9m0bNmyXNtYvny5GjRoIE9PT4WGhmrEiBFO/WrevLkGDRqk5557TiVLllRISIjGjh171TkBAFwboQwA8Jf58MMPVapUKa1bt04DBw7Uk08+qX/+859q2LChNm7cqBYtWqhr1646d+6cJOn06dO67777VLt2bW3YsEGJiYk6duyYOnbseM3jnDt3ThMnTtRHH32kVatW6fTp03r00Ucd21esWKFu3bpp8ODB2rZtm9555x3NmjUrR/AaO3asHnroIf3yyy964oknrnq8V199VTVr1tSmTZs0YsQIDR48OEeAurKt6/XBbrfr4YcfloeHh9auXau3335bw4cPv+a4Dx06pKZNm8rT01NLlixRUlKSnnjiCWVlZenZZ59Vx44d1bJlSx05ckRHjhxRw4YNc22jdevWql+/vrZs2aKZM2fqvffe04svvuhU78MPP5SPj4/Wrl2rV155RePHj88xZgBAHhkAAG7ABx98YAICAnKUd+/e3bRv397xvVmzZqZx48aO71lZWcbHx8d07drVUXbkyBEjyaxevdoYY8yECRNMixYtnNo9cOCAkWR27Nhx1f5IMmvWrHGUJScnG0lm7dq1xhhj7r//fjNp0iSn/RISEkxoaKjjuyQzZMiQ64zemLJly5qWLVs6lXXq1Mm0atXqmm1drw8LFy40bm5u5tChQ47t33//vZFkvvrqK2OMMSkpKUaS2bRpkzHGmLi4OFOuXDmTmZmZa1+vnJPc2nj++edN5cqVjd1ud9SZPn268fX1NdnZ2caYnHNpjDH169c3w4cPz/W4AIBr45kyAMBfpkaNGo4/u7q66vbbb1f16tUdZcHBwZKk48ePS5K2bNmipUuX5vp82u7du3XnnXfmehw3NzfVr1/f8b1KlSoKDAxUcnKyGjRooC1btmjVqlVOV8ays7OVkZGhc+fOqUSJEpKkevXq5Wlc0dHROb7Hx8c7lV3Z1vX6kJycrIiICIWFhV31OFfavHmzmjRpckPPqyUnJys6OtppMZdGjRrp7NmzOnjwoMqUKSPJeS4lKTQ01DFvAID8IZQBAP4yV4YFm83mVHY5CNjtdknS2bNn1bZtW7388ss52goNDS1wP86ePatx48bp4YcfzrHNy8vL8WcfH58CH+NKV7aV1z7kh7e3d4H2K4jc5vLyvAEA8odQBgAosurUqaO5c+cqMjJSbm55/7+srKwsbdiwQQ0aNJAk7dixQ6dPn1bVqlUd7e7YsUMVK1YslH6uWbMmx/fLx7qa6/WhatWqOnDggI4cOeIIoFce50o1atTQhx9+eNXVHT08PJSdnX3NNqpWraq5c+fKGOMIyatWrZKfn5/uuOOOa+4LACgYFvoAABRZ/fv316lTp9S5c2etX79eu3fv1sKFC9WzZ89rhgt3d3cNHDhQa9euVVJSknr06KF77rnHEdJGjx6tjz76SOPGjdPWrVuVnJyszz77TCNHjixQP1etWqVXXnlFv/32m6ZPn64vvvhCgwcPvuY+1+tDTEyM7rzzTnXv3l1btmzRihUr9MILL1yzzQEDBigtLU2PPvqoNmzYoJ07dyohIcHxuoLIyEj9/PPP2rFjh06ePKmLFy/maOOpp57SgQMHNHDgQG3fvl3z58/XmDFjNHToULm48GsDANwM/K8rAKDICgsL06pVq5Sdna0WLVqoevXqGjJkiAIDA68ZEEqUKKHhw4erS5cuatSokXx9fTVnzhzH9tjYWH377bf64YcfVL9+fd1zzz164403VLZs2QL185lnntGGDRtUu3Ztvfjii5oyZYpiY2Ovuc/1+uDi4qKvvvpK58+fV4MGDdS7d++rLst/2e23364lS5bo7NmzatasmerWrat3333XcdWsT58+qly5surVq6egoCCtWrUqRxvh4eFasGCB1q1bp5o1a6pfv37q1atXgQMrAOD6bMYYY3UnAAAoLLNmzdKQIUN0+vTpv+R4kZGRGjJkiIYMGfKXHA8A8PfDlTIAAAAAsBChDAAAAAAsxO2LAAAAAGAhrpQBAAAAgIUIZQAAAABgIUIZAAAAAFiIUAYAAAAAFiKUAQAAAICFCGUAAAAAYCFCGQAAAABYiFAGAAAAABb6f6FxdH/r6JpRAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install joblib\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5TILHSnxuIx1",
        "outputId": "844bfa8a-7d96-47da-860a-0da6adb3d97a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (1.3.2)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import joblib\n",
        "\n",
        "# Save the model\n",
        "joblib.dump(model_6, 'model_6.pkl')\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cD4be5vlKKz-",
        "outputId": "3a5d5208-4eb1-4ef1-b960-53b4a2a563cd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['model_6.pkl']"
            ]
          },
          "metadata": {},
          "execution_count": 132
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Save the model in SavedModel format\n",
        "model_6.save(\"model_6_savedmodel\")"
      ],
      "metadata": {
        "id": "s5jPLSlvKPza"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import shutil\n",
        "shutil.make_archive('model_6_savedmodel', 'zip', '/content/model_6_savedmodel')\n"
      ],
      "metadata": {
        "id": "-9yJuBDqSAU_",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "outputId": "533f827a-6eeb-4a2e-8e08-e3637304bdf3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'/content/model_6_savedmodel.zip'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 134
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import files\n",
        "files.download('model_6_savedmodel.zip')\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "id": "Yr5q_mGvnkRh",
        "outputId": "9ae939c7-d47a-44e8-bc96-7072b99ad151"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "download(\"download_47a1aa15-934a-42d2-b576-e36ee529b95f\", \"model_6_savedmodel.zip\", 961525145)"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "8x5uGUtYpdww"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}