Project_1_Notebook.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 113,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PXQ2rNxf41HE",
        "outputId": "f4e28c00-2f4e-4c03-ba98-76113472e22b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "# Run this cell to mount your drive to this notebook in order to read the datasets\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 114,
      "metadata": {
        "id": "9poIxsgG4uzb"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BWmM1YP7PD_J"
      },
      "source": [
        "## Read Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 115,
      "metadata": {
        "id": "cZ35nOeR43ys"
      },
      "outputs": [],
      "source": [
        "# Put the folder path where the datasets are located\n",
        "PATH = \"/content/drive/MyDrive/cs445/hw1/\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 116,
      "metadata": {
        "id": "x8buYRli5SjM"
      },
      "outputs": [],
      "source": [
        "# Read the train and test set with read_csv() method of pandas\n",
        "train = pd.read_csv(PATH + \"train.csv\")\n",
        "test = pd.read_csv(PATH + \"test.csv\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "csBleyqDPJM2"
      },
      "source": [
        "### Preprocess Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 117,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hCgu4nbNubw4",
        "outputId": "38e3ed8d-093c-47f5-d6da-2ad091245810"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ]
        }
      ],
      "source": [
        "import re\n",
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.stem import PorterStemmer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 118,
      "metadata": {
        "id": "eZekShd_vFFm"
      },
      "outputs": [],
      "source": [
        "pd.set_option(\"display.max_colwidth\", None)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 119,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 305
        },
        "id": "0bhCTJ1Juv0r",
        "outputId": "85dbe71a-cbe4-4f71-8d56-75d28f8aa622"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Unnamed: 0  \\\n",
              "0           0   \n",
              "1           1   \n",
              "2           2   \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 text  \\\n",
              "0                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          I came here and left a review before but last time I didn't get food poisoning. Unless you want to stay up all night puking I suggest you don't come here.   \n",
              "1                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     Had a very nice first visit here. The owner Ted, was very friendly from the start and the restaurant was busy on a Friday night the beginning of Chinese New Year. We ordered the Pu-pu platter and crab Rangoon and for dinner chef specials with seafood instead of Chinese American regulars and some fried rice. Everything was well prepared. The shrimp large vegetables crisp. The chili sauce sweet and tangy. \\n\\nWe will return. The owner made an effort to visit and learn our names and ask about our first visit.   \n",
              "2  This is a gorgeous and very clean hotel.  We had a room in the West Wing.  At first it was a chore to get to the room but if you look for different features of the hotel (they also have posted signs) you will easily find your way around.  Self parking was excellent and the walk to Check In was confusing at first but you'll figure it out quickly.  Check In was quick.  The walk to the room was enjoyable as the floors were shiny and clean.  The room was in almost immaculate shape.  Well decorated and very, very clean.  Loved the carpet and the dark lacquered furniture and accents.  Absolutely gorgeous.  Bed was super comfortable and pillow just right.  Towels were thick and soft.  Water pressure in sink was less than satisfactory and shower pressure too.  Bottom left hand drawer almost fell out when I opened it.  This may be nit picking but it was mark down because of water pressure and broken drawer.  Another sink would have been nice.  Yes, I would stay here again.   \n",
              "\n",
              "   label  \n",
              "0      1  \n",
              "1      4  \n",
              "2      4  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-1bf92c0d-e8e2-4f43-878f-1cf8ebf10c08\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>I came here and left a review before but last time I didn't get food poisoning. Unless you want to stay up all night puking I suggest you don't come here.</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>Had a very nice first visit here. The owner Ted, was very friendly from the start and the restaurant was busy on a Friday night the beginning of Chinese New Year. We ordered the Pu-pu platter and crab Rangoon and for dinner chef specials with seafood instead of Chinese American regulars and some fried rice. Everything was well prepared. The shrimp large vegetables crisp. The chili sauce sweet and tangy. \\n\\nWe will return. The owner made an effort to visit and learn our names and ask about our first visit.</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>This is a gorgeous and very clean hotel.  We had a room in the West Wing.  At first it was a chore to get to the room but if you look for different features of the hotel (they also have posted signs) you will easily find your way around.  Self parking was excellent and the walk to Check In was confusing at first but you'll figure it out quickly.  Check In was quick.  The walk to the room was enjoyable as the floors were shiny and clean.  The room was in almost immaculate shape.  Well decorated and very, very clean.  Loved the carpet and the dark lacquered furniture and accents.  Absolutely gorgeous.  Bed was super comfortable and pillow just right.  Towels were thick and soft.  Water pressure in sink was less than satisfactory and shower pressure too.  Bottom left hand drawer almost fell out when I opened it.  This may be nit picking but it was mark down because of water pressure and broken drawer.  Another sink would have been nice.  Yes, I would stay here again.</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1bf92c0d-e8e2-4f43-878f-1cf8ebf10c08')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-1bf92c0d-e8e2-4f43-878f-1cf8ebf10c08 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-1bf92c0d-e8e2-4f43-878f-1cf8ebf10c08');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 119
        }
      ],
      "source": [
        "train[:3]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 120,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OSiAj6atwZzi",
        "outputId": "cbf5436b-8755-41d5-d910-3963386c3c57"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ]
        }
      ],
      "source": [
        "# Define a function to perform preprocessing. This function can perform things like lowercasing, stemming, removing stopwords, etc.\n",
        "from nltk import word_tokenize\n",
        "nltk.download('punkt')\n",
        "en_stopwords = stopwords.words('english')\n",
        "from nltk.tokenize import RegexpTokenizer\n",
        "\n",
        "def preprocess(text: str): \n",
        "    text = text.lower()\n",
        "    text = \" \".join(text.split()) #Remove Extra Whitespaces\n",
        "    text = word_tokenize(text) \n",
        "\n",
        "    result = []\n",
        "    for token in text: #remove stopwords\n",
        "      if token not in en_stopwords:\n",
        "          result.append(token)\n",
        "    \n",
        "    text = result\n",
        "    \n",
        "    #if n't, replace with not. because not is important for the dataset!!!\n",
        "\n",
        "    tokenizer = RegexpTokenizer(r\"\\w+\") #remove punctuations\n",
        "    lst=tokenizer.tokenize(' '.join(text))\n",
        "\n",
        "    porter = PorterStemmer() #apply stemming\n",
        "    result=[]\n",
        "    for word in lst:\n",
        "        result.append(porter.stem(word))\n",
        "\n",
        "    return result"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 121,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "saAhAw_vyu9T",
        "outputId": "050049ea-ba7c-4e45-9d67-4d150b5190e0"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((18000, 3), (2000, 3))"
            ]
          },
          "metadata": {},
          "execution_count": 121
        }
      ],
      "source": [
        "# Apply your preprocessing function to your text fields.\n",
        "\n",
        "train.text = train.text.apply(preprocess)\n",
        "test.text = test.text.apply(preprocess)\n",
        "\n",
        "train.shape, test.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 122,
      "metadata": {
        "id": "K28dFP7Kvuwo"
      },
      "outputs": [],
      "source": [
        "to_merge = \" \" \n",
        "\n",
        "for k in range(len(train)):\n",
        "  train.text[k] = to_merge.join(train.text[k])\n",
        "\n",
        "for m in range(len(test)):\n",
        "  test.text[m] = to_merge.join(test.text[m])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 123,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 254
        },
        "id": "wEzaG65EwroF",
        "outputId": "320b03f1-83cb-461d-c3a4-0e983cc3dbcb"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Unnamed: 0  \\\n",
              "0           0   \n",
              "1           1   \n",
              "2           2   \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               text  \\\n",
              "0                                                                                                                                                                                                                                                                                                                                                                                                                                                                       came left review last time n t get food poison unless want stay night puke suggest n t come   \n",
              "1                                                                                                                                                                                                                                      nice first visit owner ted friendli start restaur busi friday night begin chines new year order pu pu platter crab rangoon dinner chef special seafood instead chines american regular fri rice everyth well prepar shrimp larg veget crisp chili sauc sweet tangi return owner made effort visit learn name ask first visit   \n",
              "2  gorgeou clean hotel room west wing first chore get room look differ featur hotel also post sign easili find way around self park excel walk check confus first ll figur quickli check quick walk room enjoy floor shini clean room almost immacul shape well decor clean love carpet dark lacquer furnitur accent absolut gorgeou bed super comfort pillow right towel thick soft water pressur sink less satisfactori shower pressur bottom left hand drawer almost fell open may nit pick mark water pressur broken drawer anoth sink would nice ye would stay   \n",
              "\n",
              "   label  \n",
              "0      1  \n",
              "1      4  \n",
              "2      4  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-7f61bf2b-0b5b-4ae4-80ba-2523eb3dc300\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>came left review last time n t get food poison unless want stay night puke suggest n t come</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>nice first visit owner ted friendli start restaur busi friday night begin chines new year order pu pu platter crab rangoon dinner chef special seafood instead chines american regular fri rice everyth well prepar shrimp larg veget crisp chili sauc sweet tangi return owner made effort visit learn name ask first visit</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>gorgeou clean hotel room west wing first chore get room look differ featur hotel also post sign easili find way around self park excel walk check confus first ll figur quickli check quick walk room enjoy floor shini clean room almost immacul shape well decor clean love carpet dark lacquer furnitur accent absolut gorgeou bed super comfort pillow right towel thick soft water pressur sink less satisfactori shower pressur bottom left hand drawer almost fell open may nit pick mark water pressur broken drawer anoth sink would nice ye would stay</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7f61bf2b-0b5b-4ae4-80ba-2523eb3dc300')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-7f61bf2b-0b5b-4ae4-80ba-2523eb3dc300 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-7f61bf2b-0b5b-4ae4-80ba-2523eb3dc300');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 123
        }
      ],
      "source": [
        "train.head(3)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 124,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 333
        },
        "id": "dgglbSugzNVx",
        "outputId": "3b0e99cd-d748-4ac6-d8a8-4fde546df568"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    Unnamed: 0  \\\n",
              "15          15   \n",
              "16          16   \n",
              "17          17   \n",
              "18          18   \n",
              "19          19   \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        text  \\\n",
              "15                                                                                                                                                                                                                                                                                                                                                                                                                                                              tucson weekend read fiamm worth visit review n t justic absolut amaz pizza itali summer better m impress liter make drive tucson pizza thank   \n",
              "16                                                                                                                                                                                                                                                                                                                                                                                                                                                           staff manag awesom food so so ca n t beat view band play around 6pm friday could hear talk move outsid grouper fish taco flavor french fri good   \n",
              "17                                                                                                                                                                                                                                                                                                                                                                    roach crawl behind shoulder seat kill napkin hungri know thing crazi storm come receiv food hair waiter came back new plate walk without say word told us manag said middl hurrican bug seek go insid place dirti bewar hurrican roach   \n",
              "18                                                               overpr low qualiti went hibachi first serv salad brown iceberg lettuc m kick take photo phone complet dead clear soup chicken broth soup base ew 7 yr old normal love clear soup would n t touch like ramen noodl chicken packet use freez dri mushroom chef friendli took 30 min come us otherwis dead restaur soy ginger sauc great good flavor veggi great well filet mignon weird kind soapi tast menu green tea ice cream fav mine tri order told rainbow sherbet disappoint 70 later two adult child knew d like back place like ever   \n",
              "19  lunch famili today aw servic first steak cook ask second ran side instead ask us brought food would like substitut wait come meal brought meal said d 15 minut rest ask manag manag trivial concern tri explain commun process n t fault told commun us first gave state flatli want n t know n t tell re go n t expect custom handl problem manag take initi fix problem free appet phone waitress took phone instead write serious peopl liabil end phone break privaci issu fail use charg anoth person appet attempt hide fact serious stay away one usual good time texa roadhous never go back one   \n",
              "\n",
              "    label  \n",
              "15      5  \n",
              "16      3  \n",
              "17      1  \n",
              "18      2  \n",
              "19      1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b713d454-8d30-4c58-af71-ed4e37c62f8c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>15</td>\n",
              "      <td>tucson weekend read fiamm worth visit review n t justic absolut amaz pizza itali summer better m impress liter make drive tucson pizza thank</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>16</td>\n",
              "      <td>staff manag awesom food so so ca n t beat view band play around 6pm friday could hear talk move outsid grouper fish taco flavor french fri good</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>17</td>\n",
              "      <td>roach crawl behind shoulder seat kill napkin hungri know thing crazi storm come receiv food hair waiter came back new plate walk without say word told us manag said middl hurrican bug seek go insid place dirti bewar hurrican roach</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>18</td>\n",
              "      <td>overpr low qualiti went hibachi first serv salad brown iceberg lettuc m kick take photo phone complet dead clear soup chicken broth soup base ew 7 yr old normal love clear soup would n t touch like ramen noodl chicken packet use freez dri mushroom chef friendli took 30 min come us otherwis dead restaur soy ginger sauc great good flavor veggi great well filet mignon weird kind soapi tast menu green tea ice cream fav mine tri order told rainbow sherbet disappoint 70 later two adult child knew d like back place like ever</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>19</td>\n",
              "      <td>lunch famili today aw servic first steak cook ask second ran side instead ask us brought food would like substitut wait come meal brought meal said d 15 minut rest ask manag manag trivial concern tri explain commun process n t fault told commun us first gave state flatli want n t know n t tell re go n t expect custom handl problem manag take initi fix problem free appet phone waitress took phone instead write serious peopl liabil end phone break privaci issu fail use charg anoth person appet attempt hide fact serious stay away one usual good time texa roadhous never go back one</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b713d454-8d30-4c58-af71-ed4e37c62f8c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-b713d454-8d30-4c58-af71-ed4e37c62f8c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-b713d454-8d30-4c58-af71-ed4e37c62f8c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 124
        }
      ],
      "source": [
        "train[15:20]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 125,
      "metadata": {
        "id": "DS7xHte-BkzB"
      },
      "outputs": [],
      "source": [
        "backup = train.copy()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 126,
      "metadata": {
        "id": "MJWiNm2ZrO8m"
      },
      "outputs": [],
      "source": [
        "# Create your binary and multiclass datasets\n",
        "binary_df = train.copy()\n",
        "multiclass_df = train.copy()\n",
        "test_binary_df = test.copy()\n",
        "test_multiclass_df = test.copy()\n",
        "\n",
        "# For binary dataset, get rid of the class 3 in the dataset and map class 1 and 2 to 0, and class 4 and 5 to 1\n",
        "binary_df.loc[train['label'] == 1,'label'] = 0\n",
        "binary_df.loc[train['label'] == 2,'label'] = 0\n",
        "binary_df.loc[train['label'] == 4,'label'] = 1\n",
        "binary_df.loc[train['label'] == 5,'label'] = 1\n",
        "binary_df.drop(train[train['label'] == 3].index, inplace = True)\n",
        "\n",
        "test_binary_df.loc[test['label'] == 1,'label'] = 0\n",
        "test_binary_df.loc[test['label'] == 2,'label'] = 0\n",
        "test_binary_df.loc[test['label'] == 4,'label'] = 1\n",
        "test_binary_df.loc[test['label'] == 5,'label'] = 1\n",
        "test_binary_df.drop(test[test['label'] == 3].index, inplace = True)\n",
        "\n",
        "\n",
        "# For multiclass dataset, make sure your classes starts from 0 and goes until 4. (5->4, 4->3, 3->2, 2->1, 1->0)\n",
        "multiclass_df.loc[train['label'] == 1,'label'] = 0\n",
        "multiclass_df.loc[train['label'] == 2,'label'] = 1\n",
        "multiclass_df.loc[train['label'] == 3,'label'] = 2\n",
        "multiclass_df.loc[train['label'] == 4,'label'] = 3\n",
        "multiclass_df.loc[train['label'] == 5,'label'] = 4\n",
        "\n",
        "test_multiclass_df.loc[test['label'] == 1,'label'] = 0\n",
        "test_multiclass_df.loc[test['label'] == 2,'label'] = 1\n",
        "test_multiclass_df.loc[test['label'] == 3,'label'] = 2\n",
        "test_multiclass_df.loc[test['label'] == 4,'label'] = 3\n",
        "test_multiclass_df.loc[test['label'] == 5,'label'] = 4"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 127,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 435
        },
        "id": "ipk_VHLs_SHF",
        "outputId": "7178d824-a635-4087-9c82-07a3dc2c289f"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    Unnamed: 0  \\\n",
              "18          18   \n",
              "19          19   \n",
              "20          20   \n",
              "21          21   \n",
              "23          23   \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               text  \\\n",
              "18                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      overpr low qualiti went hibachi first serv salad brown iceberg lettuc m kick take photo phone complet dead clear soup chicken broth soup base ew 7 yr old normal love clear soup would n t touch like ramen noodl chicken packet use freez dri mushroom chef friendli took 30 min come us otherwis dead restaur soy ginger sauc great good flavor veggi great well filet mignon weird kind soapi tast menu green tea ice cream fav mine tri order told rainbow sherbet disappoint 70 later two adult child knew d like back place like ever   \n",
              "19                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         lunch famili today aw servic first steak cook ask second ran side instead ask us brought food would like substitut wait come meal brought meal said d 15 minut rest ask manag manag trivial concern tri explain commun process n t fault told commun us first gave state flatli want n t know n t tell re go n t expect custom handl problem manag take initi fix problem free appet phone waitress took phone instead write serious peopl liabil end phone break privaci issu fail use charg anoth person appet attempt hide fact serious stay away one usual good time texa roadhous never go back one   \n",
              "20                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         ye crust littl carbon mayb cook woodfir oven simpl tasti thin crust pizza tasti salad good beer wine select probabl one stl s best pizza spot servic fine satisfi experi   \n",
              "21                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          amber tri give us old fri n t cold burnt one get togeth   \n",
              "23  look impress french german dish new orlean pleas go luke restaur know restaur locat central busi district next hilton former mason templ build histor vibe restaur front room lot grand back room seat offset view open kitchen appar want seat peopl reserv sinc one refus seat 9 15pm 6 45pm littl back forth end wait 15 minut seat howev walk restaur back room way mani tabl empti made sens act way lot tabl still remain empti throughout meal decid start meal cocktail origin want mint julep riverbend vodka lemon juic basil syrup blueberri ginger ale recommend server rel sweet cocktail ginger ale power flavor could still tast alcohol friend choos regular starter order cup crawfish bisqu ate assum like complain sour aftertast boyfriend skip appet went straight entre alway seem find basic item menu order luke burger bacon caramel onion tomato swiss chees rave one thing tell good size friend jumbo louisiana shrimp en cocott lump crab meat roast jalapeño chees grit andouil green onion sausag love obsess mussel classic moul et frite although two simpli ingredi garlic thyme broth flavor stood tast lot bolder other lot ingredi hous made fri scrumptiou best kind even food still made room dessert first bread pud delici quit sweet also fan sauc food one n t enough also order bourbon vanilla bean creme brule superb sweet plu littl cooki top ad noth dessert anyth could left good food still give prop server amaz engag front desk sour mood littl bit first made could n t ask anyon better   \n",
              "\n",
              "    label  \n",
              "18      0  \n",
              "19      0  \n",
              "20      1  \n",
              "21      0  \n",
              "23      1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e0da9b90-f36d-4528-be23-4a18acd66d7b\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>18</td>\n",
              "      <td>overpr low qualiti went hibachi first serv salad brown iceberg lettuc m kick take photo phone complet dead clear soup chicken broth soup base ew 7 yr old normal love clear soup would n t touch like ramen noodl chicken packet use freez dri mushroom chef friendli took 30 min come us otherwis dead restaur soy ginger sauc great good flavor veggi great well filet mignon weird kind soapi tast menu green tea ice cream fav mine tri order told rainbow sherbet disappoint 70 later two adult child knew d like back place like ever</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>19</td>\n",
              "      <td>lunch famili today aw servic first steak cook ask second ran side instead ask us brought food would like substitut wait come meal brought meal said d 15 minut rest ask manag manag trivial concern tri explain commun process n t fault told commun us first gave state flatli want n t know n t tell re go n t expect custom handl problem manag take initi fix problem free appet phone waitress took phone instead write serious peopl liabil end phone break privaci issu fail use charg anoth person appet attempt hide fact serious stay away one usual good time texa roadhous never go back one</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>20</td>\n",
              "      <td>ye crust littl carbon mayb cook woodfir oven simpl tasti thin crust pizza tasti salad good beer wine select probabl one stl s best pizza spot servic fine satisfi experi</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>21</td>\n",
              "      <td>amber tri give us old fri n t cold burnt one get togeth</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>23</td>\n",
              "      <td>look impress french german dish new orlean pleas go luke restaur know restaur locat central busi district next hilton former mason templ build histor vibe restaur front room lot grand back room seat offset view open kitchen appar want seat peopl reserv sinc one refus seat 9 15pm 6 45pm littl back forth end wait 15 minut seat howev walk restaur back room way mani tabl empti made sens act way lot tabl still remain empti throughout meal decid start meal cocktail origin want mint julep riverbend vodka lemon juic basil syrup blueberri ginger ale recommend server rel sweet cocktail ginger ale power flavor could still tast alcohol friend choos regular starter order cup crawfish bisqu ate assum like complain sour aftertast boyfriend skip appet went straight entre alway seem find basic item menu order luke burger bacon caramel onion tomato swiss chees rave one thing tell good size friend jumbo louisiana shrimp en cocott lump crab meat roast jalapeño chees grit andouil green onion sausag love obsess mussel classic moul et frite although two simpli ingredi garlic thyme broth flavor stood tast lot bolder other lot ingredi hous made fri scrumptiou best kind even food still made room dessert first bread pud delici quit sweet also fan sauc food one n t enough also order bourbon vanilla bean creme brule superb sweet plu littl cooki top ad noth dessert anyth could left good food still give prop server amaz engag front desk sour mood littl bit first made could n t ask anyon better</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e0da9b90-f36d-4528-be23-4a18acd66d7b')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e0da9b90-f36d-4528-be23-4a18acd66d7b button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e0da9b90-f36d-4528-be23-4a18acd66d7b');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 127
        }
      ],
      "source": [
        "binary_df[15:20]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 128,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 333
        },
        "id": "RHamMGECDVQQ",
        "outputId": "4f3ccca6-69bc-418e-8581-dacc292066e7"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    Unnamed: 0  \\\n",
              "15          15   \n",
              "16          16   \n",
              "17          17   \n",
              "18          18   \n",
              "19          19   \n",
              "\n",
              "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        text  \\\n",
              "15                                                                                                                                                                                                                                                                                                                                                                                                                                                              tucson weekend read fiamm worth visit review n t justic absolut amaz pizza itali summer better m impress liter make drive tucson pizza thank   \n",
              "16                                                                                                                                                                                                                                                                                                                                                                                                                                                           staff manag awesom food so so ca n t beat view band play around 6pm friday could hear talk move outsid grouper fish taco flavor french fri good   \n",
              "17                                                                                                                                                                                                                                                                                                                                                                    roach crawl behind shoulder seat kill napkin hungri know thing crazi storm come receiv food hair waiter came back new plate walk without say word told us manag said middl hurrican bug seek go insid place dirti bewar hurrican roach   \n",
              "18                                                               overpr low qualiti went hibachi first serv salad brown iceberg lettuc m kick take photo phone complet dead clear soup chicken broth soup base ew 7 yr old normal love clear soup would n t touch like ramen noodl chicken packet use freez dri mushroom chef friendli took 30 min come us otherwis dead restaur soy ginger sauc great good flavor veggi great well filet mignon weird kind soapi tast menu green tea ice cream fav mine tri order told rainbow sherbet disappoint 70 later two adult child knew d like back place like ever   \n",
              "19  lunch famili today aw servic first steak cook ask second ran side instead ask us brought food would like substitut wait come meal brought meal said d 15 minut rest ask manag manag trivial concern tri explain commun process n t fault told commun us first gave state flatli want n t know n t tell re go n t expect custom handl problem manag take initi fix problem free appet phone waitress took phone instead write serious peopl liabil end phone break privaci issu fail use charg anoth person appet attempt hide fact serious stay away one usual good time texa roadhous never go back one   \n",
              "\n",
              "    label  \n",
              "15      4  \n",
              "16      2  \n",
              "17      0  \n",
              "18      1  \n",
              "19      0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b5be87e6-2462-450b-92b3-3df9c24350f2\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>15</td>\n",
              "      <td>tucson weekend read fiamm worth visit review n t justic absolut amaz pizza itali summer better m impress liter make drive tucson pizza thank</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>16</td>\n",
              "      <td>staff manag awesom food so so ca n t beat view band play around 6pm friday could hear talk move outsid grouper fish taco flavor french fri good</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>17</td>\n",
              "      <td>roach crawl behind shoulder seat kill napkin hungri know thing crazi storm come receiv food hair waiter came back new plate walk without say word told us manag said middl hurrican bug seek go insid place dirti bewar hurrican roach</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>18</td>\n",
              "      <td>overpr low qualiti went hibachi first serv salad brown iceberg lettuc m kick take photo phone complet dead clear soup chicken broth soup base ew 7 yr old normal love clear soup would n t touch like ramen noodl chicken packet use freez dri mushroom chef friendli took 30 min come us otherwis dead restaur soy ginger sauc great good flavor veggi great well filet mignon weird kind soapi tast menu green tea ice cream fav mine tri order told rainbow sherbet disappoint 70 later two adult child knew d like back place like ever</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>19</td>\n",
              "      <td>lunch famili today aw servic first steak cook ask second ran side instead ask us brought food would like substitut wait come meal brought meal said d 15 minut rest ask manag manag trivial concern tri explain commun process n t fault told commun us first gave state flatli want n t know n t tell re go n t expect custom handl problem manag take initi fix problem free appet phone waitress took phone instead write serious peopl liabil end phone break privaci issu fail use charg anoth person appet attempt hide fact serious stay away one usual good time texa roadhous never go back one</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b5be87e6-2462-450b-92b3-3df9c24350f2')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-b5be87e6-2462-450b-92b3-3df9c24350f2 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-b5be87e6-2462-450b-92b3-3df9c24350f2');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 128
        }
      ],
      "source": [
        "multiclass_df[15:20]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fbz48mNfttuo"
      },
      "source": [
        "# Models"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y65GxGIFAC_K"
      },
      "source": [
        "## Non-Neural Models"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 129,
      "metadata": {
        "id": "BYhkcMWouazd"
      },
      "outputs": [],
      "source": [
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.base import TransformerMixin\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.naive_bayes import GaussianNB\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.metrics import f1_score,confusion_matrix,accuracy_score"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4MZIaJFXWScs"
      },
      "source": [
        "### Naive Bayes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 130,
      "metadata": {
        "id": "Tr32alf3WXmM"
      },
      "outputs": [],
      "source": [
        "# https://scikit-learn.org/stable/modules/generated/sklearn.naive_bayes.GaussianNB.html\n",
        "\n",
        "# Create a class for converting sparse matrix output of TfidfVectorizer to dense matrix for feeding into GaussianNB\n",
        "class DenseTransformer(TransformerMixin):\n",
        "\n",
        "    def fit(self, X, y=None, **fit_params):\n",
        "        return self\n",
        "\n",
        "    def transform(self, X, y=None, **fit_params):\n",
        "        return X.todense()\n",
        "\n",
        "\n",
        "# Initiate the pipeline with required components.You can use Pipeline class of sklearn -> https://scikit-learn.org/stable/modules/generated/sklearn.pipeline.Pipeline.html\n",
        "# There will be three components; 1) TfidfVectorizer 2) DenseTransformer 3) Naive Bayes classifier.\n",
        "\n",
        "pipeline = Pipeline([\n",
        "     ('vectorizer', TfidfVectorizer()), \n",
        "     ('to_dense', DenseTransformer()), \n",
        "     ('nb_classifier', GaussianNB())\n",
        "])\n",
        "\n",
        "\n",
        "# Set the hyperparameter space that will be scanned with GridSearchCV.\n",
        "\n",
        "parameters = {'vectorizer__min_df':[100,500,1000], 'vectorizer__ngram_range':[[1, 1],[1, 2],[1, 3]]}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0PjodOpko5pJ"
      },
      "source": [
        "### Binary"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 131,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vlZRfzqPo9iv",
        "outputId": "efdd0ef1-b423-4a6f-e8c6-229f9820e8d7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "hpyerparameter  0  : std-mean-min-max 0.00514 0.85452 0.84559 0.86047\n",
            "hpyerparameter  1  : std-mean-min-max 0.00859 0.85993 0.84629 0.87192\n",
            "hpyerparameter  2  : std-mean-min-max 0.00871 0.85993 0.84629 0.87227\n",
            "hpyerparameter  3  : std-mean-min-max 0.00483 0.81315 0.80528 0.81888\n",
            "hpyerparameter  4  : std-mean-min-max 0.0056 0.81273 0.80562 0.82096\n",
            "hpyerparameter  5  : std-mean-min-max 0.0056 0.81273 0.80562 0.82096\n",
            "hpyerparameter  6  : std-mean-min-max 0.00687 0.76886 0.75668 0.77793\n",
            "hpyerparameter  7  : std-mean-min-max 0.00687 0.76886 0.75668 0.77793\n",
            "hpyerparameter  8  : std-mean-min-max 0.00687 0.76886 0.75668 0.77793\n",
            "{'vectorizer__min_df': 100, 'vectorizer__ngram_range': [1, 2]}\n",
            "[[682  91]\n",
            " [113 707]]\n",
            "f1 score 0.8739184177997528\n",
            "accuracy score 0.871939736346516\n",
            "CPU times: user 1min 52s, sys: 1.14 s, total: 1min 53s\n",
            "Wall time: 1min 55s\n"
          ]
        }
      ],
      "source": [
        "%%time\n",
        "# Initialize and run the GridSearchCV to scan the hyperparameter and find the best hyperparameter set that will maximize the scoring option for binary classification.\n",
        "pipe  = GridSearchCV(pipeline, cv=5,param_grid=parameters)\n",
        "pipe.fit(binary_df.text, binary_df.label)\n",
        "\n",
        "\n",
        "# Report the standart deviation of split scores for each hyperparameter group.\n",
        "\n",
        "split_0 = pipe.cv_results_[\"split0_test_score\"]\n",
        "split_1 = pipe.cv_results_[\"split1_test_score\"]\n",
        "split_2 = pipe.cv_results_[\"split2_test_score\"]\n",
        "split_3 = pipe.cv_results_[\"split3_test_score\"]\n",
        "split_4 = pipe.cv_results_[\"split4_test_score\"]\n",
        "\n",
        "#we analyze the mean-std-min-max for each hyperparameter group\n",
        "for i in range(9):\n",
        "  std = np.std([split_0[i],split_1[i],split_2[i],split_3[i],split_4[i]])\n",
        "  mean = np.mean([split_0[i],split_1[i],split_2[i],split_3[i],split_4[i]])\n",
        "  min = np.min([split_0[i],split_1[i],split_2[i],split_3[i],split_4[i]])\n",
        "  max = np.max([split_0[i],split_1[i],split_2[i],split_3[i],split_4[i]])\n",
        "  print(\"hpyerparameter \", i , \" :\", \"std-mean-min-max\", round(std,5), round(mean,5), round(min,5), round(max,5))\n",
        "\n",
        "\"\"\"\n",
        "#we make sure there is no huge differences between splits\n",
        "print(\"standard deviations:\", round(np.std(split_0),4), round(np.std(split_1),4), round(np.std(split_2),4), round(np.std(split_3),4), round(np.std(split_4),4))\n",
        "print(\"mean               :\", round(np.mean(split_0),4), round(np.mean(split_1),4), round(np.mean(split_2),4), round(np.mean(split_3),4), round(np.mean(split_4),4))\n",
        "\n",
        "\"\"\"\n",
        "# Show the best parameter set for given dataset and hyperparameter space.\n",
        "print(pipe.best_params_)\n",
        "\n",
        "\n",
        "# Building the pipeline with the best parameter group and reporting Conf. Mat. and Results on the Test Set #\n",
        "# Create your Pipeline object with the best parameter set.\n",
        "best_parameters = {'vectorizer__min_df':[100], 'vectorizer__ngram_range':[[1, 2]]}\n",
        "pipe  = GridSearchCV(pipeline, cv=5,param_grid=best_parameters)\n",
        "\n",
        "# Fit your pipeline on training set.\n",
        "pipe.fit(binary_df.text, binary_df.label)\n",
        "\n",
        "# Take prediction and report the F1 and Accuracy scores for binary classification. Then show the confussion table.\n",
        "preds = pipe.predict(test_binary_df.text)\n",
        "\n",
        "print(confusion_matrix(test_binary_df.label, preds))\n",
        "print(\"f1 score\", f1_score(test_binary_df.label, preds))\n",
        "print(\"accuracy score\", accuracy_score(test_binary_df.label, preds))\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gUmILCQmpPAE"
      },
      "source": [
        "### Multi"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 132,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Od0ZnT7MpSdC",
        "outputId": "f58a5b39-e47e-4946-a827-ba3b0626ccc7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "hpyerparameter  0  : std-mean-min-max 0.00717 0.45433 0.44083 0.46083\n",
            "hpyerparameter  1  : std-mean-min-max 0.00835 0.46778 0.45528 0.47694\n",
            "hpyerparameter  2  : std-mean-min-max 0.00921 0.46744 0.45389 0.4775\n",
            "hpyerparameter  3  : std-mean-min-max 0.00852 0.46033 0.445 0.47028\n",
            "hpyerparameter  4  : std-mean-min-max 0.00846 0.46056 0.44528 0.46889\n",
            "hpyerparameter  5  : std-mean-min-max 0.00846 0.46056 0.44528 0.46889\n",
            "hpyerparameter  6  : std-mean-min-max 0.0028 0.42956 0.42472 0.43194\n",
            "hpyerparameter  7  : std-mean-min-max 0.0028 0.42956 0.42472 0.43194\n",
            "hpyerparameter  8  : std-mean-min-max 0.0028 0.42956 0.42472 0.43194\n",
            "{'vectorizer__min_df': 100, 'vectorizer__ngram_range': [1, 2]}\n",
            "[[266  88  18   5  16]\n",
            " [102 156  72  23  27]\n",
            " [ 47  76 118  99  67]\n",
            " [ 22  31  60 136 130]\n",
            " [ 30  13  20  68 310]]\n",
            "f1 score 0.4772514205660536\n",
            "accuracy score 0.493\n",
            "CPU times: user 2min 22s, sys: 1.47 s, total: 2min 23s\n",
            "Wall time: 2min 24s\n"
          ]
        }
      ],
      "source": [
        "%%time\n",
        "# Initialize and run the GridSearchCV to scan the hyperparameter and find the best hyperparameter set that will maximize the scoring option for multiclass classification.\n",
        "pipe  = GridSearchCV(pipeline, cv=5,param_grid=parameters)\n",
        "pipe.fit(multiclass_df.text, multiclass_df.label)\n",
        "\n",
        "# Report the standart deviation of split scores for each hyperparameter group.\n",
        "\n",
        "split_0 = pipe.cv_results_[\"split0_test_score\"]\n",
        "split_1 = pipe.cv_results_[\"split1_test_score\"]\n",
        "split_2 = pipe.cv_results_[\"split2_test_score\"]\n",
        "split_3 = pipe.cv_results_[\"split3_test_score\"]\n",
        "split_4 = pipe.cv_results_[\"split4_test_score\"]\n",
        "\n",
        "#we analyze the mean-std-min-max for each hyperparameter group\n",
        "for i in range(9):\n",
        "  std = np.std([split_0[i],split_1[i],split_2[i],split_3[i],split_4[i]])\n",
        "  mean = np.mean([split_0[i],split_1[i],split_2[i],split_3[i],split_4[i]])\n",
        "  min = np.min([split_0[i],split_1[i],split_2[i],split_3[i],split_4[i]])\n",
        "  max = np.max([split_0[i],split_1[i],split_2[i],split_3[i],split_4[i]])\n",
        "  print(\"hpyerparameter \", i , \" :\", \"std-mean-min-max\", round(std,5), round(mean,5), round(min,5), round(max,5))\n",
        "\n",
        "\"\"\"\n",
        "#we make sure there is no huge differences between splits\n",
        "print(\"standard deviations:\", round(np.std(split_0),4), round(np.std(split_1),4), round(np.std(split_2),4), round(np.std(split_3),4), round(np.std(split_4),4))\n",
        "print(\"mean               :\", round(np.mean(split_0),4), round(np.mean(split_1),4), round(np.mean(split_2),4), round(np.mean(split_3),4), round(np.mean(split_4),4))\n",
        "\n",
        "\"\"\"\n",
        "# Show the best parameter set for given dataset and hyperparameter space.\n",
        "print(pipe.best_params_)\n",
        "\n",
        "# Building the pipeline with the best parameter group and reporting Conf. Mat. and Results on the Test Set #\n",
        "# Create your pipeline object with the best parameter set.\n",
        "best_parameters = {'vectorizer__min_df':[100], 'vectorizer__ngram_range':[[1, 2]]}\n",
        "pipe  = GridSearchCV(pipeline, cv=5,param_grid=best_parameters)\n",
        "\n",
        "\n",
        "# Fit your pipeline on training set.\n",
        "pipe.fit(multiclass_df.text, multiclass_df.label)\n",
        "\n",
        "\n",
        "# Take prediction and report the F1 and Accuracy scores for binary classification. Then show the confussion table.\n",
        "preds = pipe.predict(test_multiclass_df.text)\n",
        "\n",
        "print(confusion_matrix(test_multiclass_df.label, preds))\n",
        "print(\"f1 score\", f1_score(test_multiclass_df.label, preds, average='macro'))\n",
        "print(\"accuracy score\", accuracy_score(test_multiclass_df.label, preds))\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EMNf8zC50lQQ"
      },
      "source": [
        "### Logistic Regression"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 136,
      "metadata": {
        "id": "M5ySIl3e552q"
      },
      "outputs": [],
      "source": [
        "# https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html\n",
        "\n",
        "# Initiate the pipeline with required components.You can use Pipeline class of sklearn -> https://scikit-learn.org/stable/modules/generated/sklearn.pipeline.Pipeline.html\n",
        "# There will be three components; 1) Word weightning 2) Logistic Regression classifier.\n",
        "\n",
        "pipeline = Pipeline([\n",
        "     ('vectorizer', TfidfVectorizer()),  \n",
        "     ('lgc', LogisticRegression(random_state=22, penalty= \"elasticnet\", solver = \"saga\"))\n",
        "])\n",
        "\n",
        "#Set the hyperparameter space that will be scanned.\n",
        "\n",
        "parameters = {'lgc__l1_ratio':[0.0,0.5,1.0], 'vectorizer__ngram_range':[[1, 1],[1, 2],[1, 3]], 'vectorizer__min_df':[100,500,1000]}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zTmaa_tHTWZ0"
      },
      "source": [
        "#### Binary"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 139,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "atCcPl1NTQWr",
        "outputId": "e950fae2-2bb1-41ea-ce43-98a78f9f052f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "hpyerparameter  0  : std-mean-min-max 0.00294 0.90539 0.90038 0.90871\n",
            "hpyerparameter  1  : std-mean-min-max 0.00397 0.90588 0.90108 0.91218\n",
            "hpyerparameter  2  : std-mean-min-max 0.00397 0.90588 0.90108 0.91218\n",
            "hpyerparameter  3  : std-mean-min-max 0.00469 0.85347 0.84832 0.85913\n",
            "hpyerparameter  4  : std-mean-min-max 0.00442 0.85368 0.84906 0.85913\n",
            "hpyerparameter  5  : std-mean-min-max 0.00442 0.85368 0.84906 0.85913\n",
            "hpyerparameter  6  : std-mean-min-max 0.00478 0.79864 0.79389 0.80666\n",
            "hpyerparameter  7  : std-mean-min-max 0.00478 0.79864 0.79389 0.80666\n",
            "hpyerparameter  8  : std-mean-min-max 0.00478 0.79864 0.79389 0.80666\n",
            "hpyerparameter  9  : std-mean-min-max 0.00266 0.90657 0.90281 0.91114\n",
            "hpyerparameter  10  : std-mean-min-max 0.00305 0.90831 0.9042 0.91357\n",
            "hpyerparameter  11  : std-mean-min-max 0.00305 0.90831 0.9042 0.91357\n",
            "hpyerparameter  12  : std-mean-min-max 0.00407 0.85479 0.85005 0.85982\n",
            "hpyerparameter  13  : std-mean-min-max 0.0038 0.85417 0.84936 0.85878\n",
            "hpyerparameter  14  : std-mean-min-max 0.0038 0.85417 0.84936 0.85878\n",
            "hpyerparameter  15  : std-mean-min-max 0.00438 0.79933 0.79528 0.80736\n",
            "hpyerparameter  16  : std-mean-min-max 0.00438 0.79933 0.79528 0.80736\n",
            "hpyerparameter  17  : std-mean-min-max 0.00438 0.79933 0.79528 0.80736\n",
            "hpyerparameter  18  : std-mean-min-max 0.00484 0.90519 0.89934 0.91218\n",
            "hpyerparameter  19  : std-mean-min-max 0.00473 0.90595 0.8983 0.91288\n",
            "hpyerparameter  20  : std-mean-min-max 0.00473 0.90595 0.8983 0.91288\n",
            "hpyerparameter  21  : std-mean-min-max 0.00435 0.85417 0.8497 0.86012\n",
            "hpyerparameter  22  : std-mean-min-max 0.00442 0.85354 0.84797 0.85977\n",
            "hpyerparameter  23  : std-mean-min-max 0.00442 0.85354 0.84797 0.85977\n",
            "hpyerparameter  24  : std-mean-min-max 0.00434 0.79864 0.7932 0.80632\n",
            "hpyerparameter  25  : std-mean-min-max 0.00434 0.79864 0.7932 0.80632\n",
            "hpyerparameter  26  : std-mean-min-max 0.00434 0.79864 0.7932 0.80632\n",
            "{'lgc__l1_ratio': 0.5, 'vectorizer__min_df': 100, 'vectorizer__ngram_range': [1, 2]}\n",
            "[[714  59]\n",
            " [ 85 735]]\n",
            "f1 score 0.9107806691449815\n",
            "accuracy score 0.9096045197740112\n",
            "CPU times: user 5min 44s, sys: 2.01 s, total: 5min 46s\n",
            "Wall time: 5min 46s\n"
          ]
        }
      ],
      "source": [
        "%%time\n",
        "# Initialize and run the GridSearchCV to scan the hyperparameter and find the best hyperparameter set that will maximize the scoring option for binary classification.\n",
        "pipe  = GridSearchCV(pipeline, cv=5,param_grid=parameters)\n",
        "pipe.fit(binary_df.text, binary_df.label)\n",
        "\n",
        "        \n",
        "# Report the standart deviation of split scores for each hyperparameter group.\n",
        "\n",
        "split_0 = pipe.cv_results_[\"split0_test_score\"]\n",
        "split_1 = pipe.cv_results_[\"split1_test_score\"]\n",
        "split_2 = pipe.cv_results_[\"split2_test_score\"]\n",
        "split_3 = pipe.cv_results_[\"split3_test_score\"]\n",
        "split_4 = pipe.cv_results_[\"split4_test_score\"]\n",
        "\n",
        "#we analyze the mean-std-min-max for each hyperparameter group\n",
        "for i in range(27):\n",
        "  std = np.std([split_0[i],split_1[i],split_2[i],split_3[i],split_4[i]])\n",
        "  mean = np.mean([split_0[i],split_1[i],split_2[i],split_3[i],split_4[i]])\n",
        "  min = np.min([split_0[i],split_1[i],split_2[i],split_3[i],split_4[i]])\n",
        "  max = np.max([split_0[i],split_1[i],split_2[i],split_3[i],split_4[i]])\n",
        "  print(\"hpyerparameter \", i , \" :\", \"std-mean-min-max\", round(std,5), round(mean,5), round(min,5), round(max,5))\n",
        "\n",
        "\"\"\"\n",
        "#we make sure there is no huge differences between splits\n",
        "print(\"standard deviations:\", round(np.std(split_0),4), round(np.std(split_1),4), round(np.std(split_2),4), round(np.std(split_3),4), round(np.std(split_4),4))\n",
        "print(\"mean               :\", round(np.mean(split_0),4), round(np.mean(split_1),4), round(np.mean(split_2),4), round(np.mean(split_3),4), round(np.mean(split_4),4))\n",
        "\n",
        "\"\"\"\n",
        "\n",
        "# Show the best parameter set for given dataset and hyperparameter space.\n",
        "print(pipe.best_params_)\n",
        "\n",
        "\n",
        "# Building the pipeline with the best parameter group and reporting Conf. Mat. and Results on the Test Set #\n",
        "# Create your Pipeline object with the best parameter set.\n",
        "best_parameters = {'vectorizer__min_df':[100], 'vectorizer__ngram_range':[[1, 2]], 'lgc__l1_ratio': [0.5]}\n",
        "pipe  = GridSearchCV(pipeline, cv=5,param_grid=best_parameters)\n",
        "\n",
        "# Fit your pipeline on training set.\n",
        "pipe.fit(binary_df.text, binary_df.label)\n",
        "\n",
        "# Take prediction and report the F1 and Accuracy scores for binary classification. Then show the confussion table.\n",
        "preds = pipe.predict(test_binary_df.text)\n",
        "\n",
        "print(confusion_matrix(test_binary_df.label, preds))\n",
        "print(\"f1 score\", f1_score(test_binary_df.label, preds))\n",
        "print(\"accuracy score\", accuracy_score(test_binary_df.label, preds))\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xgy8BLNaTRfZ"
      },
      "source": [
        "#### Multiclass"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 141,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kRYqtWiP2Gg6",
        "outputId": "0f143c6e-62b7-48d6-fb4f-b3d9fe95d1db"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "hpyerparameter  0  : std-mean-min-max 0.00665 0.53556 0.5225 0.54\n",
            "hpyerparameter  1  : std-mean-min-max 0.00451 0.53761 0.53056 0.54306\n",
            "hpyerparameter  2  : std-mean-min-max 0.00487 0.53828 0.53028 0.54389\n",
            "hpyerparameter  3  : std-mean-min-max 0.01043 0.51011 0.49083 0.52028\n",
            "hpyerparameter  4  : std-mean-min-max 0.00996 0.50961 0.49167 0.51917\n",
            "hpyerparameter  5  : std-mean-min-max 0.00996 0.50961 0.49167 0.51917\n",
            "hpyerparameter  6  : std-mean-min-max 0.00645 0.46789 0.45694 0.47639\n",
            "hpyerparameter  7  : std-mean-min-max 0.00645 0.46789 0.45694 0.47639\n",
            "hpyerparameter  8  : std-mean-min-max 0.00645 0.46789 0.45694 0.47639\n",
            "hpyerparameter  9  : std-mean-min-max 0.0072 0.53572 0.52278 0.54222\n",
            "hpyerparameter  10  : std-mean-min-max 0.00571 0.54139 0.53 0.545\n",
            "hpyerparameter  11  : std-mean-min-max 0.0056 0.54117 0.53 0.54444\n",
            "hpyerparameter  12  : std-mean-min-max 0.01076 0.51183 0.4925 0.52167\n",
            "hpyerparameter  13  : std-mean-min-max 0.011 0.51183 0.49194 0.52167\n",
            "hpyerparameter  14  : std-mean-min-max 0.011 0.51183 0.49194 0.52167\n",
            "hpyerparameter  15  : std-mean-min-max 0.00625 0.46872 0.45806 0.4775\n",
            "hpyerparameter  16  : std-mean-min-max 0.00625 0.46872 0.45806 0.4775\n",
            "hpyerparameter  17  : std-mean-min-max 0.00625 0.46872 0.45806 0.4775\n",
            "hpyerparameter  18  : std-mean-min-max 0.00714 0.53867 0.52778 0.54833\n",
            "hpyerparameter  19  : std-mean-min-max 0.0082 0.54161 0.5275 0.55\n",
            "hpyerparameter  20  : std-mean-min-max 0.00826 0.54194 0.5275 0.55056\n",
            "hpyerparameter  21  : std-mean-min-max 0.01148 0.51283 0.49222 0.52361\n",
            "hpyerparameter  22  : std-mean-min-max 0.01166 0.51328 0.49194 0.52417\n",
            "hpyerparameter  23  : std-mean-min-max 0.01166 0.51328 0.49194 0.52417\n",
            "hpyerparameter  24  : std-mean-min-max 0.00605 0.46767 0.45694 0.47472\n",
            "hpyerparameter  25  : std-mean-min-max 0.00605 0.46767 0.45694 0.47472\n",
            "hpyerparameter  26  : std-mean-min-max 0.00605 0.46767 0.45694 0.47472\n",
            "{'lgc__l1_ratio': 1.0, 'vectorizer__min_df': 100, 'vectorizer__ngram_range': [1, 3]}\n",
            "[[289  81  13   6   4]\n",
            " [ 89 179  78  22  12]\n",
            " [ 26  78 189  92  22]\n",
            " [ 14  27  73 169  96]\n",
            " [ 12  12  19 113 285]]\n",
            "f1 score 0.5512069432954926\n",
            "accuracy score 0.5555\n",
            "CPU times: user 11min 54s, sys: 3.35 s, total: 11min 57s\n",
            "Wall time: 12min\n"
          ]
        }
      ],
      "source": [
        "%%time\n",
        "# Initialize and run the GridSearchCV to scan the hyperparameter and find the best hyperparameter set that will maximize the scoring option for multiclass classification.\n",
        "pipe  = GridSearchCV(pipeline, cv=5,param_grid=parameters)\n",
        "pipe.fit(multiclass_df.text, multiclass_df.label)\n",
        "\n",
        "# Report the standart deviation of split scores for each hyperparameter group.\n",
        "\n",
        "split_0 = pipe.cv_results_[\"split0_test_score\"]\n",
        "split_1 = pipe.cv_results_[\"split1_test_score\"]\n",
        "split_2 = pipe.cv_results_[\"split2_test_score\"]\n",
        "split_3 = pipe.cv_results_[\"split3_test_score\"]\n",
        "split_4 = pipe.cv_results_[\"split4_test_score\"]\n",
        "\n",
        "#we analyze the mean-std-min-max for each hyperparameter group\n",
        "for i in range(27):\n",
        "  std = np.std([split_0[i],split_1[i],split_2[i],split_3[i],split_4[i]])\n",
        "  mean = np.mean([split_0[i],split_1[i],split_2[i],split_3[i],split_4[i]])\n",
        "  min = np.min([split_0[i],split_1[i],split_2[i],split_3[i],split_4[i]])\n",
        "  max = np.max([split_0[i],split_1[i],split_2[i],split_3[i],split_4[i]])\n",
        "  print(\"hpyerparameter \", i , \" :\", \"std-mean-min-max\", round(std,5), round(mean,5), round(min,5), round(max,5))\n",
        "\n",
        "\n",
        "\"\"\"\n",
        "#we make sure there is no huge differences between splits\n",
        "print(\"standard deviations:\", round(np.std(split_0),4), round(np.std(split_1),4), round(np.std(split_2),4), round(np.std(split_3),4), round(np.std(split_4),4))\n",
        "print(\"mean               :\", round(np.mean(split_0),4), round(np.mean(split_1),4), round(np.mean(split_2),4), round(np.mean(split_3),4), round(np.mean(split_4),4))\n",
        "\"\"\"\n",
        "\n",
        "# Show the best parameter set for given dataset and hyperparameter space.\n",
        "print(pipe.best_params_)\n",
        "\n",
        "\n",
        "# Building the pipeline with the best parameter group and reporting Conf. Mat. and Results on the Test Set #\n",
        "# Create your pipeline object with the best parameter set.\n",
        "best_parameters = {'vectorizer__min_df':[100], 'vectorizer__ngram_range':[[1, 3]], 'lgc__l1_ratio': [1.0]}\n",
        "pipe  = GridSearchCV(pipeline, cv=5,param_grid=best_parameters)\n",
        "\n",
        "\n",
        "# Fit your pipeline on training set.\n",
        "pipe.fit(multiclass_df.text, multiclass_df.label)\n",
        "\n",
        "\n",
        "# Take prediction and report the F1 and Accuracy scores for binary classification. Then show the confussion table.\n",
        "preds = pipe.predict(test_multiclass_df.text)\n",
        "\n",
        "print(confusion_matrix(test_multiclass_df.label, preds))\n",
        "print(\"f1 score\", f1_score(test_multiclass_df.label, preds, average='macro'))\n",
        "print(\"accuracy score\", accuracy_score(test_multiclass_df.label, preds))\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ISxrqFKrh8Rb"
      },
      "source": [
        "## Neural Models"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "87S_VWH9h-JQ"
      },
      "source": [
        "### Convolutional Neural Network (CNN)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 142,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gw4QZK0biCrt",
        "outputId": "b256234f-d6f7-4bde-cb09-38d1a0d3ac80"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ]
        }
      ],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import nltk,re\n",
        "import tensorflow as tf\n",
        "from sklearn.model_selection import train_test_split\n",
        "from numpy import array,asarray,zeros\n",
        "\n",
        "from nltk.stem import PorterStemmer\n",
        "from nltk.tokenize import sent_tokenize\n",
        "import nltk\n",
        "nltk.download('punkt')\n",
        "\n",
        "import keras\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "from keras.models import Sequential\n",
        "from keras.layers.convolutional import Conv1D,MaxPooling1D\n",
        "from keras.layers import Dense,Flatten,Embedding,Input,Dropout\n",
        "from keras.callbacks import ModelCheckpoint\n",
        "\n",
        "from gensim.models import Word2Vec\n",
        "import gensim.downloader as api\n",
        "\n",
        "from keras import backend as K\n",
        "\n",
        "def recall_m(y_true, y_pred):\n",
        "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
        "    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
        "    recall = true_positives / (possible_positives + K.epsilon())\n",
        "    return recall\n",
        "\n",
        "def precision_m(y_true, y_pred):\n",
        "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
        "    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
        "    precision = true_positives / (predicted_positives + K.epsilon())\n",
        "    return precision\n",
        "\n",
        "def f1_m(y_true, y_pred):\n",
        "    precision = precision_m(y_true, y_pred)\n",
        "    recall = recall_m(y_true, y_pred)\n",
        "    return 2*((precision*recall)/(precision+recall+K.epsilon()))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cg4BjaslNdaZ"
      },
      "outputs": [],
      "source": [
        "binary_df['text'] = binary_df.apply(lambda row: nltk.word_tokenize(row['text']), axis=1)\n",
        "multiclass_df['text'] = multiclass_df.apply(lambda row: nltk.word_tokenize(row['text']), axis=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iYCt5LRfbJTZ"
      },
      "outputs": [],
      "source": [
        "test_binary_df['text'] = test_binary_df.apply(lambda row: nltk.word_tokenize(row['text']), axis=1)\n",
        "test_multiclass_df['text'] = test_multiclass_df.apply(lambda row: nltk.word_tokenize(row['text']), axis=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Skg89paOvgRw",
        "outputId": "dcf194bc-53da-4ede-d75b-085cf8439b68"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(12966,) (1441,) (12966,) (1441,)\n",
            "(16200,) (1800,) (16200,) (1800,)\n"
          ]
        }
      ],
      "source": [
        "# Create a validation set from train set\n",
        "# Please use random_state of 22 and test_size of 0.1\n",
        "\n",
        "b_X_train, b_X_val, b_y_train, b_y_val = train_test_split(binary_df.text, binary_df.label, test_size=0.1, random_state=22)\n",
        "\n",
        "m_X_train, m_X_val, m_y_train, m_y_val = train_test_split(multiclass_df.text, multiclass_df.label, test_size=0.1, random_state=22)\n",
        "\n",
        "print(b_X_train.shape, b_X_val.shape, b_y_train.shape, b_y_val.shape)\n",
        "print(m_X_train.shape, m_X_val.shape, m_y_train.shape, m_y_val.shape) #since multiclass did not drop labels with 3, has more dataset."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9xlLvOCvvgNw",
        "outputId": "4eea8176-4809-4678-a1f6-6fb735efe03b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['fasttext-wiki-news-subwords-300', 'conceptnet-numberbatch-17-06-300', 'word2vec-ruscorpora-300', 'word2vec-google-news-300', 'glove-wiki-gigaword-50', 'glove-wiki-gigaword-100', 'glove-wiki-gigaword-200', 'glove-wiki-gigaword-300', 'glove-twitter-25', 'glove-twitter-50', 'glove-twitter-100', 'glove-twitter-200', '__testing_word2vec-matrix-synopsis']\n",
            "[==================================================] 100.0% 66.0/66.0MB downloaded\n"
          ]
        }
      ],
      "source": [
        "# Create your own word embeddings from scratch and load a pretrained word embeddings\n",
        "\n",
        "# You can check https://radimrehurek.com/gensim/models/word2vec.html for training a word embeddings from scratch\n",
        "\n",
        "m_wv = Word2Vec(sentences= m_X_train, size=70, window=5, min_count=1, workers=4) #multiclass word2vec\n",
        "b_wv = Word2Vec(sentences= b_X_train, size=70,window=5, min_count=1, workers=4, ) #binary word2vec\n",
        "\n",
        "# You can check https://radimrehurek.com/gensim/auto_examples/howtos/run_downloader_api.html and \n",
        "#   https://github.com/RaRe-Technologies/gensim-data for loading pretrained word embeddings. \n",
        "\n",
        "print(list(api.info()['models'].keys()))\n",
        "api_wv = api.load('glove-wiki-gigaword-50')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oK3auz1pG2oF",
        "outputId": "16a203be-072e-4099-eeab-7a65cb49f5f0"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[('surpris', 0.8139040470123291), ('disappoint', 0.7657565474510193), ('unfortun', 0.7494511008262634), ('stellar', 0.7401197552680969), ('outstand', 0.737817645072937), ('spectacular', 0.7372994422912598), ('satisfi', 0.7323342561721802), ('good', 0.7307410836219788), ('particularli', 0.7235710620880127), ('outweigh', 0.7128151059150696)]\n",
            "[('eager', 0.807209312915802), ('motivate', 0.7772501111030579), ('convince', 0.7734254598617554), ('tempted', 0.7684953808784485), ('wanting', 0.7526339888572693), ('entertain', 0.7396959662437439), ('reluctant', 0.7380232214927673), ('remind', 0.7234859466552734), ('wishing', 0.7117031812667847), ('tempt', 0.7073939442634583)]\n"
          ]
        }
      ],
      "source": [
        "#model.wv.vocab\n",
        "print(m_wv.most_similar(\"impress\"))\n",
        "print(api_wv.most_similar(\"impress\"))"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#find out average length in the data\n",
        "counter = 0\n",
        "sum = 0\n",
        "for i in b_X_train:\n",
        "  sum = sum + len(i)\n",
        "  counter = counter + 1\n",
        "\n",
        "sum/counter"
      ],
      "metadata": {
        "id": "eyalEN9fouTS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "04a0330f-af57-4962-f909-0e28e21d1d20"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "59.14915933981182"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#-------- BINARY -------"
      ],
      "metadata": {
        "id": "5taOk5Zuc2oF"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jEe1JkT9TPhF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1b81727e-51c0-4e71-bcbd-8a49ac0e490a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['nice', 'first', 'visit', 'owner', 'ted', 'friendli', 'start', 'restaur', 'busi', 'friday', 'night', 'begin', 'chines', 'new', 'year', 'order', 'pu', 'pu', 'platter', 'crab', 'rangoon', 'dinner', 'chef', 'special', 'seafood', 'instead', 'chines', 'american', 'regular', 'fri', 'rice', 'everyth', 'well', 'prepar', 'shrimp', 'larg', 'veget', 'crisp', 'chili', 'sauc', 'sweet', 'tangi', 'return', 'owner', 'made', 'effort', 'visit', 'learn', 'name', 'ask', 'first', 'visit']\n",
            "[35, 3617, 165, 228, 197, 1974, 320, 417]\n",
            "[  35 3617  165  228  197 1974  320  417    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0]\n"
          ]
        }
      ],
      "source": [
        "# Prepare your dataset for CNN classifier\n",
        "\n",
        "tokenizer = Tokenizer(num_words=50000) \n",
        "tokenizer.fit_on_texts(b_X_train)\n",
        "Xcnn_train = tokenizer.texts_to_sequences(b_X_train)\n",
        "Xcnn_val = tokenizer.texts_to_sequences(b_X_val)\n",
        "Xcnn_test = tokenizer.texts_to_sequences(test_binary_df.text)\n",
        "\n",
        "vocab_size = len(tokenizer.word_index) + 1  \n",
        "print(b_X_train[1])\n",
        "print(Xcnn_train[1]) \n",
        "\n",
        "maxlen = 70\n",
        "Xcnn_train = pad_sequences(Xcnn_train, padding='post', maxlen=maxlen)\n",
        "Xcnn_val = pad_sequences(Xcnn_val, padding='post', maxlen=maxlen)\n",
        "Xcnn_test = pad_sequences(Xcnn_test, padding='post', maxlen=maxlen)\n",
        "print(Xcnn_train[1, :]) "
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# RANDOMLY INITIALIZED"
      ],
      "metadata": {
        "id": "-LX_ZxFAaDSL"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sh01dSWqvqSx",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f546bd60-7026-4238-d7a6-39954f66d10d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_29\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding_29 (Embedding)    (None, 70, 70)            1473500   \n",
            "                                                                 \n",
            " conv1d_35 (Conv1D)          (None, 68, 128)           27008     \n",
            "                                                                 \n",
            " max_pooling1d_35 (MaxPoolin  (None, 34, 128)          0         \n",
            " g1D)                                                            \n",
            "                                                                 \n",
            " flatten_29 (Flatten)        (None, 4352)              0         \n",
            "                                                                 \n",
            " dense_58 (Dense)            (None, 16)                69648     \n",
            "                                                                 \n",
            " dense_59 (Dense)            (None, 1)                 17        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1,570,173\n",
            "Trainable params: 1,570,173\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Epoch 1/5\n",
            "811/811 [==============================] - 17s 20ms/step - loss: 0.3216 - accuracy: 0.8543 - f1_m: 0.8469 - precision_m: 0.8498 - recall_m: 0.8651 - val_loss: 0.2393 - val_accuracy: 0.9119 - val_f1_m: 0.9051 - val_precision_m: 0.9528 - val_recall_m: 0.8732\n",
            "Epoch 2/5\n",
            "811/811 [==============================] - 16s 20ms/step - loss: 0.1254 - accuracy: 0.9567 - f1_m: 0.9529 - precision_m: 0.9558 - recall_m: 0.9557 - val_loss: 0.2477 - val_accuracy: 0.8966 - val_f1_m: 0.8925 - val_precision_m: 0.8752 - val_recall_m: 0.9206\n",
            "Epoch 3/5\n",
            "811/811 [==============================] - 16s 19ms/step - loss: 0.0465 - accuracy: 0.9876 - f1_m: 0.9866 - precision_m: 0.9880 - recall_m: 0.9869 - val_loss: 0.3641 - val_accuracy: 0.8924 - val_f1_m: 0.8898 - val_precision_m: 0.8641 - val_recall_m: 0.9278\n",
            "Epoch 4/5\n",
            "811/811 [==============================] - 16s 20ms/step - loss: 0.0153 - accuracy: 0.9962 - f1_m: 0.9956 - precision_m: 0.9956 - recall_m: 0.9964 - val_loss: 0.4365 - val_accuracy: 0.8966 - val_f1_m: 0.8915 - val_precision_m: 0.8747 - val_recall_m: 0.9208\n",
            "Epoch 5/5\n",
            "811/811 [==============================] - 17s 20ms/step - loss: 0.0040 - accuracy: 0.9991 - f1_m: 0.9991 - precision_m: 0.9991 - recall_m: 0.9992 - val_loss: 0.5981 - val_accuracy: 0.8883 - val_f1_m: 0.8820 - val_precision_m: 0.8835 - val_recall_m: 0.8938\n",
            "Training Accuracy:  0.9996 Training F1:  0.9996 Training precision:  0.9997 Training recall:  0.9995\n",
            "Testing Accuracy :  0.8864 Testing F1:  0.8865 Testing precision:  0.9074 Testing recall:  0.8713\n"
          ]
        }
      ],
      "source": [
        "# Create Embedding Matrices and Layers\n",
        "\n",
        "embedding_dim = 70\n",
        "textcnnmodel = Sequential()\n",
        "textcnnmodel.add(Embedding(vocab_size, embedding_dim, input_length=maxlen)) \n",
        "\n",
        "textcnnmodel.add(Conv1D(128, 3, activation='relu'))\n",
        "textcnnmodel.add(MaxPooling1D())\n",
        "#multi_cnn.add(Dropout(0.2))\n",
        "\n",
        "textcnnmodel.add(Flatten())\n",
        "textcnnmodel.add(Dense(16, activation='relu'))\n",
        "textcnnmodel.add(Dense(1, activation='sigmoid'))\n",
        "textcnnmodel.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy',f1_m,precision_m, recall_m])\n",
        "textcnnmodel.summary() \n",
        "\n",
        "# Train models and Evaluate them for both binary and multi-class\n",
        "textcnnmodel.fit(Xcnn_train, b_y_train,\n",
        "                     epochs=5,\n",
        "                     verbose=True,\n",
        "                     validation_data=(Xcnn_val, b_y_val),\n",
        "                     batch_size=16)\n",
        "\n",
        "loss, accuracy, f1_score, precision, recall = textcnnmodel.evaluate(Xcnn_train, b_y_train, verbose=False)\n",
        "print(\"Training Accuracy:  {:.4f}\".format(accuracy),\"Training F1:  {:.4f}\".format(f1_score) ,\"Training precision:  {:.4f}\".format(precision) , \"Training recall:  {:.4f}\".format(recall))\n",
        "\n",
        "loss, accuracy, f1_score, precision, recall = textcnnmodel.evaluate(Xcnn_test, test_binary_df.label, verbose=False)\n",
        "print(\"Testing Accuracy :  {:.4f}\".format(accuracy),\"Testing F1:  {:.4f}\".format(f1_score) ,\"Testing precision:  {:.4f}\".format(precision) , \"Testing recall:  {:.4f}\".format(recall))"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "embedding_dim = 70\n",
        "textcnnmodel = Sequential()\n",
        "textcnnmodel.add(Embedding(vocab_size, embedding_dim, input_length=maxlen)) \n",
        "\n",
        "textcnnmodel.add(Conv1D(128, 5, activation='relu'))\n",
        "textcnnmodel.add(MaxPooling1D())\n",
        "\n",
        "textcnnmodel.add(Conv1D(64, 5, activation='relu'))\n",
        "textcnnmodel.add(MaxPooling1D())\n",
        "\n",
        "textcnnmodel.add(Flatten())\n",
        "textcnnmodel.add(Dense(32, activation='relu'))\n",
        "textcnnmodel.add(Dense(1, activation='sigmoid'))\n",
        "textcnnmodel.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy',f1_m,precision_m, recall_m])\n",
        "textcnnmodel.summary() \n",
        "\n",
        "# Train models and Evaluate them for both binary and multi-class\n",
        "textcnnmodel.fit(Xcnn_train, b_y_train,\n",
        "                     epochs=5,\n",
        "                     verbose=True,\n",
        "                     validation_data=(Xcnn_val, b_y_val),\n",
        "                     batch_size=32)\n",
        "\n",
        "loss, accuracy, f1_score, precision, recall = textcnnmodel.evaluate(Xcnn_train, b_y_train, verbose=False)\n",
        "print(\"Training Accuracy:  {:.4f}\".format(accuracy),\"Training F1:  {:.4f}\".format(f1_score) ,\"Training precision:  {:.4f}\".format(precision) , \"Training recall:  {:.4f}\".format(recall))\n",
        "\n",
        "loss, accuracy, f1_score, precision, recall = textcnnmodel.evaluate(Xcnn_test, test_binary_df.label, verbose=False)\n",
        "print(\"Testing Accuracy :  {:.4f}\".format(accuracy),\"Testing F1:  {:.4f}\".format(f1_score) ,\"Testing precision:  {:.4f}\".format(precision) , \"Testing recall:  {:.4f}\".format(recall))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TWZ8sUVD6cwi",
        "outputId": "b7d16090-492f-4855-b3e0-b69eebe2decc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_30\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding_30 (Embedding)    (None, 70, 70)            1473500   \n",
            "                                                                 \n",
            " conv1d_36 (Conv1D)          (None, 66, 128)           44928     \n",
            "                                                                 \n",
            " max_pooling1d_36 (MaxPoolin  (None, 33, 128)          0         \n",
            " g1D)                                                            \n",
            "                                                                 \n",
            " conv1d_37 (Conv1D)          (None, 29, 64)            41024     \n",
            "                                                                 \n",
            " max_pooling1d_37 (MaxPoolin  (None, 14, 64)           0         \n",
            " g1D)                                                            \n",
            "                                                                 \n",
            " flatten_30 (Flatten)        (None, 896)               0         \n",
            "                                                                 \n",
            " dense_60 (Dense)            (None, 32)                28704     \n",
            "                                                                 \n",
            " dense_61 (Dense)            (None, 1)                 33        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1,588,189\n",
            "Trainable params: 1,588,189\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Epoch 1/5\n",
            "406/406 [==============================] - 15s 35ms/step - loss: 0.3639 - accuracy: 0.8240 - f1_m: 0.8163 - precision_m: 0.8139 - recall_m: 0.8365 - val_loss: 0.2364 - val_accuracy: 0.9091 - val_f1_m: 0.9121 - val_precision_m: 0.8806 - val_recall_m: 0.9511\n",
            "Epoch 2/5\n",
            "406/406 [==============================] - 14s 33ms/step - loss: 0.1510 - accuracy: 0.9446 - f1_m: 0.9434 - precision_m: 0.9465 - recall_m: 0.9447 - val_loss: 0.2686 - val_accuracy: 0.8924 - val_f1_m: 0.8894 - val_precision_m: 0.9331 - val_recall_m: 0.8571\n",
            "Epoch 3/5\n",
            "406/406 [==============================] - 14s 34ms/step - loss: 0.0673 - accuracy: 0.9772 - f1_m: 0.9767 - precision_m: 0.9763 - recall_m: 0.9786 - val_loss: 0.3170 - val_accuracy: 0.8959 - val_f1_m: 0.8997 - val_precision_m: 0.8936 - val_recall_m: 0.9129\n",
            "Epoch 4/5\n",
            "406/406 [==============================] - 13s 33ms/step - loss: 0.0292 - accuracy: 0.9918 - f1_m: 0.9915 - precision_m: 0.9920 - recall_m: 0.9916 - val_loss: 0.4718 - val_accuracy: 0.9008 - val_f1_m: 0.8994 - val_precision_m: 0.9236 - val_recall_m: 0.8832\n",
            "Epoch 5/5\n",
            "406/406 [==============================] - 13s 33ms/step - loss: 0.0180 - accuracy: 0.9939 - f1_m: 0.9937 - precision_m: 0.9940 - recall_m: 0.9937 - val_loss: 0.5212 - val_accuracy: 0.8959 - val_f1_m: 0.8986 - val_precision_m: 0.8997 - val_recall_m: 0.9035\n",
            "Training Accuracy:  0.9976 Training F1:  0.9976 Training precision:  0.9971 Training recall:  0.9982\n",
            "Testing Accuracy :  0.8908 Testing F1:  0.8918 Testing precision:  0.8990 Testing recall:  0.8899\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Word embeddings trained from scratch with gensim"
      ],
      "metadata": {
        "id": "S9Fh0OcWaCcY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from gensim.models import KeyedVectors\n",
        "\n",
        "vocab_len = len(b_wv.wv.vocab) + 1\n",
        "\n",
        "b_wv.save(\"b_wv.wordvectors\")\n",
        "embedding_vector = KeyedVectors.load(\"b_wv.wordvectors\", mmap='r')\n",
        "\n",
        "embedding_matrix = np.zeros((vocab_len-1, 70))\n",
        "for word, i in tokenizer.word_index.items():\n",
        "    try:\n",
        "      #print(embedding_vector[word].shape, word)\n",
        "      embedding_matrix[i-1] = embedding_vector[word]\n",
        "    except KeyError:\n",
        "      print(\"key error\", i, word)\n",
        "  "
      ],
      "metadata": {
        "id": "FQqBGnT-_xLk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "70OcSNItVCq4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8b9a244c-07d3-4abf-cbec-6c8e7f1fb0b1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_31\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding_31 (Embedding)    (None, 70, 70)            1347290   \n",
            "                                                                 \n",
            " conv1d_38 (Conv1D)          (None, 68, 128)           27008     \n",
            "                                                                 \n",
            " max_pooling1d_38 (MaxPoolin  (None, 34, 128)          0         \n",
            " g1D)                                                            \n",
            "                                                                 \n",
            " flatten_31 (Flatten)        (None, 4352)              0         \n",
            "                                                                 \n",
            " dense_62 (Dense)            (None, 16)                69648     \n",
            "                                                                 \n",
            " dense_63 (Dense)            (None, 1)                 17        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1,443,963\n",
            "Trainable params: 1,443,963\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Epoch 1/5\n",
            "811/811 [==============================] - 16s 19ms/step - loss: 0.5869 - accuracy: 0.7043 - f1_m: 0.6490 - precision_m: 0.7160 - recall_m: 0.6449 - val_loss: 0.4106 - val_accuracy: 0.8355 - val_f1_m: 0.8059 - val_precision_m: 0.9476 - val_recall_m: 0.7253\n",
            "Epoch 2/5\n",
            "811/811 [==============================] - 18s 22ms/step - loss: 0.2524 - accuracy: 0.9041 - f1_m: 0.8955 - precision_m: 0.9067 - recall_m: 0.9000 - val_loss: 0.2369 - val_accuracy: 0.9091 - val_f1_m: 0.9049 - val_precision_m: 0.8922 - val_recall_m: 0.9258\n",
            "Epoch 3/5\n",
            "811/811 [==============================] - 16s 19ms/step - loss: 0.1439 - accuracy: 0.9477 - f1_m: 0.9445 - precision_m: 0.9494 - recall_m: 0.9478 - val_loss: 0.2563 - val_accuracy: 0.9167 - val_f1_m: 0.9083 - val_precision_m: 0.9419 - val_recall_m: 0.8879\n",
            "Epoch 4/5\n",
            "811/811 [==============================] - 14s 18ms/step - loss: 0.0857 - accuracy: 0.9712 - f1_m: 0.9688 - precision_m: 0.9704 - recall_m: 0.9720 - val_loss: 0.2965 - val_accuracy: 0.8952 - val_f1_m: 0.8939 - val_precision_m: 0.8573 - val_recall_m: 0.9452\n",
            "Epoch 5/5\n",
            "811/811 [==============================] - 14s 18ms/step - loss: 0.0465 - accuracy: 0.9850 - f1_m: 0.9834 - precision_m: 0.9839 - recall_m: 0.9854 - val_loss: 0.3284 - val_accuracy: 0.9001 - val_f1_m: 0.8945 - val_precision_m: 0.8953 - val_recall_m: 0.9065\n",
            "Training Accuracy:  0.9955 Training F1:  0.9954 Training precision:  0.9936 Training recall:  0.9976\n",
            "Testing Accuracy :  0.8933 Testing F1:  0.8935 Testing precision:  0.9123 Testing recall:  0.8806\n"
          ]
        }
      ],
      "source": [
        "wv_model = Sequential()\n",
        "#wv_model.add(b_wv.wv.get_keras_embedding(True)) \n",
        "wv_model.add(Embedding(vocab_len-1, 70, input_length=70, weights=[embedding_matrix], trainable=True))\n",
        "\n",
        "wv_model.add(Conv1D(128, 3, activation='relu'))\n",
        "wv_model.add(MaxPooling1D())\n",
        "\n",
        "wv_model.add(Flatten())\n",
        "wv_model.add(Dense(16, activation='relu'))\n",
        "wv_model.add(Dense(1, activation='sigmoid'))\n",
        "wv_model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy',f1_m,precision_m, recall_m])\n",
        "wv_model.summary() \n",
        "\n",
        "# Train models and Evaluate them for both binary and multi-class\n",
        "wv_model.fit(Xcnn_train, b_y_train,\n",
        "                     epochs=5,\n",
        "                     verbose=True,\n",
        "                     validation_data=(Xcnn_val, b_y_val),\n",
        "                     batch_size=16)\n",
        "\n",
        "\n",
        "loss, accuracy, f1_score, precision, recall = wv_model.evaluate(Xcnn_train, b_y_train, verbose=False)\n",
        "print(\"Training Accuracy:  {:.4f}\".format(accuracy),\"Training F1:  {:.4f}\".format(f1_score) ,\"Training precision:  {:.4f}\".format(precision) , \"Training recall:  {:.4f}\".format(recall))\n",
        "\n",
        "loss, accuracy, f1_score, precision, recall = wv_model.evaluate(Xcnn_test, test_binary_df.label, verbose=False)\n",
        "print(\"Testing Accuracy :  {:.4f}\".format(accuracy),\"Testing F1:  {:.4f}\".format(f1_score) ,\"Testing precision:  {:.4f}\".format(precision) , \"Testing recall:  {:.4f}\".format(recall))"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "wv_model = Sequential()\n",
        "wv_model.add(Embedding(vocab_len-1, 70, input_length=70, weights=[embedding_matrix], trainable=True))\n",
        "\n",
        "wv_model.add(Conv1D(128, 5, activation='relu'))\n",
        "wv_model.add(MaxPooling1D())\n",
        "\n",
        "wv_model.add(Conv1D(64, 5, activation='relu'))\n",
        "wv_model.add(MaxPooling1D())\n",
        "\n",
        "wv_model.add(Flatten())\n",
        "wv_model.add(Dense(10, activation='relu'))\n",
        "wv_model.add(Dense(1, activation='sigmoid'))\n",
        "wv_model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy',f1_m,precision_m, recall_m])\n",
        "wv_model.summary() \n",
        "\n",
        "# Train models and Evaluate them for both binary and multi-class\n",
        "wv_model.fit(Xcnn_train, b_y_train,\n",
        "                     epochs=5,\n",
        "                     verbose=True,\n",
        "                     validation_data=(Xcnn_val, b_y_val),\n",
        "                     batch_size=16)\n",
        "\n",
        "loss, accuracy, f1_score, precision, recall = wv_model.evaluate(Xcnn_train, b_y_train, verbose=False)\n",
        "print(\"Training Accuracy:  {:.4f}\".format(accuracy),\"Training F1:  {:.4f}\".format(f1_score) ,\"Training precision:  {:.4f}\".format(precision) , \"Training recall:  {:.4f}\".format(recall))\n",
        "\n",
        "loss, accuracy, f1_score, precision, recall = wv_model.evaluate(Xcnn_test, test_binary_df.label, verbose=False)\n",
        "print(\"Testing Accuracy :  {:.4f}\".format(accuracy),\"Testing F1:  {:.4f}\".format(f1_score) ,\"Testing precision:  {:.4f}\".format(precision) , \"Testing recall:  {:.4f}\".format(recall))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cJFKWDcvBuE3",
        "outputId": "f5049ebe-8b7c-4cbc-9f68-6e0a852d23bd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_32\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding_32 (Embedding)    (None, 70, 70)            1347290   \n",
            "                                                                 \n",
            " conv1d_39 (Conv1D)          (None, 66, 128)           44928     \n",
            "                                                                 \n",
            " max_pooling1d_39 (MaxPoolin  (None, 33, 128)          0         \n",
            " g1D)                                                            \n",
            "                                                                 \n",
            " conv1d_40 (Conv1D)          (None, 29, 64)            41024     \n",
            "                                                                 \n",
            " max_pooling1d_40 (MaxPoolin  (None, 14, 64)           0         \n",
            " g1D)                                                            \n",
            "                                                                 \n",
            " flatten_32 (Flatten)        (None, 896)               0         \n",
            "                                                                 \n",
            " dense_64 (Dense)            (None, 10)                8970      \n",
            "                                                                 \n",
            " dense_65 (Dense)            (None, 1)                 11        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1,442,223\n",
            "Trainable params: 1,442,223\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Epoch 1/5\n",
            "811/811 [==============================] - 19s 23ms/step - loss: 0.5413 - accuracy: 0.7048 - f1_m: 0.6827 - precision_m: 0.7127 - recall_m: 0.7096 - val_loss: 0.3258 - val_accuracy: 0.8723 - val_f1_m: 0.8629 - val_precision_m: 0.8899 - val_recall_m: 0.8513\n",
            "Epoch 2/5\n",
            "811/811 [==============================] - 18s 22ms/step - loss: 0.2567 - accuracy: 0.8973 - f1_m: 0.8887 - precision_m: 0.9035 - recall_m: 0.8920 - val_loss: 0.2348 - val_accuracy: 0.9084 - val_f1_m: 0.8996 - val_precision_m: 0.9165 - val_recall_m: 0.8935\n",
            "Epoch 3/5\n",
            "811/811 [==============================] - 18s 22ms/step - loss: 0.1496 - accuracy: 0.9460 - f1_m: 0.9427 - precision_m: 0.9505 - recall_m: 0.9446 - val_loss: 0.2383 - val_accuracy: 0.9119 - val_f1_m: 0.9032 - val_precision_m: 0.9264 - val_recall_m: 0.8912\n",
            "Epoch 4/5\n",
            "811/811 [==============================] - 18s 23ms/step - loss: 0.0926 - accuracy: 0.9662 - f1_m: 0.9635 - precision_m: 0.9694 - recall_m: 0.9643 - val_loss: 0.3108 - val_accuracy: 0.8994 - val_f1_m: 0.8877 - val_precision_m: 0.9349 - val_recall_m: 0.8572\n",
            "Epoch 5/5\n",
            "811/811 [==============================] - 18s 22ms/step - loss: 0.0508 - accuracy: 0.9825 - f1_m: 0.9800 - precision_m: 0.9816 - recall_m: 0.9814 - val_loss: 0.3888 - val_accuracy: 0.8786 - val_f1_m: 0.8847 - val_precision_m: 0.8368 - val_recall_m: 0.9558\n",
            "Training Accuracy:  0.9746 Training F1:  0.9746 Training precision:  0.9532 Training recall:  0.9987\n",
            "Testing Accuracy :  0.8707 Testing F1:  0.8822 Testing precision:  0.8340 Testing recall:  0.9408\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Pretrained word embeddings from gensim.api"
      ],
      "metadata": {
        "id": "aPOOnRW2aB_k"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "api_wv.save(\"api_wv.wordvectors\")\n",
        "embedding_vector = KeyedVectors.load(\"api_wv.wordvectors\", mmap='r')\n",
        "\n",
        "embedding_matrix = np.zeros((vocab_len-1, 50))\n",
        "\n",
        "unknown_counter = 0\n",
        "for word, i in tokenizer.word_index.items():\n",
        "    try:\n",
        "      #print(embedding_vector[word], word)\n",
        "      embedding_matrix[i-1] = embedding_vector[word]\n",
        "    except KeyError:\n",
        "      #print(\"key error\", i, word)\n",
        "      unknown_counter = unknown_counter + 1\n",
        "\n",
        "#print(unknown_counter) "
      ],
      "metadata": {
        "id": "GDbqXoPSCzwS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "api_model = Sequential()\n",
        "api_model.add(Embedding(vocab_len-1, 50, input_length=70, weights=[embedding_matrix], trainable=True))\n",
        "\n",
        "api_model.add(Conv1D(128, 3, activation='relu'))\n",
        "api_model.add(MaxPooling1D())\n",
        "\n",
        "api_model.add(Flatten())\n",
        "api_model.add(Dense(10, activation='relu'))\n",
        "api_model.add(Dense(1, activation='sigmoid'))\n",
        "api_model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy',f1_m,precision_m, recall_m])\n",
        "api_model.summary() \n",
        "\n",
        "# Train models and Evaluate them for both binary and multi-class\n",
        "api_model.fit(Xcnn_train, b_y_train,\n",
        "                     epochs=5,\n",
        "                     verbose=True,\n",
        "                     validation_data=(Xcnn_val, b_y_val),\n",
        "                     batch_size=32)\n",
        "\n",
        "loss, accuracy, f1_score, precision, recall = api_model.evaluate(Xcnn_train, b_y_train, verbose=False)\n",
        "print(\"Training Accuracy:  {:.4f}\".format(accuracy),\"Training F1:  {:.4f}\".format(f1_score) ,\"Training precision:  {:.4f}\".format(precision) , \"Training recall:  {:.4f}\".format(recall))\n",
        "\n",
        "loss, accuracy, f1_score, precision, recall = api_model.evaluate(Xcnn_test, test_binary_df.label, verbose=False)\n",
        "print(\"Testing Accuracy :  {:.4f}\".format(accuracy),\"Testing F1:  {:.4f}\".format(f1_score) ,\"Testing precision:  {:.4f}\".format(precision) , \"Testing recall:  {:.4f}\".format(recall))"
      ],
      "metadata": {
        "id": "thaP7iWo2aJe",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8eb345dd-e9bf-49b3-91fb-e865399494f0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_33\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding_33 (Embedding)    (None, 70, 50)            962350    \n",
            "                                                                 \n",
            " conv1d_41 (Conv1D)          (None, 68, 128)           19328     \n",
            "                                                                 \n",
            " max_pooling1d_41 (MaxPoolin  (None, 34, 128)          0         \n",
            " g1D)                                                            \n",
            "                                                                 \n",
            " flatten_33 (Flatten)        (None, 4352)              0         \n",
            "                                                                 \n",
            " dense_66 (Dense)            (None, 10)                43530     \n",
            "                                                                 \n",
            " dense_67 (Dense)            (None, 1)                 11        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1,025,219\n",
            "Trainable params: 1,025,219\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Epoch 1/5\n",
            "406/406 [==============================] - 9s 19ms/step - loss: 0.6945 - accuracy: 0.4976 - f1_m: 0.1095 - precision_m: 0.0939 - recall_m: 0.1567 - val_loss: 0.6932 - val_accuracy: 0.4969 - val_f1_m: 0.0000e+00 - val_precision_m: 0.0000e+00 - val_recall_m: 0.0000e+00\n",
            "Epoch 2/5\n",
            "406/406 [==============================] - 7s 17ms/step - loss: 0.6932 - accuracy: 0.5005 - f1_m: 0.2259 - precision_m: 0.1703 - recall_m: 0.3424 - val_loss: 0.6932 - val_accuracy: 0.4969 - val_f1_m: 0.0000e+00 - val_precision_m: 0.0000e+00 - val_recall_m: 0.0000e+00\n",
            "Epoch 3/5\n",
            "406/406 [==============================] - 7s 18ms/step - loss: 0.6932 - accuracy: 0.5022 - f1_m: 0.0000e+00 - precision_m: 0.0000e+00 - recall_m: 0.0000e+00 - val_loss: 0.6932 - val_accuracy: 0.4969 - val_f1_m: 0.0000e+00 - val_precision_m: 0.0000e+00 - val_recall_m: 0.0000e+00\n",
            "Epoch 4/5\n",
            "406/406 [==============================] - 7s 17ms/step - loss: 0.6932 - accuracy: 0.5022 - f1_m: 0.0000e+00 - precision_m: 0.0000e+00 - recall_m: 0.0000e+00 - val_loss: 0.6932 - val_accuracy: 0.4969 - val_f1_m: 0.0000e+00 - val_precision_m: 0.0000e+00 - val_recall_m: 0.0000e+00\n",
            "Epoch 5/5\n",
            "406/406 [==============================] - 7s 17ms/step - loss: 0.6932 - accuracy: 0.4966 - f1_m: 0.1263 - precision_m: 0.0945 - recall_m: 0.1946 - val_loss: 0.6932 - val_accuracy: 0.4969 - val_f1_m: 0.0000e+00 - val_precision_m: 0.0000e+00 - val_recall_m: 0.0000e+00\n",
            "Training Accuracy:  0.5022 Training F1:  0.0000 Training precision:  0.0000 Training recall:  0.0000\n",
            "Testing Accuracy :  0.4852 Testing F1:  0.0000 Testing precision:  0.0000 Testing recall:  0.0000\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "api_model = Sequential()\n",
        "api_model.add(Embedding(vocab_len-1, 50, input_length=70, weights=[embedding_matrix], trainable=True))\n",
        "\n",
        "api_model.add(Conv1D(128, 5, activation='relu'))\n",
        "api_model.add(MaxPooling1D())\n",
        "api_model.add(Dropout(0.2))\n",
        "\n",
        "api_model.add(Conv1D(64, 5, activation='relu'))\n",
        "api_model.add(MaxPooling1D())\n",
        "api_model.add(Dropout(0.2))\n",
        "\n",
        "api_model.add(Flatten())\n",
        "api_model.add(Dense(10, activation='relu'))\n",
        "api_model.add(Dense(1, activation='sigmoid'))\n",
        "api_model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy',f1_m,precision_m, recall_m])\n",
        "api_model.summary() \n",
        "\n",
        "# Train models and Evaluate them for both binary and multi-class\n",
        "\n",
        "api_model.fit(Xcnn_train, b_y_train,\n",
        "                     epochs=5,\n",
        "                     verbose=True,\n",
        "                     validation_data=(Xcnn_val, b_y_val),\n",
        "                     batch_size=32)\n",
        "\n",
        "loss, accuracy, f1_score, precision, recall = api_model.evaluate(Xcnn_train, b_y_train, verbose=False)\n",
        "print(\"Training Accuracy:  {:.4f}\".format(accuracy),\"Training F1:  {:.4f}\".format(f1_score) ,\"Training precision:  {:.4f}\".format(precision) , \"Training recall:  {:.4f}\".format(recall))\n",
        "\n",
        "loss, accuracy, f1_score, precision, recall = api_model.evaluate(Xcnn_test, test_binary_df.label, verbose=False)\n",
        "print(\"Testing Accuracy :  {:.4f}\".format(accuracy),\"Testing F1:  {:.4f}\".format(f1_score) ,\"Testing precision:  {:.4f}\".format(precision) , \"Testing recall:  {:.4f}\".format(recall))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7_vQwMZzERbW",
        "outputId": "1a9efd4d-c1a4-42fa-f84f-8e5ab413f015"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_34\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding_34 (Embedding)    (None, 70, 50)            962350    \n",
            "                                                                 \n",
            " conv1d_42 (Conv1D)          (None, 66, 128)           32128     \n",
            "                                                                 \n",
            " max_pooling1d_42 (MaxPoolin  (None, 33, 128)          0         \n",
            " g1D)                                                            \n",
            "                                                                 \n",
            " dropout_10 (Dropout)        (None, 33, 128)           0         \n",
            "                                                                 \n",
            " conv1d_43 (Conv1D)          (None, 29, 64)            41024     \n",
            "                                                                 \n",
            " max_pooling1d_43 (MaxPoolin  (None, 14, 64)           0         \n",
            " g1D)                                                            \n",
            "                                                                 \n",
            " dropout_11 (Dropout)        (None, 14, 64)            0         \n",
            "                                                                 \n",
            " flatten_34 (Flatten)        (None, 896)               0         \n",
            "                                                                 \n",
            " dense_68 (Dense)            (None, 10)                8970      \n",
            "                                                                 \n",
            " dense_69 (Dense)            (None, 1)                 11        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1,044,483\n",
            "Trainable params: 1,044,483\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Epoch 1/5\n",
            "406/406 [==============================] - 12s 28ms/step - loss: 0.6237 - accuracy: 0.6354 - f1_m: 0.5663 - precision_m: 0.5761 - recall_m: 0.5844 - val_loss: 0.3818 - val_accuracy: 0.8376 - val_f1_m: 0.8351 - val_precision_m: 0.8591 - val_recall_m: 0.8186\n",
            "Epoch 2/5\n",
            "406/406 [==============================] - 11s 27ms/step - loss: 0.3100 - accuracy: 0.8744 - f1_m: 0.8681 - precision_m: 0.8822 - recall_m: 0.8670 - val_loss: 0.2559 - val_accuracy: 0.9001 - val_f1_m: 0.8977 - val_precision_m: 0.9131 - val_recall_m: 0.8868\n",
            "Epoch 3/5\n",
            "406/406 [==============================] - 11s 27ms/step - loss: 0.1904 - accuracy: 0.9291 - f1_m: 0.9257 - precision_m: 0.9352 - recall_m: 0.9219 - val_loss: 0.2373 - val_accuracy: 0.9105 - val_f1_m: 0.9121 - val_precision_m: 0.9017 - val_recall_m: 0.9266\n",
            "Epoch 4/5\n",
            "406/406 [==============================] - 11s 28ms/step - loss: 0.1301 - accuracy: 0.9534 - f1_m: 0.9519 - precision_m: 0.9591 - recall_m: 0.9490 - val_loss: 0.2562 - val_accuracy: 0.9056 - val_f1_m: 0.9044 - val_precision_m: 0.9231 - val_recall_m: 0.8902\n",
            "Epoch 5/5\n",
            "406/406 [==============================] - 11s 28ms/step - loss: 0.0801 - accuracy: 0.9745 - f1_m: 0.9736 - precision_m: 0.9773 - recall_m: 0.9722 - val_loss: 0.3105 - val_accuracy: 0.9001 - val_f1_m: 0.8992 - val_precision_m: 0.9240 - val_recall_m: 0.8809\n",
            "Training Accuracy:  0.9895 Training F1:  0.9891 Training precision:  0.9957 Training recall:  0.9833\n",
            "Testing Accuracy :  0.8719 Testing F1:  0.8690 Testing precision:  0.9125 Testing recall:  0.8332\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# -------- MULTICLASS ---------------"
      ],
      "metadata": {
        "id": "EKhoFTNrczTp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def recall_m(y_true, y_pred):\n",
        "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 1, 5)))\n",
        "    possible_positives = K.sum(K.round(K.clip(y_true, 1, 5)))\n",
        "    recall = true_positives / (possible_positives + K.epsilon())\n",
        "    return recall\n",
        "\n",
        "def precision_m(y_true, y_pred):\n",
        "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 1, 5)))\n",
        "    predicted_positives = K.sum(K.round(K.clip(y_pred, 1, 5)))\n",
        "    precision = true_positives / (predicted_positives + K.epsilon())\n",
        "    return precision\n",
        "\n",
        "def f1_m(y_true, y_pred):\n",
        "    precision = precision_m(y_true, y_pred)\n",
        "    recall = recall_m(y_true, y_pred)\n",
        "    return 2*((precision*recall)/(precision+recall+K.epsilon()))\n"
      ],
      "metadata": {
        "id": "dJhhCP-jkXoI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bAls_pixq5Xo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8369168c-72c0-412e-8a84-793db9628f94"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['nice', 'first', 'visit', 'owner', 'ted', 'friendli', 'start', 'restaur', 'busi', 'friday', 'night', 'begin', 'chines', 'new', 'year', 'order', 'pu', 'pu', 'platter', 'crab', 'rangoon', 'dinner', 'chef', 'special', 'seafood', 'instead', 'chines', 'american', 'regular', 'fri', 'rice', 'everyth', 'well', 'prepar', 'shrimp', 'larg', 'veget', 'crisp', 'chili', 'sauc', 'sweet', 'tangi', 'return', 'owner', 'made', 'effort', 'visit', 'learn', 'name', 'ask', 'first', 'visit']\n",
            "[1234, 218, 299, 14, 169, 156, 104, 166, 53, 327, 42, 434, 827, 233, 179, 83, 765, 94, 20, 41, 21, 16, 324, 260, 161, 51, 251, 755, 162, 269, 1, 2, 10, 16, 11, 6, 1234, 717, 210, 257, 179, 823, 96]\n",
            "[   76    40   224   160    69   784    10   142   371   442   387   493\n",
            "    10  1578    61    44  4175  2301  2816 12713     9   344   487     8\n",
            "    11    44   353  1110  4176   137    11  6331   325   138     9     1\n",
            "     2    10    10  2979    76     3  2371  4696     9   123   867   927\n",
            "   784  4696   854     3   142     0     0     0     0     0     0     0\n",
            "     0     0     0     0     0     0     0     0     0     0]\n"
          ]
        }
      ],
      "source": [
        "# Prepare your dataset for CNN classifier\n",
        "\n",
        "tokenizer = Tokenizer(num_words=50000)\n",
        "tokenizer.fit_on_texts(m_X_train)\n",
        "m_Xcnn_train = tokenizer.texts_to_sequences(m_X_train)\n",
        "m_Xcnn_val = tokenizer.texts_to_sequences(m_X_val)\n",
        "m_Xcnn_test = tokenizer.texts_to_sequences(test_multiclass_df.text)\n",
        "\n",
        "vocab_size = len(tokenizer.word_index) + 1  \n",
        "print(m_X_train[1])\n",
        "print(m_Xcnn_train[1]) \n",
        "\n",
        "maxlen = 70 \n",
        "m_Xcnn_train = pad_sequences(m_Xcnn_train, padding='post', maxlen=maxlen)\n",
        "m_Xcnn_val = pad_sequences(m_Xcnn_val, padding='post', maxlen=maxlen)\n",
        "m_Xcnn_test = pad_sequences(m_Xcnn_test, padding='post', maxlen=maxlen)\n",
        "print(m_Xcnn_train[0, :]) "
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# RANDOMLY INITIALIZED"
      ],
      "metadata": {
        "id": "E0RZpu5uU6h4"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PEprgPZ5q5Od",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ab8954b2-ecaa-4d8d-8842-f074f940473e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_39\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding_39 (Embedding)    (None, 70, 70)            1473500   \n",
            "                                                                 \n",
            " conv1d_49 (Conv1D)          (None, 68, 256)           54016     \n",
            "                                                                 \n",
            " max_pooling1d_49 (MaxPoolin  (None, 34, 256)          0         \n",
            " g1D)                                                            \n",
            "                                                                 \n",
            " dropout_17 (Dropout)        (None, 34, 256)           0         \n",
            "                                                                 \n",
            " flatten_39 (Flatten)        (None, 8704)              0         \n",
            "                                                                 \n",
            " dense_78 (Dense)            (None, 20)                174100    \n",
            "                                                                 \n",
            " dense_79 (Dense)            (None, 5)                 105       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1,701,721\n",
            "Trainable params: 1,701,721\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Epoch 1/5\n",
            "1013/1013 [==============================] - 25s 24ms/step - loss: 1.2456 - accuracy: 0.4264 - f1_m: 1.4609 - precision_m: 1.0498 - recall_m: 2.4319 - val_loss: 1.0866 - val_accuracy: 0.5150 - val_f1_m: 1.4957 - val_precision_m: 1.0777 - val_recall_m: 2.4761\n",
            "Epoch 2/5\n",
            "1013/1013 [==============================] - 29s 29ms/step - loss: 0.9148 - accuracy: 0.6032 - f1_m: 1.5340 - precision_m: 1.1030 - recall_m: 2.5514 - val_loss: 1.0845 - val_accuracy: 0.5250 - val_f1_m: 1.5175 - val_precision_m: 1.0935 - val_recall_m: 2.5117\n",
            "Epoch 3/5\n",
            "1013/1013 [==============================] - 28s 27ms/step - loss: 0.6963 - accuracy: 0.7074 - f1_m: 1.5769 - precision_m: 1.1341 - recall_m: 2.6146 - val_loss: 1.2738 - val_accuracy: 0.5106 - val_f1_m: 1.5638 - val_precision_m: 1.1271 - val_recall_m: 2.5870\n",
            "Epoch 4/5\n",
            "1013/1013 [==============================] - 26s 26ms/step - loss: 0.4902 - accuracy: 0.8073 - f1_m: 1.6257 - precision_m: 1.1694 - recall_m: 2.7027 - val_loss: 1.5496 - val_accuracy: 0.5006 - val_f1_m: 1.5901 - val_precision_m: 1.1461 - val_recall_m: 2.6296\n",
            "Epoch 5/5\n",
            "1013/1013 [==============================] - 35s 35ms/step - loss: 0.3135 - accuracy: 0.8848 - f1_m: 1.6629 - precision_m: 1.1963 - recall_m: 2.7594 - val_loss: 2.1642 - val_accuracy: 0.4994 - val_f1_m: 1.6416 - val_precision_m: 1.1835 - val_recall_m: 2.7134\n",
            "\n",
            "Training Accuracy:  0.9498 Training F1:  1.6887 Training precision:  1.2148 Training recall:  2.7873\n",
            "Test Scores:\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.72      0.59      0.65       393\n",
            "           1       0.45      0.51      0.48       380\n",
            "           2       0.43      0.47      0.45       407\n",
            "           3       0.39      0.46      0.42       379\n",
            "           4       0.69      0.56      0.62       441\n",
            "\n",
            "    accuracy                           0.52      2000\n",
            "   macro avg       0.54      0.52      0.52      2000\n",
            "weighted avg       0.54      0.52      0.53      2000\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# Create Embedding Matrices and Layers - RANDOMLY INITIALIZED\n",
        "from sklearn.metrics import classification_report\n",
        "\n",
        "embedding_dim = 70\n",
        "multi_cnn = Sequential()\n",
        "multi_cnn.add(Embedding(vocab_size, embedding_dim, input_length=maxlen)) \n",
        "\n",
        "multi_cnn.add(Conv1D(256, 3, activation='relu'))\n",
        "multi_cnn.add(MaxPooling1D())\n",
        "multi_cnn.add(Dropout(0.2))\n",
        "\n",
        "multi_cnn.add(Flatten())\n",
        "multi_cnn.add(Dense(20, activation='relu'))\n",
        "multi_cnn.add(Dense(5, activation='softmax')) #added 5 neurons for output layer using softmax\n",
        "multi_cnn.compile(optimizer='adam', loss='SparseCategoricalCrossentropy', metrics=['accuracy',f1_m,precision_m, recall_m]) #changed loss\n",
        "multi_cnn.summary() \n",
        "\n",
        "\n",
        "# Train models and Evaluate them for both binary and multi-class\n",
        "multi_cnn.fit(m_Xcnn_train, m_y_train,\n",
        "                     epochs=5,\n",
        "                     verbose=True,\n",
        "                     validation_data=(m_Xcnn_val, m_y_val),\n",
        "                     batch_size=16)\n",
        "\n",
        "print(\"\")\n",
        "\n",
        "loss, accuracy, f1_score, precision, recall = multi_cnn.evaluate(m_Xcnn_train, m_y_train, verbose=False)\n",
        "print(\"Training Accuracy:  {:.4f}\".format(accuracy),\"Training F1:  {:.4f}\".format(f1_score) ,\"Training precision:  {:.4f}\".format(precision) , \"Training recall:  {:.4f}\".format(recall))\n",
        "\n",
        "print(\"Test Scores:\")\n",
        "preds = multi_cnn.predict(m_Xcnn_test)\n",
        "indices_p = np.argmax(preds, axis = 1)\n",
        "indices_p\n",
        "print(classification_report(test_multiclass_df.label,indices_p))"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Create Embedding Matrices and Layers - RANDOMLY INITIALIZED\n",
        "\n",
        "embedding_dim = 70\n",
        "multi_cnn = Sequential()\n",
        "multi_cnn.add(Embedding(vocab_size, embedding_dim, input_length=maxlen)) \n",
        "\n",
        "multi_cnn.add(Conv1D(64, 5, activation='relu'))\n",
        "multi_cnn.add(MaxPooling1D())\n",
        "multi_cnn.add(Dropout(0.2))\n",
        "\n",
        "multi_cnn.add(Conv1D(32, 5, activation='relu'))\n",
        "multi_cnn.add(MaxPooling1D())\n",
        "multi_cnn.add(Dropout(0.2))\n",
        "\n",
        "multi_cnn.add(Flatten())\n",
        "multi_cnn.add(Dense(20, activation='relu'))\n",
        "multi_cnn.add(Dense(5, activation='softmax')) #added 5 neurons for output layer using softmax\n",
        "multi_cnn.compile(optimizer='adam', loss='SparseCategoricalCrossentropy', metrics=['accuracy']) #changed loss\n",
        "multi_cnn.summary() \n",
        "\n",
        "\n",
        "# Train models and Evaluate them for both binary and multi-class\n",
        "multi_cnn.fit(m_Xcnn_train, m_y_train,\n",
        "                     epochs=5,\n",
        "                     verbose=True,\n",
        "                     validation_data=(m_Xcnn_val, m_y_val),\n",
        "                     batch_size=16)\n",
        "\n",
        "loss, accuracy = multi_cnn.evaluate(m_Xcnn_train, m_y_train, verbose=False)\n",
        "print(\"Training Accuracy: {:.4f}\".format(accuracy))\n",
        "\n",
        "print(\"Test Scores:\")\n",
        "preds = multi_cnn.predict(m_Xcnn_test)\n",
        "indices_p = np.argmax(preds, axis = 1)\n",
        "indices_p\n",
        "print(classification_report(test_multiclass_df.label,indices_p))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EP8RIURZXbNo",
        "outputId": "bfc4c374-579a-4892-b24a-e8540f20b3a3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_38\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding_38 (Embedding)    (None, 70, 70)            1473500   \n",
            "                                                                 \n",
            " conv1d_47 (Conv1D)          (None, 66, 64)            22464     \n",
            "                                                                 \n",
            " max_pooling1d_47 (MaxPoolin  (None, 33, 64)           0         \n",
            " g1D)                                                            \n",
            "                                                                 \n",
            " dropout_15 (Dropout)        (None, 33, 64)            0         \n",
            "                                                                 \n",
            " conv1d_48 (Conv1D)          (None, 29, 32)            10272     \n",
            "                                                                 \n",
            " max_pooling1d_48 (MaxPoolin  (None, 14, 32)           0         \n",
            " g1D)                                                            \n",
            "                                                                 \n",
            " dropout_16 (Dropout)        (None, 14, 32)            0         \n",
            "                                                                 \n",
            " flatten_38 (Flatten)        (None, 448)               0         \n",
            "                                                                 \n",
            " dense_76 (Dense)            (None, 20)                8980      \n",
            "                                                                 \n",
            " dense_77 (Dense)            (None, 5)                 105       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1,515,321\n",
            "Trainable params: 1,515,321\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Epoch 1/5\n",
            "1013/1013 [==============================] - 22s 21ms/step - loss: 1.3170 - accuracy: 0.3754 - val_loss: 1.1668 - val_accuracy: 0.4494\n",
            "Epoch 2/5\n",
            "1013/1013 [==============================] - 21s 21ms/step - loss: 1.0319 - accuracy: 0.5362 - val_loss: 1.0896 - val_accuracy: 0.5106\n",
            "Epoch 3/5\n",
            "1013/1013 [==============================] - 21s 20ms/step - loss: 0.8108 - accuracy: 0.6516 - val_loss: 1.2565 - val_accuracy: 0.5017\n",
            "Epoch 4/5\n",
            "1013/1013 [==============================] - 21s 20ms/step - loss: 0.6229 - accuracy: 0.7465 - val_loss: 1.3472 - val_accuracy: 0.5000\n",
            "Epoch 5/5\n",
            "1013/1013 [==============================] - 21s 21ms/step - loss: 0.4713 - accuracy: 0.8109 - val_loss: 1.6754 - val_accuracy: 0.4950\n",
            "Training Accuracy: 0.9058\n",
            "Test Scores:\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.65      0.64      0.65       393\n",
            "           1       0.42      0.49      0.45       380\n",
            "           2       0.41      0.45      0.43       407\n",
            "           3       0.38      0.38      0.38       379\n",
            "           4       0.67      0.53      0.59       441\n",
            "\n",
            "    accuracy                           0.50      2000\n",
            "   macro avg       0.51      0.50      0.50      2000\n",
            "weighted avg       0.51      0.50      0.50      2000\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Word embeddings trained from scratch with gensim"
      ],
      "metadata": {
        "id": "enI7t3s0U28J"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from gensim.models import KeyedVectors\n",
        "\n",
        "vocab_len = len(m_wv.wv.vocab) + 2\n",
        "\n",
        "m_wv.save(\"m_wv.wordvectors\")\n",
        "embedding_vector = KeyedVectors.load(\"m_wv.wordvectors\", mmap='r')\n",
        "\n",
        "embedding_matrix = np.zeros((vocab_len-1, 70))\n",
        "for word, i in tokenizer.word_index.items():\n",
        "    try:\n",
        "      #print(embedding_vector[word].shape, word)\n",
        "      embedding_matrix[i-1] = embedding_vector[word]\n",
        "    except KeyError:\n",
        "      print(\"key error\", i, word)\n",
        "  "
      ],
      "metadata": {
        "id": "KmgJLfSoGNYT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "T1LJ_mSBq5Jx",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "52ba863b-28e8-481f-eadc-ed7508577777"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_40\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding_40 (Embedding)    (None, 70, 70)            1473500   \n",
            "                                                                 \n",
            " conv1d_50 (Conv1D)          (None, 68, 256)           54016     \n",
            "                                                                 \n",
            " max_pooling1d_50 (MaxPoolin  (None, 34, 256)          0         \n",
            " g1D)                                                            \n",
            "                                                                 \n",
            " dropout_18 (Dropout)        (None, 34, 256)           0         \n",
            "                                                                 \n",
            " flatten_40 (Flatten)        (None, 8704)              0         \n",
            "                                                                 \n",
            " dense_80 (Dense)            (None, 20)                174100    \n",
            "                                                                 \n",
            " dense_81 (Dense)            (None, 5)                 105       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1,701,721\n",
            "Trainable params: 1,701,721\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Epoch 1/5\n",
            "1013/1013 [==============================] - 25s 24ms/step - loss: 1.6121 - accuracy: 0.2026 - val_loss: 1.6098 - val_accuracy: 0.1867\n",
            "Epoch 2/5\n",
            "1013/1013 [==============================] - 25s 25ms/step - loss: 1.6097 - accuracy: 0.2035 - val_loss: 1.6090 - val_accuracy: 0.2217\n",
            "Epoch 3/5\n",
            "1013/1013 [==============================] - 24s 24ms/step - loss: 1.6096 - accuracy: 0.1983 - val_loss: 1.6093 - val_accuracy: 0.2217\n",
            "Epoch 4/5\n",
            "1013/1013 [==============================] - 24s 24ms/step - loss: 1.6096 - accuracy: 0.1972 - val_loss: 1.6096 - val_accuracy: 0.1867\n",
            "Epoch 5/5\n",
            "1013/1013 [==============================] - 25s 24ms/step - loss: 1.6122 - accuracy: 0.2026 - val_loss: 1.6098 - val_accuracy: 0.2011\n",
            "Training Accuracy: 0.2003\n",
            "Test Scores:\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.20      1.00      0.33       393\n",
            "           1       0.00      0.00      0.00       380\n",
            "           2       0.00      0.00      0.00       407\n",
            "           3       0.00      0.00      0.00       379\n",
            "           4       0.00      0.00      0.00       441\n",
            "\n",
            "    accuracy                           0.20      2000\n",
            "   macro avg       0.04      0.20      0.07      2000\n",
            "weighted avg       0.04      0.20      0.06      2000\n",
            "\n"
          ]
        }
      ],
      "source": [
        "multi_cnn = Sequential()\n",
        "multi_cnn.add(Embedding(vocab_len-1, 70, input_length=70, weights=[embedding_matrix], trainable=True))\n",
        "\n",
        "multi_cnn.add(Conv1D(256, 3, activation='relu'))\n",
        "multi_cnn.add(MaxPooling1D())\n",
        "multi_cnn.add(Dropout(0.2))\n",
        "\n",
        "multi_cnn.add(Flatten())\n",
        "multi_cnn.add(Dense(20, activation='relu'))\n",
        "multi_cnn.add(Dense(5, activation='softmax')) #added 5 neurons for output layer using softmax\n",
        "\n",
        "multi_cnn.compile(optimizer='adam', loss='SparseCategoricalCrossentropy', metrics=['accuracy']) #changed loss\n",
        "multi_cnn.summary() \n",
        "\n",
        "# Train models and Evaluate them for both binary and multi-class\n",
        "multi_cnn.fit(m_Xcnn_train, m_y_train,\n",
        "                     epochs=5,\n",
        "                     verbose=True,\n",
        "                     validation_data=(m_Xcnn_val, m_y_val),\n",
        "                     batch_size=16)\n",
        "\n",
        "loss, accuracy = multi_cnn.evaluate(m_Xcnn_train, m_y_train, verbose=False)\n",
        "print(\"Training Accuracy: {:.4f}\".format(accuracy))\n",
        "\n",
        "print(\"Test Scores:\")\n",
        "preds = multi_cnn.predict(m_Xcnn_test)\n",
        "indices_p = np.argmax(preds, axis = 1)\n",
        "indices_p\n",
        "print(classification_report(test_multiclass_df.label,indices_p))"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "multi_cnn = Sequential()\n",
        "multi_cnn.add(Embedding(vocab_len-1, 70, input_length=70, weights=[embedding_matrix], trainable=True))\n",
        "\n",
        "multi_cnn.add(Conv1D(64, 5, activation='relu'))\n",
        "multi_cnn.add(MaxPooling1D())\n",
        "multi_cnn.add(Dropout(0.2))\n",
        "\n",
        "multi_cnn.add(Conv1D(32, 5, activation='relu'))\n",
        "multi_cnn.add(MaxPooling1D())\n",
        "multi_cnn.add(Dropout(0.2))\n",
        "\n",
        "multi_cnn.add(Flatten())\n",
        "multi_cnn.add(Dense(20, activation='relu'))\n",
        "multi_cnn.add(Dense(5, activation='softmax')) #added 5 neurons for output layer using softmax\n",
        "\n",
        "multi_cnn.compile(optimizer='adam', loss='SparseCategoricalCrossentropy', metrics=['accuracy']) #changed loss\n",
        "multi_cnn.summary() \n",
        "\n",
        "# Train models and Evaluate them for both binary and multi-class\n",
        "multi_cnn.fit(m_Xcnn_train, m_y_train,\n",
        "                     epochs=10,\n",
        "                     verbose=True,\n",
        "                     validation_data=(m_Xcnn_val, m_y_val),\n",
        "                     batch_size=32)\n",
        "\n",
        "loss, accuracy = multi_cnn.evaluate(m_Xcnn_train, m_y_train, verbose=False)\n",
        "print(\"Training Accuracy: {:.4f}\".format(accuracy))\n",
        "\n",
        "\n",
        "print(\"Test Scores:\")\n",
        "preds = multi_cnn.predict(m_Xcnn_test)\n",
        "indices_p = np.argmax(preds, axis = 1)\n",
        "indices_p\n",
        "print(classification_report(test_multiclass_df.label,indices_p))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PnxxsnFXXEpa",
        "outputId": "23b69dc7-8582-452d-bd93-b559953d36d3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_41\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding_41 (Embedding)    (None, 70, 70)            1473500   \n",
            "                                                                 \n",
            " conv1d_51 (Conv1D)          (None, 66, 64)            22464     \n",
            "                                                                 \n",
            " max_pooling1d_51 (MaxPoolin  (None, 33, 64)           0         \n",
            " g1D)                                                            \n",
            "                                                                 \n",
            " dropout_19 (Dropout)        (None, 33, 64)            0         \n",
            "                                                                 \n",
            " conv1d_52 (Conv1D)          (None, 29, 32)            10272     \n",
            "                                                                 \n",
            " max_pooling1d_52 (MaxPoolin  (None, 14, 32)           0         \n",
            " g1D)                                                            \n",
            "                                                                 \n",
            " dropout_20 (Dropout)        (None, 14, 32)            0         \n",
            "                                                                 \n",
            " flatten_41 (Flatten)        (None, 448)               0         \n",
            "                                                                 \n",
            " dense_82 (Dense)            (None, 20)                8980      \n",
            "                                                                 \n",
            " dense_83 (Dense)            (None, 5)                 105       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1,515,321\n",
            "Trainable params: 1,515,321\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Epoch 1/10\n",
            "507/507 [==============================] - 13s 25ms/step - loss: 1.6128 - accuracy: 0.2200 - val_loss: 1.5993 - val_accuracy: 0.2389\n",
            "Epoch 2/10\n",
            "507/507 [==============================] - 12s 24ms/step - loss: 1.5112 - accuracy: 0.2917 - val_loss: 1.3106 - val_accuracy: 0.3878\n",
            "Epoch 3/10\n",
            "507/507 [==============================] - 12s 24ms/step - loss: 1.2353 - accuracy: 0.4225 - val_loss: 1.2180 - val_accuracy: 0.4306\n",
            "Epoch 4/10\n",
            "507/507 [==============================] - 12s 24ms/step - loss: 1.0951 - accuracy: 0.4925 - val_loss: 1.1616 - val_accuracy: 0.4700\n",
            "Epoch 5/10\n",
            "507/507 [==============================] - 12s 23ms/step - loss: 0.9960 - accuracy: 0.5429 - val_loss: 1.1460 - val_accuracy: 0.4911\n",
            "Epoch 6/10\n",
            "507/507 [==============================] - 12s 24ms/step - loss: 0.9121 - accuracy: 0.5870 - val_loss: 1.1778 - val_accuracy: 0.5083\n",
            "Epoch 7/10\n",
            "507/507 [==============================] - 12s 24ms/step - loss: 0.8237 - accuracy: 0.6338 - val_loss: 1.2957 - val_accuracy: 0.4967\n",
            "Epoch 8/10\n",
            "507/507 [==============================] - 12s 24ms/step - loss: 0.7501 - accuracy: 0.6675 - val_loss: 1.3664 - val_accuracy: 0.5011\n",
            "Epoch 9/10\n",
            "507/507 [==============================] - 12s 24ms/step - loss: 0.6667 - accuracy: 0.7111 - val_loss: 1.5217 - val_accuracy: 0.5067\n",
            "Epoch 10/10\n",
            "507/507 [==============================] - 12s 24ms/step - loss: 0.5929 - accuracy: 0.7513 - val_loss: 1.6977 - val_accuracy: 0.5056\n",
            "Training Accuracy: 0.8533\n",
            "Test Scores:\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.66      0.59      0.62       393\n",
            "           1       0.37      0.41      0.39       380\n",
            "           2       0.38      0.39      0.38       407\n",
            "           3       0.38      0.47      0.42       379\n",
            "           4       0.69      0.53      0.60       441\n",
            "\n",
            "    accuracy                           0.48      2000\n",
            "   macro avg       0.49      0.48      0.48      2000\n",
            "weighted avg       0.50      0.48      0.49      2000\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Pretrained word embeddings from gensim.api"
      ],
      "metadata": {
        "id": "4toJevBkgXVV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "api_wv.save(\"api_wv.wordvectors\")\n",
        "embedding_vector = KeyedVectors.load(\"api_wv.wordvectors\", mmap='r')\n",
        "\n",
        "embedding_matrix = np.zeros((vocab_len, 50))\n",
        "\n",
        "unknown_counter = 0\n",
        "for word, i in tokenizer.word_index.items():\n",
        "    try:\n",
        "      #print(embedding_vector[word], word)\n",
        "      embedding_matrix[i-1] = embedding_vector[word]\n",
        "    except KeyError:\n",
        "      #print(\"key error\", i, word)\n",
        "      unknown_counter = unknown_counter + 1\n",
        "\n",
        "#print(unknown_counter) "
      ],
      "metadata": {
        "id": "v52sYeQPP8FL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "multi_cnn = Sequential()\n",
        "multi_cnn.add(Embedding(vocab_len, 50, input_length=70, weights=[embedding_matrix], trainable=True))\n",
        "\n",
        "multi_cnn.add(Conv1D(256, 3, activation='relu'))\n",
        "multi_cnn.add(MaxPooling1D())\n",
        "multi_cnn.add(Dropout(0.2))\n",
        "\n",
        "multi_cnn.add(Flatten())\n",
        "multi_cnn.add(Dense(10, activation='relu'))\n",
        "multi_cnn.add(Dense(5, activation='softmax')) #added 5 neurons for output layer using softmax\n",
        "multi_cnn.compile(optimizer='adam', loss='SparseCategoricalCrossentropy', metrics=['accuracy']) #changed loss\n",
        "multi_cnn.summary() \n",
        "\n",
        "\n",
        "# Train models and Evaluate them for both binary and multi-class\n",
        "multi_cnn.fit(m_Xcnn_train, m_y_train,\n",
        "                     epochs=5,\n",
        "                     verbose=True,\n",
        "                     validation_data=(m_Xcnn_val, m_y_val),\n",
        "                     batch_size=32)\n",
        "\n",
        "loss, accuracy = multi_cnn.evaluate(m_Xcnn_train, m_y_train, verbose=False)\n",
        "print(\"Training Accuracy: {:.4f}\".format(accuracy))\n",
        "\n",
        "\n",
        "print(\"Test Scores:\")\n",
        "preds = multi_cnn.predict(m_Xcnn_test)\n",
        "indices_p = np.argmax(preds, axis = 1)\n",
        "indices_p\n",
        "print(classification_report(test_multiclass_df.label,indices_p))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7L-MpR3jrkH3",
        "outputId": "8adb87ec-af93-404f-b501-7fb8741ab84b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_42\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding_42 (Embedding)    (None, 70, 50)            1052550   \n",
            "                                                                 \n",
            " conv1d_53 (Conv1D)          (None, 68, 256)           38656     \n",
            "                                                                 \n",
            " max_pooling1d_53 (MaxPoolin  (None, 34, 256)          0         \n",
            " g1D)                                                            \n",
            "                                                                 \n",
            " dropout_21 (Dropout)        (None, 34, 256)           0         \n",
            "                                                                 \n",
            " flatten_42 (Flatten)        (None, 8704)              0         \n",
            "                                                                 \n",
            " dense_84 (Dense)            (None, 10)                87050     \n",
            "                                                                 \n",
            " dense_85 (Dense)            (None, 5)                 55        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1,178,311\n",
            "Trainable params: 1,178,311\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Epoch 1/5\n",
            "507/507 [==============================] - 15s 28ms/step - loss: 1.6107 - accuracy: 0.1959 - val_loss: 1.6099 - val_accuracy: 0.2011\n",
            "Epoch 2/5\n",
            "507/507 [==============================] - 14s 27ms/step - loss: 1.6097 - accuracy: 0.1982 - val_loss: 1.6097 - val_accuracy: 0.1867\n",
            "Epoch 3/5\n",
            "507/507 [==============================] - 14s 27ms/step - loss: 1.6095 - accuracy: 0.2014 - val_loss: 1.6099 - val_accuracy: 0.1867\n",
            "Epoch 4/5\n",
            "507/507 [==============================] - 14s 27ms/step - loss: 1.6095 - accuracy: 0.2027 - val_loss: 1.6096 - val_accuracy: 0.1867\n",
            "Epoch 5/5\n",
            "507/507 [==============================] - 14s 27ms/step - loss: 1.6096 - accuracy: 0.1959 - val_loss: 1.6099 - val_accuracy: 0.1867\n",
            "Training Accuracy: 0.2027\n",
            "Test Scores:\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.00      0.00      0.00       393\n",
            "           1       0.19      1.00      0.32       380\n",
            "           2       0.00      0.00      0.00       407\n",
            "           3       0.00      0.00      0.00       379\n",
            "           4       0.00      0.00      0.00       441\n",
            "\n",
            "    accuracy                           0.19      2000\n",
            "   macro avg       0.04      0.20      0.06      2000\n",
            "weighted avg       0.04      0.19      0.06      2000\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "multi_cnn = Sequential()\n",
        "multi_cnn.add(Embedding(vocab_len, 50, input_length=70, weights=[embedding_matrix], trainable=True))\n",
        "\n",
        "multi_cnn.add(Conv1D(128, 5, activation='relu'))\n",
        "multi_cnn.add(MaxPooling1D())\n",
        "multi_cnn.add(Dropout(0.2))\n",
        "\n",
        "multi_cnn.add(Conv1D(64, 5, activation='relu'))\n",
        "multi_cnn.add(MaxPooling1D())\n",
        "multi_cnn.add(Dropout(0.2))\n",
        "\n",
        "multi_cnn.add(Conv1D(32, 5, activation='relu'))\n",
        "multi_cnn.add(MaxPooling1D())\n",
        "multi_cnn.add(Dropout(0.2))\n",
        "\n",
        "multi_cnn.add(Flatten())\n",
        "multi_cnn.add(Dense(10, activation='relu'))\n",
        "multi_cnn.add(Dense(5, activation='softmax')) #added 5 neurons for output layer using softmax\n",
        "multi_cnn.compile(optimizer='adam', loss='SparseCategoricalCrossentropy', metrics=['accuracy']) #changed loss\n",
        "multi_cnn.summary() \n",
        "\n",
        "\n",
        "# Train models and Evaluate them for both binary and multi-class\n",
        "multi_cnn.fit(m_Xcnn_train, m_y_train,\n",
        "                     epochs=5,\n",
        "                     verbose=True,\n",
        "                     validation_data=(m_Xcnn_val, m_y_val),\n",
        "                     batch_size=32)\n",
        "\n",
        "loss, accuracy = multi_cnn.evaluate(m_Xcnn_train, m_y_train, verbose=False)\n",
        "print(\"Training Accuracy: {:.4f}\".format(accuracy))\n",
        "\n",
        "print(\"Test Scores:\")\n",
        "preds = multi_cnn.predict(m_Xcnn_test)\n",
        "indices_p = np.argmax(preds, axis = 1)\n",
        "indices_p\n",
        "print(classification_report(test_multiclass_df.label,indices_p))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "brnj_balWxQQ",
        "outputId": "11dd0c46-9e06-41e3-caed-d38d7484f850"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_43\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding_43 (Embedding)    (None, 70, 50)            1052550   \n",
            "                                                                 \n",
            " conv1d_54 (Conv1D)          (None, 66, 128)           32128     \n",
            "                                                                 \n",
            " max_pooling1d_54 (MaxPoolin  (None, 33, 128)          0         \n",
            " g1D)                                                            \n",
            "                                                                 \n",
            " dropout_22 (Dropout)        (None, 33, 128)           0         \n",
            "                                                                 \n",
            " conv1d_55 (Conv1D)          (None, 29, 64)            41024     \n",
            "                                                                 \n",
            " max_pooling1d_55 (MaxPoolin  (None, 14, 64)           0         \n",
            " g1D)                                                            \n",
            "                                                                 \n",
            " dropout_23 (Dropout)        (None, 14, 64)            0         \n",
            "                                                                 \n",
            " conv1d_56 (Conv1D)          (None, 10, 32)            10272     \n",
            "                                                                 \n",
            " max_pooling1d_56 (MaxPoolin  (None, 5, 32)            0         \n",
            " g1D)                                                            \n",
            "                                                                 \n",
            " dropout_24 (Dropout)        (None, 5, 32)             0         \n",
            "                                                                 \n",
            " flatten_43 (Flatten)        (None, 160)               0         \n",
            "                                                                 \n",
            " dense_86 (Dense)            (None, 10)                1610      \n",
            "                                                                 \n",
            " dense_87 (Dense)            (None, 5)                 55        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1,137,639\n",
            "Trainable params: 1,137,639\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Epoch 1/5\n",
            "507/507 [==============================] - 16s 30ms/step - loss: 1.6017 - accuracy: 0.2391 - val_loss: 1.5648 - val_accuracy: 0.2639\n",
            "Epoch 2/5\n",
            "507/507 [==============================] - 15s 30ms/step - loss: 1.3673 - accuracy: 0.3609 - val_loss: 1.2404 - val_accuracy: 0.4300\n",
            "Epoch 3/5\n",
            "507/507 [==============================] - 15s 30ms/step - loss: 1.1755 - accuracy: 0.4519 - val_loss: 1.1965 - val_accuracy: 0.4467\n",
            "Epoch 4/5\n",
            "507/507 [==============================] - 15s 30ms/step - loss: 1.0768 - accuracy: 0.5028 - val_loss: 1.1814 - val_accuracy: 0.4633\n",
            "Epoch 5/5\n",
            "507/507 [==============================] - 15s 30ms/step - loss: 0.9813 - accuracy: 0.5555 - val_loss: 1.2425 - val_accuracy: 0.4694\n",
            "Training Accuracy: 0.6349\n",
            "Test Scores:\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.66      0.64      0.65       393\n",
            "           1       0.42      0.45      0.44       380\n",
            "           2       0.35      0.53      0.42       407\n",
            "           3       0.36      0.25      0.30       379\n",
            "           4       0.67      0.51      0.58       441\n",
            "\n",
            "    accuracy                           0.48      2000\n",
            "   macro avg       0.49      0.48      0.48      2000\n",
            "weighted avg       0.50      0.48      0.48      2000\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PCmvf-AEhxhr"
      },
      "outputs": [],
      "source": [
        "#REFERENCES:\n",
        "\n",
        "#https://www.kaggle.com/code/jagannathrk/word2vec-cnn-text-classification/notebook"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oCT-Zt3ZMocG"
      },
      "source": [
        "## My Report\n",
        "\n",
        "#***Naive Bayes***: \n",
        "\n",
        "*  Binary\n",
        "\n",
        "\n",
        "```\n",
        "Best parameters selected as:  \n",
        "\n",
        "{'vectorizer__min_df': 100, 'vectorizer__ngram_range': [1, 2]}\n",
        "\n",
        "[[682  91]\n",
        " [113 707]]\n",
        "\n",
        "f1 score 0.8739184177997528\n",
        "accuracy score 0.871939736346516\n",
        "\n",
        "```\n",
        "\n",
        "\n",
        "*  Multiclass\n",
        "\n",
        "\n",
        "```\n",
        "Best parameters selected as:  \n",
        "\n",
        "{'vectorizer__min_df': 100, 'vectorizer__ngram_range': [1, 2]}\n",
        "\n",
        "Confusion matrix: \n",
        "\n",
        "[[266  88  18   5  16]\n",
        " [102 156  72  23  27]\n",
        " [ 47  76 118  99  67]\n",
        " [ 22  31  60 136 130]\n",
        " [ 30  13  20  68 310]]\n",
        "\n",
        "\n",
        "f1 score 0.4772514205660536\n",
        "accuracy score 0.493\n",
        "\n",
        "```\n",
        "Comments on the findings:\n",
        "\n",
        "Confusion matrix gives a significant insight about our accuracy and f1 score: Multiclass classification result has been decreased significantly compared to the binary task due to the difficulty of discriminating the labels 3 and 4 as well as 0 and 1 from each other. Therefore, results do not imply that multiclass classification failed. After examining the results of hyperparameter combinations, there is approximately 0.1 difference for the binary task and around 0.05 difference for the multiclass task. All in all we obtained, 0.88 f1 score for the binary task given that the simple baseline would be 0.5 while we obtained 0.5 for the multiclass task given that simple baseline would be 0.20. \n",
        "\n",
        "\n",
        "\n",
        "#***Logistic regression:***\n",
        "\n",
        "*   Binary\n",
        "\n",
        "```\n",
        "Best parameters selected as:  \n",
        "\n",
        "{'lgc__l1_ratio': 0.5, 'vectorizer__min_df': 100, 'vectorizer__ngram_range': [1, 2]}\n",
        "\n",
        "[[714  59]\n",
        " [ 85 735]]\n",
        "\n",
        "f1 score 0.9107806691449815\n",
        "accuracy score 0.9096045197740112\n",
        "\n",
        "```\n",
        "\n",
        "\n",
        "*   Multiclass\n",
        "\n",
        "```\n",
        "Best parameters selected as: \n",
        "\n",
        "{'lgc__l1_ratio': 1.0, 'vectorizer__min_df': 100, 'vectorizer__ngram_range': [1, 3]}\n",
        "\n",
        "[[289  81  13   6   4]\n",
        " [ 89 179  78  22  12]\n",
        " [ 26  78 189  92  22]\n",
        " [ 14  27  73 169  96]\n",
        " [ 12  12  19 113 285]]\n",
        "\n",
        "f1 score 0.5512069432954926\n",
        "accuracy score 0.5555\n",
        "\n",
        "```\n",
        "\n",
        "Comments on the findings:\n",
        "\n",
        "We obtained much better results compared to naive bayes multiclass model. In detail, our f1 score and accuracy score has been increased around 0.08 with 0.55. There is a similar confusion matrix just like we had in naive bayes case. Model is very successful if we could merge 2 positive and 2 negative classes into 1 positive and 1 negative. Again, our model outperformed simple baselines by 0.40 and 0.35 for the binary task and multiclass task respectively.\n",
        "\n",
        "\n",
        "#***CNN:***\n",
        "\n",
        "In the cnn part, we have tried 3 different embedding strategies: Randomly initialized word embeddings, Word embeddings trained from scratch with gensim, Pretrained word embeddings from gensim.api. In order to do so:\n",
        "\n",
        "\n",
        "For the Randomly initialized word embeddings, we set embedding dimension, vocabulary size and input length parameters in the embedding layer of our network. So that, weights in the embedding matrix will be initialized randomly.\n",
        "\n",
        "\n",
        "For the Word embeddings trained from scratch with gensim, we  import our trained word2vec model using keyedvectors feature of gensim.models and form the embedding vector. Using the embedding vector, we create the embedding matrix which will be given as a parameter in the embedding layer of our network.\n",
        "\n",
        "\n",
        "For the Pretrained word embeddings from gensim.api, we  import our pre-trained api model using keyedvectors feature of gensim.models and form the embedding vector: Using the embedding vector, we create the embedding matrix which will be given as a parameter in the embedding layer of our network. \n",
        "\n",
        "\n",
        "*   Binary\n",
        "\n",
        "We prepared our dataset for cnn model by fitting our tokenizer to the binary dataset. Then, we transform each text in texts to a sequence of integers using texts_to_sequences method. We set a maxlen parameter after taking the average length of each text and we apply padding.   \n",
        "\n",
        "After preparing the dataset, we continue by building our model: \n",
        "\n",
        "For the output layer, we set our activation function as sigmoid since our task is binary classification and we want our output 0 or 1. Then, we set loss function binary_crossentropy for this task. \n",
        "\n",
        "```\n",
        "binary_model.add(Dense(1, activation='sigmoid'))\n",
        "binary_model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n",
        "```\n",
        "\n",
        "We have built 2 different models for each embedding strategy -> in total 6 models for the binary task. Using randomly initialized embedding layer, we obtained around 88 percent accuracy in the test dataset. Despite adding another convolutional layer and changing filter size, we do not observe a significant difference between trained models. Using word embeddings trained from scratch with gensim, we obtained around 89 percent accuracy in the test set which is slightly better than randomly initiliazed word embedding results. Last but not least, we obtained around 89 percent accuracy in the test dataset using pretrained word embeddings from gensim.api. However, we come across an interesting output for the first trained model: model was not able to decrease the loss, thus did not learn. Increasing the number of convolutional layer, kernel size and filter size definitely helped model. In a nutshell, we conlude that each models produced decent scores with the well-prepared models and parameters.  \n",
        "\n",
        "**Results:**\n",
        "\n",
        "```\n",
        "Randomly initiliazed Test Scores:\n",
        "Testing Accuracy :  0.8908 Testing F1:  0.8918 Testing precision:  0.8990 Testing recall:  0.8899\n",
        "\n",
        "\n",
        "Word embeddings trained from scratch Test Scores:\n",
        "Testing Accuracy :  0.8933 Testing F1:  0.8935 Testing precision:  0.9123 Testing recall:  0.8806\n",
        "\n",
        "\n",
        "Pre-trained API Test Scores:\n",
        "Testing Accuracy :  0.8719 Testing F1:  0.8690 Testing precision:  0.9125 Testing recall:  0.8332\n",
        "```\n",
        "\n",
        "\n",
        "\n",
        "*   Multiclass\n",
        "\n",
        "We prepared our dataset for cnn model by fitting our tokenizer to the multiclass dataset. Then, we transform each text in texts to a sequence of integers using texts_to_sequences method. We set a maxlen parameter after taking the average length of each text and we apply padding.   \n",
        "\n",
        "After preparing the dataset, we continue by building our model: \n",
        "\n",
        "Apart from a few differences, we built our model just like binary case. What differs in this case is:  the output layer. We use softmax activation function instead of sigmoid and we use 5 neurons in the output layer. Furthermore, we modified the loss function as sparse categorical crossentropy. \n",
        "\n",
        "```\n",
        "multi_cnn.add(Dense(5, activation='softmax')) #added 5 neurons for output layer using softmax\n",
        "multi_cnn.compile(optimizer='adam', loss='SparseCategoricalCrossentropy', metrics=['accuracy']) \n",
        "```\n",
        "We have built 2 different models for each embedding strategy -> in total 6 models for the multiclass task. Using randomly initialized embedding layer, we obtained around 50 percent accuracy in the test dataset. Despite adding another convolutional layer and changing filter size, we do not observe a significant difference between trained models. Using word embeddings trained from scratch with gensim, we obtained around 52 percent accuracy in the test set which is slightly better than randomly initiliazed word embedding results. Interestingly, first trained model did not end up learning and resulted around 20 percent accuracy in the test set. Probably, model complexity was not sufficient to handle multiclass classification task. Because, increasing the number of convolutional layer, kernel size and filter size definitely helped model. Last but not least, we obtained around 45 percent accuracy in the test dataset using pretrained word embeddings from gensim.api. However, we come across an interesting output for the first trained model: model was not able to decrease the loss, thus did not learn and end up 19 percent accuracy in the test set. Similarly, increasing the number of convolutional layer, kernel size and filter size definitely helped model. In a nutshell, we conlude that each models produced decent scores with the well-prepared models and parameters. Taking into account the baseline is 20 percent for a 5 class classification task, models did a good job. In terms of embedding layer effect, we can conclude that, using a pre-trained embedding layer is not good as much as training a model from scratch as expected.  \n",
        "\n",
        "\n",
        "\n",
        "**Results:**\n",
        "\n",
        "```\n",
        "Randomly initiliazed Test Scores:\n",
        "             precision    recall  f1-score   support\n",
        "\n",
        "           0       0.72      0.59      0.65       393\n",
        "           1       0.45      0.51      0.48       380\n",
        "           2       0.43      0.47      0.45       407\n",
        "           3       0.39      0.46      0.42       379\n",
        "           4       0.69      0.56      0.62       441\n",
        "\n",
        "    accuracy                           0.52      2000\n",
        "   macro avg       0.54      0.52      0.52      2000\n",
        "weighted avg       0.54      0.52      0.53      2000\n",
        "\n",
        "```\n",
        "\n",
        "\n",
        "\n",
        "```\n",
        "Word embeddings trained from scratch Test Scores:\n",
        "              precision    recall  f1-score   support\n",
        "\n",
        "           0       0.66      0.59      0.62       393\n",
        "           1       0.37      0.41      0.39       380\n",
        "           2       0.38      0.39      0.38       407\n",
        "           3       0.38      0.47      0.42       379\n",
        "           4       0.69      0.53      0.60       441\n",
        "\n",
        "    accuracy                           0.48      2000\n",
        "   macro avg       0.49      0.48      0.48      2000\n",
        "weighted avg       0.50      0.48      0.49      2000\n",
        "\n",
        "```\n",
        "\n",
        "\n",
        "\n",
        "```\n",
        "Pre-trained API Test Scores:\n",
        "              precision    recall  f1-score   support\n",
        "\n",
        "           0       0.66      0.64      0.65       393\n",
        "           1       0.42      0.45      0.44       380\n",
        "           2       0.35      0.53      0.42       407\n",
        "           3       0.36      0.25      0.30       379\n",
        "           4       0.67      0.51      0.58       441\n",
        "\n",
        "    accuracy                           0.48      2000\n",
        "   macro avg       0.49      0.48      0.48      2000\n",
        "weighted avg       0.50      0.48      0.48      2000\n",
        "\n",
        "```\n",
        "\n"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "name": "Project 1 Notebook.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}