project_classifier.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "project_classifier.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "mount_file_id": "11Gx4SSjoVQyT9HJF-CE_OL7Qh_GP21V9",
      "authorship_tag": "ABX9TyNh1IFvhw3yahwMdqOJC7lj",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jabanto22/NLP-Project/blob/main/project_classifier.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qq5q4_iAT9hr",
        "outputId": "239a674b-8190-4180-af31-a490fdd5c474"
      },
      "source": [
        "!pip install torch \n",
        "!pip install torchtext \n",
        "!pip install torchvision \n",
        "!pip install transformers\n",
        "!pip install tweet-preprocessor"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (1.8.1+cu101)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch) (3.7.4.3)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torch) (1.19.5)\n",
            "Requirement already satisfied: torchtext in /usr/local/lib/python3.7/dist-packages (0.9.1)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from torchtext) (4.41.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torchtext) (1.19.5)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from torchtext) (2.23.0)\n",
            "Requirement already satisfied: torch==1.8.1 in /usr/local/lib/python3.7/dist-packages (from torchtext) (1.8.1+cu101)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->torchtext) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->torchtext) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->torchtext) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->torchtext) (2020.12.5)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch==1.8.1->torchtext) (3.7.4.3)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.7/dist-packages (0.9.1+cu101)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torchvision) (1.19.5)\n",
            "Requirement already satisfied: torch==1.8.1 in /usr/local/lib/python3.7/dist-packages (from torchvision) (1.8.1+cu101)\n",
            "Requirement already satisfied: pillow>=4.1.1 in /usr/local/lib/python3.7/dist-packages (from torchvision) (7.1.2)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch==1.8.1->torchvision) (3.7.4.3)\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.5.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.44)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (3.10.1)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.10.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: tweet-preprocessor in /usr/local/lib/python3.7/dist-packages (0.6.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QCuVquksUmId"
      },
      "source": [
        "# Libraries\n",
        "import pandas as pd\n",
        "import preprocessor as p\n",
        "import numpy as np\n",
        "import random\n",
        "import copy\n",
        "import os\n",
        "import time, datetime\n",
        "import json\n",
        "import torch\n",
        "\n",
        "# Models\n",
        "import torch.nn as nn\n",
        "from transformers import AutoModel, AutoTokenizer, AdamW, get_linear_schedule_with_warmup\n",
        "\n",
        "# Training\n",
        "from torch.utils.data import DataLoader, Dataset\n",
        "from torch.cuda.amp import autocast, GradScaler\n",
        "from tqdm import tqdm\n",
        "\n",
        "# Evaluation\n",
        "from sklearn.metrics import classification_report, accuracy_score\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Global definitions\n",
        "source_folder = '/content/drive/MyDrive/Colab Notebooks/NLP_Project/project-data/'"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bHQeDATpVO4E"
      },
      "source": [
        "def read_data(filename):\n",
        "    \"\"\"\n",
        "    Read twitter datasets.\n",
        "    \"\"\"\n",
        "    data = pd.DataFrame()\n",
        "    with open(filename, 'r', encoding=\"utf8\") as f:\n",
        "        for line in f:\n",
        "            line = json.loads(line)\n",
        "            tweet_id = line[0][\"id_str\"]\n",
        "            tweet = p.clean(line[0][\"text\"])\n",
        "            comments = \"\"\n",
        "            for row in line:\n",
        "                # use tweet preprocessor to clean text\n",
        "                comments += \" \" + p.clean(row[\"text\"]) + \".\"\n",
        "            data = data.append({\"id\":tweet_id,\"text\":tweet,\"comments\":comments}, ignore_index=True)\n",
        "    f.close()\n",
        "\n",
        "    return data"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hj36xwPfVSk3"
      },
      "source": [
        "def read_label(filename):\n",
        "    \"\"\"\n",
        "    Read class labels.\n",
        "    \"\"\"\n",
        "    label = pd.DataFrame()\n",
        "\n",
        "    with open(filename, 'r', encoding=\"utf8\") as f:\n",
        "        label = pd.DataFrame.from_dict(json.load(f), orient=\"index\").reset_index()\n",
        "        label.columns = [\"id\", \"label\"]\n",
        "    f.close()\n",
        "\n",
        "    return label"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l7Kde4JpVV6D"
      },
      "source": [
        "def merge_data_label(data, label, extract_label=False):\n",
        "    \"\"\"\n",
        "    Merge train data with class labels and class label codes for prediction.\n",
        "    \"\"\"\n",
        "    data = pd.merge(data, label, on=\"id\", how=\"outer\")\n",
        "    data.label = pd.Categorical(data.label)\n",
        "    class_labels = dict(enumerate(data.label.cat.categories))\n",
        "    data['label'] = data.label.cat.codes\n",
        "\n",
        "    # write predicted labels to json file\n",
        "    with open(source_folder + 'labels.json', 'w') as f:\n",
        "        json.dump(class_labels, f, separators=(',', ':'))\n",
        "    f.close()\n",
        "\n",
        "    return data     "
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yyfa5WUsVY-v"
      },
      "source": [
        "def extract_class_labels():\n",
        "    # read class labels from json file\n",
        "    label = pd.DataFrame()\n",
        "    with open(source_folder + 'labels.json', 'r', encoding=\"utf8\") as f:\n",
        "        label = json.load(f)\n",
        "    f.close()\n",
        "    return label"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8P34cEC5Vd2L"
      },
      "source": [
        "def save_data_to_csv():\n",
        "    \"\"\"\n",
        "    Read and extract datasets from files.\n",
        "    \"\"\"\n",
        "    # read data (jsonl files)\n",
        "    train_data = read_data(source_folder + 'train.data.jsonl')\n",
        "    dev_data = read_data(source_folder + 'dev.data.jsonl')\n",
        "    test_data = read_data(source_folder + 'test.data.jsonl')\n",
        "    covid_data = read_data(source_folder + 'covid.data.jsonl')\n",
        "\n",
        "    # read labels (json files)\n",
        "    train_label = read_label(source_folder + 'train.label.json')\n",
        "    dev_label = read_label(source_folder + 'dev.label.json')\n",
        "\n",
        "    # merge data with class labels\n",
        "    train_data = merge_data_label(train_data, train_label)\n",
        "    dev_data = merge_data_label(dev_data, dev_label)\n",
        "\n",
        "    # write filetered data to csv\n",
        "    open(source_folder + 'train.csv','w', newline='').write(train_data.to_csv(index=False))\n",
        "    open(source_folder + 'dev.csv','w', newline='').write(dev_data.to_csv(index=False))\n",
        "    open(source_folder + 'test.csv','w', newline='').write(test_data.to_csv(index=False))\n",
        "    open(source_folder + 'covid.csv','w', newline='').write(covid_data.to_csv(index=False))\n",
        "\n",
        "\n",
        "def check_input_files(filename):\n",
        "    \"\"\"\n",
        "    Check input files if they exist.\n",
        "    \"\"\"\n",
        "    try:\n",
        "        f = open(filename,'r')\n",
        "        f.close()\n",
        "    except:\n",
        "        save_data_to_csv()\n",
        "\n",
        "\n",
        "def read_csv_datasets():\n",
        "    # check if input files exist\n",
        "    check_input_files(source_folder + 'train.csv')\n",
        "    check_input_files(source_folder + 'dev.csv')\n",
        "    check_input_files(source_folder + 'test.csv')\n",
        "    check_input_files(source_folder + 'covid.csv')\n",
        "\n",
        "    # read datasets\n",
        "    train_df = pd.read_csv(source_folder + 'train.csv')\n",
        "    dev_df = pd.read_csv(source_folder + 'dev.csv')\n",
        "    test_df = pd.read_csv(source_folder + 'test.csv')\n",
        "\n",
        "    return train_df, dev_df, test_df"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "demyfc8jVmIe"
      },
      "source": [
        "class CustomDataset(Dataset):\n",
        "\n",
        "    def __init__(self, data, maxlen, with_labels=True, bert_model='bert-base-uncased'):\n",
        "\n",
        "        self.data = data  # pandas dataframe\n",
        "        #Initialize the tokenizer\n",
        "        self.tokenizer = AutoTokenizer.from_pretrained(bert_model)  \n",
        "\n",
        "        self.maxlen = maxlen\n",
        "        self.with_labels = with_labels \n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.data)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "\n",
        "        # Selecting sentence1 and sentence2 at the specified index in the data frame\n",
        "        sent1 = str(self.data.loc[index, 'text'])\n",
        "        sent2 = str(self.data.loc[index, 'comments'])\n",
        "\n",
        "        # Tokenize the pair of sentences to get token ids, attention masks and token type ids\n",
        "        encoded_pair = self.tokenizer(sent1, sent2, \n",
        "                                      padding='max_length',  # Pad to max_length\n",
        "                                      truncation=True,  # Truncate to max_length\n",
        "                                      max_length=self.maxlen,  \n",
        "                                      return_tensors='pt')  # Return torch.Tensor objects\n",
        "        \n",
        "        token_ids = encoded_pair['input_ids'].squeeze(0)  # tensor of token ids\n",
        "        attn_masks = encoded_pair['attention_mask'].squeeze(0)  # binary tensor with \"0\" for padded values and \"1\" for the other values\n",
        "        token_type_ids = encoded_pair['token_type_ids'].squeeze(0)  # binary tensor with \"0\" for the 1st sentence tokens & \"1\" for the 2nd sentence tokens\n",
        "\n",
        "        if self.with_labels:  # True if the dataset has labels\n",
        "            label = self.data.loc[index, 'label']\n",
        "            return token_ids, attn_masks, token_type_ids, label  \n",
        "        else:\n",
        "            return token_ids, attn_masks, token_type_ids"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gogTJTYXVpE7"
      },
      "source": [
        "class SentencePairClassifier(nn.Module):\n",
        "\n",
        "    def __init__(self, bert_model=\"bert-base-uncased\", freeze_bert=False):\n",
        "        super(SentencePairClassifier, self).__init__()\n",
        "        #  Instantiating BERT-based model object\n",
        "        self.bert_layer = AutoModel.from_pretrained(bert_model)\n",
        "\n",
        "        #  Fix the hidden-state size of the encoder outputs (If you want to add other pre-trained models here, search for the encoder output size)\n",
        "        if bert_model == \"albert-base-v2\":  # 12M parameters\n",
        "            hidden_size = 768\n",
        "        elif bert_model == \"albert-large-v2\":  # 18M parameters\n",
        "            hidden_size = 1024\n",
        "        elif bert_model == \"albert-xlarge-v2\":  # 60M parameters\n",
        "            hidden_size = 2048\n",
        "        elif bert_model == \"albert-xxlarge-v2\":  # 235M parameters\n",
        "            hidden_size = 4096\n",
        "        elif bert_model == \"bert-base-uncased\": # 110M parameters\n",
        "            hidden_size = 768\n",
        "\n",
        "        # Freeze bert layers and only train the classification layer weights\n",
        "        if freeze_bert:\n",
        "            for p in self.bert_layer.parameters():\n",
        "                p.requires_grad = False\n",
        "\n",
        "        # Classification layer\n",
        "        self.cls_layer = nn.Linear(hidden_size, 1)\n",
        "\n",
        "        self.dropout = nn.Dropout(p=0.1)\n",
        "\n",
        "    @autocast()  # run in mixed precision\n",
        "    def forward(self, input_ids, attn_masks, token_type_ids):\n",
        "        '''\n",
        "        Inputs:\n",
        "            -input_ids : Tensor  containing token ids\n",
        "            -attn_masks : Tensor containing attention masks to be used to focus on non-padded values\n",
        "            -token_type_ids : Tensor containing token type ids to be used to identify sentence1 and sentence2\n",
        "        '''\n",
        "\n",
        "        # Feeding the inputs to the BERT-based model to obtain contextualized representations\n",
        "        output = self.bert_layer(input_ids, attn_masks, token_type_ids)\n",
        "        \n",
        "        # Feeding to the classifier layer the last layer hidden-state of the [CLS] token further processed by a\n",
        "        # Linear Layer and a Tanh activation. The Linear layer weights were trained from the sentence order prediction (ALBERT) or next sentence prediction (BERT)\n",
        "        # objective during pre-training.\n",
        "        logits = self.cls_layer(self.dropout(output['pooler_output']))\n",
        "\n",
        "        return logits"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8lbH3xznVsD0"
      },
      "source": [
        "def set_seed(seed):\n",
        "    \"\"\" Set all seeds to make results reproducible \"\"\"\n",
        "    torch.manual_seed(seed)\n",
        "    torch.cuda.manual_seed_all(seed)\n",
        "    torch.backends.cudnn.deterministic = True\n",
        "    torch.backends.cudnn.benchmark = False\n",
        "    np.random.seed(seed)\n",
        "    random.seed(seed)\n",
        "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
        "\n",
        "\n",
        "def evaluate_loss(net, device, criterion, dataloader):\n",
        "    net.eval()\n",
        "\n",
        "    mean_loss = 0\n",
        "    count = 0\n",
        "\n",
        "    with torch.no_grad():\n",
        "        for it, (seq, attn_masks, token_type_ids, labels) in enumerate(tqdm(dataloader)):\n",
        "            seq, attn_masks, token_type_ids, labels = \\\n",
        "                seq.to(device), attn_masks.to(device), token_type_ids.to(device), labels.to(device)\n",
        "            logits = net(seq, attn_masks, token_type_ids)\n",
        "            mean_loss += criterion(logits.squeeze(-1), labels.float()).item()\n",
        "            count += 1\n",
        "\n",
        "    return mean_loss / count"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e80rkr_BVs92"
      },
      "source": [
        "def train_bert(net, criterion, opti, lr, lr_scheduler, train_loader, val_loader, epochs, iters_to_accumulate):\n",
        "\n",
        "    best_loss = np.Inf\n",
        "    best_ep = 1\n",
        "    nb_iterations = len(train_loader)\n",
        "    print_every = nb_iterations // 5  # print the training loss 5 times per epoch\n",
        "    train_losses = []\n",
        "    val_losses = []\n",
        "\n",
        "    scaler = GradScaler()\n",
        "\n",
        "    for ep in range(epochs):\n",
        "\n",
        "        net.train()\n",
        "        running_loss = 0.0\n",
        "        for it, (seq, attn_masks, token_type_ids, labels) in enumerate(tqdm(train_loader)):\n",
        "\n",
        "            # Converting to cuda tensors\n",
        "            seq, attn_masks, token_type_ids, labels = \\\n",
        "                seq.to(device), attn_masks.to(device), token_type_ids.to(device), labels.to(device)\n",
        "    \n",
        "            # Enables autocasting for the forward pass (model + loss)\n",
        "            with autocast():\n",
        "                # Obtaining the logits from the model\n",
        "                logits = net(seq, attn_masks, token_type_ids)\n",
        "\n",
        "                # Computing loss\n",
        "                loss = criterion(logits.squeeze(-1), labels.float())\n",
        "                loss = loss / iters_to_accumulate  # Normalize the loss because it is averaged\n",
        "\n",
        "            # Backpropagating the gradients\n",
        "            # Scales loss.  Calls backward() on scaled loss to create scaled gradients.\n",
        "            scaler.scale(loss).backward()\n",
        "\n",
        "            if (it + 1) % iters_to_accumulate == 0:\n",
        "                # Optimization step\n",
        "                # scaler.step() first unscales the gradients of the optimizer's assigned params.\n",
        "                # If these gradients do not contain infs or NaNs, opti.step() is then called,\n",
        "                # otherwise, opti.step() is skipped.\n",
        "                scaler.step(opti)\n",
        "                # Updates the scale for next iteration.\n",
        "                scaler.update()\n",
        "                # Adjust the learning rate based on the number of iterations.\n",
        "                lr_scheduler.step()\n",
        "                # Clear gradients\n",
        "                opti.zero_grad()\n",
        "\n",
        "            running_loss += loss.item()\n",
        "            train_losses.append(loss)\n",
        "        \n",
        "            if (it + 1) % print_every == 0:  # Print training loss information\n",
        "                print()\n",
        "                print(\"Iteration {}/{} of epoch {} complete. Loss : {} \"\n",
        "                      .format(it+1, nb_iterations, ep+1, running_loss / print_every))\n",
        "\n",
        "                running_loss = 0.0\n",
        "\n",
        "\n",
        "        val_loss = evaluate_loss(net, device, criterion, val_loader)  # Compute validation loss\n",
        "        val_losses.append(val_loss)\n",
        "        print()\n",
        "        print(\"Epoch {} complete! Validation Loss : {}\".format(ep+1, val_loss))\n",
        "\n",
        "        if val_loss < best_loss:\n",
        "            print(\"Best validation loss improved from {} to {}\".format(best_loss, val_loss))\n",
        "            print()\n",
        "            net_copy = copy.deepcopy(net)  # save a copy of the model\n",
        "            best_loss = val_loss\n",
        "            best_ep = ep + 1\n",
        "\n",
        "    # Saving the model\n",
        "    path_to_model='/content/drive/MyDrive/Colab Notebooks/NLP_Project/models/{}_lr_{}_val_loss_{}_ep_{}.pt'.format(bert_model, lr, round(best_loss, 5), best_ep)\n",
        "    torch.save(net_copy.state_dict(), path_to_model)\n",
        "    print(\"The model has been saved in {}\".format(path_to_model))\n",
        "\n",
        "    del loss\n",
        "    torch.cuda.empty_cache()"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YyJFIE6hVy3J"
      },
      "source": [
        "def get_probs_from_logits(logits):\n",
        "    \"\"\"\n",
        "    Converts a tensor of logits into an array of probabilities by applying the sigmoid function\n",
        "    \"\"\"\n",
        "    probs = torch.sigmoid(logits.unsqueeze(-1))\n",
        "\n",
        "    return probs.detach().cpu().numpy()\n",
        "\n",
        "\n",
        "def test_prediction(net, device, dataloader, with_labels=True, result_file=\"/content/drive/MyDrive/Colab Notebooks/NLP_Project/results/output.txt\"):\n",
        "    \"\"\"\n",
        "    Predict the probabilities on a dataset with or without labels and print the result in a file\n",
        "    \"\"\"\n",
        "    net.eval()\n",
        "    w = open(result_file, 'w')\n",
        "    probs_all = []\n",
        "\n",
        "    with torch.no_grad():\n",
        "        if with_labels:\n",
        "            for seq, attn_masks, token_type_ids, _ in tqdm(dataloader):\n",
        "                seq, attn_masks, token_type_ids = seq.to(device), attn_masks.to(device), token_type_ids.to(device)\n",
        "                logits = net(seq, attn_masks, token_type_ids)\n",
        "                probs = get_probs_from_logits(logits.squeeze(-1)).squeeze(-1)\n",
        "                probs_all += probs.tolist()\n",
        "        else:\n",
        "            for seq, attn_masks, token_type_ids in tqdm(dataloader):\n",
        "                seq, attn_masks, token_type_ids = seq.to(device), attn_masks.to(device), token_type_ids.to(device)\n",
        "                logits = net(seq, attn_masks, token_type_ids)\n",
        "                probs = get_probs_from_logits(logits.squeeze(-1)).squeeze(-1)\n",
        "                probs_all += probs.tolist()\n",
        "\n",
        "    w.writelines(str(prob)+'\\n' for prob in probs_all)\n",
        "    w.close()"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VaShlP2HV044",
        "outputId": "01fed2dd-3faa-4cb2-e936-470b47799cb4"
      },
      "source": [
        "model = '/content/drive/MyDrive/Colab Notebooks/NLP_Project/models'\n",
        "try:\n",
        "    os.makedirs(model)\n",
        "    print(\"Directory:\", model, \"created.\")\n",
        "except:\n",
        "    print(\"Directory:\", model, \"already exists.\")"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Directory: /content/drive/MyDrive/Colab Notebooks/NLP_Project/models already exists.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fzj-uZdfV-pa"
      },
      "source": [
        "# retrieve train, dev, and test datasets\n",
        "train_df, dev_df, test_df = read_csv_datasets()"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3KtPcqg8WFoY"
      },
      "source": [
        "# training parameters\n",
        "bert_model = \"bert-base-uncased\"  # 'albert-base-v2', 'albert-large-v2', 'albert-xlarge-v2', 'albert-xxlarge-v2', 'bert-base-uncased', ...\n",
        "freeze_bert = False  # if True, freeze the encoder weights and only update the classification layer weights\n",
        "maxlen = 64  # maximum length of the tokenized input sentence pair : if greater than \"maxlen\", the input is truncated and else if smaller, the input is padded\n",
        "bs = 16  # batch size\n",
        "iters_to_accumulate = 2  # the gradient accumulation adds gradients over an effective batch of size : bs * iters_to_accumulate. If set to \"1\", you get the usual batch size\n",
        "lr = 2e-5  # learning rate\n",
        "epochs = 4  # number of training epochs"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gryuyn2ZWLQc"
      },
      "source": [
        "#  Set all seeds to make reproducible results\n",
        "set_seed(1)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7MdWWM6kWPUB",
        "outputId": "785440cc-3569-46e2-a6ed-bc6991bbd3ca"
      },
      "source": [
        "# Creating instances of training and validation set\n",
        "print(\"Reading training data...\")\n",
        "train_set = CustomDataset(train_df, maxlen, bert_model)\n",
        "print(\"Reading validation data...\")\n",
        "val_set = CustomDataset(dev_df, maxlen, bert_model)\n",
        "\n",
        "# Creating instances of training and validation dataloaders\n",
        "train_loader = DataLoader(train_set, batch_size=bs, num_workers=5)\n",
        "val_loader = DataLoader(val_set, batch_size=bs, num_workers=5)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading training data...\n",
            "Reading validation data...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:477: UserWarning: This DataLoader will create 5 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  cpuset_checked))\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Dm4w9EouWTwO",
        "outputId": "79b9e076-16d3-43a1-c544-99ae32702ed8"
      },
      "source": [
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "net = SentencePairClassifier(bert_model, freeze_bert=freeze_bert)\n",
        "\n",
        "if torch.cuda.device_count() > 1:  # if multiple GPUs\n",
        "    print(\"Let's use\", torch.cuda.device_count(), \"GPUs!\")\n",
        "    net = nn.DataParallel(net)\n",
        "\n",
        "net.to(device)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "SentencePairClassifier(\n",
              "  (bert_layer): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (pooler): BertPooler(\n",
              "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "      (activation): Tanh()\n",
              "    )\n",
              "  )\n",
              "  (cls_layer): Linear(in_features=768, out_features=1, bias=True)\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cpSH_r3pWZUQ",
        "outputId": "24e33316-2c6b-44bc-ae72-d829c53d2aa1"
      },
      "source": [
        "# model parameters\n",
        "criterion = nn.BCEWithLogitsLoss()\n",
        "opti = AdamW(net.parameters(), lr=lr, weight_decay=1e-2)\n",
        "num_warmup_steps = 0 # The number of steps for the warmup phase.\n",
        "num_training_steps = epochs * len(train_loader)  # The total number of training steps\n",
        "t_total = (len(train_loader) // iters_to_accumulate) * epochs  # Necessary to take into account Gradient accumulation\n",
        "lr_scheduler = get_linear_schedule_with_warmup(optimizer=opti, num_warmup_steps=num_warmup_steps, num_training_steps=t_total)\n",
        "\n",
        "train_bert(net, criterion, opti, lr, lr_scheduler, train_loader, val_loader, epochs, iters_to_accumulate)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r  0%|          | 0/291 [00:00<?, ?it/s]/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:477: UserWarning: This DataLoader will create 5 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  cpuset_checked))\n",
            " 20%|█▉        | 58/291 [00:05<00:20, 11.58it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Iteration 58/291 of epoch 1 complete. Loss : 0.31356283842489635 \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 41%|████      | 118/291 [00:10<00:15, 10.82it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Iteration 116/291 of epoch 1 complete. Loss : 0.24180220208805184 \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 60%|██████    | 176/291 [00:15<00:09, 11.52it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Iteration 174/291 of epoch 1 complete. Loss : 0.209068580819615 \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 80%|████████  | 234/291 [00:20<00:05, 11.40it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Iteration 232/291 of epoch 1 complete. Loss : 0.19216701844385986 \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 291/291 [00:26<00:00, 11.19it/s]\n",
            "  0%|          | 0/37 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Iteration 290/291 of epoch 1 complete. Loss : 0.1867364129886545 \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 37/37 [00:01<00:00, 24.16it/s]\n",
            "  0%|          | 0/291 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch 1 complete! Validation Loss : 0.32860503969965754\n",
            "Best validation loss improved from inf to 0.32860503969965754\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 21%|██        | 60/291 [00:05<00:20, 11.41it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Iteration 58/291 of epoch 2 complete. Loss : 0.181085374848596 \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 41%|████      | 118/291 [00:10<00:15, 11.39it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Iteration 116/291 of epoch 2 complete. Loss : 0.14206970499121938 \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 60%|██████    | 176/291 [00:15<00:10, 11.29it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Iteration 174/291 of epoch 2 complete. Loss : 0.13594611580001897 \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 80%|████████  | 234/291 [00:20<00:05, 11.19it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Iteration 232/291 of epoch 2 complete. Loss : 0.1376969989324952 \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 291/291 [00:26<00:00, 11.15it/s]\n",
            "  0%|          | 0/37 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Iteration 290/291 of epoch 2 complete. Loss : 0.1201564109852088 \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 37/37 [00:01<00:00, 23.72it/s]\n",
            "  0%|          | 0/291 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch 2 complete! Validation Loss : 0.2985529512972445\n",
            "Best validation loss improved from 0.32860503969965754 to 0.2985529512972445\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 21%|██        | 60/291 [00:05<00:20, 11.26it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Iteration 58/291 of epoch 3 complete. Loss : 0.12133852931959875 \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 41%|████      | 118/291 [00:10<00:15, 11.15it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Iteration 116/291 of epoch 3 complete. Loss : 0.09134575413090401 \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 60%|██████    | 176/291 [00:16<00:10, 11.20it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Iteration 174/291 of epoch 3 complete. Loss : 0.07610811721854682 \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 80%|████████  | 234/291 [00:21<00:05, 11.22it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Iteration 232/291 of epoch 3 complete. Loss : 0.09529936355378094 \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 291/291 [00:26<00:00, 11.05it/s]\n",
            "  0%|          | 0/37 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Iteration 290/291 of epoch 3 complete. Loss : 0.08517805183434794 \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 37/37 [00:01<00:00, 23.98it/s]\n",
            "  0%|          | 0/291 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch 3 complete! Validation Loss : 0.311886233252448\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 21%|██        | 60/291 [00:05<00:20, 11.16it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Iteration 58/291 of epoch 4 complete. Loss : 0.07865392224028192 \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 40%|███▉      | 116/291 [00:10<00:16, 10.89it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Iteration 116/291 of epoch 4 complete. Loss : 0.059146523202673114 \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 60%|██████    | 176/291 [00:16<00:10, 11.08it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Iteration 174/291 of epoch 4 complete. Loss : 0.04629435878375481 \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 80%|████████  | 234/291 [00:21<00:05, 11.14it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Iteration 232/291 of epoch 4 complete. Loss : 0.05762256585575383 \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 291/291 [00:26<00:00, 10.94it/s]\n",
            "  0%|          | 0/37 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Iteration 290/291 of epoch 4 complete. Loss : 0.05528651693322022 \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 37/37 [00:01<00:00, 24.38it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch 4 complete! Validation Loss : 0.3257690700324806\n",
            "The model has been saved in /content/drive/MyDrive/Colab Notebooks/NLP_Project/models/bert-base-uncased_lr_2e-05_val_loss_0.29855_ep_2.pt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KEM1VEwrbIMM",
        "outputId": "f05d6a6d-730c-469c-e450-a64bd718e16c"
      },
      "source": [
        "results = '/content/drive/MyDrive/Colab Notebooks/NLP_Project/results'\n",
        "try:\n",
        "    os.makedirs(results)\n",
        "    print(\"Directory:\", results, \"created.\")\n",
        "except:\n",
        "    print(\"Directory:\", results, \"already exists.\")"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Directory: /content/drive/MyDrive/Colab Notebooks/NLP_Project/results already exists.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T2RDJ17Bbg_P",
        "outputId": "f46c3d98-92c7-40e1-9377-490599b43223"
      },
      "source": [
        "path_to_model = '/content/drive/MyDrive/Colab Notebooks/NLP_Project/models/bert-base-uncased_lr_2e-05_val_loss_0.29855_ep_2.pt'  \n",
        "\n",
        "path_to_output_file = '/content/drive/MyDrive/Colab Notebooks/NLP_Project/results/output.txt'\n",
        "\n",
        "print(\"Reading test data...\")\n",
        "test_set = CustomDataset(test_df, maxlen, False, bert_model)\n",
        "test_loader = DataLoader(test_set, batch_size=bs, num_workers=5)\n",
        "\n",
        "model = SentencePairClassifier(bert_model)\n",
        "if torch.cuda.device_count() > 1:  # if multiple GPUs\n",
        "    print(\"Let's use\", torch.cuda.device_count(), \"GPUs!\")\n",
        "    model = nn.DataParallel(model)\n",
        "\n",
        "print()\n",
        "print(\"Loading the weights of the model...\")\n",
        "model.load_state_dict(torch.load(path_to_model))\n",
        "model.to(device)\n",
        "\n",
        "print(\"Predicting on test data...\")\n",
        "test_prediction(net=model, device=device, dataloader=test_loader, with_labels=False,\n",
        "                result_file=path_to_output_file)\n",
        "print()\n",
        "print(\"Predictions are available in : {}\".format(path_to_output_file))"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading test data...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:477: UserWarning: This DataLoader will create 5 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  cpuset_checked))\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Loading the weights of the model...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r  0%|          | 0/37 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Predicting on test data...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 37/37 [00:01<00:00, 22.90it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Predictions are available in : /content/drive/MyDrive/Colab Notebooks/NLP_Project/results/output.txt\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NCuBnyqJd6YN"
      },
      "source": [
        "def extract_class_labels():\n",
        "    # read class labels from json file\n",
        "    label = pd.DataFrame()\n",
        "    with open(source_folder + 'labels.json', 'r', encoding=\"utf8\") as f:\n",
        "        label = json.load(f)\n",
        "    f.close()\n",
        "    return label\n",
        "\n",
        "labels = extract_class_labels()"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wx69HtGedodr",
        "outputId": "5acfaf30-93ce-410d-d720-919e1c9bfe99"
      },
      "source": [
        "probs_test = pd.read_csv(path_to_output_file, header=None)[0]  # prediction probabilities\n",
        "threshold = 0.5   # you can adjust this threshold for your own dataset\n",
        "preds_test=(probs_test>=threshold).astype('uint8') # predicted labels using the above fixed threshold\n",
        "\n",
        "preds_test\n",
        "\n",
        "pred_label = {}\n",
        "for i in range(len(preds_test)):\n",
        "    code = str(preds_test[i])\n",
        "    text_id = str(test_df.iloc[i]['id'])\n",
        "    pred_label[text_id] = labels[code]\n",
        "\n",
        "# write predicted labels to json file\n",
        "path_to_results = source_folder + 'test-output.json'\n",
        "with open(source_folder + 'test-output.json', 'w') as f:\n",
        "    json.dump(pred_label, f, separators=(',', ':'))\n",
        "f.close()\n",
        "\n",
        "print(\"Predictions are available in : {}\".format(path_to_results))"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Predictions are available in : /content/drive/MyDrive/Colab Notebooks/NLP_Project/project-data/test-output.json\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}