3_XLNet_Model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XLNET \n",
    "\n",
    "Paper - https://arxiv.org/abs/1906.08237\n",
    "\n",
    "Transformer XL - https://arxiv.org/abs/1901.02860\n",
    "\n",
    "### Permutation Language Modeling\n",
    "\n",
    "\n",
    "![](https://i0.wp.com/mlexplained.com/wp-content/uploads/2019/06/Screen-Shot-2019-06-22-at-5.38.12-PM.png?w=1680)\n",
    "[Image Courtesy](https://mlexplained.com/2019/06/30/paper-dissected-xlnet-generalized-autoregressive-pretraining-for-language-understanding-explained/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I1106 05:39:07.322217 139812953790208 file_utils.py:39] PyTorch version 1.1.0 available.\n",
      "I1106 05:39:07.354457 139812953790208 modeling_xlnet.py:194] Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex .\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"  \n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"1\"\n",
    "\n",
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "from sklearn import model_selection\n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch.utils import data\n",
    "from transformers import (WEIGHTS_NAME, \n",
    "                          BertConfig, BertForSequenceClassification, BertTokenizer,\n",
    "                          RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer,\n",
    "                          XLMConfig, XLMForSequenceClassification, XLMTokenizer, \n",
    "                          XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer,\n",
    "                          DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer)\n",
    "from transformers import AdamW, WarmupLinearSchedule\n",
    "from transformers import AutoModelForSequenceClassification, AutoTokenizer\n",
    "\n",
    "MODEL_CLASSES = {\n",
    "    'bert': (BertConfig, BertForSequenceClassification, BertTokenizer),\n",
    "    'xlnet': (XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer),\n",
    "    'xlm': (XLMConfig, XLMForSequenceClassification, XLMTokenizer),\n",
    "    'roberta': (RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer),\n",
    "    'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer)\n",
    "}\n",
    "\n",
    "df = pd.read_csv(\"../input/ClothingReviews.csv\")\n",
    "train_df, test_df = model_selection.train_test_split(df, test_size=0.3, random_state=2019)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Custom Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_text_to_features(examples, tokenizer,\n",
    "                                      max_length=512,\n",
    "                                      pad_on_left=False,\n",
    "                                      pad_token=0,\n",
    "                                      pad_token_segment_id=4,\n",
    "                                      mask_padding_with_zero=True):\n",
    "    \"\"\"\n",
    "    Loads a data file into a list of ``InputFeatures``\n",
    "    Args:\n",
    "        examples: List of ``InputExamples`` or ``tf.data.Dataset`` containing the examples.\n",
    "        tokenizer: Instance of a tokenizer that will tokenize the examples\n",
    "        max_length: Maximum example length\n",
    "        task: GLUE task\n",
    "        label_list: List of labels. Can be obtained from the processor using the ``processor.get_labels()`` method\n",
    "        output_mode: String indicating the output mode. Either ``regression`` or ``classification``\n",
    "        pad_on_left: If set to ``True``, the examples will be padded on the left rather than on the right (default)\n",
    "        pad_token: Padding token\n",
    "        pad_token_segment_id: The segment ID for the padding token (It is usually 0, but can vary such as for XLNet where it is 4)\n",
    "        mask_padding_with_zero: If set to ``True``, the attention mask will be filled by ``1`` for actual values\n",
    "            and by ``0`` for padded values. If set to ``False``, inverts it (``1`` for padded values, ``0`` for\n",
    "            actual values)\n",
    "    Returns:\n",
    "        If the ``examples`` input is a ``tf.data.Dataset``, will return a ``tf.data.Dataset``\n",
    "        containing the task-specific features. If the input is a list of ``InputExamples``, will return\n",
    "        a list of task-specific ``InputFeatures`` which can be fed to the model.\n",
    "    \"\"\"\n",
    "    features = [[],[],[]]\n",
    "    for (ex_index, example) in enumerate(examples):\n",
    "#         if ex_index % 10000 == 0:\n",
    "#             print(\"Writing example %d\" % (ex_index))\n",
    "\n",
    "        inputs = tokenizer.encode_plus(\n",
    "            example,\n",
    "            None,\n",
    "            add_special_tokens=True,\n",
    "            max_length=max_length,\n",
    "        )\n",
    "        input_ids, token_type_ids = inputs[\"input_ids\"], inputs[\"token_type_ids\"]\n",
    "\n",
    "        # The mask has 1 for real tokens and 0 for padding tokens. Only real\n",
    "        # tokens are attended to.\n",
    "        attention_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)\n",
    "\n",
    "        # Zero-pad up to the sequence length.\n",
    "        padding_length = max_length - len(input_ids)\n",
    "        if pad_on_left:\n",
    "            input_ids = ([pad_token] * padding_length) + input_ids\n",
    "            attention_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + attention_mask\n",
    "            token_type_ids = ([pad_token_segment_id] * padding_length) + token_type_ids\n",
    "        else:\n",
    "            input_ids = input_ids + ([pad_token] * padding_length)\n",
    "            attention_mask = attention_mask + ([0 if mask_padding_with_zero else 1] * padding_length)\n",
    "            token_type_ids = token_type_ids + ([pad_token_segment_id] * padding_length)\n",
    "\n",
    "        assert len(input_ids) == max_length, \"Error with input length {} vs {}\".format(len(input_ids), max_length)\n",
    "        assert len(attention_mask) == max_length, \"Error with input length {} vs {}\".format(len(attention_mask), max_length)\n",
    "        assert len(token_type_ids) == max_length, \"Error with input length {} vs {}\".format(len(token_type_ids), max_length)\n",
    "\n",
    "#         if ex_index < 1:\n",
    "#             print(\"*** Example ***\")\n",
    "#             print(\"input_ids: %s\" % \" \".join([str(x) for x in input_ids]))\n",
    "#             print(\"attention_mask: %s\" % \" \".join([str(x) for x in attention_mask]))\n",
    "#             print(\"token_type_ids: %s\" % \" \".join([str(x) for x in token_type_ids]))\n",
    "\n",
    "        features[0].append(input_ids)\n",
    "        features[1].append(attention_mask)\n",
    "        features[2].append(token_type_ids)\n",
    "\n",
    "    return features\n",
    "\n",
    "def seed_everything(seed=123):\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    if torch.cuda.is_available():\n",
    "        torch.cuda.manual_seed(seed)\n",
    "        torch.backends.cudnn.deterministic = True\n",
    "\n",
    "def sigmoid(x):\n",
    "    return 1 / (1 + np.exp(-x))\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BERT Model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I1106 05:39:07.972162 139812953790208 configuration_utils.py:151] loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/xlnet-base-cased-config.json from cache at /home/srk/.cache/torch/transformers/c9cc6e53904f7f3679a31ec4af244f4419e25ebc8e71ebf8c558a31cbcf07fc8.ef1824921bc0786e97dc88d55eb17aabf18aac90f24bd34c0650529e7ba27d6f\n",
      "I1106 05:39:07.975797 139812953790208 configuration_utils.py:168] Model config {\n",
      "  \"attn_type\": \"bi\",\n",
      "  \"bi_data\": false,\n",
      "  \"clamp_len\": -1,\n",
      "  \"d_head\": 64,\n",
      "  \"d_inner\": 3072,\n",
      "  \"d_model\": 768,\n",
      "  \"dropout\": 0.1,\n",
      "  \"end_n_top\": 5,\n",
      "  \"ff_activation\": \"gelu\",\n",
      "  \"finetuning_task\": null,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"mem_len\": null,\n",
      "  \"n_head\": 12,\n",
      "  \"n_layer\": 12,\n",
      "  \"n_token\": 32000,\n",
      "  \"num_labels\": 2,\n",
      "  \"output_attentions\": false,\n",
      "  \"output_hidden_states\": false,\n",
      "  \"output_past\": true,\n",
      "  \"pruned_heads\": {},\n",
      "  \"reuse_len\": null,\n",
      "  \"same_length\": false,\n",
      "  \"start_n_top\": 5,\n",
      "  \"summary_activation\": \"tanh\",\n",
      "  \"summary_last_dropout\": 0.1,\n",
      "  \"summary_type\": \"last\",\n",
      "  \"summary_use_proj\": true,\n",
      "  \"torchscript\": false,\n",
      "  \"untie_r\": true,\n",
      "  \"use_bfloat16\": false\n",
      "}\n",
      "\n",
      "I1106 05:39:08.301557 139812953790208 tokenization_utils.py:374] loading file https://s3.amazonaws.com/models.huggingface.co/bert/xlnet-base-cased-spiece.model from cache at /home/srk/.cache/torch/transformers/dad589d582573df0293448af5109cb6981ca77239ed314e15ca63b7b8a318ddd.8b10bd978b5d01c21303cc761fc9ecd464419b3bf921864a355ba807cfbfafa8\n",
      "I1106 05:39:08.698660 139812953790208 configuration_utils.py:151] loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/xlnet-base-cased-config.json from cache at /home/srk/.cache/torch/transformers/c9cc6e53904f7f3679a31ec4af244f4419e25ebc8e71ebf8c558a31cbcf07fc8.ef1824921bc0786e97dc88d55eb17aabf18aac90f24bd34c0650529e7ba27d6f\n",
      "I1106 05:39:08.702148 139812953790208 configuration_utils.py:168] Model config {\n",
      "  \"attn_type\": \"bi\",\n",
      "  \"bi_data\": false,\n",
      "  \"clamp_len\": -1,\n",
      "  \"d_head\": 64,\n",
      "  \"d_inner\": 3072,\n",
      "  \"d_model\": 768,\n",
      "  \"dropout\": 0.1,\n",
      "  \"end_n_top\": 5,\n",
      "  \"ff_activation\": \"gelu\",\n",
      "  \"finetuning_task\": null,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"mem_len\": null,\n",
      "  \"n_head\": 12,\n",
      "  \"n_layer\": 12,\n",
      "  \"n_token\": 32000,\n",
      "  \"num_labels\": 1,\n",
      "  \"output_attentions\": false,\n",
      "  \"output_hidden_states\": false,\n",
      "  \"output_past\": true,\n",
      "  \"pruned_heads\": {},\n",
      "  \"reuse_len\": null,\n",
      "  \"same_length\": false,\n",
      "  \"start_n_top\": 5,\n",
      "  \"summary_activation\": \"tanh\",\n",
      "  \"summary_last_dropout\": 0.1,\n",
      "  \"summary_type\": \"last\",\n",
      "  \"summary_use_proj\": true,\n",
      "  \"torchscript\": false,\n",
      "  \"untie_r\": true,\n",
      "  \"use_bfloat16\": false\n",
      "}\n",
      "\n",
      "I1106 05:39:09.031414 139812953790208 modeling_utils.py:337] loading weights file https://s3.amazonaws.com/models.huggingface.co/bert/xlnet-base-cased-pytorch_model.bin from cache at /home/srk/.cache/torch/transformers/24197ba0ce5dbfe23924431610704c88e2c0371afa49149360e4c823219ab474.7eac4fe898a021204e63c88c00ea68c60443c57f94b4bc3c02adbde6465745ac\n",
      "I1106 05:39:12.378334 139812953790208 modeling_utils.py:405] Weights of XLNetForSequenceClassification not initialized from pretrained model: ['sequence_summary.summary.weight', 'sequence_summary.summary.bias', 'logits_proj.weight', 'logits_proj.bias']\n",
      "I1106 05:39:12.380427 139812953790208 modeling_utils.py:408] Weights from pretrained model not used in XLNetForSequenceClassification: ['lm_loss.weight', 'lm_loss.bias']\n",
      "/home/srk/env/DS2/lib/python3.6/site-packages/ipykernel_launcher.py:13: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  del sys.path[0]\n"
     ]
    }
   ],
   "source": [
    "model_start_time = time.time()\n",
    "# Model Config and initialize\n",
    "model_name = \"xlnet\"\n",
    "pretrained_model_name = \"xlnet-base-cased\"\n",
    "n_classes = 1\n",
    "config_class, model_class, tokenizer_class = MODEL_CLASSES[model_name]\n",
    "config = config_class.from_pretrained(pretrained_model_name)\n",
    "tokenizer = tokenizer_class.from_pretrained(pretrained_model_name, do_lower_case=False)\n",
    "model = model_class.from_pretrained(pretrained_model_name, num_labels=1)\n",
    "\n",
    "# Dataset Preparation\n",
    "max_length = 128\n",
    "train_df[\"text\"] = train_df[\"text\"].astype(str).fillna(\"NA\")\n",
    "train_features = convert_text_to_features(train_df[\"text\"], tokenizer, max_length=max_length)\n",
    "X = torch.tensor(train_features[0], dtype=torch.long)\n",
    "X_mask = torch.tensor(train_features[1], dtype=torch.long)\n",
    "X_seg_ids = torch.tensor(train_features[2], dtype=torch.long)\n",
    "y = train_df[\"recommended\"].values\n",
    "y = torch.tensor(y[:,np.newaxis], dtype=torch.float32)\n",
    "\n",
    "batch_size = 8\n",
    "train_dataset = data.TensorDataset(X, X_mask, X_seg_ids, y)\n",
    "train_loader = data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
    "\n",
    "# Model Building\n",
    "n_epochs = 1\n",
    "accumulation_steps = 1\n",
    "param_optimizer = list(model.named_parameters())\n",
    "no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n",
    "optimizer_grouped_parameters = [\n",
    "    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},\n",
    "    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n",
    "    ]\n",
    "num_train_optimization_steps = int(n_epochs*len(train_dataset)/batch_size/accumulation_steps)\n",
    "num_warmup_steps = int(0.05*num_train_optimization_steps)\n",
    "optimizer = AdamW(optimizer_grouped_parameters, lr=2e-5, correct_bias=False)\n",
    "scheduler = WarmupLinearSchedule(optimizer,\n",
    "                                 warmup_steps=num_warmup_steps,\n",
    "                                 t_total=num_train_optimization_steps)\n",
    "\n",
    "seed_everything()\n",
    "model.to(device)\n",
    "loss_fn = nn.BCEWithLogitsLoss()\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    model.train()\n",
    "    for x_batch, x_mask, x_seg_ids, y_batch in train_loader:\n",
    "        outputs = model(x_batch.to(device),\n",
    "                        attention_mask=x_mask.to(device),\n",
    "                        token_type_ids=x_seg_ids.to(device),\n",
    "                        labels=None)\n",
    "        y_pred = outputs[0]\n",
    "        loss = loss_fn(y_pred, y_batch.to(device))\n",
    "        loss.backward()\n",
    "        scheduler.step()\n",
    "        optimizer.step()\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "model_end_time = time.time()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preparation of test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/srk/env/DS2/lib/python3.6/site-packages/ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9632808396274816\n"
     ]
    }
   ],
   "source": [
    "pred_start_time = time.time()\n",
    "# Data Preparation\n",
    "test_df[\"text\"] = test_df[\"text\"].astype(str).fillna(\"NA\")\n",
    "test_features = convert_text_to_features(test_df[\"text\"], tokenizer, max_length=max_length)\n",
    "\n",
    "test_X = torch.tensor(test_features[0], dtype=torch.long)\n",
    "test_X_mask = torch.tensor(test_features[1], dtype=torch.long)\n",
    "test_X_seg_ids = torch.tensor(test_features[2], dtype=torch.long)\n",
    "test_y = test_df[\"recommended\"].values\n",
    "\n",
    "test_dataset = data.TensorDataset(test_X, test_X_mask, test_X_seg_ids)\n",
    "test_loader = data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n",
    "\n",
    "# Prediction\n",
    "preds = np.zeros([len(test_dataset), 1])\n",
    "model.eval()\n",
    "for i, (x_batch, x_mask, x_seg_ids) in enumerate(test_loader):\n",
    "    outputs = model(x_batch.to(device),\n",
    "                    attention_mask=x_mask.to(device),\n",
    "                    token_type_ids=x_seg_ids.to(device),\n",
    "                    labels=None)\n",
    "    y_pred = sigmoid(outputs[0].detach().cpu().numpy())\n",
    "    preds[i*batch_size:(i+1)*batch_size, :] = y_pred\n",
    "    \n",
    "from sklearn import metrics\n",
    "print(metrics.roc_auc_score(test_y, preds))\n",
    "pred_end_time = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "404.6116931438446\n",
      "55.29689908027649\n"
     ]
    }
   ],
   "source": [
    "print(model_end_time - model_start_time)\n",
    "print(pred_end_time - pred_start_time)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### References\n",
    "\n",
    "[MLExplained - XLNET](https://mlexplained.com/2019/06/30/paper-dissected-xlnet-generalized-autoregressive-pretraining-for-language-understanding-explained/)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}