1-data_sampling_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Downsampling and Preprocessing\n",
    "\n",
    "This Jupyter Notebook contains the first step of the entire ML pipeline for this project: data pre-processing and downsampling. Data pre-processing is done to clean up any messy text in reviews and preserve only the words from the review that preserve the most information (removing redundancy). This is essential to gain reliable insights when conducting EDA. \n",
    "\n",
    "Downsampling is done in order to create a dataset which is smaller than the original dataset but is still representative of it as a whole. It is required as my computer is not powerful enough to operate on all the data from the dataset.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#General Imports\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle \n",
    "from os.path import join\n",
    "\n",
    "#Preprocessing related imports \n",
    "from nltk.stem import WordNetLemmatizer\n",
    "import gensim.parsing.preprocessing as gpp\n",
    "import gensim.utils as gu\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Rating</th>\n",
       "      <th>Title</th>\n",
       "      <th>Review</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>more like funchuck</td>\n",
       "      <td>Gave this to my dad for a gag gift after direc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>Inspiring</td>\n",
       "      <td>I hope a lot of people hear this cd. We need m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>The best soundtrack ever to anything.</td>\n",
       "      <td>I'm reading a lot of reviews saying that this ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Chrono Cross OST</td>\n",
       "      <td>The music of Yasunori Misuda is without questi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Too good to be true</td>\n",
       "      <td>Probably the greatest soundtrack in history! U...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2999995</th>\n",
       "      <td>1</td>\n",
       "      <td>Don't do it!!</td>\n",
       "      <td>The high chair looks great when it first comes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2999996</th>\n",
       "      <td>2</td>\n",
       "      <td>Looks nice, low functionality</td>\n",
       "      <td>I have used this highchair for 2 kids now and ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2999997</th>\n",
       "      <td>2</td>\n",
       "      <td>compact, but hard to clean</td>\n",
       "      <td>We have a small house, and really wanted two o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2999998</th>\n",
       "      <td>3</td>\n",
       "      <td>Hard to clean!</td>\n",
       "      <td>I agree with everyone else who says this chair...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2999999</th>\n",
       "      <td>1</td>\n",
       "      <td>what is it saying?</td>\n",
       "      <td>not sure what this book is supposed to be. It ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3000000 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Rating                                  Title  \\\n",
       "0             3                     more like funchuck   \n",
       "1             5                              Inspiring   \n",
       "2             5  The best soundtrack ever to anything.   \n",
       "3             4                       Chrono Cross OST   \n",
       "4             5                    Too good to be true   \n",
       "...         ...                                    ...   \n",
       "2999995       1                          Don't do it!!   \n",
       "2999996       2          Looks nice, low functionality   \n",
       "2999997       2             compact, but hard to clean   \n",
       "2999998       3                         Hard to clean!   \n",
       "2999999       1                     what is it saying?   \n",
       "\n",
       "                                                    Review  \n",
       "0        Gave this to my dad for a gag gift after direc...  \n",
       "1        I hope a lot of people hear this cd. We need m...  \n",
       "2        I'm reading a lot of reviews saying that this ...  \n",
       "3        The music of Yasunori Misuda is without questi...  \n",
       "4        Probably the greatest soundtrack in history! U...  \n",
       "...                                                    ...  \n",
       "2999995  The high chair looks great when it first comes...  \n",
       "2999996  I have used this highchair for 2 kids now and ...  \n",
       "2999997  We have a small house, and really wanted two o...  \n",
       "2999998  I agree with everyone else who says this chair...  \n",
       "2999999  not sure what this book is supposed to be. It ...  \n",
       "\n",
       "[3000000 rows x 3 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Load full dataset \n",
    "data_dir = \"data/\"\n",
    "data = pd.read_csv(join(data_dir, \"train.csv\"), header=None, names=['Rating', 'Title', 'Review'])\n",
    "display(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    600000\n",
       "2    600000\n",
       "3    600000\n",
       "4    600000\n",
       "5    600000\n",
       "Name: Rating, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check distribution of rating values as this is likely our target variable\n",
    "data[\"Rating\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We see that the rating values are distributed equally with an equal number of data points for each label value. We want to preserve this property when we downsample our dataset. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing\n",
    "We first preprocess our entire dataset by applying the following transformations to the textual data:\n",
    "- Stripping HTML Tags (`gpp.strip_tags`)\n",
    "- Removing all Punctuation (`gpp.strip_punctuation`)\n",
    "- Removing all extra whitespaces (`gpp.strip_multiple_whitespaces`)\n",
    "- Removing all numerics (`gpp.strip_numeric`)\n",
    "- Removing stopwords(`gpp.remove_stopwords`)\n",
    "- Removing words shorter than 3 letters (`gpp.strip_short`)\n",
    "\n",
    "Following this initial pre-processing, we also then lemmatize all the words in the reviews to produce lemmatized strings.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_text(text):\n",
    "    \"\"\"Preprocesses a given string text input\"\"\"\n",
    "    preprocs = [\n",
    "        gpp.strip_tags, \n",
    "        gpp.strip_punctuation,\n",
    "        gpp.strip_multiple_whitespaces,\n",
    "        gpp.strip_numeric,\n",
    "        gpp.remove_stopwords, \n",
    "        gpp.strip_short, \n",
    "    ]\n",
    "    text = gu.to_unicode(text.lower().strip())\n",
    "    for preproc in preprocs:\n",
    "        text = preproc(text)\n",
    "    return text\n",
    "\n",
    "def lemmatize(text):\n",
    "    \"\"\"Lemmatizes a given string text input\"\"\"\n",
    "    wnl = WordNetLemmatizer()\n",
    "    return wnl.lemmatize(text)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Combining both the above functions into a single preprocessing function\n",
    "preprocess = lambda text: lemmatize(preprocess_text(str(text)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Before we apply the preprocessing, we notice that the dataset has two columns with textial data: the title of the review and the review itself. As the title of the data also indicates the feelings of the user towards the product and is essentially a summarization of the review it is also informative for predicting user rating. Therefore, we create a new feature \"ReviewFull\" which is a concatenation of the review title as well as the review itself, and use this as our primary data for EDA and model training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Rating</th>\n",
       "      <th>ReviewFull</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>like funchuck gave dad gag gift directing nuns...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>inspiring hope lot people hear need strong pos...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>best soundtrack reading lot reviews saying bes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>chrono cross ost music yasunori misuda questio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>good true probably greatest soundtrack history...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Rating                                         ReviewFull\n",
       "0       3  like funchuck gave dad gag gift directing nuns...\n",
       "1       5  inspiring hope lot people hear need strong pos...\n",
       "2       5  best soundtrack reading lot reviews saying bes...\n",
       "3       4  chrono cross ost music yasunori misuda questio...\n",
       "4       5  good true probably greatest soundtrack history..."
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create the ReviewFull data column\n",
    "data[\"ReviewFull\"] = data[\"Title\"] + \" \" + data[\"Review\"]\n",
    "data = data.drop([\"Title\", \"Review\"], axis=1)\n",
    "\n",
    "# Apply the preprocessing to the textual data\n",
    "data[\"ReviewFull\"] = data[\"ReviewFull\"].apply(preprocess)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the data\n",
    "data.to_csv(join(data_dir, \"preprocessed_train.csv\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Downsampling\n",
    "\n",
    "We now create a smaller dataset which my computer can process when doing EDA and modeling. We downsample to a dataset size of 50000 data points, ensuring that there is an even distribution of ratings by grouping by the \"Rating\" column when sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Rating</th>\n",
       "      <th>ReviewFull</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>260774</th>\n",
       "      <td>1</td>\n",
       "      <td>diappointed service order thing direct address...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1339843</th>\n",
       "      <td>1</td>\n",
       "      <td>allsop scratch repair kit product according pa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099824</th>\n",
       "      <td>1</td>\n",
       "      <td>angry requirements years old ibook running tig...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355715</th>\n",
       "      <td>1</td>\n",
       "      <td>yawn years heard film scary disturbing long ag...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2880294</th>\n",
       "      <td>1</td>\n",
       "      <td>terrible waste time bother switches past prese...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1319052</th>\n",
       "      <td>5</td>\n",
       "      <td>loved book finished reading flying seat pants ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040680</th>\n",
       "      <td>5</td>\n",
       "      <td>beautiful marriage photographs text photograph...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>819639</th>\n",
       "      <td>5</td>\n",
       "      <td>great came soon days functions compared compet...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1673128</th>\n",
       "      <td>5</td>\n",
       "      <td>perfectly written novel think safe read book g...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2768635</th>\n",
       "      <td>5</td>\n",
       "      <td>great wwii game game great wwii game filled ac...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>50000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Rating                                         ReviewFull\n",
       "260774        1  diappointed service order thing direct address...\n",
       "1339843       1  allsop scratch repair kit product according pa...\n",
       "1099824       1  angry requirements years old ibook running tig...\n",
       "355715        1  yawn years heard film scary disturbing long ag...\n",
       "2880294       1  terrible waste time bother switches past prese...\n",
       "...         ...                                                ...\n",
       "1319052       5  loved book finished reading flying seat pants ...\n",
       "1040680       5  beautiful marriage photographs text photograph...\n",
       "819639        5  great came soon days functions compared compet...\n",
       "1673128       5  perfectly written novel think safe read book g...\n",
       "2768635       5  great wwii game game great wwii game filled ac...\n",
       "\n",
       "[50000 rows x 2 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "downsampled = data.groupby(\"Rating\").sample(10000)\n",
    "display(downsampled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    10000\n",
       "2    10000\n",
       "3    10000\n",
       "4    10000\n",
       "5    10000\n",
       "Name: Rating, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Ensure equal distribution of targets\n",
    "downsampled[\"Rating\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save data\n",
    "downsampled.to_csv(join(data_dir, \"downsampled_preprocessed_train_50000.csv\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.5 64-bit ('askoski': conda)",
   "language": "python",
   "name": "python37564bitaskoskicondaf89acb10eb5d40d0a2b1becabd7f59e8"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}