ML_Pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ML Pipeline\n",
    "\n",
    "### *Importing Libraries and Loading data from database.*\n",
    "- Importing Python libraries\n",
    "- Loading dataset from database with [`read_sql_table`](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.read_sql_table.html)\n",
    "- Defining feature and target variables X and Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\Shubham\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\Shubham\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\Shubham\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     C:\\Users\\Shubham\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n"
     ]
    }
   ],
   "source": [
    "# import libraries\n",
    "import pickle\n",
    "import string\n",
    "import unittest\n",
    "\n",
    "import time\n",
    "import progressbar\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sqlalchemy import create_engine\n",
    "import nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')\n",
    "nltk.download('wordnet')\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "\n",
    "from nltk.tokenize import word_tokenize, sent_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "\n",
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "from sklearn.metrics import classification_report, f1_score, make_scorer\n",
    "from sklearn.base import BaseEstimator, TransformerMixin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>message</th>\n",
       "      <th>original</th>\n",
       "      <th>genre</th>\n",
       "      <th>related</th>\n",
       "      <th>request</th>\n",
       "      <th>offer</th>\n",
       "      <th>aid_related</th>\n",
       "      <th>medical_help</th>\n",
       "      <th>medical_products</th>\n",
       "      <th>...</th>\n",
       "      <th>aid_centers</th>\n",
       "      <th>other_infrastructure</th>\n",
       "      <th>weather_related</th>\n",
       "      <th>floods</th>\n",
       "      <th>storm</th>\n",
       "      <th>fire</th>\n",
       "      <th>earthquake</th>\n",
       "      <th>cold</th>\n",
       "      <th>other_weather</th>\n",
       "      <th>direct_report</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8181</th>\n",
       "      <td>9181</td>\n",
       "      <td>we didn't made nothing at all for change the c...</td>\n",
       "      <td>cher de n'avoir rien fait pour changer ce pauv...</td>\n",
       "      <td>direct</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16665</th>\n",
       "      <td>19596</td>\n",
       "      <td>One of the main difficulties in fighting the f...</td>\n",
       "      <td>None</td>\n",
       "      <td>news</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11971</th>\n",
       "      <td>13832</td>\n",
       "      <td>http://twitpic.com/18el95 - Daughters's bedroo...</td>\n",
       "      <td>None</td>\n",
       "      <td>social</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20681</th>\n",
       "      <td>24065</td>\n",
       "      <td>Critics point out that what Somalia will have ...</td>\n",
       "      <td>None</td>\n",
       "      <td>news</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10909</th>\n",
       "      <td>12541</td>\n",
       "      <td>We have a small mass of clothing , non-perisha...</td>\n",
       "      <td>None</td>\n",
       "      <td>direct</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17000</th>\n",
       "      <td>19970</td>\n",
       "      <td>No rainfall was recorded in the main river bas...</td>\n",
       "      <td>None</td>\n",
       "      <td>news</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17115</th>\n",
       "      <td>20100</td>\n",
       "      <td>There's also volatility, of a different kind, ...</td>\n",
       "      <td>None</td>\n",
       "      <td>news</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21340</th>\n",
       "      <td>24789</td>\n",
       "      <td>The programme focused on hiring, training and ...</td>\n",
       "      <td>None</td>\n",
       "      <td>news</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20654</th>\n",
       "      <td>24033</td>\n",
       "      <td>Heavy portable pump (petrol driven) or tractor...</td>\n",
       "      <td>None</td>\n",
       "      <td>news</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25239</th>\n",
       "      <td>29151</td>\n",
       "      <td>Reports of retaliatory violence by the nationa...</td>\n",
       "      <td>None</td>\n",
       "      <td>news</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id                                            message  \\\n",
       "8181    9181  we didn't made nothing at all for change the c...   \n",
       "16665  19596  One of the main difficulties in fighting the f...   \n",
       "11971  13832  http://twitpic.com/18el95 - Daughters's bedroo...   \n",
       "20681  24065  Critics point out that what Somalia will have ...   \n",
       "10909  12541  We have a small mass of clothing , non-perisha...   \n",
       "17000  19970  No rainfall was recorded in the main river bas...   \n",
       "17115  20100  There's also volatility, of a different kind, ...   \n",
       "21340  24789  The programme focused on hiring, training and ...   \n",
       "20654  24033  Heavy portable pump (petrol driven) or tractor...   \n",
       "25239  29151  Reports of retaliatory violence by the nationa...   \n",
       "\n",
       "                                                original   genre  related  \\\n",
       "8181   cher de n'avoir rien fait pour changer ce pauv...  direct        0   \n",
       "16665                                               None    news        1   \n",
       "11971                                               None  social        1   \n",
       "20681                                               None    news        0   \n",
       "10909                                               None  direct        1   \n",
       "17000                                               None    news        1   \n",
       "17115                                               None    news        1   \n",
       "21340                                               None    news        1   \n",
       "20654                                               None    news        0   \n",
       "25239                                               None    news        1   \n",
       "\n",
       "       request  offer  aid_related  medical_help  medical_products  ...  \\\n",
       "8181         0      0            0             0                 0  ...   \n",
       "16665        0      0            0             0                 0  ...   \n",
       "11971        0      0            0             0                 0  ...   \n",
       "20681        0      0            0             0                 0  ...   \n",
       "10909        0      0            1             0                 1  ...   \n",
       "17000        0      0            0             0                 0  ...   \n",
       "17115        0      0            1             1                 0  ...   \n",
       "21340        0      0            0             0                 0  ...   \n",
       "20654        0      0            0             0                 0  ...   \n",
       "25239        0      0            1             0                 0  ...   \n",
       "\n",
       "       aid_centers  other_infrastructure  weather_related  floods  storm  \\\n",
       "8181             0                     0                0       0      0   \n",
       "16665            0                     0                1       0      0   \n",
       "11971            0                     0                1       0      0   \n",
       "20681            0                     0                0       0      0   \n",
       "10909            0                     0                0       0      0   \n",
       "17000            0                     0                0       0      0   \n",
       "17115            0                     1                0       0      0   \n",
       "21340            0                     0                0       0      0   \n",
       "20654            0                     0                0       0      0   \n",
       "25239            0                     0                0       0      0   \n",
       "\n",
       "       fire  earthquake  cold  other_weather  direct_report  \n",
       "8181      0           0     0              0              0  \n",
       "16665     1           0     0              0              0  \n",
       "11971     0           1     0              0              1  \n",
       "20681     0           0     0              0              0  \n",
       "10909     0           0     0              0              0  \n",
       "17000     0           0     0              0              0  \n",
       "17115     0           0     0              0              0  \n",
       "21340     0           0     0              0              0  \n",
       "20654     0           0     0              0              0  \n",
       "25239     0           0     0              0              0  \n",
       "\n",
       "[10 rows x 40 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# loading data from database\n",
    "engine = create_engine('sqlite:///data/disaster_messages.db')\n",
    "data = pd.read_sql_table('messages', engine)\n",
    "data.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assigning predictors and prediction variables\n",
    "X = data.message\n",
    "Y = data.loc[:, 'related':'direct_report']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### *Writing a Tokenization Function to process Text Data*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(text):\n",
    "    # Making table with mapping of whether string.punctuations present or not\n",
    "    table = text.maketrans(dict.fromkeys(string.punctuation))\n",
    "    \n",
    "    # Word Tokenizer\n",
    "    words = word_tokenize(text.lower().strip().translate(table))\n",
    "    \n",
    "    # removing stopwords\n",
    "    words = [word for word in words if word not in stopwords.words('english')]\n",
    "    \n",
    "    # Word lemmetizer\n",
    "    lemmed = [WordNetLemmatizer().lemmatize(word) for word in words]\n",
    "    \n",
    "    # Word lemmetazier with verb\n",
    "    lemmed = [WordNetLemmatizer().lemmatize(word, pos = 'v') for word in lemmed]\n",
    "    # Stemming words\n",
    "    stemmed = [PorterStemmer().stem(word) for word in lemmed]\n",
    "    \n",
    "    return stemmed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build a Machine Learning Pipeline\n",
    "\n",
    "*This machine learning pipeline should take in the `message` column as input and output classification results on the other 36 categories in the dataset.* \n",
    "\n",
    "*You may find the [MultiOutputClassifier](http://scikit-learn.org/stable/modules/generated/sklearn.multioutput.MultiOutputClassifier.html) helpful for predicting multiple target variables.*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Making pipeline\n",
    "pipeline = Pipeline([\n",
    "    # Using tfidf vectorizer\n",
    "    ('text', TfidfVectorizer(tokenizer = tokenize)),\n",
    "    # Using MultipleOutfitClassifier\n",
    "    ('clf', MultiOutputClassifier(RandomForestClassifier(random_state = 42, verbose = 1)))\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train Pipeline\n",
    "- *Splitting data into train and test sets*\n",
    "- *Training pipeline*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Spliting Completed\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:   23.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:   17.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    2.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:   25.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:   11.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    8.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    6.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    5.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    5.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    6.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    9.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    9.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    3.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    4.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    4.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    6.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    6.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:   17.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:   10.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    8.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    7.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    4.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    2.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    3.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    2.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    3.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    7.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:   18.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    9.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    9.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    3.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    9.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    4.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    8.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting Completed, time taken : 426.5052945613861\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:   20.0s finished\n"
     ]
    }
   ],
   "source": [
    "# Spliting data in training and test data\n",
    "\n",
    "train_X, test_X, train_Y, test_Y = train_test_split(X, Y, random_state = 42)\n",
    "print('Spliting Completed')\n",
    "\n",
    "# Fitting data\n",
    "start_time = time.time()\n",
    "pipeline = pipeline.fit(train_X, train_Y)\n",
    "end_time = time.time()\n",
    "\n",
    "print(\"Fitting Completed, time taken : {}\".format(end_time - start_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing model\n",
    "\n",
    "- *Reporting the **f1 score, precision and recall** for each output category of the dataset.* \n",
    "- *We can do this by iterating through the columns and calling sklearn's `classification_report` on each.*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.6s finished\n"
     ]
    }
   ],
   "source": [
    "# Predicting values from the pipeline \n",
    "y_preds = pipeline.predict(test_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Shubham\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1221: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "related\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.70      0.39      0.50      1563\n",
      "           1       0.83      0.95      0.89      4991\n",
      "\n",
      "    accuracy                           0.81      6554\n",
      "   macro avg       0.77      0.67      0.69      6554\n",
      "weighted avg       0.80      0.81      0.79      6554\n",
      "\n",
      "request\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.98      0.94      5443\n",
      "           1       0.85      0.47      0.61      1111\n",
      "\n",
      "    accuracy                           0.90      6554\n",
      "   macro avg       0.88      0.73      0.77      6554\n",
      "weighted avg       0.89      0.90      0.88      6554\n",
      "\n",
      "offer\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00      6521\n",
      "           1       0.00      0.00      0.00        33\n",
      "\n",
      "    accuracy                           0.99      6554\n",
      "   macro avg       0.50      0.50      0.50      6554\n",
      "weighted avg       0.99      0.99      0.99      6554\n",
      "\n",
      "aid_related\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.80      0.84      0.82      3884\n",
      "           1       0.75      0.69      0.72      2670\n",
      "\n",
      "    accuracy                           0.78      6554\n",
      "   macro avg       0.77      0.77      0.77      6554\n",
      "weighted avg       0.78      0.78      0.78      6554\n",
      "\n",
      "medical_help\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      1.00      0.96      6019\n",
      "           1       0.69      0.07      0.13       535\n",
      "\n",
      "    accuracy                           0.92      6554\n",
      "   macro avg       0.81      0.53      0.54      6554\n",
      "weighted avg       0.90      0.92      0.89      6554\n",
      "\n",
      "medical_products\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.97      6210\n",
      "           1       0.87      0.08      0.14       344\n",
      "\n",
      "    accuracy                           0.95      6554\n",
      "   macro avg       0.91      0.54      0.56      6554\n",
      "weighted avg       0.95      0.95      0.93      6554\n",
      "\n",
      "search_and_rescue\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6395\n",
      "           1       0.61      0.07      0.12       159\n",
      "\n",
      "    accuracy                           0.98      6554\n",
      "   macro avg       0.79      0.53      0.56      6554\n",
      "weighted avg       0.97      0.98      0.97      6554\n",
      "\n",
      "security\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6438\n",
      "           1       0.25      0.01      0.02       116\n",
      "\n",
      "    accuracy                           0.98      6554\n",
      "   macro avg       0.62      0.50      0.50      6554\n",
      "weighted avg       0.97      0.98      0.97      6554\n",
      "\n",
      "military\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      1.00      0.98      6354\n",
      "           1       0.62      0.08      0.14       200\n",
      "\n",
      "    accuracy                           0.97      6554\n",
      "   macro avg       0.79      0.54      0.56      6554\n",
      "weighted avg       0.96      0.97      0.96      6554\n",
      "\n",
      "child_alone\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      6554\n",
      "\n",
      "    accuracy                           1.00      6554\n",
      "   macro avg       1.00      1.00      1.00      6554\n",
      "weighted avg       1.00      1.00      1.00      6554\n",
      "\n",
      "water\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      6136\n",
      "           1       0.85      0.38      0.52       418\n",
      "\n",
      "    accuracy                           0.96      6554\n",
      "   macro avg       0.90      0.69      0.75      6554\n",
      "weighted avg       0.95      0.96      0.95      6554\n",
      "\n",
      "food\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.99      0.97      5809\n",
      "           1       0.86      0.60      0.71       745\n",
      "\n",
      "    accuracy                           0.94      6554\n",
      "   macro avg       0.91      0.79      0.84      6554\n",
      "weighted avg       0.94      0.94      0.94      6554\n",
      "\n",
      "shelter\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.99      0.96      5973\n",
      "           1       0.80      0.33      0.46       581\n",
      "\n",
      "    accuracy                           0.93      6554\n",
      "   macro avg       0.87      0.66      0.71      6554\n",
      "weighted avg       0.93      0.93      0.92      6554\n",
      "\n",
      "clothing\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6456\n",
      "           1       0.74      0.17      0.28        98\n",
      "\n",
      "    accuracy                           0.99      6554\n",
      "   macro avg       0.86      0.59      0.64      6554\n",
      "weighted avg       0.98      0.99      0.98      6554\n",
      "\n",
      "money\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6421\n",
      "           1       0.78      0.05      0.10       133\n",
      "\n",
      "    accuracy                           0.98      6554\n",
      "   macro avg       0.88      0.53      0.54      6554\n",
      "weighted avg       0.98      0.98      0.97      6554\n",
      "\n",
      "missing_people\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6481\n",
      "           1       1.00      0.01      0.03        73\n",
      "\n",
      "    accuracy                           0.99      6554\n",
      "   macro avg       0.99      0.51      0.51      6554\n",
      "weighted avg       0.99      0.99      0.98      6554\n",
      "\n",
      "refugees\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      1.00      0.98      6339\n",
      "           1       0.40      0.03      0.05       215\n",
      "\n",
      "    accuracy                           0.97      6554\n",
      "   macro avg       0.68      0.51      0.52      6554\n",
      "weighted avg       0.95      0.97      0.95      6554\n",
      "\n",
      "death\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      6257\n",
      "           1       0.84      0.13      0.22       297\n",
      "\n",
      "    accuracy                           0.96      6554\n",
      "   macro avg       0.90      0.56      0.60      6554\n",
      "weighted avg       0.95      0.96      0.94      6554\n",
      "\n",
      "other_aid\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.87      1.00      0.93      5690\n",
      "           1       0.56      0.03      0.06       864\n",
      "\n",
      "    accuracy                           0.87      6554\n",
      "   macro avg       0.72      0.51      0.50      6554\n",
      "weighted avg       0.83      0.87      0.82      6554\n",
      "\n",
      "infrastructure_related\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      1.00      0.97      6143\n",
      "           1       0.00      0.00      0.00       411\n",
      "\n",
      "    accuracy                           0.94      6554\n",
      "   macro avg       0.47      0.50      0.48      6554\n",
      "weighted avg       0.88      0.94      0.91      6554\n",
      "\n",
      "transport\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      6251\n",
      "           1       0.71      0.05      0.09       303\n",
      "\n",
      "    accuracy                           0.96      6554\n",
      "   macro avg       0.84      0.52      0.53      6554\n",
      "weighted avg       0.94      0.96      0.94      6554\n",
      "\n",
      "buildings\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.98      6231\n",
      "           1       0.82      0.08      0.15       323\n",
      "\n",
      "    accuracy                           0.95      6554\n",
      "   macro avg       0.89      0.54      0.56      6554\n",
      "weighted avg       0.95      0.95      0.94      6554\n",
      "\n",
      "electricity\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6407\n",
      "           1       1.00      0.02      0.04       147\n",
      "\n",
      "    accuracy                           0.98      6554\n",
      "   macro avg       0.99      0.51      0.51      6554\n",
      "weighted avg       0.98      0.98      0.97      6554\n",
      "\n",
      "tools\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00      6511\n",
      "           1       0.00      0.00      0.00        43\n",
      "\n",
      "    accuracy                           0.99      6554\n",
      "   macro avg       0.50      0.50      0.50      6554\n",
      "weighted avg       0.99      0.99      0.99      6554\n",
      "\n",
      "hospitals\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00      6498\n",
      "           1       0.00      0.00      0.00        56\n",
      "\n",
      "    accuracy                           0.99      6554\n",
      "   macro avg       0.50      0.50      0.50      6554\n",
      "weighted avg       0.98      0.99      0.99      6554\n",
      "\n",
      "shops\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      6530\n",
      "           1       0.00      0.00      0.00        24\n",
      "\n",
      "    accuracy                           1.00      6554\n",
      "   macro avg       0.50      0.50      0.50      6554\n",
      "weighted avg       0.99      1.00      0.99      6554\n",
      "\n",
      "aid_centers\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6473\n",
      "           1       0.00      0.00      0.00        81\n",
      "\n",
      "    accuracy                           0.99      6554\n",
      "   macro avg       0.49      0.50      0.50      6554\n",
      "weighted avg       0.98      0.99      0.98      6554\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "other_infrastructure\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      6271\n",
      "           1       0.00      0.00      0.00       283\n",
      "\n",
      "    accuracy                           0.96      6554\n",
      "   macro avg       0.48      0.50      0.49      6554\n",
      "weighted avg       0.92      0.96      0.94      6554\n",
      "\n",
      "weather_related\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.95      0.92      4781\n",
      "           1       0.84      0.71      0.77      1773\n",
      "\n",
      "    accuracy                           0.88      6554\n",
      "   macro avg       0.87      0.83      0.84      6554\n",
      "weighted avg       0.88      0.88      0.88      6554\n",
      "\n",
      "floods\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.99      0.98      6035\n",
      "           1       0.88      0.47      0.61       519\n",
      "\n",
      "    accuracy                           0.95      6554\n",
      "   macro avg       0.92      0.73      0.79      6554\n",
      "weighted avg       0.95      0.95      0.95      6554\n",
      "\n",
      "storm\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.98      0.97      5949\n",
      "           1       0.75      0.53      0.62       605\n",
      "\n",
      "    accuracy                           0.94      6554\n",
      "   macro avg       0.85      0.76      0.79      6554\n",
      "weighted avg       0.93      0.94      0.94      6554\n",
      "\n",
      "fire\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6488\n",
      "           1       0.00      0.00      0.00        66\n",
      "\n",
      "    accuracy                           0.99      6554\n",
      "   macro avg       0.49      0.50      0.50      6554\n",
      "weighted avg       0.98      0.99      0.98      6554\n",
      "\n",
      "earthquake\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.99      0.98      5964\n",
      "           1       0.88      0.78      0.83       590\n",
      "\n",
      "    accuracy                           0.97      6554\n",
      "   macro avg       0.93      0.88      0.90      6554\n",
      "weighted avg       0.97      0.97      0.97      6554\n",
      "\n",
      "cold\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6413\n",
      "           1       0.83      0.07      0.13       141\n",
      "\n",
      "    accuracy                           0.98      6554\n",
      "   macro avg       0.91      0.54      0.56      6554\n",
      "weighted avg       0.98      0.98      0.97      6554\n",
      "\n",
      "other_weather\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.97      6219\n",
      "           1       0.65      0.03      0.06       335\n",
      "\n",
      "    accuracy                           0.95      6554\n",
      "   macro avg       0.80      0.52      0.52      6554\n",
      "weighted avg       0.93      0.95      0.93      6554\n",
      "\n",
      "direct_report\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.86      0.98      0.91      5282\n",
      "           1       0.77      0.34      0.47      1272\n",
      "\n",
      "    accuracy                           0.85      6554\n",
      "   macro avg       0.82      0.66      0.69      6554\n",
      "weighted avg       0.84      0.85      0.83      6554\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Creating a table with precision, recall, f1-score for all the columns\n",
    "results_dict = {}\n",
    "\n",
    "for pred, label, col in zip(y_preds.transpose(), test_Y.values.transpose(), test_Y.columns):\n",
    "    print(col)\n",
    "    print(classification_report(label, pred))\n",
    "    results_dict[col] = classification_report(label, pred, output_dict = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are number of categories with small numbers of messages where none were picked up:\n",
    "- aid_centers\n",
    "- shops\n",
    "- hospitals\n",
    "- tools\n",
    "- offer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***What is the overall performance of the model?***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>related</th>\n",
       "      <td>0.801030</td>\n",
       "      <td>0.814922</td>\n",
       "      <td>0.794653</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>request</th>\n",
       "      <td>0.893249</td>\n",
       "      <td>0.896857</td>\n",
       "      <td>0.884284</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>offer</th>\n",
       "      <td>0.989955</td>\n",
       "      <td>0.994965</td>\n",
       "      <td>0.992454</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>aid_related</th>\n",
       "      <td>0.777840</td>\n",
       "      <td>0.779371</td>\n",
       "      <td>0.777901</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>medical_help</th>\n",
       "      <td>0.904538</td>\n",
       "      <td>0.921575</td>\n",
       "      <td>0.891176</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>medical_products</th>\n",
       "      <td>0.946817</td>\n",
       "      <td>0.950870</td>\n",
       "      <td>0.930851</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>search_and_rescue</th>\n",
       "      <td>0.968471</td>\n",
       "      <td>0.976350</td>\n",
       "      <td>0.967059</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>security</th>\n",
       "      <td>0.969479</td>\n",
       "      <td>0.981996</td>\n",
       "      <td>0.973671</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>military</th>\n",
       "      <td>0.960937</td>\n",
       "      <td>0.970400</td>\n",
       "      <td>0.959205</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>child_alone</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>water</th>\n",
       "      <td>0.952072</td>\n",
       "      <td>0.956057</td>\n",
       "      <td>0.948123</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>food</th>\n",
       "      <td>0.940491</td>\n",
       "      <td>0.943546</td>\n",
       "      <td>0.938990</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shelter</th>\n",
       "      <td>0.925703</td>\n",
       "      <td>0.933018</td>\n",
       "      <td>0.919927</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>clothing</th>\n",
       "      <td>0.983882</td>\n",
       "      <td>0.986726</td>\n",
       "      <td>0.982650</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>money</th>\n",
       "      <td>0.976630</td>\n",
       "      <td>0.980470</td>\n",
       "      <td>0.972036</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>missing_people</th>\n",
       "      <td>0.989135</td>\n",
       "      <td>0.989014</td>\n",
       "      <td>0.983700</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>refugees</th>\n",
       "      <td>0.949404</td>\n",
       "      <td>0.966738</td>\n",
       "      <td>0.952534</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>death</th>\n",
       "      <td>0.954963</td>\n",
       "      <td>0.959414</td>\n",
       "      <td>0.944862</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other_aid</th>\n",
       "      <td>0.830404</td>\n",
       "      <td>0.869088</td>\n",
       "      <td>0.815162</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>infrastructure_related</th>\n",
       "      <td>0.878504</td>\n",
       "      <td>0.937138</td>\n",
       "      <td>0.906874</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transport</th>\n",
       "      <td>0.944745</td>\n",
       "      <td>0.955142</td>\n",
       "      <td>0.936115</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>buildings</th>\n",
       "      <td>0.947885</td>\n",
       "      <td>0.953921</td>\n",
       "      <td>0.935677</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>electricity</th>\n",
       "      <td>0.978512</td>\n",
       "      <td>0.978029</td>\n",
       "      <td>0.967605</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tools</th>\n",
       "      <td>0.986921</td>\n",
       "      <td>0.993439</td>\n",
       "      <td>0.990169</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hospitals</th>\n",
       "      <td>0.982984</td>\n",
       "      <td>0.991456</td>\n",
       "      <td>0.987202</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shops</th>\n",
       "      <td>0.992690</td>\n",
       "      <td>0.996338</td>\n",
       "      <td>0.994511</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>aid_centers</th>\n",
       "      <td>0.975435</td>\n",
       "      <td>0.987641</td>\n",
       "      <td>0.981500</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other_infrastructure</th>\n",
       "      <td>0.915499</td>\n",
       "      <td>0.956668</td>\n",
       "      <td>0.935631</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weather_related</th>\n",
       "      <td>0.881580</td>\n",
       "      <td>0.884040</td>\n",
       "      <td>0.880665</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>floods</th>\n",
       "      <td>0.950399</td>\n",
       "      <td>0.953158</td>\n",
       "      <td>0.946535</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>storm</th>\n",
       "      <td>0.934731</td>\n",
       "      <td>0.940189</td>\n",
       "      <td>0.935660</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fire</th>\n",
       "      <td>0.979960</td>\n",
       "      <td>0.989777</td>\n",
       "      <td>0.984844</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>earthquake</th>\n",
       "      <td>0.969360</td>\n",
       "      <td>0.970400</td>\n",
       "      <td>0.969553</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cold</th>\n",
       "      <td>0.976821</td>\n",
       "      <td>0.979707</td>\n",
       "      <td>0.971253</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other_weather</th>\n",
       "      <td>0.934929</td>\n",
       "      <td>0.949649</td>\n",
       "      <td>0.927533</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>direct_report</th>\n",
       "      <td>0.842606</td>\n",
       "      <td>0.852304</td>\n",
       "      <td>0.828366</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        precision    recall  f1-score  support\n",
       "related                  0.801030  0.814922  0.794653   6554.0\n",
       "request                  0.893249  0.896857  0.884284   6554.0\n",
       "offer                    0.989955  0.994965  0.992454   6554.0\n",
       "aid_related              0.777840  0.779371  0.777901   6554.0\n",
       "medical_help             0.904538  0.921575  0.891176   6554.0\n",
       "medical_products         0.946817  0.950870  0.930851   6554.0\n",
       "search_and_rescue        0.968471  0.976350  0.967059   6554.0\n",
       "security                 0.969479  0.981996  0.973671   6554.0\n",
       "military                 0.960937  0.970400  0.959205   6554.0\n",
       "child_alone              1.000000  1.000000  1.000000   6554.0\n",
       "water                    0.952072  0.956057  0.948123   6554.0\n",
       "food                     0.940491  0.943546  0.938990   6554.0\n",
       "shelter                  0.925703  0.933018  0.919927   6554.0\n",
       "clothing                 0.983882  0.986726  0.982650   6554.0\n",
       "money                    0.976630  0.980470  0.972036   6554.0\n",
       "missing_people           0.989135  0.989014  0.983700   6554.0\n",
       "refugees                 0.949404  0.966738  0.952534   6554.0\n",
       "death                    0.954963  0.959414  0.944862   6554.0\n",
       "other_aid                0.830404  0.869088  0.815162   6554.0\n",
       "infrastructure_related   0.878504  0.937138  0.906874   6554.0\n",
       "transport                0.944745  0.955142  0.936115   6554.0\n",
       "buildings                0.947885  0.953921  0.935677   6554.0\n",
       "electricity              0.978512  0.978029  0.967605   6554.0\n",
       "tools                    0.986921  0.993439  0.990169   6554.0\n",
       "hospitals                0.982984  0.991456  0.987202   6554.0\n",
       "shops                    0.992690  0.996338  0.994511   6554.0\n",
       "aid_centers              0.975435  0.987641  0.981500   6554.0\n",
       "other_infrastructure     0.915499  0.956668  0.935631   6554.0\n",
       "weather_related          0.881580  0.884040  0.880665   6554.0\n",
       "floods                   0.950399  0.953158  0.946535   6554.0\n",
       "storm                    0.934731  0.940189  0.935660   6554.0\n",
       "fire                     0.979960  0.989777  0.984844   6554.0\n",
       "earthquake               0.969360  0.970400  0.969553   6554.0\n",
       "cold                     0.976821  0.979707  0.971253   6554.0\n",
       "other_weather            0.934929  0.949649  0.927533   6554.0\n",
       "direct_report            0.842606  0.852304  0.828366   6554.0"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Finding weighted average of all metrics\n",
    "weighted_avg = {}\n",
    "\n",
    "for key in results_dict.keys():\n",
    "    weighted_avg[key] = results_dict[key]['weighted avg']\n",
    "\n",
    "df_weight_avg = pd.DataFrame(weighted_avg).transpose()\n",
    "df_weight_avg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    36.000000\n",
       "mean      0.936359\n",
       "std       0.056942\n",
       "min       0.777901\n",
       "25%       0.925631\n",
       "50%       0.947329\n",
       "75%       0.975629\n",
       "max       1.000000\n",
       "Name: f1-score, dtype: float64"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Summary of the metrics\n",
    "df_weight_avg['f1-score'].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***What are the lowest f1-scores?***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>related</th>\n",
       "      <td>0.801030</td>\n",
       "      <td>0.814922</td>\n",
       "      <td>0.794653</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>request</th>\n",
       "      <td>0.893249</td>\n",
       "      <td>0.896857</td>\n",
       "      <td>0.884284</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>aid_related</th>\n",
       "      <td>0.777840</td>\n",
       "      <td>0.779371</td>\n",
       "      <td>0.777901</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>medical_help</th>\n",
       "      <td>0.904538</td>\n",
       "      <td>0.921575</td>\n",
       "      <td>0.891176</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shelter</th>\n",
       "      <td>0.925703</td>\n",
       "      <td>0.933018</td>\n",
       "      <td>0.919927</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other_aid</th>\n",
       "      <td>0.830404</td>\n",
       "      <td>0.869088</td>\n",
       "      <td>0.815162</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>infrastructure_related</th>\n",
       "      <td>0.878504</td>\n",
       "      <td>0.937138</td>\n",
       "      <td>0.906874</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weather_related</th>\n",
       "      <td>0.881580</td>\n",
       "      <td>0.884040</td>\n",
       "      <td>0.880665</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>direct_report</th>\n",
       "      <td>0.842606</td>\n",
       "      <td>0.852304</td>\n",
       "      <td>0.828366</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        precision    recall  f1-score  support\n",
       "related                  0.801030  0.814922  0.794653   6554.0\n",
       "request                  0.893249  0.896857  0.884284   6554.0\n",
       "aid_related              0.777840  0.779371  0.777901   6554.0\n",
       "medical_help             0.904538  0.921575  0.891176   6554.0\n",
       "shelter                  0.925703  0.933018  0.919927   6554.0\n",
       "other_aid                0.830404  0.869088  0.815162   6554.0\n",
       "infrastructure_related   0.878504  0.937138  0.906874   6554.0\n",
       "weather_related          0.881580  0.884040  0.880665   6554.0\n",
       "direct_report            0.842606  0.852304  0.828366   6554.0"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Lowest score that is less than 25\n",
    "df_weight_avg[df_weight_avg['f1-score'] <= df_weight_avg['f1-score'].quantile(0.25)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['related',\n",
       " 'request',\n",
       " 'aid_related',\n",
       " 'medical_help',\n",
       " 'shelter',\n",
       " 'other_aid',\n",
       " 'infrastructure_related',\n",
       " 'weather_related',\n",
       " 'direct_report']"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "key_categories = (df_weight_avg[df_weight_avg['f1-score'] <= df_weight_avg['f1-score'].quantile(0.25)].index).tolist()\n",
    "key_categories"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***What are the highest f1-scores?***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>offer</th>\n",
       "      <td>0.989955</td>\n",
       "      <td>0.994965</td>\n",
       "      <td>0.992454</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>child_alone</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>clothing</th>\n",
       "      <td>0.983882</td>\n",
       "      <td>0.986726</td>\n",
       "      <td>0.982650</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>missing_people</th>\n",
       "      <td>0.989135</td>\n",
       "      <td>0.989014</td>\n",
       "      <td>0.983700</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tools</th>\n",
       "      <td>0.986921</td>\n",
       "      <td>0.993439</td>\n",
       "      <td>0.990169</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hospitals</th>\n",
       "      <td>0.982984</td>\n",
       "      <td>0.991456</td>\n",
       "      <td>0.987202</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shops</th>\n",
       "      <td>0.992690</td>\n",
       "      <td>0.996338</td>\n",
       "      <td>0.994511</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>aid_centers</th>\n",
       "      <td>0.975435</td>\n",
       "      <td>0.987641</td>\n",
       "      <td>0.981500</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fire</th>\n",
       "      <td>0.979960</td>\n",
       "      <td>0.989777</td>\n",
       "      <td>0.984844</td>\n",
       "      <td>6554.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                precision    recall  f1-score  support\n",
       "offer            0.989955  0.994965  0.992454   6554.0\n",
       "child_alone      1.000000  1.000000  1.000000   6554.0\n",
       "clothing         0.983882  0.986726  0.982650   6554.0\n",
       "missing_people   0.989135  0.989014  0.983700   6554.0\n",
       "tools            0.986921  0.993439  0.990169   6554.0\n",
       "hospitals        0.982984  0.991456  0.987202   6554.0\n",
       "shops            0.992690  0.996338  0.994511   6554.0\n",
       "aid_centers      0.975435  0.987641  0.981500   6554.0\n",
       "fire             0.979960  0.989777  0.984844   6554.0"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Lowest score that is less than 75\n",
    "df_weight_avg[df_weight_avg['f1-score'] >= df_weight_avg['f1-score'].quantile(0.75)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    9.000000\n",
       "mean     0.855445\n",
       "std      0.051976\n",
       "min      0.777901\n",
       "25%      0.815162\n",
       "50%      0.880665\n",
       "75%      0.891176\n",
       "max      0.919927\n",
       "Name: f1-score, dtype: float64"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_weight_avg.loc[key_categories]['f1-score'].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***The average f1-score for the key paramaters with the base model is 0.8556 with a std of 0.0519.***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***Creating a function to easily assess a model's performance.***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_performance(labels, preds):\n",
    "    # Printing results\n",
    "    results_dict = {}\n",
    "    \n",
    "    for pred, label, col in zip(preds.transpose(), labels.values.transpose(), labels.columns):\n",
    "        print(col)\n",
    "        print(classification_report(label, pred))\n",
    "        results_dict[col] = classification_report(label, pred, output_dict = True)\n",
    "    \n",
    "    # Converting to a dataframe\n",
    "    weighted_avg = {}\n",
    "    \n",
    "    for key in results_dict.keys():\n",
    "        weighted_avg[key] = results_dict[key]['weighted avg']\n",
    "\n",
    "    df_weight_avg = pd.DataFrame(weighted_avg).transpose()\n",
    "    \n",
    "    # Showing metrics\n",
    "    key_categories = ['related', 'request', 'aid_related', 'medical_help',\n",
    "                      'shelter', 'other_aid', 'infrastructure_related',\n",
    "                      'weather_related', 'direct_report']\n",
    "    \n",
    "    print(df_weight_avg.loc[key_categories]['f1-score'].describe()) \n",
    "    return df_weight_avg"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Improving Model\n",
    "Using grid search to find better parameters. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*Because of the imbalance in the classes, it is possible to get high accuracy scores even with all positive messages in a class falsely labeled as negative.* \n",
    "\n",
    "*As a result, we would like to use the f1-score when evaluating my models, with the micro average used, based on this [StackExchange answer](https://datascience.stackexchange.com/questions/15989/micro-average-vs-macro-average-performance-in-a-multiclass-classification-settin).*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Performing grid search\n",
    "make_scorer(f1_score, average = 'micro')\n",
    "\n",
    "parameters = {\n",
    "    'text__ngram_range': [(1, 2)],\n",
    "    'text__max_df': [0.5],\n",
    "    'clf__estimator__max_depth': [100],\n",
    "    'clf__estimator__min_samples_split': [25], \n",
    "    'clf__estimator__n_estimators': [200]\n",
    "}\n",
    "\n",
    "cv = GridSearchCV(pipeline, parameters)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing  Model\n",
    "*Displaying the accuracy, precision, and recall of the tuned model.*\n",
    "\n",
    "#### *Trying random changes to check if it works or worth changing.*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    9.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    7.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:   11.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    5.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    5.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    5.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    9.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    7.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    9.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    6.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:   11.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    2.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    5.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    5.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    5.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    9.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    5.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    5.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    7.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    9.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    7.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:   11.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    5.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    5.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    6.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    7.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    5.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    5.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    5.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:   10.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    5.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    5.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    8.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    9.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    9.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:   11.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    5.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    5.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    5.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:   12.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    7.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    7.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    6.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    8.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    6.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    2.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:   10.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    2.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    2.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    2.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    8.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    5.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    5.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    9.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:   11.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    7.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:   13.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    5.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    5.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    5.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    5.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    3.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:   13.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    7.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    5.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    7.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    4.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken to fit : 1812.5776765346527\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    9.2s finished\n"
     ]
    }
   ],
   "source": [
    "# Fitting \n",
    "start_time = time.time()\n",
    "cv.fit(train_X, train_Y)\n",
    "end_time = time.time()\n",
    "print(\"time taken to fit : {}\".format(end_time - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'clf__estimator__max_depth': 100,\n",
       " 'clf__estimator__min_samples_split': 25,\n",
       " 'clf__estimator__n_estimators': 200,\n",
       " 'text__max_df': 0.5,\n",
       " 'text__ngram_range': (1, 2)}"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.20318419746418845"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 200 out of 200 | elapsed:    0.5s finished\n"
     ]
    }
   ],
   "source": [
    "# predict\n",
    "start_time = time.time()\n",
    "y_preds = cv.predict(test_X)\n",
    "end_time = time.time()\n",
    "print(\"time taken to predict : {}\".format(end_time - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Shubham\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1221: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "related\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.79      0.10      0.17      1563\n",
      "           1       0.78      0.99      0.87      4991\n",
      "\n",
      "    accuracy                           0.78      6554\n",
      "   macro avg       0.78      0.54      0.52      6554\n",
      "weighted avg       0.78      0.78      0.71      6554\n",
      "\n",
      "request\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.87      1.00      0.93      5443\n",
      "           1       0.93      0.27      0.42      1111\n",
      "\n",
      "    accuracy                           0.87      6554\n",
      "   macro avg       0.90      0.63      0.67      6554\n",
      "weighted avg       0.88      0.87      0.84      6554\n",
      "\n",
      "offer\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00      6521\n",
      "           1       0.00      0.00      0.00        33\n",
      "\n",
      "    accuracy                           0.99      6554\n",
      "   macro avg       0.50      0.50      0.50      6554\n",
      "weighted avg       0.99      0.99      0.99      6554\n",
      "\n",
      "aid_related\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.75      0.92      0.82      3884\n",
      "           1       0.83      0.54      0.65      2670\n",
      "\n",
      "    accuracy                           0.77      6554\n",
      "   macro avg       0.79      0.73      0.74      6554\n",
      "weighted avg       0.78      0.77      0.76      6554\n",
      "\n",
      "medical_help\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      1.00      0.96      6019\n",
      "           1       0.71      0.01      0.02       535\n",
      "\n",
      "    accuracy                           0.92      6554\n",
      "   macro avg       0.82      0.50      0.49      6554\n",
      "weighted avg       0.90      0.92      0.88      6554\n",
      "\n",
      "medical_products\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.97      6210\n",
      "           1       0.75      0.01      0.02       344\n",
      "\n",
      "    accuracy                           0.95      6554\n",
      "   macro avg       0.85      0.50      0.50      6554\n",
      "weighted avg       0.94      0.95      0.92      6554\n",
      "\n",
      "search_and_rescue\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6395\n",
      "           1       0.00      0.00      0.00       159\n",
      "\n",
      "    accuracy                           0.98      6554\n",
      "   macro avg       0.49      0.50      0.49      6554\n",
      "weighted avg       0.95      0.98      0.96      6554\n",
      "\n",
      "security\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6438\n",
      "           1       0.00      0.00      0.00       116\n",
      "\n",
      "    accuracy                           0.98      6554\n",
      "   macro avg       0.49      0.50      0.50      6554\n",
      "weighted avg       0.96      0.98      0.97      6554\n",
      "\n",
      "military\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      1.00      0.98      6354\n",
      "           1       1.00      0.01      0.02       200\n",
      "\n",
      "    accuracy                           0.97      6554\n",
      "   macro avg       0.98      0.51      0.50      6554\n",
      "weighted avg       0.97      0.97      0.96      6554\n",
      "\n",
      "child_alone\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      6554\n",
      "\n",
      "    accuracy                           1.00      6554\n",
      "   macro avg       1.00      1.00      1.00      6554\n",
      "weighted avg       1.00      1.00      1.00      6554\n",
      "\n",
      "water\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      1.00      0.97      6136\n",
      "           1       0.95      0.10      0.17       418\n",
      "\n",
      "    accuracy                           0.94      6554\n",
      "   macro avg       0.95      0.55      0.57      6554\n",
      "weighted avg       0.94      0.94      0.92      6554\n",
      "\n",
      "food\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      1.00      0.95      5809\n",
      "           1       0.94      0.14      0.24       745\n",
      "\n",
      "    accuracy                           0.90      6554\n",
      "   macro avg       0.92      0.57      0.59      6554\n",
      "weighted avg       0.91      0.90      0.87      6554\n",
      "\n",
      "shelter\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      1.00      0.96      5973\n",
      "           1       0.85      0.06      0.11       581\n",
      "\n",
      "    accuracy                           0.92      6554\n",
      "   macro avg       0.88      0.53      0.53      6554\n",
      "weighted avg       0.91      0.92      0.88      6554\n",
      "\n",
      "clothing\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6456\n",
      "           1       1.00      0.02      0.04        98\n",
      "\n",
      "    accuracy                           0.99      6554\n",
      "   macro avg       0.99      0.51      0.52      6554\n",
      "weighted avg       0.99      0.99      0.98      6554\n",
      "\n",
      "money\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6421\n",
      "           1       1.00      0.01      0.01       133\n",
      "\n",
      "    accuracy                           0.98      6554\n",
      "   macro avg       0.99      0.50      0.50      6554\n",
      "weighted avg       0.98      0.98      0.97      6554\n",
      "\n",
      "missing_people\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6481\n",
      "           1       0.00      0.00      0.00        73\n",
      "\n",
      "    accuracy                           0.99      6554\n",
      "   macro avg       0.49      0.50      0.50      6554\n",
      "weighted avg       0.98      0.99      0.98      6554\n",
      "\n",
      "refugees\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      1.00      0.98      6339\n",
      "           1       0.00      0.00      0.00       215\n",
      "\n",
      "    accuracy                           0.97      6554\n",
      "   macro avg       0.48      0.50      0.49      6554\n",
      "weighted avg       0.94      0.97      0.95      6554\n",
      "\n",
      "death\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      6257\n",
      "           1       1.00      0.02      0.04       297\n",
      "\n",
      "    accuracy                           0.96      6554\n",
      "   macro avg       0.98      0.51      0.51      6554\n",
      "weighted avg       0.96      0.96      0.93      6554\n",
      "\n",
      "other_aid\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.87      1.00      0.93      5690\n",
      "           1       1.00      0.00      0.00       864\n",
      "\n",
      "    accuracy                           0.87      6554\n",
      "   macro avg       0.93      0.50      0.47      6554\n",
      "weighted avg       0.89      0.87      0.81      6554\n",
      "\n",
      "infrastructure_related\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      1.00      0.97      6143\n",
      "           1       0.00      0.00      0.00       411\n",
      "\n",
      "    accuracy                           0.94      6554\n",
      "   macro avg       0.47      0.50      0.48      6554\n",
      "weighted avg       0.88      0.94      0.91      6554\n",
      "\n",
      "transport\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.98      6251\n",
      "           1       0.00      0.00      0.00       303\n",
      "\n",
      "    accuracy                           0.95      6554\n",
      "   macro avg       0.48      0.50      0.49      6554\n",
      "weighted avg       0.91      0.95      0.93      6554\n",
      "\n",
      "buildings\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.97      6231\n",
      "           1       1.00      0.01      0.01       323\n",
      "\n",
      "    accuracy                           0.95      6554\n",
      "   macro avg       0.98      0.50      0.49      6554\n",
      "weighted avg       0.95      0.95      0.93      6554\n",
      "\n",
      "electricity\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6407\n",
      "           1       1.00      0.01      0.01       147\n",
      "\n",
      "    accuracy                           0.98      6554\n",
      "   macro avg       0.99      0.50      0.50      6554\n",
      "weighted avg       0.98      0.98      0.97      6554\n",
      "\n",
      "tools\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00      6511\n",
      "           1       0.00      0.00      0.00        43\n",
      "\n",
      "    accuracy                           0.99      6554\n",
      "   macro avg       0.50      0.50      0.50      6554\n",
      "weighted avg       0.99      0.99      0.99      6554\n",
      "\n",
      "hospitals\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00      6498\n",
      "           1       0.00      0.00      0.00        56\n",
      "\n",
      "    accuracy                           0.99      6554\n",
      "   macro avg       0.50      0.50      0.50      6554\n",
      "weighted avg       0.98      0.99      0.99      6554\n",
      "\n",
      "shops\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      6530\n",
      "           1       0.00      0.00      0.00        24\n",
      "\n",
      "    accuracy                           1.00      6554\n",
      "   macro avg       0.50      0.50      0.50      6554\n",
      "weighted avg       0.99      1.00      0.99      6554\n",
      "\n",
      "aid_centers\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6473\n",
      "           1       0.00      0.00      0.00        81\n",
      "\n",
      "    accuracy                           0.99      6554\n",
      "   macro avg       0.49      0.50      0.50      6554\n",
      "weighted avg       0.98      0.99      0.98      6554\n",
      "\n",
      "other_infrastructure\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      6271\n",
      "           1       0.00      0.00      0.00       283\n",
      "\n",
      "    accuracy                           0.96      6554\n",
      "   macro avg       0.48      0.50      0.49      6554\n",
      "weighted avg       0.92      0.96      0.94      6554\n",
      "\n",
      "weather_related\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.79      0.99      0.88      4781\n",
      "           1       0.92      0.31      0.46      1773\n",
      "\n",
      "    accuracy                           0.81      6554\n",
      "   macro avg       0.86      0.65      0.67      6554\n",
      "weighted avg       0.83      0.81      0.77      6554\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "floods\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      1.00      0.96      6035\n",
      "           1       0.92      0.06      0.12       519\n",
      "\n",
      "    accuracy                           0.93      6554\n",
      "   macro avg       0.92      0.53      0.54      6554\n",
      "weighted avg       0.92      0.93      0.89      6554\n",
      "\n",
      "storm\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      1.00      0.95      5949\n",
      "           1       0.87      0.09      0.16       605\n",
      "\n",
      "    accuracy                           0.91      6554\n",
      "   macro avg       0.89      0.54      0.56      6554\n",
      "weighted avg       0.91      0.91      0.88      6554\n",
      "\n",
      "fire\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6488\n",
      "           1       0.00      0.00      0.00        66\n",
      "\n",
      "    accuracy                           0.99      6554\n",
      "   macro avg       0.49      0.50      0.50      6554\n",
      "weighted avg       0.98      0.99      0.98      6554\n",
      "\n",
      "earthquake\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      1.00      0.96      5964\n",
      "           1       0.92      0.25      0.39       590\n",
      "\n",
      "    accuracy                           0.93      6554\n",
      "   macro avg       0.93      0.62      0.68      6554\n",
      "weighted avg       0.93      0.93      0.91      6554\n",
      "\n",
      "cold\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6413\n",
      "           1       0.00      0.00      0.00       141\n",
      "\n",
      "    accuracy                           0.98      6554\n",
      "   macro avg       0.49      0.50      0.49      6554\n",
      "weighted avg       0.96      0.98      0.97      6554\n",
      "\n",
      "other_weather\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.97      6219\n",
      "           1       1.00      0.00      0.01       335\n",
      "\n",
      "    accuracy                           0.95      6554\n",
      "   macro avg       0.97      0.50      0.49      6554\n",
      "weighted avg       0.95      0.95      0.92      6554\n",
      "\n",
      "direct_report\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.83      0.99      0.91      5282\n",
      "           1       0.83      0.18      0.30      1272\n",
      "\n",
      "    accuracy                           0.83      6554\n",
      "   macro avg       0.83      0.59      0.60      6554\n",
      "weighted avg       0.83      0.83      0.79      6554\n",
      "\n",
      "count    9.000000\n",
      "mean     0.814871\n",
      "std      0.067540\n",
      "min      0.705402\n",
      "25%      0.767797\n",
      "50%      0.807280\n",
      "75%      0.880699\n",
      "max      0.906950\n",
      "Name: f1-score, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "results_cv = model_performance(test_Y, y_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Additional Features:\n",
    "\n",
    "*Features that are common in improving NLP Classification include ([Reference](https://www.analyticsvidhya.com/blog/2018/04/a-comprehensive-guide-to-understand-and-implement-text-classification-in-python/)):*\n",
    "- Number of words in sentence\n",
    "- Number of characters in message\n",
    "- Parts of speech count\n",
    "\n",
    "**These additional features will be added to the pipeline.**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### *Number of Words*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating class for performing word count operation \n",
    "class WordCount(BaseEstimator, TransformerMixin):\n",
    "    # Tokenizing words\n",
    "    def word_count(self, text):\n",
    "        table = text.maketrans(dict.fromkeys(string.punctuation))\n",
    "        words = word_tokenize(text.lower().strip().translate(table))\n",
    "        return len(words)\n",
    "    # For fitting\n",
    "    def fit(self, x, y=None):\n",
    "        return self\n",
    "    # Transformation ,Creating series\n",
    "    def transform(self, x):\n",
    "        count = pd.Series(x).apply(self.word_count)\n",
    "        return pd.DataFrame(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "....\n",
      "----------------------------------------------------------------------\n",
      "Ran 4 tests in 0.017s\n",
      "\n",
      "OK\n"
     ]
    }
   ],
   "source": [
    "class Test_WordCount_Class(unittest.TestCase):\n",
    "    # Testing the above made class, referred from stackoverflow \n",
    "    def setUp(self):\n",
    "        self.sample_1 = WordCount().fit_transform('I went to the shops today.')\n",
    "        self.sample_2 = WordCount().fit_transform(\"They've been very good to me, you know?\")\n",
    "        self.sample_3 = WordCount().fit_transform(\"In they went. Two by two.\")\n",
    "        self.sample_4 = WordCount().fit_transform(\"It takes a lot to love someone. \\\n",
    "            You would think that loving someone for 20 years would count for something.\")\n",
    "    \n",
    "    def test_sample_1(self):\n",
    "        self.assertEqual(self.sample_1[0][0], 6, \"Word count should be 6.\")\n",
    "        \n",
    "    def test_sample_2(self):\n",
    "        self.assertEqual(self.sample_2[0][0], 8, \"Word count should be 8.\")\n",
    "    \n",
    "    def test_sample_3(self):\n",
    "        self.assertEqual(self.sample_3[0][0], 6, \"Word count should be 6.\")\n",
    "        \n",
    "    def test_sample_4(self):\n",
    "        self.assertEqual(self.sample_4[0][0], 20, \"Word count should be 20.\")\n",
    "\n",
    "tests = Test_WordCount_Class()\n",
    "\n",
    "tests_loaded = unittest.TestLoader().loadTestsFromModule(tests)\n",
    "\n",
    "unittest.TextTestRunner().run(tests_loaded);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### *Number of Characters*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Example\n",
    "len('I went to sydney last year.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# COunting number of characters\n",
    "class CharacterCount(BaseEstimator, TransformerMixin):\n",
    "    # Function to count characters\n",
    "    def character_count(self, text):\n",
    "        return len(text)\n",
    "    \n",
    "    # Fitting function\n",
    "    def fit(self, x, y = None):\n",
    "        return self\n",
    "    \n",
    "    # Transformation function\n",
    "    def transform(self, x):\n",
    "        count = pd.Series(x).apply(self.character_count)\n",
    "        return pd.DataFrame(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "....\n",
      "----------------------------------------------------------------------\n",
      "Ran 4 tests in 0.018s\n",
      "\n",
      "OK\n"
     ]
    }
   ],
   "source": [
    "# Checking and testing above class, referred from stackoverflow \n",
    "class Test_CharacterCount_Class(unittest.TestCase):\n",
    "    def setUp(self):\n",
    "        self.sample_1 = CharacterCount().fit_transform('I went to the shops today.')\n",
    "        self.sample_2 = CharacterCount().fit_transform(\"They've been very good to me, you know?\")\n",
    "        self.sample_3 = CharacterCount().fit_transform(\"In they went. Two by two.\")\n",
    "        self.sample_4 = CharacterCount().fit_transform(\"It takes a lot to love someone. You would think that loving someone for 20 years would count for something.\")\n",
    "        \n",
    "    def test_sample_1(self):\n",
    "        self.assertEqual(self.sample_1[0][0], 26, 'Count should be 26.')\n",
    "    \n",
    "    def test_sample_2(self):\n",
    "        self.assertEqual(self.sample_2[0][0], 39, 'Count should be 39.')\n",
    "    \n",
    "    def test_sample_3(self):\n",
    "        self.assertEqual(self.sample_3[0][0], 25, 'Count should be 25.')\n",
    "    \n",
    "    def test_sample_4(self):\n",
    "        self.assertEqual(self.sample_4[0][0], 107, 'Count should be 107.')\n",
    "\n",
    "tests = Test_CharacterCount_Class()\n",
    "tests_loaded = unittest.TestLoader().loadTestsFromModule(tests)\n",
    "unittest.TextTestRunner().run(tests_loaded);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### *Number of Nouns*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of Nouns\n",
    "class NounCount(BaseEstimator, TransformerMixin):\n",
    "    # Counting number of nouns\n",
    "    def noun_count(self, text):\n",
    "        count = 0\n",
    "        sentence_list = sent_tokenize(text)\n",
    "        for sentence in sentence_list:\n",
    "            pos_tags = nltk.pos_tag(tokenize(sentence))\n",
    "            for _, tag in pos_tags:\n",
    "                if tag in ['PRP', 'NN']:\n",
    "                    count += 1\n",
    "        \n",
    "        return count\n",
    "            \n",
    "    # Fitting function\n",
    "    def fit(self, x, y = None):\n",
    "        return self\n",
    "    \n",
    "    # Transformation\n",
    "    def transform(self, x):\n",
    "        count = pd.Series(x).apply(self.noun_count)\n",
    "        return pd.DataFrame(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "....\n",
      "----------------------------------------------------------------------\n",
      "Ran 4 tests in 0.154s\n",
      "\n",
      "OK\n"
     ]
    }
   ],
   "source": [
    "# Testing the noun class\n",
    "class Test_NounCount_Class(unittest.TestCase):\n",
    "    def setUp(self):\n",
    "        self.sample_1 = NounCount().fit_transform('I went to the shops today.')\n",
    "        self.sample_2 = NounCount().fit_transform(\"They've been very good to me, you know?\")\n",
    "        self.sample_3 = NounCount().fit_transform(\"In they went; two by two.\")\n",
    "        self.sample_4 = NounCount().fit_transform(\"It takes a lot to love someone. You would think that loving someone for 20 years would count for something.\")\n",
    "        \n",
    "    def test_sample_1(self):\n",
    "        self.assertEqual(self.sample_1[0][0], 2, 'Count should be 2.')\n",
    "    \n",
    "    def test_sample_2(self):\n",
    "        self.assertEqual(self.sample_2[0][0], 0, 'Count should be 0.')\n",
    "    \n",
    "    def test_sample_3(self):\n",
    "        self.assertEqual(self.sample_3[0][0], 0, 'Count should be 0.')\n",
    "    \n",
    "    def test_sample_4(self):\n",
    "        self.assertEqual(self.sample_4[0][0], 5, 'Count should be 5.')\n",
    "\n",
    "tests = Test_NounCount_Class()\n",
    "tests_loaded = unittest.TestLoader().loadTestsFromModule(tests)\n",
    "unittest.TextTestRunner().run(tests_loaded);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### *Number of Verbs*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Getting verb Counts\n",
    "class VerbCount(BaseEstimator, TransformerMixin):\n",
    "    def verb_count(self, text):\n",
    "        count = 0\n",
    "        sentence_list = sent_tokenize(text)\n",
    "        for sentence in sentence_list:\n",
    "            pos_tags = nltk.pos_tag(tokenize(sentence))\n",
    "            for word, tag in pos_tags:\n",
    "                if tag in ['VB', 'VBP']:\n",
    "                    count += 1\n",
    "        \n",
    "        return count\n",
    "            \n",
    "    \n",
    "    def fit(self, x, y=None):\n",
    "        return self\n",
    "    \n",
    "    def transform(self, x):\n",
    "        count = pd.Series(x).apply(self.verb_count)\n",
    "        return pd.DataFrame(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Example\n",
    "nltk.pos_tag(tokenize('Did you did it?'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "....\n",
      "----------------------------------------------------------------------\n",
      "Ran 4 tests in 0.080s\n",
      "\n",
      "OK\n"
     ]
    }
   ],
   "source": [
    "class Test_VerbCount_Class(unittest.TestCase):\n",
    "    def setUp(self):\n",
    "        self.sample_1 = VerbCount().fit_transform('I went to the shops today.')\n",
    "        self.sample_2 = VerbCount().fit_transform(\"They've been very good to me, you know?\")\n",
    "        self.sample_3 = VerbCount().fit_transform(\"In they went; two by two.\")\n",
    "        self.sample_4 = VerbCount().fit_transform(\"It takes a lot to love someone. You would think that loving someone for 20 years would count for something.\")\n",
    "        \n",
    "    def test_sample_1(self):\n",
    "        self.assertEqual(self.sample_1[0][0], 1, 'Count should be 1.')\n",
    "    \n",
    "    def test_sample_2(self):\n",
    "        self.assertEqual(self.sample_2[0][0], 0, 'Count should be 0.')\n",
    "        \n",
    "    def test_sample_3(self):\n",
    "        self.assertEqual(self.sample_3[0][0], 1, 'Count should be 1.')\n",
    "    \n",
    "    def test_sample_4(self):\n",
    "        self.assertEqual(self.sample_4[0][0], 4, 'Count should be 4.')\n",
    "\n",
    "tests = Test_VerbCount_Class()\n",
    "tests_loaded = unittest.TestLoader().loadTestsFromModule(tests)\n",
    "unittest.TextTestRunner().run(tests_loaded);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Making pipeline\n",
    "pipeline = Pipeline([\n",
    "    (\"features\", FeatureUnion([\n",
    "        (\"text\", TfidfVectorizer(tokenizer = tokenize, max_df = 0.5, ngram_range = (1, 2))),\n",
    "        (\"word_count\", WordCount()),\n",
    "        (\"character_count\", CharacterCount()),\n",
    "        (\"noun_count\", NounCount()),\n",
    "        (\"verb_count\", VerbCount())\n",
    "    ])),\n",
    "    (\"clf\", MultiOutputClassifier(RandomForestClassifier(min_samples_split = 25, random_state = 42, verbose = 1)))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "scorer = make_scorer(f1_score, average = 'micro')\n",
    "\n",
    "parameters = {\n",
    "    'clf__estimator__max_depth': [500],\n",
    "    'clf__estimator__n_estimators': [300]\n",
    "}\n",
    "\n",
    "cv = GridSearchCV(pipeline, parameters, scoring = scorer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   41.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   31.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    6.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   45.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   22.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   19.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   13.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   10.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   11.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   16.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   23.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   20.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    9.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   10.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    7.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   14.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   14.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   27.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   19.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   16.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   16.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    9.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    5.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    7.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    5.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    7.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   15.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   34.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   19.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   20.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    7.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   19.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   10.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   17.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   31.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   39.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   30.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    5.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   46.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   20.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   16.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   13.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   10.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   11.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   16.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   21.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   21.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    8.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   10.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    8.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   13.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   14.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   34.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   20.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   16.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   23.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    9.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    5.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    7.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    5.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    8.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   15.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   36.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   20.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   20.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    7.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   19.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   10.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   17.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   32.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   40.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   30.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    5.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   45.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   20.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   16.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   12.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   10.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   10.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   16.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   21.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   21.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    8.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   10.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    8.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   12.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   14.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   27.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   20.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   16.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   16.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    9.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    5.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    7.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    5.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    7.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   16.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   35.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   22.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   27.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   10.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   28.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   12.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   24.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   46.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    2.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    2.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    2.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   53.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   31.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    5.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:  1.1min finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   57.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   46.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   36.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   29.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   31.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   45.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:  1.0min finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   59.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   24.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   26.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    9.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   12.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   14.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   26.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   19.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   17.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   22.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   10.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    9.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    8.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    5.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    8.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   18.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   37.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   20.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   22.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    7.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   23.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   11.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   18.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   33.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   38.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   29.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    5.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   43.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   19.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   16.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   14.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   12.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   11.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   17.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   23.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   20.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    8.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   14.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    9.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   13.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   15.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   30.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   26.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   18.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   20.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   10.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    6.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   10.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    4.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    8.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   25.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   38.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   20.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   20.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    7.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   20.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    9.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   17.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   31.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   55.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   41.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    7.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:  1.0min finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   26.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   21.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   18.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   13.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   14.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   21.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   28.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   27.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   11.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   14.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   11.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   17.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   19.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   44.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   28.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   22.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   22.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   13.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    9.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   12.4s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    7.7s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   12.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   23.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   46.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   26.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   26.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   10.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   25.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   13.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   23.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken : 6950.132976055145\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:   48.1s finished\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "cv.fit(train_X, train_Y)\n",
    "end_time = time.time()\n",
    "\n",
    "print(\"time taken : {}\".format(end_time - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6311097953418664"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'clf__estimator__max_depth': 500, 'clf__estimator__n_estimators': 300}"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    2.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    2.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    2.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    2.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    2.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    2.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.6s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    2.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    2.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    2.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    2.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    0.9s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    2.0s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.8s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    2.1s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    2.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    2.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.5s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    2.3s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 300 out of 300 | elapsed:    1.9s finished\n",
      "C:\\Users\\Shubham\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1221: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken to predict : 192.67354893684387\n",
      "related\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.75      0.30      0.43      1563\n",
      "           1       0.82      0.97      0.89      4991\n",
      "\n",
      "    accuracy                           0.81      6554\n",
      "   macro avg       0.78      0.64      0.66      6554\n",
      "weighted avg       0.80      0.81      0.78      6554\n",
      "\n",
      "request\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.99      0.94      5443\n",
      "           1       0.87      0.46      0.60      1111\n",
      "\n",
      "    accuracy                           0.90      6554\n",
      "   macro avg       0.88      0.72      0.77      6554\n",
      "weighted avg       0.89      0.90      0.88      6554\n",
      "\n",
      "offer\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00      6521\n",
      "           1       0.00      0.00      0.00        33\n",
      "\n",
      "    accuracy                           0.99      6554\n",
      "   macro avg       0.50      0.50      0.50      6554\n",
      "weighted avg       0.99      0.99      0.99      6554\n",
      "\n",
      "aid_related\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.80      0.85      0.82      3884\n",
      "           1       0.76      0.68      0.72      2670\n",
      "\n",
      "    accuracy                           0.78      6554\n",
      "   macro avg       0.78      0.77      0.77      6554\n",
      "weighted avg       0.78      0.78      0.78      6554\n",
      "\n",
      "medical_help\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      1.00      0.96      6019\n",
      "           1       0.75      0.06      0.11       535\n",
      "\n",
      "    accuracy                           0.92      6554\n",
      "   macro avg       0.84      0.53      0.54      6554\n",
      "weighted avg       0.91      0.92      0.89      6554\n",
      "\n",
      "medical_products\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.97      6210\n",
      "           1       0.88      0.06      0.11       344\n",
      "\n",
      "    accuracy                           0.95      6554\n",
      "   macro avg       0.91      0.53      0.54      6554\n",
      "weighted avg       0.95      0.95      0.93      6554\n",
      "\n",
      "search_and_rescue\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6395\n",
      "           1       0.85      0.07      0.13       159\n",
      "\n",
      "    accuracy                           0.98      6554\n",
      "   macro avg       0.91      0.53      0.56      6554\n",
      "weighted avg       0.97      0.98      0.97      6554\n",
      "\n",
      "security\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6438\n",
      "           1       0.33      0.01      0.02       116\n",
      "\n",
      "    accuracy                           0.98      6554\n",
      "   macro avg       0.66      0.50      0.50      6554\n",
      "weighted avg       0.97      0.98      0.97      6554\n",
      "\n",
      "military\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      1.00      0.98      6354\n",
      "           1       0.70      0.04      0.07       200\n",
      "\n",
      "    accuracy                           0.97      6554\n",
      "   macro avg       0.84      0.52      0.53      6554\n",
      "weighted avg       0.96      0.97      0.96      6554\n",
      "\n",
      "child_alone\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      6554\n",
      "\n",
      "    accuracy                           1.00      6554\n",
      "   macro avg       1.00      1.00      1.00      6554\n",
      "weighted avg       1.00      1.00      1.00      6554\n",
      "\n",
      "water\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      6136\n",
      "           1       0.91      0.40      0.56       418\n",
      "\n",
      "    accuracy                           0.96      6554\n",
      "   macro avg       0.93      0.70      0.77      6554\n",
      "weighted avg       0.96      0.96      0.95      6554\n",
      "\n",
      "food\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.99      0.96      5809\n",
      "           1       0.86      0.52      0.65       745\n",
      "\n",
      "    accuracy                           0.94      6554\n",
      "   macro avg       0.90      0.75      0.81      6554\n",
      "weighted avg       0.93      0.94      0.93      6554\n",
      "\n",
      "shelter\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      1.00      0.96      5973\n",
      "           1       0.85      0.29      0.43       581\n",
      "\n",
      "    accuracy                           0.93      6554\n",
      "   macro avg       0.89      0.64      0.70      6554\n",
      "weighted avg       0.93      0.93      0.92      6554\n",
      "\n",
      "clothing\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6456\n",
      "           1       0.56      0.05      0.09        98\n",
      "\n",
      "    accuracy                           0.99      6554\n",
      "   macro avg       0.77      0.53      0.54      6554\n",
      "weighted avg       0.98      0.99      0.98      6554\n",
      "\n",
      "money\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6421\n",
      "           1       0.88      0.05      0.10       133\n",
      "\n",
      "    accuracy                           0.98      6554\n",
      "   macro avg       0.93      0.53      0.54      6554\n",
      "weighted avg       0.98      0.98      0.97      6554\n",
      "\n",
      "missing_people\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6481\n",
      "           1       0.00      0.00      0.00        73\n",
      "\n",
      "    accuracy                           0.99      6554\n",
      "   macro avg       0.49      0.50      0.50      6554\n",
      "weighted avg       0.98      0.99      0.98      6554\n",
      "\n",
      "refugees\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      1.00      0.98      6339\n",
      "           1       0.00      0.00      0.00       215\n",
      "\n",
      "    accuracy                           0.97      6554\n",
      "   macro avg       0.48      0.50      0.49      6554\n",
      "weighted avg       0.94      0.97      0.95      6554\n",
      "\n",
      "death\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      6257\n",
      "           1       0.86      0.14      0.25       297\n",
      "\n",
      "    accuracy                           0.96      6554\n",
      "   macro avg       0.91      0.57      0.61      6554\n",
      "weighted avg       0.96      0.96      0.95      6554\n",
      "\n",
      "other_aid\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.87      1.00      0.93      5690\n",
      "           1       0.72      0.02      0.03       864\n",
      "\n",
      "    accuracy                           0.87      6554\n",
      "   macro avg       0.80      0.51      0.48      6554\n",
      "weighted avg       0.85      0.87      0.81      6554\n",
      "\n",
      "infrastructure_related\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      1.00      0.97      6143\n",
      "           1       0.00      0.00      0.00       411\n",
      "\n",
      "    accuracy                           0.94      6554\n",
      "   macro avg       0.47      0.50      0.48      6554\n",
      "weighted avg       0.88      0.94      0.91      6554\n",
      "\n",
      "transport\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      6251\n",
      "           1       0.75      0.03      0.06       303\n",
      "\n",
      "    accuracy                           0.95      6554\n",
      "   macro avg       0.85      0.51      0.52      6554\n",
      "weighted avg       0.95      0.95      0.93      6554\n",
      "\n",
      "buildings\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      6231\n",
      "           1       0.87      0.11      0.19       323\n",
      "\n",
      "    accuracy                           0.96      6554\n",
      "   macro avg       0.91      0.55      0.58      6554\n",
      "weighted avg       0.95      0.96      0.94      6554\n",
      "\n",
      "electricity\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6407\n",
      "           1       1.00      0.01      0.03       147\n",
      "\n",
      "    accuracy                           0.98      6554\n",
      "   macro avg       0.99      0.51      0.51      6554\n",
      "weighted avg       0.98      0.98      0.97      6554\n",
      "\n",
      "tools\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00      6511\n",
      "           1       0.00      0.00      0.00        43\n",
      "\n",
      "    accuracy                           0.99      6554\n",
      "   macro avg       0.50      0.50      0.50      6554\n",
      "weighted avg       0.99      0.99      0.99      6554\n",
      "\n",
      "hospitals\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00      6498\n",
      "           1       0.00      0.00      0.00        56\n",
      "\n",
      "    accuracy                           0.99      6554\n",
      "   macro avg       0.50      0.50      0.50      6554\n",
      "weighted avg       0.98      0.99      0.99      6554\n",
      "\n",
      "shops\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      6530\n",
      "           1       0.00      0.00      0.00        24\n",
      "\n",
      "    accuracy                           1.00      6554\n",
      "   macro avg       0.50      0.50      0.50      6554\n",
      "weighted avg       0.99      1.00      0.99      6554\n",
      "\n",
      "aid_centers\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6473\n",
      "           1       0.00      0.00      0.00        81\n",
      "\n",
      "    accuracy                           0.99      6554\n",
      "   macro avg       0.49      0.50      0.50      6554\n",
      "weighted avg       0.98      0.99      0.98      6554\n",
      "\n",
      "other_infrastructure\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      6271\n",
      "           1       0.00      0.00      0.00       283\n",
      "\n",
      "    accuracy                           0.96      6554\n",
      "   macro avg       0.48      0.50      0.49      6554\n",
      "weighted avg       0.92      0.96      0.94      6554\n",
      "\n",
      "weather_related\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.87      0.96      0.91      4781\n",
      "           1       0.85      0.62      0.72      1773\n",
      "\n",
      "    accuracy                           0.87      6554\n",
      "   macro avg       0.86      0.79      0.82      6554\n",
      "weighted avg       0.87      0.87      0.86      6554\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "floods\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.97      6035\n",
      "           1       0.89      0.45      0.59       519\n",
      "\n",
      "    accuracy                           0.95      6554\n",
      "   macro avg       0.92      0.72      0.78      6554\n",
      "weighted avg       0.95      0.95      0.94      6554\n",
      "\n",
      "storm\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.99      0.96      5949\n",
      "           1       0.77      0.34      0.48       605\n",
      "\n",
      "    accuracy                           0.93      6554\n",
      "   macro avg       0.85      0.67      0.72      6554\n",
      "weighted avg       0.92      0.93      0.92      6554\n",
      "\n",
      "fire\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6488\n",
      "           1       0.00      0.00      0.00        66\n",
      "\n",
      "    accuracy                           0.99      6554\n",
      "   macro avg       0.49      0.50      0.50      6554\n",
      "weighted avg       0.98      0.99      0.98      6554\n",
      "\n",
      "earthquake\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.99      0.98      5964\n",
      "           1       0.89      0.76      0.82       590\n",
      "\n",
      "    accuracy                           0.97      6554\n",
      "   macro avg       0.94      0.88      0.90      6554\n",
      "weighted avg       0.97      0.97      0.97      6554\n",
      "\n",
      "cold\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6413\n",
      "           1       0.80      0.03      0.05       141\n",
      "\n",
      "    accuracy                           0.98      6554\n",
      "   macro avg       0.89      0.51      0.52      6554\n",
      "weighted avg       0.98      0.98      0.97      6554\n",
      "\n",
      "other_weather\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.97      6219\n",
      "           1       0.67      0.01      0.01       335\n",
      "\n",
      "    accuracy                           0.95      6554\n",
      "   macro avg       0.81      0.50      0.49      6554\n",
      "weighted avg       0.93      0.95      0.92      6554\n",
      "\n",
      "direct_report\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.86      0.98      0.91      5282\n",
      "           1       0.79      0.32      0.45      1272\n",
      "\n",
      "    accuracy                           0.85      6554\n",
      "   macro avg       0.82      0.65      0.68      6554\n",
      "weighted avg       0.84      0.85      0.82      6554\n",
      "\n",
      "count    9.000000\n",
      "mean     0.849839\n",
      "std      0.053507\n",
      "min      0.777465\n",
      "25%      0.811273\n",
      "50%      0.861071\n",
      "75%      0.890073\n",
      "max      0.916466\n",
      "Name: f1-score, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "y_preds = cv.predict(test_X)\n",
    "end_time = time.time()\n",
    "print(\"time taken to predict : {}\".format(end_time - start_time))\n",
    "\n",
    "results_cv = model_performance(test_Y, y_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exporting model as a pickle file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Approximately 1.75GB\n",
    "with open('data/model.pkl', 'wb') as file:\n",
    "    pickle.dump(cv, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}