nb2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/m09170/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import os\n",
    "import warnings\n",
    "\n",
    "from fancyimpute import SimpleFill\n",
    "\n",
    "from sklearn.pipeline import make_pipeline, make_union\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.preprocessing import LabelEncoder, OneHotEncoder, StandardScaler\n",
    "from sklearn.decomposition import PCA, LatentDirichletAllocation\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "#product data\n",
    "train = pd.read_csv('../input/train.csv')\n",
    "test = pd.read_csv('../input/test.csv')\n",
    "\n",
    "#sales, exchange rates, social network data\n",
    "sales = pd.read_csv('../input/sales.csv')\n",
    "\n",
    "#website navigation data\n",
    "navigation = pd.read_csv('../input/navigation.csv')\n",
    "\n",
    "#product images vectorized with ResNet50\n",
    "vimages = pd.read_csv('../input/vimages.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "bda1637e7052353ab9acad765f0e870f8078b066"
   },
   "source": [
    "# Data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "_uuid": "66ca5a8fc9b5449d4417a85d5b13e71aaf85b924"
   },
   "outputs": [],
   "source": [
    "currency_and_social_columns = sales.columns[9:].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "_uuid": "3037dd2f0cf7ecd05b0b0e60648464573e18c277"
   },
   "outputs": [],
   "source": [
    "first_day = sales.loc[sales.Date == 'Day_1',:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "_uuid": "8f75e5524d09fe87b0a5c4db1576d75e7b26e4bf"
   },
   "outputs": [],
   "source": [
    "all_currency_and_social = sales.groupby('sku_hash').mean()[currency_and_social_columns]\n",
    "first_day_currency_and_social = first_day.groupby('sku_hash').mean()[currency_and_social_columns]\n",
    "first_day_currency_and_social.columns = ['first_day_' + col for col in first_day_currency_and_social.columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "_uuid": "1b4a7b1f23753fef7f6f08f225406933b8dd0cc8"
   },
   "outputs": [],
   "source": [
    "all_sales = sales.groupby('sku_hash').sum()['sales_quantity']\n",
    "all_sales = pd.DataFrame(all_sales)\n",
    "first_day_sales = first_day.groupby(['sku_hash', 'day_transaction_date', 'Month_transaction']).sum()['sales_quantity']\n",
    "first_day_sales = pd.DataFrame(first_day_sales)\n",
    "first_day_sales.columns = ['first_day_sales']\n",
    "first_day_sales.reset_index(inplace=True)\n",
    "first_day_sales.set_index('sku_hash', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Features for number of sales from each day from 1 to 7\n",
    "\n",
    "sku_features = []\n",
    "for dayid in range(1, 8):\n",
    "    day = sales.loc[sales.Date == 'Day_{}'.format(dayid),:]\n",
    "    day_sales = day.groupby(['sku_hash'])['sales_quantity'].sum()\n",
    "\n",
    "    sku_features.append(day_sales)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "sku_features.append(sales.groupby('sku_hash')['sku_hash'].count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "_uuid": "d8f2ab70598dbb879912b15f29b5ac259d377f29"
   },
   "outputs": [],
   "source": [
    "sales_data = pd.merge(all_sales, first_day_sales, left_index=True, right_index=True)\n",
    "sales_data = pd.merge(sales_data, all_currency_and_social, left_index=True, right_index=True)\n",
    "sales_data = pd.merge(sales_data, first_day_currency_and_social, left_index=True, right_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "_uuid": "6ffaf9b7a26c4b59cce5cf0b2558c1af1dc8b40e"
   },
   "outputs": [],
   "source": [
    "# Convert month_transaction to numeric\n",
    "monthDict = {1:'Jan', 2:'Feb', 3:'Mar', 4:'Apr', 5:'May', 6:'Jun', 7:'Jul', 8:'Aug', 9:'Sep', 10:'Oct', 11:'Nov', 12:'Dec'}\n",
    "sales_data.Month_transaction = sales_data.Month_transaction.astype('object').map(monthDict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "_uuid": "47555ae4986bb0076458b16ba79c128e185aaeb1"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sales_quantity</th>\n",
       "      <th>day_transaction_date</th>\n",
       "      <th>Month_transaction</th>\n",
       "      <th>first_day_sales</th>\n",
       "      <th>currency_rate_USD</th>\n",
       "      <th>currency_rate_GBP</th>\n",
       "      <th>currency_rate_CNY</th>\n",
       "      <th>currency_rate_JPY</th>\n",
       "      <th>currency_rate_KRW</th>\n",
       "      <th>currency_rate_USD_1_day_before</th>\n",
       "      <th>...</th>\n",
       "      <th>first_day_NetSentiment_6_day_before</th>\n",
       "      <th>first_day_PositiveSentiment_6_day_before</th>\n",
       "      <th>first_day_NegativeSentiment_6_day_before</th>\n",
       "      <th>first_day_Impressions_6_day_before</th>\n",
       "      <th>first_day_TotalBuzzPost_7_day_before</th>\n",
       "      <th>first_day_TotalBuzz_7_day_before</th>\n",
       "      <th>first_day_NetSentiment_7_day_before</th>\n",
       "      <th>first_day_PositiveSentiment_7_day_before</th>\n",
       "      <th>first_day_NegativeSentiment_7_day_before</th>\n",
       "      <th>first_day_Impressions_7_day_before</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sku_hash</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>000cb631113e2f54ca5512139a6592e9584957aa</th>\n",
       "      <td>114</td>\n",
       "      <td>Friday</td>\n",
       "      <td>Jun</td>\n",
       "      <td>15</td>\n",
       "      <td>1.126212</td>\n",
       "      <td>0.874773</td>\n",
       "      <td>7.661876</td>\n",
       "      <td>123.874618</td>\n",
       "      <td>1258.195660</td>\n",
       "      <td>1.126400</td>\n",
       "      <td>...</td>\n",
       "      <td>71.497052</td>\n",
       "      <td>17888.0</td>\n",
       "      <td>2973.0</td>\n",
       "      <td>514585381.0</td>\n",
       "      <td>41276.0</td>\n",
       "      <td>64550.0</td>\n",
       "      <td>68.741355</td>\n",
       "      <td>20130.0</td>\n",
       "      <td>3729.0</td>\n",
       "      <td>4.917362e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0020d561eab8b88ab55dfde84a2f12b865e5e0b4</th>\n",
       "      <td>365</td>\n",
       "      <td>Friday</td>\n",
       "      <td>Apr</td>\n",
       "      <td>282</td>\n",
       "      <td>1.081008</td>\n",
       "      <td>0.841606</td>\n",
       "      <td>7.447166</td>\n",
       "      <td>119.015813</td>\n",
       "      <td>1223.698506</td>\n",
       "      <td>1.081296</td>\n",
       "      <td>...</td>\n",
       "      <td>69.491343</td>\n",
       "      <td>15711.0</td>\n",
       "      <td>2828.0</td>\n",
       "      <td>271425144.0</td>\n",
       "      <td>42770.0</td>\n",
       "      <td>60152.0</td>\n",
       "      <td>68.060358</td>\n",
       "      <td>18711.0</td>\n",
       "      <td>3556.0</td>\n",
       "      <td>3.521520e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0026e7a0fcfe5999a44b70b1acaff00fc1ad3ac2</th>\n",
       "      <td>110</td>\n",
       "      <td>Friday</td>\n",
       "      <td>Nov</td>\n",
       "      <td>15</td>\n",
       "      <td>1.161866</td>\n",
       "      <td>0.886239</td>\n",
       "      <td>7.708679</td>\n",
       "      <td>132.288017</td>\n",
       "      <td>1295.056829</td>\n",
       "      <td>1.161648</td>\n",
       "      <td>...</td>\n",
       "      <td>69.026640</td>\n",
       "      <td>24459.0</td>\n",
       "      <td>4482.0</td>\n",
       "      <td>542601613.0</td>\n",
       "      <td>61979.0</td>\n",
       "      <td>88755.0</td>\n",
       "      <td>55.200683</td>\n",
       "      <td>25463.0</td>\n",
       "      <td>7350.0</td>\n",
       "      <td>1.189237e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>00287bbb94c12066df6491dccd744ee87ff01a90</th>\n",
       "      <td>45</td>\n",
       "      <td>Friday</td>\n",
       "      <td>Nov</td>\n",
       "      <td>5</td>\n",
       "      <td>1.189247</td>\n",
       "      <td>0.889329</td>\n",
       "      <td>7.851854</td>\n",
       "      <td>132.787147</td>\n",
       "      <td>1290.424868</td>\n",
       "      <td>1.189655</td>\n",
       "      <td>...</td>\n",
       "      <td>70.311199</td>\n",
       "      <td>22493.0</td>\n",
       "      <td>3921.0</td>\n",
       "      <td>801621926.0</td>\n",
       "      <td>54173.0</td>\n",
       "      <td>78972.0</td>\n",
       "      <td>74.279561</td>\n",
       "      <td>25491.0</td>\n",
       "      <td>3762.0</td>\n",
       "      <td>5.129485e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>003f8a76cb823eb7c58b6d052c57a8933f9275fd</th>\n",
       "      <td>398</td>\n",
       "      <td>Friday</td>\n",
       "      <td>Oct</td>\n",
       "      <td>95</td>\n",
       "      <td>1.181248</td>\n",
       "      <td>0.892284</td>\n",
       "      <td>7.793914</td>\n",
       "      <td>132.539294</td>\n",
       "      <td>1331.520249</td>\n",
       "      <td>1.181057</td>\n",
       "      <td>...</td>\n",
       "      <td>64.672699</td>\n",
       "      <td>22477.0</td>\n",
       "      <td>4822.0</td>\n",
       "      <td>419612863.0</td>\n",
       "      <td>55415.0</td>\n",
       "      <td>80913.0</td>\n",
       "      <td>67.001526</td>\n",
       "      <td>23523.0</td>\n",
       "      <td>4648.0</td>\n",
       "      <td>4.983092e+08</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 180 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          sales_quantity day_transaction_date  \\\n",
       "sku_hash                                                                        \n",
       "000cb631113e2f54ca5512139a6592e9584957aa             114               Friday   \n",
       "0020d561eab8b88ab55dfde84a2f12b865e5e0b4             365               Friday   \n",
       "0026e7a0fcfe5999a44b70b1acaff00fc1ad3ac2             110               Friday   \n",
       "00287bbb94c12066df6491dccd744ee87ff01a90              45               Friday   \n",
       "003f8a76cb823eb7c58b6d052c57a8933f9275fd             398               Friday   \n",
       "\n",
       "                                         Month_transaction  first_day_sales  \\\n",
       "sku_hash                                                                      \n",
       "000cb631113e2f54ca5512139a6592e9584957aa               Jun               15   \n",
       "0020d561eab8b88ab55dfde84a2f12b865e5e0b4               Apr              282   \n",
       "0026e7a0fcfe5999a44b70b1acaff00fc1ad3ac2               Nov               15   \n",
       "00287bbb94c12066df6491dccd744ee87ff01a90               Nov                5   \n",
       "003f8a76cb823eb7c58b6d052c57a8933f9275fd               Oct               95   \n",
       "\n",
       "                                          currency_rate_USD  \\\n",
       "sku_hash                                                      \n",
       "000cb631113e2f54ca5512139a6592e9584957aa           1.126212   \n",
       "0020d561eab8b88ab55dfde84a2f12b865e5e0b4           1.081008   \n",
       "0026e7a0fcfe5999a44b70b1acaff00fc1ad3ac2           1.161866   \n",
       "00287bbb94c12066df6491dccd744ee87ff01a90           1.189247   \n",
       "003f8a76cb823eb7c58b6d052c57a8933f9275fd           1.181248   \n",
       "\n",
       "                                          currency_rate_GBP  \\\n",
       "sku_hash                                                      \n",
       "000cb631113e2f54ca5512139a6592e9584957aa           0.874773   \n",
       "0020d561eab8b88ab55dfde84a2f12b865e5e0b4           0.841606   \n",
       "0026e7a0fcfe5999a44b70b1acaff00fc1ad3ac2           0.886239   \n",
       "00287bbb94c12066df6491dccd744ee87ff01a90           0.889329   \n",
       "003f8a76cb823eb7c58b6d052c57a8933f9275fd           0.892284   \n",
       "\n",
       "                                          currency_rate_CNY  \\\n",
       "sku_hash                                                      \n",
       "000cb631113e2f54ca5512139a6592e9584957aa           7.661876   \n",
       "0020d561eab8b88ab55dfde84a2f12b865e5e0b4           7.447166   \n",
       "0026e7a0fcfe5999a44b70b1acaff00fc1ad3ac2           7.708679   \n",
       "00287bbb94c12066df6491dccd744ee87ff01a90           7.851854   \n",
       "003f8a76cb823eb7c58b6d052c57a8933f9275fd           7.793914   \n",
       "\n",
       "                                          currency_rate_JPY  \\\n",
       "sku_hash                                                      \n",
       "000cb631113e2f54ca5512139a6592e9584957aa         123.874618   \n",
       "0020d561eab8b88ab55dfde84a2f12b865e5e0b4         119.015813   \n",
       "0026e7a0fcfe5999a44b70b1acaff00fc1ad3ac2         132.288017   \n",
       "00287bbb94c12066df6491dccd744ee87ff01a90         132.787147   \n",
       "003f8a76cb823eb7c58b6d052c57a8933f9275fd         132.539294   \n",
       "\n",
       "                                          currency_rate_KRW  \\\n",
       "sku_hash                                                      \n",
       "000cb631113e2f54ca5512139a6592e9584957aa        1258.195660   \n",
       "0020d561eab8b88ab55dfde84a2f12b865e5e0b4        1223.698506   \n",
       "0026e7a0fcfe5999a44b70b1acaff00fc1ad3ac2        1295.056829   \n",
       "00287bbb94c12066df6491dccd744ee87ff01a90        1290.424868   \n",
       "003f8a76cb823eb7c58b6d052c57a8933f9275fd        1331.520249   \n",
       "\n",
       "                                          currency_rate_USD_1_day_before  ...  \\\n",
       "sku_hash                                                                  ...   \n",
       "000cb631113e2f54ca5512139a6592e9584957aa                        1.126400  ...   \n",
       "0020d561eab8b88ab55dfde84a2f12b865e5e0b4                        1.081296  ...   \n",
       "0026e7a0fcfe5999a44b70b1acaff00fc1ad3ac2                        1.161648  ...   \n",
       "00287bbb94c12066df6491dccd744ee87ff01a90                        1.189655  ...   \n",
       "003f8a76cb823eb7c58b6d052c57a8933f9275fd                        1.181057  ...   \n",
       "\n",
       "                                          first_day_NetSentiment_6_day_before  \\\n",
       "sku_hash                                                                        \n",
       "000cb631113e2f54ca5512139a6592e9584957aa                            71.497052   \n",
       "0020d561eab8b88ab55dfde84a2f12b865e5e0b4                            69.491343   \n",
       "0026e7a0fcfe5999a44b70b1acaff00fc1ad3ac2                            69.026640   \n",
       "00287bbb94c12066df6491dccd744ee87ff01a90                            70.311199   \n",
       "003f8a76cb823eb7c58b6d052c57a8933f9275fd                            64.672699   \n",
       "\n",
       "                                          first_day_PositiveSentiment_6_day_before  \\\n",
       "sku_hash                                                                             \n",
       "000cb631113e2f54ca5512139a6592e9584957aa                                   17888.0   \n",
       "0020d561eab8b88ab55dfde84a2f12b865e5e0b4                                   15711.0   \n",
       "0026e7a0fcfe5999a44b70b1acaff00fc1ad3ac2                                   24459.0   \n",
       "00287bbb94c12066df6491dccd744ee87ff01a90                                   22493.0   \n",
       "003f8a76cb823eb7c58b6d052c57a8933f9275fd                                   22477.0   \n",
       "\n",
       "                                          first_day_NegativeSentiment_6_day_before  \\\n",
       "sku_hash                                                                             \n",
       "000cb631113e2f54ca5512139a6592e9584957aa                                    2973.0   \n",
       "0020d561eab8b88ab55dfde84a2f12b865e5e0b4                                    2828.0   \n",
       "0026e7a0fcfe5999a44b70b1acaff00fc1ad3ac2                                    4482.0   \n",
       "00287bbb94c12066df6491dccd744ee87ff01a90                                    3921.0   \n",
       "003f8a76cb823eb7c58b6d052c57a8933f9275fd                                    4822.0   \n",
       "\n",
       "                                          first_day_Impressions_6_day_before  \\\n",
       "sku_hash                                                                       \n",
       "000cb631113e2f54ca5512139a6592e9584957aa                         514585381.0   \n",
       "0020d561eab8b88ab55dfde84a2f12b865e5e0b4                         271425144.0   \n",
       "0026e7a0fcfe5999a44b70b1acaff00fc1ad3ac2                         542601613.0   \n",
       "00287bbb94c12066df6491dccd744ee87ff01a90                         801621926.0   \n",
       "003f8a76cb823eb7c58b6d052c57a8933f9275fd                         419612863.0   \n",
       "\n",
       "                                          first_day_TotalBuzzPost_7_day_before  \\\n",
       "sku_hash                                                                         \n",
       "000cb631113e2f54ca5512139a6592e9584957aa                               41276.0   \n",
       "0020d561eab8b88ab55dfde84a2f12b865e5e0b4                               42770.0   \n",
       "0026e7a0fcfe5999a44b70b1acaff00fc1ad3ac2                               61979.0   \n",
       "00287bbb94c12066df6491dccd744ee87ff01a90                               54173.0   \n",
       "003f8a76cb823eb7c58b6d052c57a8933f9275fd                               55415.0   \n",
       "\n",
       "                                          first_day_TotalBuzz_7_day_before  \\\n",
       "sku_hash                                                                     \n",
       "000cb631113e2f54ca5512139a6592e9584957aa                           64550.0   \n",
       "0020d561eab8b88ab55dfde84a2f12b865e5e0b4                           60152.0   \n",
       "0026e7a0fcfe5999a44b70b1acaff00fc1ad3ac2                           88755.0   \n",
       "00287bbb94c12066df6491dccd744ee87ff01a90                           78972.0   \n",
       "003f8a76cb823eb7c58b6d052c57a8933f9275fd                           80913.0   \n",
       "\n",
       "                                          first_day_NetSentiment_7_day_before  \\\n",
       "sku_hash                                                                        \n",
       "000cb631113e2f54ca5512139a6592e9584957aa                            68.741355   \n",
       "0020d561eab8b88ab55dfde84a2f12b865e5e0b4                            68.060358   \n",
       "0026e7a0fcfe5999a44b70b1acaff00fc1ad3ac2                            55.200683   \n",
       "00287bbb94c12066df6491dccd744ee87ff01a90                            74.279561   \n",
       "003f8a76cb823eb7c58b6d052c57a8933f9275fd                            67.001526   \n",
       "\n",
       "                                          first_day_PositiveSentiment_7_day_before  \\\n",
       "sku_hash                                                                             \n",
       "000cb631113e2f54ca5512139a6592e9584957aa                                   20130.0   \n",
       "0020d561eab8b88ab55dfde84a2f12b865e5e0b4                                   18711.0   \n",
       "0026e7a0fcfe5999a44b70b1acaff00fc1ad3ac2                                   25463.0   \n",
       "00287bbb94c12066df6491dccd744ee87ff01a90                                   25491.0   \n",
       "003f8a76cb823eb7c58b6d052c57a8933f9275fd                                   23523.0   \n",
       "\n",
       "                                          first_day_NegativeSentiment_7_day_before  \\\n",
       "sku_hash                                                                             \n",
       "000cb631113e2f54ca5512139a6592e9584957aa                                    3729.0   \n",
       "0020d561eab8b88ab55dfde84a2f12b865e5e0b4                                    3556.0   \n",
       "0026e7a0fcfe5999a44b70b1acaff00fc1ad3ac2                                    7350.0   \n",
       "00287bbb94c12066df6491dccd744ee87ff01a90                                    3762.0   \n",
       "003f8a76cb823eb7c58b6d052c57a8933f9275fd                                    4648.0   \n",
       "\n",
       "                                          first_day_Impressions_7_day_before  \n",
       "sku_hash                                                                      \n",
       "000cb631113e2f54ca5512139a6592e9584957aa                        4.917362e+08  \n",
       "0020d561eab8b88ab55dfde84a2f12b865e5e0b4                        3.521520e+08  \n",
       "0026e7a0fcfe5999a44b70b1acaff00fc1ad3ac2                        1.189237e+09  \n",
       "00287bbb94c12066df6491dccd744ee87ff01a90                        5.129485e+08  \n",
       "003f8a76cb823eb7c58b6d052c57a8933f9275fd                        4.983092e+08  \n",
       "\n",
       "[5 rows x 180 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sales_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "_uuid": "c079fa8c7e5fbf025992978fda132052d7b2c86b"
   },
   "outputs": [],
   "source": [
    "first_day_navigation = navigation.loc[navigation.Date == 'Day 1',:]\n",
    "first_day_views = first_day_navigation.groupby('sku_hash').sum()[['page_views', 'addtocart']]\n",
    "first_day_views.columns = ['first_day_page_views', 'first_day_addtocart']\n",
    "views = navigation.groupby('sku_hash').sum()[['page_views', 'addtocart']]\n",
    "navigation_data = pd.merge(views, first_day_views, left_index=True, right_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "_uuid": "e1e9390dd178b3027e67c6d014f06b4384ca7c4a"
   },
   "outputs": [],
   "source": [
    "sales_data.sales_quantity = sales_data.sales_quantity.astype('float64')\n",
    "sales_data.first_day_sales = sales_data.first_day_sales.astype('float64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "_uuid": "8ac1d88343fc901fab020553bf35089283c86500"
   },
   "outputs": [],
   "source": [
    "sales_data['sales_quantity_log'] = (sales_data.sales_quantity + 1).apply(np.log)\n",
    "sales_data['first_day_sales_log'] = (sales_data.first_day_sales + 1).apply(np.log)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load in raddar's features from the first kernel\n",
    "\n",
    "darius_train = pd.read_csv('raddar-features-train.csv')\n",
    "darius_test = pd.read_csv('raddar-features-test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "_uuid": "c3a8340e1694ce077c8e9d28dd46ca451f0e0ea9"
   },
   "outputs": [],
   "source": [
    "train_data = pd.merge(train, sales_data, left_on='sku_hash', right_index=True)\n",
    "train_data = pd.merge(train_data, navigation_data, how='left', left_on='sku_hash', right_index=True)\n",
    "train_data = pd.merge(train_data, vimages, left_on='sku_hash', right_on='sku_hash')\n",
    "\n",
    "# Add raddar's features to my dataframe\n",
    "\n",
    "cols = darius_train.columns\n",
    "drop_cols = [c for c in darius_train.columns if (c in train_data.columns and c != 'ID')]\n",
    "\n",
    "train_data = pd.merge(train_data.drop(drop_cols, axis=1), darius_train, how='left', left_on='ID', right_on='ID')\n",
    "\n",
    "# Add sales per day features\n",
    "extra_features = []\n",
    "for i, feature in enumerate(sku_features):\n",
    "    feature = pd.DataFrame(feature)\n",
    "    feature.columns=['extra{}'.format(i)]\n",
    "    train_data = pd.merge(train_data, feature, how='left', left_on='sku_hash', right_index=True)\n",
    "    extra_features.append('extra{}'.format(i))\n",
    "\n",
    "# Add feature which is ratio between 7th day sales and 1st day sales\n",
    "# i.e. how quickly sales dropped off\n",
    "train_data['extraratio'] = train_data['extra6'] / train_data['extra0']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "_uuid": "4a90d1563fb6bd221613c23ea1c499177f7fe8d1"
   },
   "outputs": [],
   "source": [
    "test_data = pd.merge(test, sales_data, left_on='sku_hash', right_index=True)\n",
    "test_data = pd.merge(test_data, navigation_data, how='left', left_on='sku_hash', right_index=True)\n",
    "test_data = pd.merge(test_data, vimages, left_on='sku_hash', right_on='sku_hash')\n",
    "\n",
    "# Add raddar's feature\n",
    "test_data = pd.merge(test_data.drop(drop_cols, axis=1), darius_test, how='left', left_on='ID', right_on='ID')\n",
    "\n",
    "extra_features = []\n",
    "for i, feature in enumerate(sku_features):\n",
    "#     print(feature)\n",
    "    feature = pd.DataFrame(feature)\n",
    "    feature.columns=['extra{}'.format(i)]\n",
    "#     print(feature)\n",
    "    test_data = pd.merge(test_data, feature, how='left', left_on='sku_hash', right_index=True)\n",
    "    extra_features.append('extra{}'.format(i))\n",
    "    \n",
    "test_data['extraratio'] = test_data['extra6'] / test_data['extra0']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data[navigation_data.columns] = train_data[navigation_data.columns].fillna(0)\n",
    "test_data[navigation_data.columns] = test_data[navigation_data.columns].fillna(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "e92ecde8be29c103afbef5861a6a5b55751697ec"
   },
   "source": [
    "# Modeling\n",
    "## utils\n",
    "from https://github.com/pjankiewicz/PandasSelector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "_uuid": "5aa824605618927c2bff72a54db96f7a66c572d7"
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "class PandasSelector(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, columns=None, dtype=None, inverse=False,\n",
    "                 return_vector=True):\n",
    "        self.dtype = dtype\n",
    "        self.columns = columns\n",
    "        self.inverse = inverse\n",
    "        self.return_vector = return_vector\n",
    "\n",
    "        if isinstance(self.columns, str):\n",
    "            self.columns = [self.columns]\n",
    "\n",
    "    def check_condition(self, x, col):\n",
    "        cond = (self.dtype is not None and x[col].dtype == self.dtype) or \\\n",
    "               (self.columns is not None and col in self.columns)\n",
    "        return self.inverse ^ cond\n",
    "\n",
    "    def fit(self, x, y=None):\n",
    "        return self\n",
    "\n",
    "    def _check_if_all_columns_present(self, x):\n",
    "        if not self.inverse and self.columns is not None:\n",
    "            missing_columns = set(self.columns) - set(x.columns)\n",
    "            if len(missing_columns) > 0:\n",
    "                missing_columns_ = ','.join(col for col in missing_columns)\n",
    "                raise KeyError('Keys are missing in the record: %s' %\n",
    "                               missing_columns_)\n",
    "\n",
    "    def transform(self, x):\n",
    "        # check if x is a pandas DataFrame\n",
    "        if not isinstance(x, pd.DataFrame):\n",
    "            raise KeyError('Input is not a pandas DataFrame')\n",
    "\n",
    "        selected_cols = []\n",
    "        for col in x.columns:\n",
    "            if self.check_condition(x, col):\n",
    "                selected_cols.append(col)\n",
    "\n",
    "        # if the column was selected and inversed = False make sure the column\n",
    "        # is in the DataFrame\n",
    "        self._check_if_all_columns_present(x)\n",
    "\n",
    "        # if only 1 column is returned return a vector instead of a dataframe\n",
    "        if len(selected_cols) == 1 and self.return_vector:\n",
    "            return list(x[selected_cols[0]])\n",
    "        else:\n",
    "            return x[selected_cols]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "107def94e3025396616fa08411e8260d6e7b4fb6"
   },
   "source": [
    "## separate models for each prediction month"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "_uuid": "816b8f0cb0fbadf9328c28df8795e3ac1d69b582"
   },
   "outputs": [],
   "source": [
    "train_data1 = train_data.loc[train_data.month == 1, :].copy()\n",
    "train_data1.drop(['month', 'sku_hash', 'ID'], axis=1, inplace=True)\n",
    "\n",
    "X_test1 = test_data.loc[test_data.month == 1, :].copy()\n",
    "X_test1.drop(['month', 'sku_hash'], axis=1, inplace=True)\n",
    "X_test1.set_index('ID', inplace=True)\n",
    "\n",
    "y_train1 = (train_data1.target + 1).apply(np.log)\n",
    "X_train1 = train_data1.drop('target', axis=1)\n",
    "\n",
    "train_data2 = train_data.loc[train_data.month == 2, :].copy()\n",
    "train_data2.drop(['month', 'sku_hash', 'ID'], axis=1, inplace=True)\n",
    "\n",
    "X_test2 = test_data.loc[test_data.month == 2, :].copy()\n",
    "X_test2.drop(['month', 'sku_hash'], axis=1, inplace=True)\n",
    "X_test2.set_index('ID', inplace=True)\n",
    "\n",
    "y_train2 = (train_data2.target + 1).apply(np.log)\n",
    "X_train2 = train_data2.drop('target', axis=1)\n",
    "\n",
    "\n",
    "train_data3 = train_data.loc[train_data.month == 3, :].copy()\n",
    "train_data3.drop(['month', 'sku_hash', 'ID'], axis=1, inplace=True)\n",
    "\n",
    "X_test3 = test_data.loc[test_data.month == 3, :].copy()\n",
    "X_test3.drop(['month', 'sku_hash'], axis=1, inplace=True)\n",
    "X_test3.set_index('ID', inplace=True)\n",
    "\n",
    "y_train3 = (train_data3.target + 1).apply(np.log)\n",
    "X_train3 = train_data3.drop('target', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "_uuid": "614637c1e10a48604fedb614c472815b08954642"
   },
   "outputs": [],
   "source": [
    "images_cols = vimages.columns[1:].tolist()\n",
    "float_cols = X_train1.dtypes[X_train1.dtypes == 'float64'].index.tolist()\n",
    "float_cols = list(set(float_cols) - set(images_cols))\n",
    "float_cols.remove('sales_quantity_log')\n",
    "float_cols.remove('first_day_sales_log')\n",
    "# float_cols.remove('sales_quantity')\n",
    "float_cols.remove('first_day_sales')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "_uuid": "6f9e5a39469aec53f970042015788144ab4432d8"
   },
   "outputs": [],
   "source": [
    "categorical_cols = X_train1.dtypes[X_train1.dtypes == 'object'].index.tolist()\n",
    "categorical_cols.remove('en_US_description')\n",
    "categorical_cols.remove('color')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = pd.DataFrame()\n",
    "x_test = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "split_points = [0]\n",
    "combined_data = pd.concat([train_data1, train_data2, train_data3, X_test1, X_test2, X_test3], axis=0)\n",
    "for df in train_data1, train_data2, train_data3, X_test1, X_test2, X_test3:\n",
    "    split_points.append(split_points[-1] + len(df))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "description features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "_uuid": "64c46d264f4c14828e399da1e5c01ead65ee1ec6"
   },
   "outputs": [],
   "source": [
    "descs = combined_data['en_US_description']\n",
    "vect = TfidfVectorizer(stop_words='english', max_features=5000)\n",
    "\n",
    "descs = vect.fit_transform(descs)\n",
    "vect_pca = LatentDirichletAllocation(n_components=16)\n",
    "descs = vect_pca.fit_transform(descs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "im_data = combined_data[images_cols]\n",
    "im_pca = PCA(n_components=16)\n",
    "im_data = im_pca.fit_transform(im_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "float_cols_pca = PCA(n_components=16)\n",
    "float_data = float_cols_pca.fit_transform(SimpleFill().fit_transform(combined_data[float_cols]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import TruncatedSVD\n",
    "cat_pca = TruncatedSVD(n_components=32)\n",
    "cat_ohe = OneHotEncoder(handle_unknown='ignore')\n",
    "cat_data = cat_pca.fit_transform(cat_ohe.fit_transform(combined_data[categorical_cols]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "#months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']\n",
    "#extra_columns = pd.DataFrame()\n",
    "#extra_columns['1'] = combined_data['Month_transaction'].apply(months.index)\n",
    "#extra_columns['2'] = combined_data['product_gender'].apply(['Women', 'Unisex', 'Men'].index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sep    344\n",
       "Jan    236\n",
       "Jun    197\n",
       "May    153\n",
       "Nov    152\n",
       "Oct    150\n",
       "Mar    137\n",
       "Apr    120\n",
       "Feb    112\n",
       "Jul    107\n",
       "Dec     51\n",
       "Aug      7\n",
       "Name: Month_transaction, dtype: int64"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data1['Month_transaction'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 439,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['ID', 'product_type', 'product_gender', 'page_views', 'sales_quantity',\n",
       "       'TotalBuzzPost', 'TotalBuzz', 'NetSentiment', 'PositiveSentiment',\n",
       "       'NegativeSentiment', 'Impressions', 'fr_FR_price',\n",
       "       'macro_function_count', 'function_count', 'sub_function_count',\n",
       "       'model_count', 'aesthetic_sub_line_count', 'macro_material_count',\n",
       "       'color_count', 'page_views_nav1', 'page_views_nav2', 'page_views_nav3',\n",
       "       'page_views_nav4', 'page_views_nav5', 'page_views_nav6',\n",
       "       'sales_quantity_type1', 'sales_quantity_type2', 'sales_quantity_zone1',\n",
       "       'sales_quantity_zone2', 'sales_quantity_zone3', 'sales_quantity_zone4',\n",
       "       'sales_quantity_zone5', 'mean_target'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 439,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "darius_train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_all = np.concatenate([descs, im_data, float_data, cat_data, combined_data[['sales_quantity_log', 'first_day_sales_log', 'sales_quantity', 'first_day_sales', 'page_views', 'fr_FR_price']].values, combined_data[extra_features + darius_train.columns[1:].tolist()].values], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = split_points\n",
    "x_train_1 = x_all[s[0]:s[1]]\n",
    "x_train_2 = x_all[s[1]:s[2]]\n",
    "x_train_3 = x_all[s[2]:s[3]]\n",
    "x_test_1 = x_all[s[3]:s[4]]\n",
    "x_test_2 = x_all[s[4]:s[5]]\n",
    "x_test_3 = x_all[s[5]:s[6]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Taken from mlcrate\n",
    "# https://github.com/mxbi/mlcrate\n",
    "from mlcrate.time import Timer\n",
    "from mlcrate.xgb import get_importances\n",
    "def train_kfold(params, x_train, y_train, x_test=None, folds=5, stratify=None, random_state=1337, skip_checks=False, print_imp='final'):\n",
    "    \"\"\"Trains a set of XGBoost models with chosen parameters on a KFold split dataset, returning full out-of-fold\n",
    "    training set predictions (useful for stacking) as well as test set predictions and the models themselves.\n",
    "    Test set predictions are generated by averaging predictions from all the individual fold models - this means\n",
    "    1 model fewer has to be trained and from my experience performs better than retraining a single model on the full set.\n",
    "    Optionally, the split can be stratified along a passed array. Feature importances are also computed and summed across all folds for convenience.\n",
    "    Keyword arguments:\n",
    "    params -- Parameters passed to the xgboost model, as well as ['early_stopping_rounds', 'nrounds', 'verbose_eval'], which are passed to xgb.train()\n",
    "              Defaults: early_stopping_rounds = 50, nrounds = 100000, verbose_eval = 1\n",
    "    x_train -- The training set features\n",
    "    y_train -- The training set labels\n",
    "    x_test (optional) -- The test set features\n",
    "    folds (default: 5) -- The number of folds to perform\n",
    "    stratify (optional) -- An array to stratify the splits along\n",
    "    random_state (default: 1337) -- Random seed for splitting folds\n",
    "    skip_checks -- By default, this function tries to reorder the test set columns to match the order of the training set columns. Set this to disable this behaviour.\n",
    "    print_imp -- One of ['every', 'final', None] - 'every' prints importances for every fold, 'final' prints combined importances at the end, None does not print importance\n",
    "    Returns:\n",
    "    models -- a list of trained xgboost.Booster objects\n",
    "    p_train -- Out-of-fold training set predictions (shaped like y_train)\n",
    "    p_test -- Mean of test set predictions from the models\n",
    "    imps -- dict with \\{feature: importance\\} pairs representing the sum feature importance from all the models.\n",
    "    \"\"\"\n",
    "\n",
    "    from sklearn.model_selection import KFold, StratifiedKFold, RepeatedKFold  # Optional dependencies\n",
    "    from collections import defaultdict\n",
    "    import numpy as np\n",
    "    import xgboost as xgb\n",
    "\n",
    "    assert print_imp in ['every', 'final', None]\n",
    "\n",
    "    # If it's a dataframe, we can take column names, otherwise just use column indices (eg. for printing importances).\n",
    "    if hasattr(x_train, 'columns'):\n",
    "        columns = x_train.columns.values\n",
    "        columns_exists = True\n",
    "    else:\n",
    "        columns = np.arange(x_train.shape[1]).astype(str)\n",
    "        columns_exists = False\n",
    "#     print(columns)\n",
    "\n",
    "    x_train = np.asarray(x_train)\n",
    "    y_train = np.array(y_train)\n",
    "\n",
    "    if x_test is not None:\n",
    "        if columns_exists and not skip_checks:\n",
    "            try:\n",
    "                x_test = x_test[columns]\n",
    "            except Exception as e:\n",
    "                print('[mlcrate] Could not coerce x_test columns to match x_train columns. Set skip_checks=True to run anyway.')\n",
    "                raise e\n",
    "\n",
    "        x_test = np.asarray(x_test)\n",
    "        d_test = xgb.DMatrix(x_test)\n",
    "\n",
    "    if not skip_checks:\n",
    "        assert x_train.shape[1] == x_test.shape[1], \"x_train and x_test have different numbers of features.\"\n",
    "\n",
    "    print('[mlcrate] Training {} {}XGBoost models on training set {} {}'.format(folds, 'stratified ' if stratify is not None else '',\n",
    "            x_train.shape, 'with test set {}'.format(x_test.shape) if x_test is not None else 'without a test set'))\n",
    "\n",
    "    # Init a timer to get fold durations\n",
    "    t = Timer()\n",
    "\n",
    "    if stratify is not None:\n",
    "        kf = StratifiedKFold(n_splits=folds, shuffle=True, random_state=random_state)\n",
    "        splits = kf.split(x_train, stratify)\n",
    "    else:\n",
    "        # MODIFIED\n",
    "        kf = RepeatedKFold(n_repeats=10, n_splits=folds, random_state=4242)\n",
    "        splits = kf.split(x_train)\n",
    "\n",
    "    p_train = np.zeros_like(y_train, dtype=np.float32)\n",
    "    ps_test = []\n",
    "    models = []\n",
    "    scores = []\n",
    "    imps = defaultdict(int)\n",
    "\n",
    "    fold_i = 0\n",
    "    for train_kf, valid_kf in splits:\n",
    "        print('[mlcrate] Running fold {}, {} train samples, {} validation samples'.format(fold_i, len(train_kf), len(valid_kf)))\n",
    "        d_train = xgb.DMatrix(x_train[train_kf], label=y_train[train_kf])\n",
    "        d_valid = xgb.DMatrix(x_train[valid_kf], label=y_train[valid_kf])\n",
    "\n",
    "        # Start a timer for the fold\n",
    "        t.add('fold{}'.format(fold_i))\n",
    "\n",
    "        # Metrics to print\n",
    "        watchlist = [(d_train, 'train'), (d_valid, 'valid')]\n",
    "\n",
    "        mdl = xgb.train(params, d_train, params.get('nrounds', 100000), watchlist,\n",
    "                        early_stopping_rounds=params.get('early_stopping_rounds', 50), verbose_eval=params.get('verbose_eval', 1))\n",
    "\n",
    "        scores.append(mdl.best_score)\n",
    "\n",
    "        print('[mlcrate] Finished training fold {} - took {} - running score {}'.format(fold_i, t.format_elapsed('fold{}'.format(fold_i)), np.mean(scores)))\n",
    "\n",
    "        # Get importances for this model and add to global importance\n",
    "#         print(mdl, columns)\n",
    "        imp = get_importances(mdl, columns)\n",
    "        if print_imp == 'every':\n",
    "            print('Fold {} importances:'.format(fold_i), imp)\n",
    "\n",
    "        for f, i in imp:\n",
    "            imps[f] += i\n",
    "\n",
    "        # Get predictions from the model\n",
    "        p_valid = mdl.predict(d_valid, ntree_limit=mdl.best_ntree_limit)\n",
    "        if x_test is not None:\n",
    "            p_test = mdl.predict(d_test, ntree_limit=mdl.best_ntree_limit)\n",
    "\n",
    "        p_train[valid_kf] = p_valid\n",
    "\n",
    "        ps_test.append(p_test)\n",
    "        models.append(mdl)\n",
    "\n",
    "        fold_i += 1\n",
    "\n",
    "    if x_test is not None:\n",
    "        p_test = np.mean(ps_test, axis=0)\n",
    "\n",
    "    print('[mlcrate] Finished training {} XGBoost models, took {}'.format(folds, t.format_elapsed(0)))\n",
    "\n",
    "    if print_imp in ['every', 'final']:\n",
    "        print('[mlcrate] Overall feature importances:', sorted(imps.items(), key=lambda x: x[1], reverse=True))\n",
    "\n",
    "    if x_test is None:\n",
    "        p_test = None\n",
    "\n",
    "    return models, p_train, p_test, imps, np.mean(scores)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[mlcrate] Training 3 XGBoost models on training set (1766, 126) with test set (1722, 126)\n",
      "[mlcrate] Running fold 0, 1177 train samples, 589 validation samples\n",
      "[0]\ttrain-rmse:4.82397\tvalid-rmse:4.77384\n",
      "Multiple eval metrics have been passed: 'valid-rmse' will be used for early stopping.\n",
      "\n",
      "Will train until valid-rmse hasn't improved in 500 rounds.\n",
      "[2000]\ttrain-rmse:0.090203\tvalid-rmse:0.373127\n",
      "Stopping. Best iteration:\n",
      "[1670]\ttrain-rmse:0.10829\tvalid-rmse:0.372846\n",
      "\n",
      "[mlcrate] Finished training fold 0 - took 17s - running score 0.372846\n",
      "[mlcrate] Running fold 1, 1177 train samples, 589 validation samples\n",
      "[0]\ttrain-rmse:4.77795\tvalid-rmse:4.86585\n",
      "Multiple eval metrics have been passed: 'valid-rmse' will be used for early stopping.\n",
      "\n",
      "Will train until valid-rmse hasn't improved in 500 rounds.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/m09170/anaconda3/lib/python3.6/site-packages/mlcrate/backend.py:7: UserWarning: Timer.format_elapsed() has been deprecated in favour of Timer.fsince() and will be removed soon\n",
      "  warn(message)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2000]\ttrain-rmse:0.087407\tvalid-rmse:0.372753\n",
      "[4000]\ttrain-rmse:0.032126\tvalid-rmse:0.370279\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-45-4c366a1bfc27>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;31m# Train 30 model folds per month\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;31m# folds=3 not 30 because I modified the train_kfold function from the mlcrate source\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp_train1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp_test1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscore1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_kfold\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_train_1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_test_1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfolds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     17\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp_train2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp_test2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscore2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_kfold\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_train_2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_test_2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfolds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp_train3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp_test3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscore3\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_kfold\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_train_3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_test_3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfolds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-42-1640186a62eb>\u001b[0m in \u001b[0;36mtrain_kfold\u001b[0;34m(params, x_train, y_train, x_test, folds, stratify, random_state, skip_checks, print_imp)\u001b[0m\n\u001b[1;32m     93\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m         mdl = xgb.train(params, d_train, params.get('nrounds', 100000), watchlist,\n\u001b[0;32m---> 95\u001b[0;31m                         early_stopping_rounds=params.get('early_stopping_rounds', 50), verbose_eval=params.get('verbose_eval', 1))\n\u001b[0m\u001b[1;32m     96\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     97\u001b[0m         \u001b[0mscores\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmdl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbest_score\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/xgboost-0.6-py3.6.egg/xgboost/training.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(params, dtrain, num_boost_round, evals, obj, feval, maximize, early_stopping_rounds, evals_result, verbose_eval, xgb_model, callbacks, learning_rates)\u001b[0m\n\u001b[1;32m    202\u001b[0m                            \u001b[0mevals\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mevals\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    203\u001b[0m                            \u001b[0mobj\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeval\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 204\u001b[0;31m                            xgb_model=xgb_model, callbacks=callbacks)\n\u001b[0m\u001b[1;32m    205\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    206\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/xgboost-0.6-py3.6.egg/xgboost/training.py\u001b[0m in \u001b[0;36m_train_internal\u001b[0;34m(params, dtrain, num_boost_round, evals, obj, feval, xgb_model, callbacks)\u001b[0m\n\u001b[1;32m     82\u001b[0m         \u001b[0;31m# check evaluation result.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     83\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevals\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 84\u001b[0;31m             \u001b[0mbst_eval_set\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbst\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval_set\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevals\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeval\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     85\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbst_eval_set\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSTRING_TYPES\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     86\u001b[0m                 \u001b[0mmsg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbst_eval_set\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/xgboost-0.6-py3.6.egg/xgboost/core.py\u001b[0m in \u001b[0;36meval_set\u001b[0;34m(self, evals, iteration, feval)\u001b[0m\n\u001b[1;32m    957\u001b[0m                                               \u001b[0mdmats\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevnames\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    958\u001b[0m                                               \u001b[0mc_bst_ulong\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevals\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 959\u001b[0;31m                                               ctypes.byref(msg)))\n\u001b[0m\u001b[1;32m    960\u001b[0m         \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    961\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mfeval\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "params = {}\n",
    "params['objective'] = 'reg:linear'\n",
    "params['max_depth'] = 3\n",
    "params['subsample'] = 0.7\n",
    "params['colsample_bylevel'] = 0.8\n",
    "#params['gamma'] = 5\n",
    "# params['min_child_weight'] = 25\n",
    "params['eta'] = 0.02\n",
    "params['silent'] = 1\n",
    "params['eval_metric'] = 'rmse'\n",
    "params['verbose_eval'] = 2000\n",
    "params['early_stopping_rounds'] = 500\n",
    "\n",
    "# Train 30 model folds per month\n",
    "# folds=3 not 30 because I modified the train_kfold function from the mlcrate source\n",
    "_, p_train1, p_test1, _, score1 = train_kfold(params, x_train_1, y_train1, x_test_1, folds=3)\n",
    "_, p_train2, p_test2, _, score2 = train_kfold(params, x_train_2, y_train2, x_test_2, folds=3)\n",
    "_, p_train3, p_test3, _, score3 = train_kfold(params, x_train_3, y_train3, x_test_3, folds=3)\n",
    "\n",
    "print('{:.4f} {:.4f} {:.4f} {:.4f}'.format(score1, score2, score3, np.mean([score1, score2, score3])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 515,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Do exp(target) before submission\n",
    "p_test1 = pd.Series(p_test1)\n",
    "p_test1 = (p_test1).apply(np.exp)  - 1\n",
    "p_test1.index = X_test1.index\n",
    "\n",
    "p_test2 = pd.Series(p_test2)\n",
    "p_test2 = (p_test2).apply(np.exp)  - 1\n",
    "p_test2.index = X_test2.index\n",
    "\n",
    "p_test3 = pd.Series(p_test3)\n",
    "p_test3 = (p_test3).apply(np.exp) - 1\n",
    "p_test3.index = X_test3.index\n",
    "\n",
    "submission = pd.DataFrame(pd.concat([p_test1, p_test2, p_test3]))\n",
    "submission.columns = ['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 516,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save submission\n",
    "submission.to_csv('submission_xgb6_bag30.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}