model_cat.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import packages\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import featuretools as ft\n",
    "import lightgbm as lgb\n",
    "import catboost as cat\n",
    "from catboost import CatBoostClassifier,Pool\n",
    "from lightgbm import plot_tree\n",
    "from graphviz import Digraph\n",
    "import seaborn as sns\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import time\n",
    "import pickle\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Original Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "feat_num = 457"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_total_raw = pd.read_csv('./data/features%s.csv'%(feat_num))\n",
    "with open('./data/features%s.pickle'%(feat_num), 'rb') as handle:\n",
    "    df_total_raw = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#to_drop = ['card1_count','addr2','V117','V118','V120','V122','id_25','id_21','id_26',\n",
    "#           'V305','id_17','V27','V28','V88','V89','id_14','V107','V240','V241','V41','id_24','V68']\n",
    "#to_drop = ['C8']\n",
    "#with open('./data/feat%s_rm_pm_importance0.pickle'%(feat_num), 'rb') as handle:\n",
    "#    to_drop = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_total = df_total_raw.drop(to_drop,axis=1)\n",
    "df_total = df_total_raw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "features_train = df_total[df_total['isFraud'].notnull()]\n",
    "features_test = df_total[df_total['isFraud'].isnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels_train = features_train['isFraud']\n",
    "features_train = features_train.drop(columns = ['isFraud', 'TransactionID'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(590540, 455)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical = ['ProductCD', 'card2', 'card3', 'card4', 'card5','card6',\n",
    "              'addr1','addr2','P_email','R_email','M1','M2','M3',\n",
    "              'M4','M5','M6','M7','M8','M9','DeviceType','DeviceInfo','dow','hour',\n",
    "              'Device_name','Device_version','screen_width','screen_height',\n",
    "               'P_email_suffix','R_email_suffix','id_30_OS','id_30_version',\n",
    "              'is_card_freq_Device','is_wide','is_long','is_zero','is_win8_vista',\n",
    "              'is_windows_otheros','is_card_freq_pdc','is_card_freq_addr1'] \n",
    "ids = [ 'id_%s'%(i) for i in range(12,39)]\n",
    "categorical = categorical + ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical = list(set(categorical).intersection(df_total.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "features_test_new = features_test.drop(columns = ['isFraud', 'TransactionID'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {'learning_rate': 0.01,\n",
    "          \"boosting_type\":\"Plain\" ,\n",
    "          'loss_function': 'CrossEntropy',#'Logloss',\n",
    "          \"eval_metric\": 'AUC',\n",
    "          'random_state': 47,\n",
    "          'task_type' :'GPU',\n",
    "          'grow_policy':'Lossguide',\n",
    "          #'reg_lambda': 0.6485237330340494,\n",
    "          #'min_data_in_leaf':100,\n",
    "          #'max_leaves':1400\n",
    "          #'is_unbalance':True\n",
    "          #'scale_pos_weight':9\n",
    "         }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TransactionDT</th>\n",
       "      <th>TransactionAmt</th>\n",
       "      <th>ProductCD</th>\n",
       "      <th>card1</th>\n",
       "      <th>card2</th>\n",
       "      <th>card3</th>\n",
       "      <th>card4</th>\n",
       "      <th>card5</th>\n",
       "      <th>card6</th>\n",
       "      <th>addr1</th>\n",
       "      <th>...</th>\n",
       "      <th>id_02_to_std_card1</th>\n",
       "      <th>D15_to_mean_card1</th>\n",
       "      <th>D15_to_std_card1</th>\n",
       "      <th>D15_to_mean_addr1</th>\n",
       "      <th>D15_to_std_addr1</th>\n",
       "      <th>TransactionAmt_decimal</th>\n",
       "      <th>Device_name</th>\n",
       "      <th>Device_version</th>\n",
       "      <th>screen_width</th>\n",
       "      <th>screen_height</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>86400</td>\n",
       "      <td>68.5</td>\n",
       "      <td>0</td>\n",
       "      <td>13926</td>\n",
       "      <td>NaN</td>\n",
       "      <td>150.0</td>\n",
       "      <td>0</td>\n",
       "      <td>142.0</td>\n",
       "      <td>0</td>\n",
       "      <td>315.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>500</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>86401</td>\n",
       "      <td>29.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2755</td>\n",
       "      <td>404.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>1</td>\n",
       "      <td>102.0</td>\n",
       "      <td>0</td>\n",
       "      <td>325.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>86469</td>\n",
       "      <td>59.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4663</td>\n",
       "      <td>490.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>2</td>\n",
       "      <td>166.0</td>\n",
       "      <td>1</td>\n",
       "      <td>330.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.188598</td>\n",
       "      <td>1.578656</td>\n",
       "      <td>1.420777</td>\n",
       "      <td>1.251501</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>86499</td>\n",
       "      <td>50.0</td>\n",
       "      <td>0</td>\n",
       "      <td>18132</td>\n",
       "      <td>567.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>1</td>\n",
       "      <td>117.0</td>\n",
       "      <td>1</td>\n",
       "      <td>476.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.471748</td>\n",
       "      <td>0.434842</td>\n",
       "      <td>0.581165</td>\n",
       "      <td>0.474049</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>86506</td>\n",
       "      <td>50.0</td>\n",
       "      <td>1</td>\n",
       "      <td>4497</td>\n",
       "      <td>514.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>1</td>\n",
       "      <td>102.0</td>\n",
       "      <td>0</td>\n",
       "      <td>420.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.9245</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 455 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   TransactionDT  TransactionAmt  ProductCD  card1  card2  card3  card4  \\\n",
       "0          86400            68.5          0  13926    NaN  150.0      0   \n",
       "1          86401            29.0          0   2755  404.0  150.0      1   \n",
       "2          86469            59.0          0   4663  490.0  150.0      2   \n",
       "3          86499            50.0          0  18132  567.0  150.0      1   \n",
       "4          86506            50.0          1   4497  514.0  150.0      1   \n",
       "\n",
       "   card5  card6  addr1  ...  id_02_to_std_card1  D15_to_mean_card1  \\\n",
       "0  142.0      0  315.0  ...                 NaN           0.000000   \n",
       "1  102.0      0  325.0  ...                 NaN           0.000000   \n",
       "2  166.0      1  330.0  ...                 NaN           2.188598   \n",
       "3  117.0      1  476.0  ...                 NaN           0.471748   \n",
       "4  102.0      0  420.0  ...              0.9245                NaN   \n",
       "\n",
       "   D15_to_std_card1  D15_to_mean_addr1  D15_to_std_addr1  \\\n",
       "0          0.000000           0.000000          0.000000   \n",
       "1          0.000000           0.000000          0.000000   \n",
       "2          1.578656           1.420777          1.251501   \n",
       "3          0.434842           0.581165          0.474049   \n",
       "4               NaN                NaN               NaN   \n",
       "\n",
       "   TransactionAmt_decimal  Device_name  Device_version  screen_width  \\\n",
       "0                     500           -1              -1            -1   \n",
       "1                       0           -1              -1            -1   \n",
       "2                       0           -1              -1            -1   \n",
       "3                       0           -1              -1            -1   \n",
       "4                       0            0               0             0   \n",
       "\n",
       "   screen_height  \n",
       "0             -1  \n",
       "1             -1  \n",
       "2             -1  \n",
       "3             -1  \n",
       "4              0  \n",
       "\n",
       "[5 rows x 455 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "cbc_1 = CatBoostClassifier(**params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 0.7805921\ttest: 0.7690242\tbest: 0.7690242 (0)\ttotal: 76.8ms\tremaining: 12m 48s\n",
      "500:\tlearn: 0.9099354\ttest: 0.8797977\tbest: 0.8797977 (500)\ttotal: 23.6s\tremaining: 7m 28s\n",
      "1000:\tlearn: 0.9291734\ttest: 0.8955471\tbest: 0.8955471 (1000)\ttotal: 46.4s\tremaining: 6m 56s\n",
      "1500:\tlearn: 0.9404129\ttest: 0.9020657\tbest: 0.9020657 (1500)\ttotal: 1m 8s\tremaining: 6m 30s\n",
      "2000:\tlearn: 0.9493622\ttest: 0.9075140\tbest: 0.9075140 (2000)\ttotal: 1m 31s\tremaining: 6m 4s\n",
      "2500:\tlearn: 0.9559695\ttest: 0.9110211\tbest: 0.9110211 (2500)\ttotal: 1m 53s\tremaining: 5m 40s\n",
      "3000:\tlearn: 0.9610814\ttest: 0.9143843\tbest: 0.9143843 (3000)\ttotal: 2m 15s\tremaining: 5m 15s\n",
      "3500:\tlearn: 0.9652727\ttest: 0.9160654\tbest: 0.9160906 (3495)\ttotal: 2m 37s\tremaining: 4m 52s\n",
      "4000:\tlearn: 0.9686460\ttest: 0.9173258\tbest: 0.9173595 (3984)\ttotal: 2m 59s\tremaining: 4m 29s\n",
      "4500:\tlearn: 0.9717069\ttest: 0.9182107\tbest: 0.9182107 (4500)\ttotal: 3m 21s\tremaining: 4m 6s\n",
      "5000:\tlearn: 0.9742119\ttest: 0.9192885\tbest: 0.9192987 (4991)\ttotal: 3m 43s\tremaining: 3m 43s\n",
      "5500:\tlearn: 0.9764746\ttest: 0.9196888\tbest: 0.9197186 (5495)\ttotal: 4m 6s\tremaining: 3m 21s\n",
      "6000:\tlearn: 0.9785380\ttest: 0.9202790\tbest: 0.9202821 (5995)\ttotal: 4m 28s\tremaining: 2m 59s\n",
      "6500:\tlearn: 0.9802800\ttest: 0.9203541\tbest: 0.9203622 (6498)\ttotal: 4m 51s\tremaining: 2m 36s\n",
      "7000:\tlearn: 0.9819420\ttest: 0.9206212\tbest: 0.9206339 (6994)\ttotal: 5m 14s\tremaining: 2m 14s\n",
      "7500:\tlearn: 0.9834141\ttest: 0.9207319\tbest: 0.9207388 (7452)\ttotal: 5m 36s\tremaining: 1m 52s\n",
      "8000:\tlearn: 0.9847196\ttest: 0.9207106\tbest: 0.9207525 (7552)\ttotal: 5m 59s\tremaining: 1m 29s\n",
      "bestTest = 0.9207524955\n",
      "bestIteration = 7552\n",
      "Shrink model to first 7553 iterations.\n",
      "366.33359003067017\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "train_dataset = Pool(data=features_train.iloc[0:472432,:],\n",
    "                     label=labels_train.values[0:472432])#,cat_features=categorical\n",
    "\n",
    "eval_dataset = Pool(data=features_train.iloc[472432:,:],\n",
    "                    label=labels_train.values[472432:])#,cat_features=categorical\n",
    "\n",
    "\n",
    "model = cat.train(dtrain = train_dataset,\n",
    "                  params=params,\n",
    "                  num_boost_round = 10000, \n",
    "                  eval_set = eval_dataset,\n",
    "                  verbose_eval=500,\n",
    "                  early_stopping_rounds = 500)\n",
    "print(time.time()-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'learn': {'CrossEntropy': 0.04049573784560529, 'AUC': 0.9848528504371643}, 'validation': {'CrossEntropy': 0.08383661067889135, 'AUC': 0.9207524955272675}}\n"
     ]
    }
   ],
   "source": [
    "print(model.get_best_score())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "splits = 5\n",
    "folds = KFold(n_splits = splits,random_state=50)\n",
    "predictions = np.zeros(len(features_test_new))\n",
    "ave_auc = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 0\n",
      "0:\tlearn: 0.7853286\ttest: 0.7695125\tbest: 0.7695125 (0)\ttotal: 58.4ms\tremaining: 9m 43s\n",
      "500:\tlearn: 0.9097171\ttest: 0.8743044\tbest: 0.8743044 (500)\ttotal: 23.9s\tremaining: 7m 33s\n",
      "1000:\tlearn: 0.9280349\ttest: 0.8925769\tbest: 0.8925769 (1000)\ttotal: 46.7s\tremaining: 6m 59s\n",
      "1500:\tlearn: 0.9393933\ttest: 0.9023674\tbest: 0.9023842 (1495)\ttotal: 1m 9s\tremaining: 6m 35s\n",
      "2000:\tlearn: 0.9481987\ttest: 0.9076049\tbest: 0.9076049 (2000)\ttotal: 1m 33s\tremaining: 6m 12s\n",
      "2500:\tlearn: 0.9550296\ttest: 0.9117606\tbest: 0.9117606 (2500)\ttotal: 1m 55s\tremaining: 5m 45s\n",
      "3000:\tlearn: 0.9606536\ttest: 0.9138484\tbest: 0.9138484 (3000)\ttotal: 2m 19s\tremaining: 5m 24s\n",
      "3500:\tlearn: 0.9649511\ttest: 0.9160973\tbest: 0.9161022 (3497)\ttotal: 2m 42s\tremaining: 5m 2s\n",
      "4000:\tlearn: 0.9685147\ttest: 0.9175644\tbest: 0.9175644 (4000)\ttotal: 3m 8s\tremaining: 4m 42s\n",
      "4500:\tlearn: 0.9714721\ttest: 0.9186276\tbest: 0.9186365 (4491)\ttotal: 3m 30s\tremaining: 4m 16s\n",
      "5000:\tlearn: 0.9740893\ttest: 0.9193466\tbest: 0.9193490 (4999)\ttotal: 3m 53s\tremaining: 3m 53s\n",
      "5500:\tlearn: 0.9763451\ttest: 0.9197591\tbest: 0.9197591 (5500)\ttotal: 4m 15s\tremaining: 3m 29s\n",
      "6000:\tlearn: 0.9784129\ttest: 0.9201080\tbest: 0.9201080 (6000)\ttotal: 4m 39s\tremaining: 3m 6s\n",
      "6500:\tlearn: 0.9802193\ttest: 0.9201774\tbest: 0.9202442 (6451)\ttotal: 5m 2s\tremaining: 2m 42s\n",
      "bestTest = 0.9202441573\n",
      "bestIteration = 6451\n",
      "Shrink model to first 6452 iterations.\n",
      "Fold 1\n",
      "0:\tlearn: 0.7859373\ttest: 0.7958635\tbest: 0.7958635 (0)\ttotal: 57.7ms\tremaining: 9m 37s\n",
      "500:\tlearn: 0.9098832\ttest: 0.8922737\tbest: 0.8922737 (500)\ttotal: 24.3s\tremaining: 7m 40s\n",
      "1000:\tlearn: 0.9275982\ttest: 0.9022652\tbest: 0.9022652 (1000)\ttotal: 48.3s\tremaining: 7m 14s\n",
      "1500:\tlearn: 0.9384006\ttest: 0.9080811\tbest: 0.9080849 (1499)\ttotal: 1m 10s\tremaining: 6m 37s\n",
      "2000:\tlearn: 0.9471734\ttest: 0.9125400\tbest: 0.9125412 (1999)\ttotal: 1m 33s\tremaining: 6m 12s\n",
      "2500:\tlearn: 0.9541568\ttest: 0.9162270\tbest: 0.9162270 (2499)\ttotal: 1m 55s\tremaining: 5m 47s\n",
      "3000:\tlearn: 0.9594790\ttest: 0.9190036\tbest: 0.9190036 (3000)\ttotal: 2m 18s\tremaining: 5m 22s\n",
      "3500:\tlearn: 0.9640299\ttest: 0.9213679\tbest: 0.9213679 (3500)\ttotal: 2m 41s\tremaining: 5m\n",
      "4000:\tlearn: 0.9676398\ttest: 0.9228166\tbest: 0.9228196 (3999)\ttotal: 3m 6s\tremaining: 4m 39s\n",
      "4500:\tlearn: 0.9707465\ttest: 0.9236351\tbest: 0.9236351 (4500)\ttotal: 3m 28s\tremaining: 4m 14s\n",
      "5000:\tlearn: 0.9733261\ttest: 0.9244623\tbest: 0.9245161 (4981)\ttotal: 3m 51s\tremaining: 3m 50s\n",
      "5500:\tlearn: 0.9757656\ttest: 0.9251611\tbest: 0.9251680 (5499)\ttotal: 4m 13s\tremaining: 3m 27s\n",
      "6000:\tlearn: 0.9778708\ttest: 0.9257798\tbest: 0.9257798 (6000)\ttotal: 4m 36s\tremaining: 3m 4s\n",
      "6500:\tlearn: 0.9798139\ttest: 0.9262230\tbest: 0.9262310 (6405)\ttotal: 4m 59s\tremaining: 2m 41s\n",
      "7000:\tlearn: 0.9815350\ttest: 0.9265323\tbest: 0.9265373 (6974)\ttotal: 5m 22s\tremaining: 2m 17s\n",
      "7500:\tlearn: 0.9830772\ttest: 0.9270258\tbest: 0.9270258 (7500)\ttotal: 5m 45s\tremaining: 1m 55s\n",
      "8000:\tlearn: 0.9843919\ttest: 0.9273587\tbest: 0.9273831 (7986)\ttotal: 6m 9s\tremaining: 1m 32s\n",
      "8500:\tlearn: 0.9855967\ttest: 0.9276452\tbest: 0.9276842 (8373)\ttotal: 6m 32s\tremaining: 1m 9s\n",
      "9000:\tlearn: 0.9867335\ttest: 0.9278868\tbest: 0.9278980 (8966)\ttotal: 6m 55s\tremaining: 46.1s\n",
      "9500:\tlearn: 0.9877724\ttest: 0.9281000\tbest: 0.9281021 (9429)\ttotal: 7m 19s\tremaining: 23.1s\n",
      "9999:\tlearn: 0.9887305\ttest: 0.9282283\tbest: 0.9282302 (9750)\ttotal: 7m 42s\tremaining: 0us\n",
      "bestTest = 0.9282301664\n",
      "bestIteration = 9750\n",
      "Shrink model to first 9751 iterations.\n",
      "Fold 2\n",
      "0:\tlearn: 0.7899265\ttest: 0.7741911\tbest: 0.7741911 (0)\ttotal: 58.8ms\tremaining: 9m 47s\n",
      "500:\tlearn: 0.9125598\ttest: 0.8872411\tbest: 0.8872486 (498)\ttotal: 23.3s\tremaining: 7m 20s\n",
      "1000:\tlearn: 0.9304249\ttest: 0.8979834\tbest: 0.8979834 (1000)\ttotal: 45.7s\tremaining: 6m 51s\n",
      "1500:\tlearn: 0.9410916\ttest: 0.9043108\tbest: 0.9043108 (1500)\ttotal: 1m 7s\tremaining: 6m 24s\n",
      "2000:\tlearn: 0.9490536\ttest: 0.9093006\tbest: 0.9093037 (1998)\ttotal: 1m 30s\tremaining: 6m\n",
      "2500:\tlearn: 0.9557808\ttest: 0.9133599\tbest: 0.9133599 (2500)\ttotal: 1m 52s\tremaining: 5m 35s\n",
      "3000:\tlearn: 0.9612387\ttest: 0.9159917\tbest: 0.9160277 (2988)\ttotal: 2m 14s\tremaining: 5m 12s\n",
      "3500:\tlearn: 0.9657710\ttest: 0.9175318\tbest: 0.9176623 (3492)\ttotal: 2m 35s\tremaining: 4m 48s\n",
      "4000:\tlearn: 0.9690622\ttest: 0.9187967\tbest: 0.9187967 (4000)\ttotal: 2m 58s\tremaining: 4m 27s\n",
      "4500:\tlearn: 0.9717845\ttest: 0.9193880\tbest: 0.9194193 (4418)\ttotal: 3m 20s\tremaining: 4m 4s\n",
      "5000:\tlearn: 0.9742676\ttest: 0.9201826\tbest: 0.9201826 (5000)\ttotal: 3m 42s\tremaining: 3m 42s\n",
      "5500:\tlearn: 0.9766770\ttest: 0.9209098\tbest: 0.9209370 (5498)\ttotal: 4m 4s\tremaining: 3m 20s\n",
      "6000:\tlearn: 0.9786388\ttest: 0.9215479\tbest: 0.9215817 (5996)\ttotal: 4m 26s\tremaining: 2m 57s\n",
      "6500:\tlearn: 0.9804386\ttest: 0.9220306\tbest: 0.9220476 (6419)\ttotal: 4m 48s\tremaining: 2m 35s\n",
      "7000:\tlearn: 0.9820956\ttest: 0.9225134\tbest: 0.9225385 (6929)\ttotal: 5m 11s\tremaining: 2m 13s\n",
      "7500:\tlearn: 0.9835622\ttest: 0.9228802\tbest: 0.9229047 (7476)\ttotal: 5m 33s\tremaining: 1m 50s\n",
      "8000:\tlearn: 0.9848545\ttest: 0.9232076\tbest: 0.9232333 (7974)\ttotal: 5m 55s\tremaining: 1m 28s\n",
      "8500:\tlearn: 0.9860563\ttest: 0.9235128\tbest: 0.9235128 (8500)\ttotal: 6m 18s\tremaining: 1m 6s\n",
      "9000:\tlearn: 0.9871891\ttest: 0.9235631\tbest: 0.9235752 (8995)\ttotal: 6m 40s\tremaining: 44.4s\n",
      "9500:\tlearn: 0.9882061\ttest: 0.9237250\tbest: 0.9237250 (9500)\ttotal: 7m 4s\tremaining: 22.3s\n",
      "9999:\tlearn: 0.9891454\ttest: 0.9239644\tbest: 0.9239686 (9970)\ttotal: 7m 26s\tremaining: 0us\n",
      "bestTest = 0.9239685535\n",
      "bestIteration = 9970\n",
      "Shrink model to first 9971 iterations.\n",
      "Fold 3\n",
      "0:\tlearn: 0.7777233\ttest: 0.7803959\tbest: 0.7803959 (0)\ttotal: 75.3ms\tremaining: 12m 32s\n",
      "500:\tlearn: 0.9078701\ttest: 0.9035538\tbest: 0.9035538 (500)\ttotal: 23.5s\tremaining: 7m 25s\n",
      "1000:\tlearn: 0.9262786\ttest: 0.9163050\tbest: 0.9163050 (1000)\ttotal: 45.8s\tremaining: 6m 52s\n",
      "1500:\tlearn: 0.9380443\ttest: 0.9232609\tbest: 0.9232609 (1500)\ttotal: 1m 8s\tremaining: 6m 28s\n",
      "2000:\tlearn: 0.9467627\ttest: 0.9284037\tbest: 0.9284037 (2000)\ttotal: 1m 32s\tremaining: 6m 9s\n",
      "2500:\tlearn: 0.9539366\ttest: 0.9323545\tbest: 0.9323545 (2500)\ttotal: 1m 54s\tremaining: 5m 44s\n",
      "3000:\tlearn: 0.9593655\ttest: 0.9347867\tbest: 0.9347867 (3000)\ttotal: 2m 18s\tremaining: 5m 22s\n",
      "4000:\tlearn: 0.9676398\ttest: 0.9379576\tbest: 0.9379576 (4000)\ttotal: 3m 4s\tremaining: 4m 35s\n",
      "5500:\tlearn: 0.9755633\ttest: 0.9396522\tbest: 0.9396522 (5500)\ttotal: 4m 13s\tremaining: 3m 27s\n",
      "7000:\tlearn: 0.9810144\ttest: 0.9410868\tbest: 0.9410977 (6983)\ttotal: 5m 21s\tremaining: 2m 17s\n",
      "8500:\tlearn: 0.9852272\ttest: 0.9419714\tbest: 0.9419822 (8383)\ttotal: 6m 30s\tremaining: 1m 8s\n",
      "9500:\tlearn: 0.9874865\ttest: 0.9423306\tbest: 0.9423325 (9498)\ttotal: 7m 16s\tremaining: 22.9s\n",
      "500:\tlearn: 0.9096906\ttest: 0.8791796\tbest: 0.8791796 (500)\ttotal: 23.9s\tremaining: 7m 32s\n",
      "2000:\tlearn: 0.9494293\ttest: 0.9074122\tbest: 0.9074215 (1997)\ttotal: 1m 29s\tremaining: 5m 58s\n",
      "3500:\tlearn: 0.9655457\ttest: 0.9159151\tbest: 0.9159394 (3496)\ttotal: 2m 36s\tremaining: 4m 50s\n",
      "5000:\tlearn: 0.9743156\ttest: 0.9188692\tbest: 0.9189235 (4993)\ttotal: 3m 42s\tremaining: 3m 42s\n",
      "6500:\tlearn: 0.9803577\ttest: 0.9200912\tbest: 0.9201131 (6006)\ttotal: 4m 49s\tremaining: 2m 35s\n",
      "bestTest = 0.9201130867\n",
      "bestIteration = 6006\n",
      "Shrink model to first 6007 iterations.\n"
     ]
    }
   ],
   "source": [
    "for fold_num, (trn_idx, val_idx) in enumerate(folds.split(features_train.values, \n",
    "                                                          labels_train.values)):\n",
    "    print(\"Fold {}\".format(fold_num))\n",
    "    train_df, y_train_df = features_train.iloc[trn_idx], labels_train.iloc[trn_idx]\n",
    "    valid_df, y_valid_df = features_train.iloc[val_idx], labels_train.iloc[val_idx]\n",
    "    \n",
    "    train_dataset = Pool(data=train_df,label=y_train_df)#,cat_features=categorical\n",
    "    eval_dataset = Pool(data=valid_df,label=y_valid_df)#,cat_features=categorical\n",
    "    \n",
    "    model = cat.train(dtrain = train_dataset,\n",
    "                  params=params,\n",
    "                  num_boost_round = 10000, \n",
    "                  eval_set = eval_dataset,\n",
    "                  verbose_eval=500,\n",
    "                  early_stopping_rounds = 500)\n",
    "\n",
    "    \n",
    "    pred = model.predict(valid_df)\n",
    "    auc_score = roc_auc_score(y_valid_df, pred)\n",
    "    ave_auc += auc_score / splits\n",
    "    predictions += model.predict(features_test_new) / splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9269963400097325"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# with all features\n",
    "ave_auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "id_test = features_test['TransactionID']\n",
    "submission = pd.DataFrame({'TransactionID': id_test, \n",
    "                            'isFraud': predictions})\n",
    "submission.to_csv('./data/sub_cat_feat%s.csv'%(feat_num), index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}