kaggle.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\hnpathak\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import normalize\n",
    "import os \n",
    "os.environ['OMP_NUM_THREADS'] = '4'\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import LabelEncoder, OneHotEncoder\n",
    "from sklearn.cross_validation import train_test_split\n",
    "TRAIN_SAMPLE_SIZE = 800000\n",
    "from datetime import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "morning_start = time(6)\n",
    "morning_end = time(12)\n",
    "afternoon_start = time(13)\n",
    "afternoon_end = time(17)\n",
    "evening_start = time(18)\n",
    "evening_end = time(22)\n",
    "night_start= time(23)\n",
    "night_end= time(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>click_id</th>\n",
       "      <th>ip</th>\n",
       "      <th>app</th>\n",
       "      <th>device</th>\n",
       "      <th>os</th>\n",
       "      <th>channel</th>\n",
       "      <th>click_time</th>\n",
       "      <th>time_of_day</th>\n",
       "      <th>morning</th>\n",
       "      <th>afternoon</th>\n",
       "      <th>evening</th>\n",
       "      <th>night</th>\n",
       "      <th>late_night</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>5744</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>107</td>\n",
       "      <td>2017-11-10 04:00:00</td>\n",
       "      <td>04:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>119901</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>466</td>\n",
       "      <td>2017-11-10 04:00:00</td>\n",
       "      <td>04:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>72287</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>128</td>\n",
       "      <td>2017-11-10 04:00:00</td>\n",
       "      <td>04:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>78477</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>111</td>\n",
       "      <td>2017-11-10 04:00:00</td>\n",
       "      <td>04:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>123080</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>328</td>\n",
       "      <td>2017-11-10 04:00:00</td>\n",
       "      <td>04:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   click_id      ip  app  device  os  channel          click_time time_of_day  \\\n",
       "0         0    5744    9       1   3      107 2017-11-10 04:00:00    04:00:00   \n",
       "1         1  119901    9       1   3      466 2017-11-10 04:00:00    04:00:00   \n",
       "2         2   72287   21       1  19      128 2017-11-10 04:00:00    04:00:00   \n",
       "3         3   78477   15       1  13      111 2017-11-10 04:00:00    04:00:00   \n",
       "4         4  123080   12       1  13      328 2017-11-10 04:00:00    04:00:00   \n",
       "\n",
       "   morning  afternoon  evening  night  late_night  \n",
       "0      0.0        0.0      0.0    0.0         1.0  \n",
       "1      0.0        0.0      0.0    0.0         1.0  \n",
       "2      0.0        0.0      0.0    0.0         1.0  \n",
       "3      0.0        0.0      0.0    0.0         1.0  \n",
       "4      0.0        0.0      0.0    0.0         1.0  "
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_test = pd.read_csv('./data/test/test.csv')\n",
    "dff = final_test\n",
    "dff.iloc[:,6] = pd.to_datetime(dff.iloc[:,6])\n",
    "dff['time_of_day'] = dff.iloc[:,6].apply(lambda x: x.time())\n",
    "#df['time_of_day'].head\n",
    "dff['morning'] = (dff['time_of_day'] >= time(8,0)) & (dff['time_of_day'] <= time(12,0))\n",
    "dff['afternoon'] = (dff['time_of_day'] >= time(12,1)) & (dff['time_of_day'] <= time(17,0))\n",
    "dff['evening'] = (dff['time_of_day'] >= time(17,1)) & (dff['time_of_day'] <= time(20,0))\n",
    "dff['night'] = (dff['time_of_day'] >= time(20,1)) & (dff['time_of_day'] <= time(23,59))\n",
    "dff['late_night'] = (dff['time_of_day'] >= time(0,1)) & (dff['time_of_day'] <= time(8,0))\n",
    "dff['night'] = dff['night'].astype(float)\n",
    "dff['afternoon'] = dff['afternoon'].astype(float)\n",
    "dff['late_night'] = dff['late_night'].astype(float)\n",
    "dff['morning'] = dff['morning'].astype(float)\n",
    "dff['evening'] = dff['evening'].astype(float)\n",
    "test_df = dff\n",
    "#del df\n",
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "del dff, test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ip</th>\n",
       "      <th>app</th>\n",
       "      <th>device</th>\n",
       "      <th>os</th>\n",
       "      <th>channel</th>\n",
       "      <th>morning</th>\n",
       "      <th>afternoon</th>\n",
       "      <th>evening</th>\n",
       "      <th>night</th>\n",
       "      <th>late_night</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5744</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>107</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>119901</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>466</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>72287</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>128</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>78477</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>111</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>123080</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>328</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       ip  app  device  os  channel  morning  afternoon  evening  night  \\\n",
       "0    5744    9       1   3      107      0.0        0.0      0.0    0.0   \n",
       "1  119901    9       1   3      466      0.0        0.0      0.0    0.0   \n",
       "2   72287   21       1  19      128      0.0        0.0      0.0    0.0   \n",
       "3   78477   15       1  13      111      0.0        0.0      0.0    0.0   \n",
       "4  123080   12       1  13      328      0.0        0.0      0.0    0.0   \n",
       "\n",
       "   late_night  \n",
       "0         1.0  \n",
       "1         1.0  \n",
       "2         1.0  \n",
       "3         1.0  \n",
       "4         1.0  "
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df2 = test_df.iloc[:,[1,2,3,4,5,8,9,10,11,12]]\n",
    "test_df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_LABEL = 'y'                                   \t\t\t        # Name of the variable to be predicted\n",
    "#KEYS = [i for i in raw_data.keys().tolist() if i != Y_LABEL]\t# Name of predictors\n",
    "N_INSTANCES = df2.shape[0]                     \t\t\t    # Number of instances\n",
    "N_INPUT = x_train.shape[1]                      \t\t\t    # Input size\n",
    "N_CLASSES = 2                                    \t\t\t    # Number of classes (output size)\n",
    "TEST_SIZE = 0.1                                    \t\t\t      # Test set size (% of dataset)\n",
    "TRAIN_SIZE = int(N_INSTANCES * (1 - TEST_SIZE))     \t\t\t    # Train size\n",
    "LEARNING_RATE = 0.001                               \t\t\t    # Learning rate\n",
    "TRAINING_EPOCHS =  20                             \t\t\t    # Number of epochs\n",
    "BATCH_SIZE = 100                                    \t\t\t    # Batch size\n",
    "DISPLAY_STEP = 1                                    \t\t\t    # Display progress each x epochs\n",
    "HIDDEN_SIZE = 10          \t                                   \t\t      # Number of hidden neurons 256\n",
    "ACTIVATION_FUNCTION_OUT = tf.nn.sigmoid                          # Last layer act fct\n",
    "STDDEV = 0.1                                        \t\t\t    # Standard deviation (for weights random init)\n",
    "RANDOM_STATE = 100\n",
    "\n",
    "n_input = N_INPUT                   # input n labels\n",
    "n_hidden_1 = 20            # 1st layer\n",
    "n_hidden_2 = 40            # 2nd layer\n",
    "n_hidden_3 = 30            # 3rd layer\n",
    "n_hidden_4 = 10           # 4th layer\n",
    "n_classes = N_CLASSES               # output m classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = tf.placeholder(tf.float32, [None, n_input],name='X')\n",
    "y = tf.placeholder(tf.float32, [None, n_classes],name='y')\n",
    "dropout_keep_prob = tf.placeholder(tf.float32)\n",
    "\n",
    "\n",
    "weights = {\n",
    "    'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1],stddev=STDDEV)),\n",
    "    'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2],stddev=STDDEV)),\n",
    "    'h3': tf.Variable(tf.random_normal([n_hidden_2, n_hidden_3],stddev=STDDEV)),\n",
    "    'h4': tf.Variable(tf.random_normal([n_hidden_3, n_hidden_4],stddev=STDDEV)),\n",
    "    'out': tf.Variable(tf.random_normal([n_hidden_4, n_classes],stddev=STDDEV)),                                   \n",
    "}\n",
    "\n",
    "biases = {\n",
    "    'b1': tf.Variable(tf.random_normal([n_hidden_1])),\n",
    "    'b2': tf.Variable(tf.random_normal([n_hidden_2])),\n",
    "    'b3': tf.Variable(tf.random_normal([n_hidden_3])),\n",
    "    'b4': tf.Variable(tf.random_normal([n_hidden_4])),\n",
    "    'out': tf.Variable(tf.random_normal([n_classes]))\n",
    "}    \n",
    "   \n",
    "def mlp(_X, _weights, _biases, dropout_keep_prob):\n",
    "    layer1 = tf.nn.dropout(tf.nn.relu(tf.add(tf.matmul(_X, _weights['h1']), _biases['b1'])), dropout_keep_prob)\n",
    "    layer2 = tf.nn.dropout(tf.nn.relu(tf.add(tf.matmul(layer1, _weights['h2']), _biases['b2'])), dropout_keep_prob)\n",
    "    layer3 = tf.nn.dropout(tf.nn.relu(tf.add(tf.matmul(layer2, _weights['h3']), _biases['b3'])), dropout_keep_prob)\n",
    "    layer4 = tf.nn.dropout(tf.nn.relu(tf.add(tf.matmul(layer3, _weights['h4']), _biases['b4'])), dropout_keep_prob)\n",
    "    out = ACTIVATION_FUNCTION_OUT(tf.add(tf.matmul(layer4, _weights['out']), _biases['out']))\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = mlp(X, weights, biases, dropout_keep_prob)\n",
    "\n",
    "# Loss and optimizer\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y)) # softmax loss\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate = LEARNING_RATE).minimize(cost)\n",
    "\n",
    "# Accuracy\n",
    "correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "confusion_matrix = tf.confusion_matrix(labels= tf.argmax(y, 1),predictions = tf.argmax(pred, 1), num_classes=2)\n",
    "predictions =  tf.argmax(pred,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Negative samples\n",
      "(737196, 8)\n",
      "data_combined:  (1194042, 8)\n",
      "Epoch: 000/005 cost: 0.695697863\n",
      "Training accuracy: 0.610\n",
      "Epoch: 001/005 cost: 0.695888526\n",
      "Training accuracy: 0.610\n",
      "Epoch: 002/005 cost: 0.695866950\n",
      "Training accuracy: 0.610\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-180-56dc4c6473b0>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     65\u001b[0m                 \u001b[0msess\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m{\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mbatch_xs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mbatch_ys\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdropout_keep_prob\u001b[0m\u001b[1;33m:\u001b[0m \u001b[1;36m0.9\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     66\u001b[0m                 \u001b[1;31m# Calculate average cost\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 67\u001b[1;33m                 \u001b[0mavg_cost\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0msess\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcost\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m{\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mbatch_xs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mbatch_ys\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdropout_keep_prob\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m1.\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m/\u001b[0m\u001b[0mtotal_batch\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     68\u001b[0m             \u001b[1;31m# Display progress\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     69\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mepoch\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mDISPLAY_STEP\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    787\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    788\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 789\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    790\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    791\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    995\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    996\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m--> 997\u001b[1;33m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[0;32m    998\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    999\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1130\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1131\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[1;32m-> 1132\u001b[1;33m                            target_list, options, run_metadata)\n\u001b[0m\u001b[0;32m   1133\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1134\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1137\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1138\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1139\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1140\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1141\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1119\u001b[0m         return tf_session.TF_Run(session, options,\n\u001b[0;32m   1120\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1121\u001b[1;33m                                  status, run_metadata)\n\u001b[0m\u001b[0;32m   1122\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1123\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msession\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "data_pos = pd.read_csv('data/positives/positives.csv',header=None)\n",
    "print('Negative samples')\n",
    "items = os.listdir('./data/negatives/')\n",
    "count = 0\n",
    "for item in items:\n",
    "    count = count +1\n",
    "    if count == 3:\n",
    "        break\n",
    "    #NEGATIVE data\n",
    "    data_neg = pd.read_csv('data/negatives/'+str(item), header=None)\n",
    "    print(data_neg.shape)\n",
    "    #Combine one section of positive and negative\n",
    "    data_combined =  pd.concat([data_pos, data_neg])\n",
    "    data_combined = data_combined.sample(frac=1)\n",
    "    print('data_combined: ', data_combined.shape)\n",
    "    #keep interested columns\n",
    "    data_f = data_combined.iloc[:,0:6]\n",
    "    #data_f.astype(float)\n",
    "    labels = data_combined.iloc[:,7:8]\n",
    "    labels.astype(float)\n",
    "    #n_values = np.max(labels) \n",
    "    labels_ = np.eye(2)[(labels.values).reshape(-1)]\n",
    "    #convert the date and time\n",
    "    df = data_f\n",
    "    df.iloc[:,5] = pd.to_datetime(df.iloc[:,5])\n",
    "    df['time_of_day'] = df.iloc[:,5].apply(lambda x: x.time())\n",
    "    #df['time_of_day'].head\n",
    "    df['morning'] = (df['time_of_day'] >= time(8,0)) & (df['time_of_day'] <= time(12,0))\n",
    "    df['afternoon'] = (df['time_of_day'] >= time(12,1)) & (df['time_of_day'] <= time(17,0))\n",
    "    df['evening'] = (df['time_of_day'] >= time(17,1)) & (df['time_of_day'] <= time(20,0))\n",
    "    df['night'] = (df['time_of_day'] >= time(20,1)) & (df['time_of_day'] <= time(23,59))\n",
    "    df['late_night'] = (df['time_of_day'] >= time(0,1)) & (df['time_of_day'] <= time(8,0))\n",
    "    df['night'] = df['night'].astype(float)\n",
    "    df['afternoon'] = df['afternoon'].astype(float)\n",
    "    df['late_night'] = df['late_night'].astype(float)\n",
    "    df['morning'] = df['morning'].astype(float)\n",
    "    df['evening'] = df['evening'].astype(float)\n",
    "    ###########################\n",
    "    df2 = df.iloc[:,[0,1,2,3,4,7,8,9,10,11]]\n",
    "    df2.head()\n",
    "    df2.astype(float).head()\n",
    "    #split the data set\n",
    "    x_train, x_test, y_train, y_test = train_test_split(df2,labels_,test_size = 0.01,random_state = 100)\n",
    "    x_train.astype(float)\n",
    "    x_test.astype(float)\n",
    "    y_train.astype(float)\n",
    "    y_test.astype(float)\n",
    "    ##################################\n",
    "    ##################################\n",
    "    ###################################\n",
    "    tf.set_random_seed(1234)\n",
    "    #saver = tf.train.Saver()\n",
    "    with tf.Session() as sess:\n",
    "        tf.global_variables_initializer().run()\n",
    "\n",
    "        for epoch in range(TRAINING_EPOCHS):\n",
    "            avg_cost = 0.\n",
    "            total_batch = int(x_train.shape[0] / BATCH_SIZE)\n",
    "            X_batches = np.array_split(x_train, total_batch)\n",
    "            Y_batches = np.array_split(y_train, total_batch)\n",
    "            # Loop over all batches\n",
    "            for i in range(total_batch):\n",
    "                batch_xs, batch_ys =  X_batches[i], Y_batches[i]\n",
    "                # Fit using batched data\n",
    "                sess.run(optimizer, feed_dict={X: batch_xs, y: batch_ys, dropout_keep_prob: 0.9})\n",
    "                # Calculate average cost\n",
    "                avg_cost += sess.run(cost, feed_dict={X: batch_xs, y: batch_ys, dropout_keep_prob:1.})/total_batch\n",
    "            # Display progress\n",
    "            if epoch % DISPLAY_STEP == 0:\n",
    "                print(\"Epoch: %03d/%03d cost: %.9f\" % (epoch, TRAINING_EPOCHS, avg_cost))\n",
    "                train_acc = sess.run(accuracy, feed_dict={X: batch_xs, y: batch_ys, dropout_keep_prob:1.})\n",
    "                print(\"Training accuracy: %.3f\" % (train_acc))\n",
    "        test_acc = sess.run(accuracy, feed_dict={X: x_test, y: y_test, dropout_keep_prob:1.})\n",
    "        confusion = sess.run(confusion_matrix, feed_dict={X: x_test, y: y_test, dropout_keep_prob:1.})\n",
    "        print(\"Test accuracy: %.3f\" % (test_acc))\n",
    "        answer  = sess.run(predictions, feed_dict={X: x_test, y: y_test, dropout_keep_prob:1.})\n",
    "        final_test_acc = sess.run(predictions, feed_dict={X: test_df2.values, y: y_test, dropout_keep_prob:1.})\n",
    "        #print(answer)\n",
    "        np.savetxt('./m1/final_test_'+item+'.csv',final_test_acc)\n",
    "        #np.savetxt('./m1/answer', answer)\n",
    "        print(confusion)\n",
    "        #saver.save(sess, './m1/model/')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Negative samples\n",
      "        0   1  2   3    4                    5   6  7\n",
      "0   83230   3  1  13  379  2017-11-06 14:32:21 NaN  0\n",
      "1   17357   3  1  19  379  2017-11-06 14:33:34 NaN  0\n",
      "2   35810   3  1  13  379  2017-11-06 14:34:12 NaN  0\n",
      "3   45745  14  1  13  478  2017-11-06 14:34:52 NaN  0\n",
      "4  161007   3  1  13  379  2017-11-06 14:35:08 NaN  0\n",
      "(737196, 8)\n",
      "\n",
      " Positive samples\n",
      "        0   1  2   3    4                    5                    6  7\n",
      "0  204158  35  1  13   21  2017-11-06 15:41:07  2017-11-07 08:17:19  1\n",
      "1   29692   9  1  22  215  2017-11-06 16:00:02  2017-11-07 10:05:22  1\n",
      "2   64516  35  1  13   21  2017-11-06 16:00:02  2017-11-06 23:40:50  1\n",
      "3  172429  35  1  46  274  2017-11-06 16:00:03  2017-11-07 00:55:29  1\n",
      "4  199085  35  1  13  274  2017-11-06 16:00:04  2017-11-06 23:04:54  1\n",
      "(456846, 8)\n"
     ]
    }
   ],
   "source": [
    "# data is divided into positive negative samples and 7 gb file of negatives is further divided into 250 splits. \n",
    "# $ split -n l/250 negatives.csv\n",
    "print('Negative samples')\n",
    "data_neg = pd.read_csv('data/negatives/xaa', header=None)\n",
    "print(data_neg.head())\n",
    "print(data_neg.shape)\n",
    "print('\\n Positive samples')\n",
    "data_pos = pd.read_csv('data/positives/positives.csv',header=None)\n",
    "print(data_pos.head())\n",
    "print(data_pos.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data_combined:  (1194042, 8)\n"
     ]
    }
   ],
   "source": [
    "# Build x_train, y_tarin, x_test, y_test\n",
    "data_combined =  pd.concat([data_pos, data_neg])\n",
    "data_combined = data_combined.sample(frac=1)\n",
    "print('data_combined: ', data_combined.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1194042, 2)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_f = data_combined.iloc[:,0:6]\n",
    "\n",
    "#data_f.astype(float)\n",
    "labels = data_combined.iloc[:,7:8]\n",
    "labels.astype(float)\n",
    "type(labels)\n",
    "#n_values = np.max(labels) \n",
    "labels_ = np.eye(2)[(labels.values).reshape(-1)]\n",
    "labels_.shape\n",
    "# labels = labels.values\n",
    "# enc = OneHotEncoder()\n",
    "# enc.fit(labels)\n",
    "# labels= enc.transform(labels)\n",
    "# labels.shape\n",
    "#del data_combined, data_neg, data_pos\n",
    "# y = np.zeros((data_combined.shape[0], 2))\n",
    "# y[np.arange(data_combined.shape[0]), y] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>time_of_day</th>\n",
       "      <th>morning</th>\n",
       "      <th>afternoon</th>\n",
       "      <th>evening</th>\n",
       "      <th>night</th>\n",
       "      <th>late_night</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>392605</th>\n",
       "      <td>42424</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>466</td>\n",
       "      <td>2017-11-09 07:58:17</td>\n",
       "      <td>07:58:17</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>276471</th>\n",
       "      <td>45745</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>259</td>\n",
       "      <td>2017-11-08 12:08:02</td>\n",
       "      <td>12:08:02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>258022</th>\n",
       "      <td>46542</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>489</td>\n",
       "      <td>2017-11-06 16:05:05</td>\n",
       "      <td>16:05:05</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>672377</th>\n",
       "      <td>131715</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>121</td>\n",
       "      <td>2017-11-06 16:14:00</td>\n",
       "      <td>16:14:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327700</th>\n",
       "      <td>80975</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>53</td>\n",
       "      <td>463</td>\n",
       "      <td>2017-11-06 16:06:16</td>\n",
       "      <td>16:06:16</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             0   1  2   3    4                   5 time_of_day  morning  \\\n",
       "392605   42424   9  1  17  466 2017-11-09 07:58:17    07:58:17      0.0   \n",
       "276471   45745  20  1  13  259 2017-11-08 12:08:02    12:08:02      0.0   \n",
       "258022   46542  14  1  13  489 2017-11-06 16:05:05    16:05:05      0.0   \n",
       "672377  131715  18  1  17  121 2017-11-06 16:14:00    16:14:00      0.0   \n",
       "327700   80975  14  1  53  463 2017-11-06 16:06:16    16:06:16      0.0   \n",
       "\n",
       "        afternoon  evening  night  late_night  \n",
       "392605        0.0      0.0    0.0         1.0  \n",
       "276471        1.0      0.0    0.0         0.0  \n",
       "258022        1.0      0.0    0.0         0.0  \n",
       "672377        1.0      0.0    0.0         0.0  \n",
       "327700        1.0      0.0    0.0         0.0  "
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datetime import time\n",
    "df = data_f\n",
    "morning_start = time(6)\n",
    "morning_end = time(12)\n",
    "afternoon_start = time(13)\n",
    "afternoon_end = time(17)\n",
    "evening_start = time(18)\n",
    "evening_end = time(22)\n",
    "night_start= time(23)\n",
    "night_end= time(5)\n",
    "df.head()\n",
    "df.iloc[:,5] = pd.to_datetime(df.iloc[:,5])\n",
    "df['time_of_day'] = df.iloc[:,5].apply(lambda x: x.time())\n",
    "#df['time_of_day'].head\n",
    "\n",
    "df['morning'] = (df['time_of_day'] >= time(8,0)) & (df['time_of_day'] <= time(12,0))\n",
    "df['afternoon'] = (df['time_of_day'] >= time(12,1)) & (df['time_of_day'] <= time(17,0))\n",
    "df['evening'] = (df['time_of_day'] >= time(17,1)) & (df['time_of_day'] <= time(20,0))\n",
    "df['night'] = (df['time_of_day'] >= time(20,1)) & (df['time_of_day'] <= time(23,59))\n",
    "df['late_night'] = (df['time_of_day'] >= time(0,1)) & (df['time_of_day'] <= time(8,0))\n",
    "df['night'] = df['night'].astype(float)\n",
    "df['afternoon'] = df['afternoon'].astype(float)\n",
    "df['late_night'] = df['late_night'].astype(float)\n",
    "df['morning'] = df['morning'].astype(float)\n",
    "df['evening'] = df['evening'].astype(float)\n",
    "df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>morning</th>\n",
       "      <th>afternoon</th>\n",
       "      <th>evening</th>\n",
       "      <th>night</th>\n",
       "      <th>late_night</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>392605</th>\n",
       "      <td>42424.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>466.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>276471</th>\n",
       "      <td>45745.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>259.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>258022</th>\n",
       "      <td>46542.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>489.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>672377</th>\n",
       "      <td>131715.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>121.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327700</th>\n",
       "      <td>80975.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>463.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               0     1    2     3      4  morning  afternoon  evening  night  \\\n",
       "392605   42424.0   9.0  1.0  17.0  466.0      0.0        0.0      0.0    0.0   \n",
       "276471   45745.0  20.0  1.0  13.0  259.0      0.0        1.0      0.0    0.0   \n",
       "258022   46542.0  14.0  1.0  13.0  489.0      0.0        1.0      0.0    0.0   \n",
       "672377  131715.0  18.0  1.0  17.0  121.0      0.0        1.0      0.0    0.0   \n",
       "327700   80975.0  14.0  1.0  53.0  463.0      0.0        1.0      0.0    0.0   \n",
       "\n",
       "        late_night  \n",
       "392605         1.0  \n",
       "276471         0.0  \n",
       "258022         0.0  \n",
       "672377         0.0  \n",
       "327700         0.0  "
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2 = df.iloc[:,[0,1,2,3,4,7,8,9,10,11]]\n",
    "df2.head()\n",
    "df2.astype(float).head()\n",
    "del df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 0.],\n",
       "       [0., 1.],\n",
       "       [0., 1.],\n",
       "       ...,\n",
       "       [1., 0.],\n",
       "       [1., 0.],\n",
       "       [1., 0.]])"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train, x_test, y_train, y_test = train_test_split(df2,labels_,test_size = 0.1,random_state = 100)\n",
    "x_train.astype(float)\n",
    "x_test.astype(float)\n",
    "y_train.astype(float)\n",
    "y_test.astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>click_id</th>\n",
       "      <th>ip</th>\n",
       "      <th>app</th>\n",
       "      <th>device</th>\n",
       "      <th>os</th>\n",
       "      <th>channel</th>\n",
       "      <th>click_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>5744</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>107</td>\n",
       "      <td>2017-11-10 04:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>119901</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>466</td>\n",
       "      <td>2017-11-10 04:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>72287</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>128</td>\n",
       "      <td>2017-11-10 04:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>78477</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>111</td>\n",
       "      <td>2017-11-10 04:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>123080</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>328</td>\n",
       "      <td>2017-11-10 04:00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   click_id      ip  app  device  os  channel           click_time\n",
       "0         0    5744    9       1   3      107  2017-11-10 04:00:00\n",
       "1         1  119901    9       1   3      466  2017-11-10 04:00:00\n",
       "2         2   72287   21       1  19      128  2017-11-10 04:00:00\n",
       "3         3   78477   15       1  13      111  2017-11-10 04:00:00\n",
       "4         4  123080   12       1  13      328  2017-11-10 04:00:00"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_test = pd.read_csv('./data/test/test.csv')\n",
    "final_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   click_id      ip  app  device  os  channel          click_time time_of_day  \\\n",
      "0         0    5744    9       1   3      107 2017-11-10 04:00:00    04:00:00   \n",
      "1         1  119901    9       1   3      466 2017-11-10 04:00:00    04:00:00   \n",
      "2         2   72287   21       1  19      128 2017-11-10 04:00:00    04:00:00   \n",
      "3         3   78477   15       1  13      111 2017-11-10 04:00:00    04:00:00   \n",
      "4         4  123080   12       1  13      328 2017-11-10 04:00:00    04:00:00   \n",
      "\n",
      "   morning  afternoon  evening  night  late_night  \n",
      "0      0.0        0.0      0.0    0.0         1.0  \n",
      "1      0.0        0.0      0.0    0.0         1.0  \n",
      "2      0.0        0.0      0.0    0.0         1.0  \n",
      "3      0.0        0.0      0.0    0.0         1.0  \n",
      "4      0.0        0.0      0.0    0.0         1.0  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>click_id</th>\n",
       "      <th>ip</th>\n",
       "      <th>app</th>\n",
       "      <th>device</th>\n",
       "      <th>os</th>\n",
       "      <th>channel</th>\n",
       "      <th>click_time</th>\n",
       "      <th>time_of_day</th>\n",
       "      <th>morning</th>\n",
       "      <th>afternoon</th>\n",
       "      <th>evening</th>\n",
       "      <th>night</th>\n",
       "      <th>late_night</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>5744</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>107</td>\n",
       "      <td>2017-11-10 04:00:00</td>\n",
       "      <td>04:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>119901</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>466</td>\n",
       "      <td>2017-11-10 04:00:00</td>\n",
       "      <td>04:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>72287</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>128</td>\n",
       "      <td>2017-11-10 04:00:00</td>\n",
       "      <td>04:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>78477</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>111</td>\n",
       "      <td>2017-11-10 04:00:00</td>\n",
       "      <td>04:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>123080</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>328</td>\n",
       "      <td>2017-11-10 04:00:00</td>\n",
       "      <td>04:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   click_id      ip  app  device  os  channel          click_time time_of_day  \\\n",
       "0         0    5744    9       1   3      107 2017-11-10 04:00:00    04:00:00   \n",
       "1         1  119901    9       1   3      466 2017-11-10 04:00:00    04:00:00   \n",
       "2         2   72287   21       1  19      128 2017-11-10 04:00:00    04:00:00   \n",
       "3         3   78477   15       1  13      111 2017-11-10 04:00:00    04:00:00   \n",
       "4         4  123080   12       1  13      328 2017-11-10 04:00:00    04:00:00   \n",
       "\n",
       "   morning  afternoon  evening  night  late_night  \n",
       "0      0.0        0.0      0.0    0.0         1.0  \n",
       "1      0.0        0.0      0.0    0.0         1.0  \n",
       "2      0.0        0.0      0.0    0.0         1.0  \n",
       "3      0.0        0.0      0.0    0.0         1.0  \n",
       "4      0.0        0.0      0.0    0.0         1.0  "
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#del df\n",
    "from datetime import time\n",
    "df = final_test\n",
    "morning_start = time(6)\n",
    "morning_end = time(12)\n",
    "afternoon_start = time(13)\n",
    "afternoon_end = time(17)\n",
    "evening_start = time(18)\n",
    "evening_end = time(22)\n",
    "night_start= time(23)\n",
    "night_end= time(5)\n",
    "\n",
    "df.iloc[:,6] = pd.to_datetime(df.iloc[:,6])\n",
    "df['time_of_day'] = df.iloc[:,6].apply(lambda x: x.time())\n",
    "#df['time_of_day'].head\n",
    "df['morning'] = (df['time_of_day'] >= time(8,0)) & (df['time_of_day'] <= time(12,0))\n",
    "df['afternoon'] = (df['time_of_day'] >= time(12,1)) & (df['time_of_day'] <= time(17,0))\n",
    "df['evening'] = (df['time_of_day'] >= time(17,1)) & (df['time_of_day'] <= time(20,0))\n",
    "df['night'] = (df['time_of_day'] >= time(20,1)) & (df['time_of_day'] <= time(23,59))\n",
    "df['late_night'] = (df['time_of_day'] >= time(0,1)) & (df['time_of_day'] <= time(8,0))\n",
    "df['night'] = df['night'].astype(float)\n",
    "df['afternoon'] = df['afternoon'].astype(float)\n",
    "df['late_night'] = df['late_night'].astype(float)\n",
    "df['morning'] = df['morning'].astype(float)\n",
    "df['evening'] = df['evening'].astype(float)\n",
    "print(df.head())\n",
    "test_df = df\n",
    "#del df\n",
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ip</th>\n",
       "      <th>app</th>\n",
       "      <th>device</th>\n",
       "      <th>os</th>\n",
       "      <th>channel</th>\n",
       "      <th>morning</th>\n",
       "      <th>afternoon</th>\n",
       "      <th>evening</th>\n",
       "      <th>night</th>\n",
       "      <th>late_night</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5744</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>107</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>119901</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>466</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>72287</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>128</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>78477</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>111</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>123080</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>328</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       ip  app  device  os  channel  morning  afternoon  evening  night  \\\n",
       "0    5744    9       1   3      107      0.0        0.0      0.0    0.0   \n",
       "1  119901    9       1   3      466      0.0        0.0      0.0    0.0   \n",
       "2   72287   21       1  19      128      0.0        0.0      0.0    0.0   \n",
       "3   78477   15       1  13      111      0.0        0.0      0.0    0.0   \n",
       "4  123080   12       1  13      328      0.0        0.0      0.0    0.0   \n",
       "\n",
       "   late_night  \n",
       "0         1.0  \n",
       "1         1.0  \n",
       "2         1.0  \n",
       "3         1.0  \n",
       "4         1.0  "
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df2 = test_df.iloc[:,[1,2,3,4,5,8,9,10,11,12]]\n",
    "test_df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0           1.0\n",
       "1           1.0\n",
       "2           1.0\n",
       "3           1.0\n",
       "4           1.0\n",
       "5           1.0\n",
       "6           1.0\n",
       "7           1.0\n",
       "8           1.0\n",
       "9           1.0\n",
       "10          1.0\n",
       "11          1.0\n",
       "12          1.0\n",
       "13          1.0\n",
       "14          1.0\n",
       "15          1.0\n",
       "16          1.0\n",
       "17          1.0\n",
       "18          1.0\n",
       "19          1.0\n",
       "20          1.0\n",
       "21          1.0\n",
       "22          1.0\n",
       "23          1.0\n",
       "24          1.0\n",
       "25          1.0\n",
       "26          1.0\n",
       "27          1.0\n",
       "28          1.0\n",
       "29          1.0\n",
       "           ... \n",
       "18790439    1.0\n",
       "18790440    1.0\n",
       "18790441    1.0\n",
       "18790442    1.0\n",
       "18790443    1.0\n",
       "18790444    1.0\n",
       "18790445    1.0\n",
       "18790446    1.0\n",
       "18790447    1.0\n",
       "18790448    1.0\n",
       "18790449    1.0\n",
       "18790450    1.0\n",
       "18790451    1.0\n",
       "18790452    1.0\n",
       "18790453    1.0\n",
       "18790454    1.0\n",
       "18790455    1.0\n",
       "18790456    1.0\n",
       "18790457    1.0\n",
       "18790458    1.0\n",
       "18790459    1.0\n",
       "18790460    1.0\n",
       "18790461    1.0\n",
       "18790462    1.0\n",
       "18790463    1.0\n",
       "18790464    1.0\n",
       "18790465    1.0\n",
       "18790466    1.0\n",
       "18790467    1.0\n",
       "18790468    1.0\n",
       "Length: 18790469, dtype: float64"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df2.iloc[:,6:10].sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "del df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_LABEL = 'y'                                   \t\t\t        # Name of the variable to be predicted\n",
    "#KEYS = [i for i in raw_data.keys().tolist() if i != Y_LABEL]\t# Name of predictors\n",
    "N_INSTANCES = df2.shape[0]                     \t\t\t    # Number of instances\n",
    "N_INPUT = x_train.shape[1]                      \t\t\t    # Input size\n",
    "N_CLASSES = 2                                    \t\t\t    # Number of classes (output size)\n",
    "TEST_SIZE = 0.1                                    \t\t\t      # Test set size (% of dataset)\n",
    "TRAIN_SIZE = int(N_INSTANCES * (1 - TEST_SIZE))     \t\t\t    # Train size\n",
    "LEARNING_RATE = 0.001                               \t\t\t    # Learning rate\n",
    "TRAINING_EPOCHS =   1                             \t\t\t    # Number of epochs\n",
    "BATCH_SIZE = 100                                    \t\t\t    # Batch size\n",
    "DISPLAY_STEP = 1                                    \t\t\t    # Display progress each x epochs\n",
    "HIDDEN_SIZE = 10          \t                                   \t\t      # Number of hidden neurons 256\n",
    "ACTIVATION_FUNCTION_OUT = tf.nn.sigmoid                          # Last layer act fct\n",
    "STDDEV = 0.1                                        \t\t\t    # Standard deviation (for weights random init)\n",
    "RANDOM_STATE = 100   \t\t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_input = N_INPUT                   # input n labels\n",
    "n_hidden_1 = 20            # 1st layer\n",
    "n_hidden_2 = 40            # 2nd layer\n",
    "n_hidden_3 = 30            # 3rd layer\n",
    "n_hidden_4 = 10           # 4th layer\n",
    "n_classes = N_CLASSES               # output m classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_input],name='X')\n",
    "y = tf.placeholder(tf.float32, [None, n_classes],name='y')\n",
    "dropout_keep_prob = tf.placeholder(tf.float32)\n",
    "\n",
    "\n",
    "weights = {\n",
    "    'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1],stddev=STDDEV)),\n",
    "    'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2],stddev=STDDEV)),\n",
    "    'h3': tf.Variable(tf.random_normal([n_hidden_2, n_hidden_3],stddev=STDDEV)),\n",
    "    'h4': tf.Variable(tf.random_normal([n_hidden_3, n_hidden_4],stddev=STDDEV)),\n",
    "    'out': tf.Variable(tf.random_normal([n_hidden_4, n_classes],stddev=STDDEV)),                                   \n",
    "}\n",
    "\n",
    "biases = {\n",
    "    'b1': tf.Variable(tf.random_normal([n_hidden_1])),\n",
    "    'b2': tf.Variable(tf.random_normal([n_hidden_2])),\n",
    "    'b3': tf.Variable(tf.random_normal([n_hidden_3])),\n",
    "    'b4': tf.Variable(tf.random_normal([n_hidden_4])),\n",
    "    'out': tf.Variable(tf.random_normal([n_classes]))\n",
    "}    \n",
    "   \n",
    "def mlp(_X, _weights, _biases, dropout_keep_prob):\n",
    "    layer1 = tf.nn.dropout(tf.nn.relu(tf.add(tf.matmul(_X, _weights['h1']), _biases['b1'])), dropout_keep_prob)\n",
    "    layer2 = tf.nn.dropout(tf.nn.relu(tf.add(tf.matmul(layer1, _weights['h2']), _biases['b2'])), dropout_keep_prob)\n",
    "    layer3 = tf.nn.dropout(tf.nn.relu(tf.add(tf.matmul(layer2, _weights['h3']), _biases['b3'])), dropout_keep_prob)\n",
    "    layer4 = tf.nn.dropout(tf.nn.relu(tf.add(tf.matmul(layer3, _weights['h4']), _biases['b4'])), dropout_keep_prob)\n",
    "    out = ACTIVATION_FUNCTION_OUT(tf.add(tf.matmul(layer4, _weights['out']), _biases['out']))\n",
    "    return out\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = mlp(X, weights, biases, dropout_keep_prob)\n",
    "\n",
    "# Loss and optimizer\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y)) # softmax loss\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate = LEARNING_RATE).minimize(cost)\n",
    "\n",
    "# Accuracy\n",
    "correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "confusion_matrix = tf.confusion_matrix(labels= tf.argmax(y, 1),predictions = tf.argmax(pred, 1), num_classes=2)\n",
    "predictions =  tf.argmax(pred,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 000/001 cost: 0.583111460\n",
      "Training accuracy: 0.810\n",
      "Test accuracy: 0.787\n",
      "[[66803  6909]\n",
      " [18532 27161]]\n"
     ]
    }
   ],
   "source": [
    "tf.set_random_seed(1234)\n",
    "saver = tf.train.Saver()\n",
    "with tf.Session() as sess:\n",
    "    tf.global_variables_initializer().run()\n",
    "    \n",
    "    for epoch in range(TRAINING_EPOCHS):\n",
    "        avg_cost = 0.\n",
    "        total_batch = int(x_train.shape[0] / BATCH_SIZE)\n",
    "        X_batches = np.array_split(x_train, total_batch)\n",
    "        Y_batches = np.array_split(y_train, total_batch)\n",
    "        # Loop over all batches\n",
    "        for i in range(total_batch):\n",
    "#             randidx = np.random.randint(int(TRAIN_SIZE), size = BATCH_SIZE)\n",
    "#             batch_xs = x_tr[randidx, :]\n",
    "#             batch_ys = labels[randidx, :]\n",
    "            batch_xs, batch_ys =  X_batches[i], Y_batches[i]\n",
    "            # Fit using batched data\n",
    "            sess.run(optimizer, feed_dict={X: batch_xs, y: batch_ys, dropout_keep_prob: 0.9})\n",
    "            # Calculate average cost\n",
    "            avg_cost += sess.run(cost, feed_dict={X: batch_xs, y: batch_ys, dropout_keep_prob:1.})/total_batch\n",
    "        # Display progress\n",
    "        if epoch % DISPLAY_STEP == 0:\n",
    "            print(\"Epoch: %03d/%03d cost: %.9f\" % (epoch, TRAINING_EPOCHS, avg_cost))\n",
    "            train_acc = sess.run(accuracy, feed_dict={X: batch_xs, y: batch_ys, dropout_keep_prob:1.})\n",
    "            print(\"Training accuracy: %.3f\" % (train_acc))\n",
    "    test_acc = sess.run(accuracy, feed_dict={X: x_test, y: y_test, dropout_keep_prob:1.})\n",
    "    confusion = sess.run(confusion_matrix, feed_dict={X: x_test, y: y_test, dropout_keep_prob:1.})\n",
    "    print(\"Test accuracy: %.3f\" % (test_acc))\n",
    "    answer  = sess.run(predictions, feed_dict={X: x_test, y: y_test, dropout_keep_prob:1.})\n",
    "    final_test_acc = sess.run(predictions, feed_dict={X: test_df2.values, y: y_test, dropout_keep_prob:1.})\n",
    "    #print(answer)\n",
    "    np.savetxt('./m1/final_test_acc', final_test_acc)\n",
    "    np.savetxt('./m1/answer', answer)\n",
    "    print(confusion)\n",
    "    saver.save(sess, './m1/model/')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = np.loadtxt('./m1/final_test_acc')\n",
    "np.savetxt('./m1/final_test_acc.csv', temp, delimiter=\",\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = temp.astype(np.int32)\n",
    "final_test['click_id'] = final_test['click_id'].astype(np.int32)\n",
    "a = np.vstack((final_test['click_id'].values,temp))\n",
    "a = a.T\n",
    "a = np.int32(a)\n",
    "#a[:5]\n",
    "np.savetxt('./m1/final_test_acc1.csv', a, fmt='%i', delimiter=\",\")"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "import numpy\n",
    "import pandas\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.pipeline import Pipeline\n",
    "# fix random seed for reproducibility\n",
    "seed = 7\n",
    "numpy.random.seed(seed)\n",
    "# load dataset\n",
    "#dataframe = pandas.read_csv(\"sonar.csv\", header=None)\n",
    "#dataset = dataframe.values\n",
    "# split into input (X) and output (Y) variables\n",
    "X = x_train\n",
    "Y = y_train\n",
    "# encode class values as integers\n",
    "encoder = LabelEncoder()\n",
    "encoder.fit(Y)\n",
    "encoded_Y = encoder.transform(Y)\n",
    "# larger model\n",
    "def create_larger():\n",
    "\t# create model\n",
    "\tmodel = Sequential()\n",
    "\tmodel.add(Dense(10, input_dim=x_train.shape[1], init='normal', activation='relu'))\n",
    "\tmodel.add(Dense(5, init='normal', activation='relu'))\n",
    "\tmodel.add(Dense(1, init='normal', activation='sigmoid'))\n",
    "\t# Compile model\n",
    "\tmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "\treturn model\n",
    "numpy.random.seed(seed)\n",
    "estimators = []\n",
    "estimators.append(('standardize', StandardScaler()))\n",
    "estimators.append(('mlp', KerasClassifier(build_fn=create_larger, nb_epoch=100, batch_size=5, verbose=1)))\n",
    "pipeline = Pipeline(estimators)\n",
    "kfold = StratifiedKFold(y=encoded_Y, n_folds=10, shuffle=True, random_state=seed)\n",
    "results = cross_val_score(pipeline, X, encoded_Y, cv=kfold)\n",
    "print(\"Larger: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}