Neural_Network_Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "pcxEynIzRAOm",
    "outputId": "162c88c9-953b-4498-c218-a630536cb24b"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from keras.wrappers.scikit_learn import KerasRegressor\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Dropout\n",
    "\n",
    "from sklearn.metrics import mean_absolute_error "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 221
    },
    "colab_type": "code",
    "id": "0bSoxhNx7ZFr",
    "outputId": "03928636-9756-4e8a-f280-dd4f462f3dd8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1591497, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathroomcnt</th>\n",
       "      <th>bedroomcnt</th>\n",
       "      <th>calculatedfinishedsquarefeet</th>\n",
       "      <th>yearbuilt</th>\n",
       "      <th>taxvaluedollarcnt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3660.0</td>\n",
       "      <td>2003.0</td>\n",
       "      <td>655280.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1152.0</td>\n",
       "      <td>1911.0</td>\n",
       "      <td>196726.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1230.0</td>\n",
       "      <td>1977.0</td>\n",
       "      <td>218275.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1464.0</td>\n",
       "      <td>1958.0</td>\n",
       "      <td>57901.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1185.0</td>\n",
       "      <td>1949.0</td>\n",
       "      <td>270354.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathroomcnt  bedroomcnt  calculatedfinishedsquarefeet  yearbuilt  \\\n",
       "0          3.0         6.0                        3660.0     2003.0   \n",
       "1          2.0         3.0                        1152.0     1911.0   \n",
       "2          2.0         2.0                        1230.0     1977.0   \n",
       "3          1.0         3.0                        1464.0     1958.0   \n",
       "4          2.0         2.0                        1185.0     1949.0   \n",
       "\n",
       "   taxvaluedollarcnt  \n",
       "0           655280.0  \n",
       "1           196726.0  \n",
       "2           218275.0  \n",
       "3            57901.0  \n",
       "4           270354.0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv(\"./Data/train_2017.csv\")\n",
    "test = pd.read_csv(\"./Data/test_2017.csv\")\n",
    "\n",
    "print(train.shape)\n",
    "\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 221
    },
    "colab_type": "code",
    "id": "ovdZs4VN_vmg",
    "outputId": "4058fb15-7700-4844-f67a-f9f68f214d30"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(530499, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathroomcnt</th>\n",
       "      <th>bedroomcnt</th>\n",
       "      <th>calculatedfinishedsquarefeet</th>\n",
       "      <th>yearbuilt</th>\n",
       "      <th>taxvaluedollarcnt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1900.0</td>\n",
       "      <td>1985.0</td>\n",
       "      <td>293880.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2236.0</td>\n",
       "      <td>1987.0</td>\n",
       "      <td>798691.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1291.0</td>\n",
       "      <td>1956.0</td>\n",
       "      <td>419180.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.5</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4425.0</td>\n",
       "      <td>2010.0</td>\n",
       "      <td>1687000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1620.0</td>\n",
       "      <td>1953.0</td>\n",
       "      <td>293830.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathroomcnt  bedroomcnt  calculatedfinishedsquarefeet  yearbuilt  \\\n",
       "0          2.0         3.0                        1900.0     1985.0   \n",
       "1          2.5         3.0                        2236.0     1987.0   \n",
       "2          2.0         3.0                        1291.0     1956.0   \n",
       "3          4.5         4.0                        4425.0     2010.0   \n",
       "4          2.0         3.0                        1620.0     1953.0   \n",
       "\n",
       "   taxvaluedollarcnt  \n",
       "0           293880.0  \n",
       "1           798691.0  \n",
       "2           419180.0  \n",
       "3          1687000.0  \n",
       "4           293830.0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(test.shape)\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lh6C3Hx97ZIr"
   },
   "outputs": [],
   "source": [
    "X_train = train[['bathroomcnt', 'bedroomcnt', 'calculatedfinishedsquarefeet', \n",
    "        'yearbuilt']].values\n",
    "y_train = train['taxvaluedollarcnt'].values\n",
    "\n",
    "X_test = test[['bathroomcnt', 'bedroomcnt', 'calculatedfinishedsquarefeet', \n",
    "        'yearbuilt']].values\n",
    "y_test = test['taxvaluedollarcnt'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 198
    },
    "colab_type": "code",
    "id": "d5XOOV0g7pG_",
    "outputId": "7cf7829d-be89-4e82-9641-bbfc71f1cf66"
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "scaler = StandardScaler()\n",
    "\n",
    "#X = scaler.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "5EQFLwHO7pLh",
    "outputId": "ae9984fe-709e-442f-ac51-dbe581450e89"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1591497, 4), (530499, 4), (1591497,), (530499,))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape, X_test.shape, y_train.shape, y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "O-fWVPdt7pNl"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 255
    },
    "colab_type": "code",
    "id": "gNbou5w5RLxM",
    "outputId": "126c9b35-6c16-4133-8f47-55e82f992c31"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0925 11:58:45.993171 4478809536 deprecation.py:506] From /anaconda3/envs/U4-S2-NNF/lib/python3.7/site-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense (Dense)                (None, 32)                160       \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 32)                1056      \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 33        \n",
      "=================================================================\n",
      "Total params: 1,249\n",
      "Trainable params: 1,249\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "\n",
    "nnmodel = Sequential()\n",
    "# Input => Hidden\n",
    "nnmodel.add(Dense(32, input_dim=4, activation='relu'))\n",
    "# Hidden\n",
    "nnmodel.add(Dense(32, activation='relu'))\n",
    "#nnmodel.add(Dense(32, activation='relu'))\n",
    "#nnmodel.add(Dense(32, activation='relu'))\n",
    "# Output Layer\n",
    "nnmodel.add(Dense(1, activation='linear'))\n",
    "\n",
    "# Compile\n",
    "nnmodel.compile(loss='mean_absolute_error',\n",
    "                    optimizer='adam',\n",
    "                    metrics=['mean_absolute_error'])\n",
    "\n",
    "nnmodel.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 392
    },
    "colab_type": "code",
    "id": "T0NZCyb6RP6F",
    "outputId": "461bbebb-4a33-4e32-e659-9d8822e349cb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1114047 samples, validate on 477450 samples\n",
      "Epoch 1/10\n",
      "1114047/1114047 [==============================] - 27s 24us/sample - loss: 213000.8385 - mean_absolute_error: 213000.8438 - val_loss: 208528.2338 - val_mean_absolute_error: 208527.7969\n",
      "Epoch 2/10\n",
      "1114047/1114047 [==============================] - 28s 25us/sample - loss: 208502.9721 - mean_absolute_error: 208501.5625 - val_loss: 208269.6272 - val_mean_absolute_error: 208269.4688\n",
      "Epoch 3/10\n",
      "1114047/1114047 [==============================] - 28s 25us/sample - loss: 208412.3327 - mean_absolute_error: 208412.8281 - val_loss: 208760.6680 - val_mean_absolute_error: 208759.7812\n",
      "Epoch 4/10\n",
      "1114047/1114047 [==============================] - 28s 25us/sample - loss: 208356.6067 - mean_absolute_error: 208356.3594 - val_loss: 208001.8678 - val_mean_absolute_error: 208001.5625\n",
      "Epoch 5/10\n",
      "1114047/1114047 [==============================] - 27s 25us/sample - loss: 208316.9387 - mean_absolute_error: 208317.3906 - val_loss: 207958.5350 - val_mean_absolute_error: 207958.4844\n",
      "Epoch 6/10\n",
      "1114047/1114047 [==============================] - 27s 24us/sample - loss: 208261.1845 - mean_absolute_error: 208260.5000 - val_loss: 208068.3471 - val_mean_absolute_error: 208067.7344\n",
      "Epoch 7/10\n",
      "1114047/1114047 [==============================] - 27s 24us/sample - loss: 208197.1289 - mean_absolute_error: 208197.2656 - val_loss: 207890.9126 - val_mean_absolute_error: 207890.6719\n",
      "Epoch 8/10\n",
      "1114047/1114047 [==============================] - 28s 25us/sample - loss: 208126.0919 - mean_absolute_error: 208126.3438 - val_loss: 207837.1162 - val_mean_absolute_error: 207836.9062\n",
      "Epoch 9/10\n",
      "1114047/1114047 [==============================] - 28s 25us/sample - loss: 208040.8385 - mean_absolute_error: 208040.8750 - val_loss: 208218.1386 - val_mean_absolute_error: 208217.9844\n",
      "Epoch 10/10\n",
      "1114047/1114047 [==============================] - 27s 25us/sample - loss: 207943.1488 - mean_absolute_error: 207943.2031 - val_loss: 207599.9379 - val_mean_absolute_error: 207599.5312\n",
      "530499/530499 [==============================] - 4s 7us/sample - loss: 207718.6455 - mean_absolute_error: 207718.8438\n",
      "mean_absolute_error: 20771884.375\n"
     ]
    }
   ],
   "source": [
    "nnmodel.fit(X_train, y_train, epochs=10, batch_size=32, validation_split=.3, verbose=1)\n",
    "scores = nnmodel.evaluate(X_test, y_test)\n",
    "print(f'{nnmodel.metrics_names[1]}: {scores[1]*100}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 463
    },
    "colab_type": "code",
    "id": "iFqjmheFRP8W",
    "outputId": "7e4c6403-7ca3-49a7-f57e-22050d8670b2"
   },
   "outputs": [],
   "source": [
    "# Function to create model\n",
    "def create_model():\n",
    "    # create model\n",
    "    nnmodel = Sequential()\n",
    "    # Input => Hidden\n",
    "    nnmodel.add(Dense(32, input_dim=4, activation='relu'))\n",
    "    # Hidden\n",
    "    nnmodel.add(Dense(32, activation='relu'))\n",
    "    nnmodel.add(Dense(32, activation='relu'))\n",
    "    nnmodel.add(Dense(32, activation='relu'))\n",
    "    # Output Layer\n",
    "    nnmodel.add(Dense(1, activation='linear'))\n",
    "\n",
    "    # Compile\n",
    "    nnmodel.compile(loss='mean_absolute_error',\n",
    "                    optimizer='adam',\n",
    "                    metrics=['mean_absolute_error'])\n",
    "    return model\n",
    "  \n",
    "model = KerasRegressor(build_fn=create_model, verbose=1)\n",
    "\n",
    "# define the grid search parameters\n",
    "param_grid = {'batch_size': [10, 20, 40, 60, 80, 100],\n",
    "              'epochs': [10,20,40,60,80]}\n",
    "\n",
    "# Create Grid Search\n",
    "grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=5)\n",
    "grid_result = grid.fit(X_train, y_train)\n",
    "\n",
    "# Report Results\n",
    "print(f\"Best: {grid_result.best_score_} using {grid_result.best_params_}\")\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(f\"Means: {mean}, Stdev: {stdev} with: {param}\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "w8WinkXiRP-d"
   },
   "outputs": [],
   "source": [
    "\n",
    "def baseline_model(optimizer='adam'):\n",
    "    # create model\n",
    "    model = Sequential()\n",
    "    model.add(Dense(32, activation='relu', \n",
    "                    kernel_regularizer = 'l2', \n",
    "                    kernel_initializer = 'normal', \n",
    "                    input_shape=(4,)))\n",
    "   # model.add(BatchNormalization())\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(32, activation='relu', \n",
    "                    kernel_regularizer = 'l2',\n",
    "                    kernel_initializer = 'normal'))\n",
    "    #model.add(BatchNormalization())\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(1, activation='linear',\n",
    "                    kernel_regularizer = 'l2', \n",
    "                    kernel_initializer='normal'))\n",
    "    model.compile(loss='mean_absolute_error', optimizer=optimizer, metrics=['mean_absolute_error'])\n",
    "    return model\n",
    "\n",
    "def gridSearch_neural_network(X_train, y_train):\n",
    "    \n",
    "    print(\"Train Data:\", X_train.shape)\n",
    "    print(\"Train label:\", y_train.shape)\n",
    "    # evaluate model with standardized dataset\n",
    "    estimator = KerasRegressor(build_fn=baseline_model, nb_epoch=10, batch_size=5, verbose=1)\n",
    "    \n",
    "    # grid search epochs, batch size and optimizer\n",
    "    optimizers = ['rmsprop', 'adam']\n",
    "    #dropout_rate = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]\n",
    "    init = ['glorot_uniform', 'normal', 'uniform']\n",
    "    epochs = [50, 100, 150]\n",
    "    batches = [5, 10, 20]\n",
    "    weight_constraint = [1, 2, 3, 4, 5]\n",
    "    param_grid = dict(optimizer=optimizers, \n",
    "                      #dropout_rate=dropout_rate, \n",
    "                      epochs=epochs, \n",
    "                      batch_size=batches, \n",
    "                      #weight_constraint=weight_constraint, \n",
    "                      #init=init\n",
    "                     )\n",
    "    \n",
    "    grid = GridSearchCV(estimator=estimator, param_grid=param_grid)\n",
    "    grid_result = grid.fit(X_train, y_train)\n",
    "    # summarize results\n",
    "    print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "    means = grid_result.cv_results_['mean_test_score']\n",
    "    stds = grid_result.cv_results_['std_test_score']\n",
    "    params = grid_result.cv_results_['params']\n",
    "    for mean, stdev, param in zip(means, stds, params):\n",
    "        print(\"%f (%f) with: %r\" % (mean, stdev, param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Data: (1591497, 4)\n",
      "Train label: (1591497,)\n",
      "WARNING:tensorflow:From /anaconda3/envs/U4-S2-NNF/lib/python3.7/site-packages/tensorflow/python/keras/initializers.py:143: calling RandomNormal.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/envs/U4-S2-NNF/lib/python3.7/site-packages/sklearn/model_selection/_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "gridSearch_neural_network(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zjw9V6UARQCI"
   },
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gPW59MTv73yv"
   },
   "outputs": [],
   "source": [
    "pickle.dump(nnmodel, open('zillow_nn_model.pkl','wb'))"
   ]
  }
 ],
 "metadata": {
  "accelerator": "TPU",
  "colab": {
   "name": "Neural_Network_Model.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "U4-S2-NNF (Python3)",
   "language": "python",
   "name": "u4-s2-nnf"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}