3.2. Decision Trees.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('cleaned_data/Cleaned Data.csv', index_col=[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Assigning values\n",
    "X = df.drop('Signal', axis=1)\n",
    "y = df['Signal']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1        Sell\n",
       "2         Buy\n",
       "3         Buy\n",
       "4         Buy\n",
       "5         Buy\n",
       "         ... \n",
       "11575     Buy\n",
       "11576    Sell\n",
       "11577    Sell\n",
       "11578    Sell\n",
       "11579     Buy\n",
       "Name: Signal, Length: 11579, dtype: object"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Checking data\n",
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Train test split\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0, stratify=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#scale our data to avoid issues (X_scaled = X_std * (max - min) + min)\n",
    "from sklearn.preprocessing import MinMaxScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MinMaxScaler()"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#creating an instance\n",
    "scaler = MinMaxScaler()\n",
    "\n",
    "#adjusting scaler only based on the training set (without test values to prevent data leakage)\n",
    "scaler.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#perform the transformation\n",
    "X_train = scaler.transform(X_train)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(max_depth=4)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import Decision Tree Classifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "tree = DecisionTreeClassifier(max_depth=4)\n",
    "tree.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Max depth\n",
    "maxDepth = np.array([1, 2, 5, 10])\n",
    "\n",
    "# Minimum number of samples required to split any internal node \n",
    "minSamplesNode = np.array([2, 5, 10])\n",
    "\n",
    "# The minimum number of samples required to be at a leaf/terminal node\n",
    "minSamplesLeaf = np.array([10, 20, 30])\n",
    "\n",
    "# Import necessary functions\n",
    "from sklearn.model_selection import StratifiedKFold, cross_val_score\n",
    "\n",
    "# Create k-Fold CV object\n",
    "kFold = StratifiedKFold(n_splits=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5576815697736371\n",
      "{'criterion': 'entropy', 'max_depth': 5, 'min_samples_leaf': 10, 'min_samples_split': 2}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "# Define the hyperparameter values to be tested\n",
    "param_grid = {'criterion': ['gini', 'entropy'],\n",
    "              'max_depth': maxDepth,\n",
    "              'min_samples_split': minSamplesNode,\n",
    "              'min_samples_leaf': minSamplesLeaf}\n",
    "\n",
    "# Run brute-force grid search\n",
    "gs = GridSearchCV(estimator=DecisionTreeClassifier(random_state=0),\n",
    "                  param_grid=param_grid,\n",
    "                  scoring='accuracy',\n",
    "                  cv=kFold, n_jobs=-1)\n",
    "gs = gs.fit(X_train, y_train)\n",
    "print(gs.best_score_)\n",
    "print(gs.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy:  0.5489\n"
     ]
    }
   ],
   "source": [
    "# Extract best parameter\n",
    "clf = gs.best_estimator_\n",
    "\n",
    "# Fit model given best parameter\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "# Print out score on Test dataset\n",
    "print('Test accuracy: {0: .4f}'.format(clf.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix: \n",
      " [[ 470    0  934]\n",
      " [  83    0  141]\n",
      " [ 466    0 1380]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "\n",
    "y_pred = tree.predict(X_test)\n",
    "print('Confusion matrix: \\n', \n",
    "      metrics.confusion_matrix(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5187      Buy\n",
       "1490     Hold\n",
       "9159     Sell\n",
       "4083      Buy\n",
       "7318      Buy\n",
       "         ... \n",
       "3141      Buy\n",
       "3648     Sell\n",
       "1019     Sell\n",
       "11245    Sell\n",
       "9795      Buy\n",
       "Name: Signal, Length: 3474, dtype: object"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Sell', 'Buy', 'Sell', ..., 'Sell', 'Buy', 'Sell'], dtype=object)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         Buy       0.52      0.41      0.46      1404\n",
      "        Hold       0.00      0.00      0.00       224\n",
      "        Sell       0.59      0.76      0.66      1846\n",
      "\n",
      "    accuracy                           0.57      3474\n",
      "   macro avg       0.37      0.39      0.37      3474\n",
      "weighted avg       0.52      0.57      0.54      3474\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_test,y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5313930391225494"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#checking if our model learned something\n",
    "6153/len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#We see that our model has similar accuracy as a model which is just guessing on sell\n",
    "#Without any adjustments, this decision tree would perform as good as a model which is just guessing on Sell"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forest "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(max_depth=10, n_jobs=-1, random_state=0)"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# Create classifier object and fit it to data\n",
    "forest = RandomForestClassifier(criterion='gini', random_state=0, n_jobs=-1, max_depth=10)\n",
    "forest.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = forest.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         Buy       0.54      0.37      0.44      1404\n",
      "        Hold       0.00      0.00      0.00       224\n",
      "        Sell       0.59      0.81      0.68      1846\n",
      "\n",
      "    accuracy                           0.58      3474\n",
      "   macro avg       0.38      0.39      0.37      3474\n",
      "weighted avg       0.53      0.58      0.54      3474\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Christian Rauhut\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1221: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_test,y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix: \n",
      " [[ 513    0  891]\n",
      " [  78    0  146]\n",
      " [ 355    0 1491]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "print('Confusion matrix: \\n', \n",
    "      metrics.confusion_matrix(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#With the Random Forest Adjustment, however, we can see that our model is able to increase its accuracy\n",
    "#We see without any adjustments, our model shows skills"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimizing Random Forest with GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5795188077514425\n",
      "{'criterion': 'gini', 'max_depth': 10, 'max_features': 'auto'}\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         Buy       0.52      0.41      0.46      1404\n",
      "        Hold       0.00      0.00      0.00       224\n",
      "        Sell       0.59      0.76      0.66      1846\n",
      "\n",
      "    accuracy                           0.57      3474\n",
      "   macro avg       0.37      0.39      0.37      3474\n",
      "weighted avg       0.52      0.57      0.54      3474\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Define the hyperparameter values to be tested\n",
    "param_grid = {'criterion': ['gini'],\n",
    "              'max_depth': [1, 2, 5, 10],\n",
    "              'max_features': ['auto']}\n",
    "\n",
    "# Run brute-force grid search\n",
    "gs = GridSearchCV(estimator=RandomForestClassifier(random_state=0),\n",
    "                  param_grid=param_grid,\n",
    "                  scoring='accuracy',\n",
    "                  cv=kFold, n_jobs=-1)\n",
    "gs = gs.fit(X_train, y_train)\n",
    "print(gs.best_score_)\n",
    "print(gs.best_params_)\n",
    "\n",
    "\n",
    "y_pred = forest.predict(X_test)\n",
    "\n",
    "print(classification_report(y_test,y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix: \n",
      " [[ 572    1  831]\n",
      " [  77    0  147]\n",
      " [ 448    2 1396]]\n"
     ]
    }
   ],
   "source": [
    "print('Confusion matrix: \\n', \n",
    "      metrics.confusion_matrix(y_test, y_pred))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}