Data preprocessing .ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "13a88d8c",
   "metadata": {},
   "source": [
    "# Data Pre-processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8e5a5508",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import os\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a18ca52",
   "metadata": {},
   "source": [
    "Load Data Set "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "461070a0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Country   Age   Salary  Purchased \n",
      "0   France   44.0  72000.0         No\n",
      "1    Spain   27.0  48000.0        Yes\n",
      "2  Germany    NaN  54000.0         No\n",
      "3    Spain   39.0      NaN         No\n",
      "4  Germany    NaN  64000.0        Yes\n",
      "5   France   35.0  58000.0        Yes\n",
      "6    Spain   39.0  52000.0         No\n",
      "7   France   48.0      NaN        Yes\n",
      "8  Germany   50.0  83000.0         No\n",
      "9   France   37.0  67000.0        Yes\n"
     ]
    }
   ],
   "source": [
    "os.chdir (\"E:\\Data science\\Data preprocessing\")\n",
    "df1= pd.read_excel(\"Data Preprocessing Data File.xlsx\")\t\n",
    "print (df1)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c26b1a28",
   "metadata": {},
   "source": [
    "Load independent variables and dependent variables to two separate arrays "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "980bc918",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['France ' 44.0 72000.0]\n",
      " ['Spain ' 27.0 48000.0]\n",
      " ['Germany ' nan 54000.0]\n",
      " ['Spain ' 39.0 nan]\n",
      " ['Germany ' nan 64000.0]\n",
      " ['France ' 35.0 58000.0]\n",
      " ['Spain ' 39.0 52000.0]\n",
      " ['France ' 48.0 nan]\n",
      " ['Germany ' 50.0 83000.0]\n",
      " ['France ' 37.0 67000.0]]\n"
     ]
    }
   ],
   "source": [
    "x = df1.iloc[:,:-1].values\n",
    "print (x)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fc435b67",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['No' 'Yes' 'No' 'No' 'Yes' 'Yes' 'No' 'Yes' 'No' 'Yes']\n"
     ]
    }
   ],
   "source": [
    "y = df1.iloc[:,3].values\n",
    "print (y)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "970510f2",
   "metadata": {},
   "source": [
    "Missing value treatment – Impute Values "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "33b1b360",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['France ' 44.0 72000.0]\n",
      " ['Spain ' 27.0 48000.0]\n",
      " ['Germany ' 39.875 54000.0]\n",
      " ['Spain ' 39.0 62250.0]\n",
      " ['Germany ' 39.875 64000.0]\n",
      " ['France ' 35.0 58000.0]\n",
      " ['Spain ' 39.0 52000.0]\n",
      " ['France ' 48.0 62250.0]\n",
      " ['Germany ' 50.0 83000.0]\n",
      " ['France ' 37.0 67000.0]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.impute import SimpleImputer\n",
    "imputer = SimpleImputer(missing_values=np.nan,strategy='mean')\n",
    "imputer = imputer.fit(x[:,1:3])             #perform fit\n",
    "x [:,1:3]= imputer.transform(x[:,1:3])      #perform transform-extract column 1 &2 as it contain nan values\n",
    "print (x)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7db6240c",
   "metadata": {},
   "source": [
    "Label Encoding #as system not understand the text data so need to convert into label encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e099ec01",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 44.0 72000.0]\n",
      " [2 27.0 48000.0]\n",
      " [1 39.875 54000.0]\n",
      " [2 39.0 62250.0]\n",
      " [1 39.875 64000.0]\n",
      " [0 35.0 58000.0]\n",
      " [2 39.0 52000.0]\n",
      " [0 48.0 62250.0]\n",
      " [1 50.0 83000.0]\n",
      " [0 37.0 67000.0]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "label_x = LabelEncoder()\n",
    "x[:,0]= label_x.fit_transform(x[:,0])\n",
    "print (x)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7b9e983",
   "metadata": {},
   "source": [
    "One hot encoding /Column Transformation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e565a9b3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1.0 0.0 0.0 44.0 72000.0]\n",
      " [0.0 0.0 1.0 27.0 48000.0]\n",
      " [0.0 1.0 0.0 39.875 54000.0]\n",
      " [0.0 0.0 1.0 39.0 62250.0]\n",
      " [0.0 1.0 0.0 39.875 64000.0]\n",
      " [1.0 0.0 0.0 35.0 58000.0]\n",
      " [0.0 0.0 1.0 39.0 52000.0]\n",
      " [1.0 0.0 0.0 48.0 62250.0]\n",
      " [0.0 1.0 0.0 50.0 83000.0]\n",
      " [1.0 0.0 0.0 37.0 67000.0]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "ct = ColumnTransformer(transformers=[('encoder', OneHotEncoder(), [0])], remainder='passthrough')\n",
    "x = np.array(ct.fit_transform(x))\n",
    "print (x)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3acd84b3",
   "metadata": {},
   "source": [
    "Label Encoding Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f90f4f65",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 1 0 0 1 1 0 1 0 1]\n"
     ]
    }
   ],
   "source": [
    "label_y = LabelEncoder()\n",
    "y = label_y.fit_transform(y)\n",
    "print (y)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74d4b749",
   "metadata": {},
   "source": [
    "Standardization to have data in same digit format-values converted to -3 to +3 range"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "24dddd21",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          0         1         2         3         4\n",
      "0  1.224745 -0.654654 -0.654654  0.664912  1.002707\n",
      "1 -0.816497 -0.654654  1.527525 -2.075331 -1.465494\n",
      "2 -0.816497  1.527525 -0.654654  0.000000 -0.848444\n",
      "3 -0.816497 -0.654654  1.527525 -0.141042  0.000000\n",
      "4 -0.816497  1.527525 -0.654654  0.000000  0.179973\n",
      "5  1.224745 -0.654654 -0.654654 -0.785805 -0.437077\n",
      "6 -0.816497 -0.654654  1.527525 -0.141042 -1.054127\n",
      "7  1.224745 -0.654654 -0.654654  1.309675  0.000000\n",
      "8 -0.816497  1.527525 -0.654654  1.632056  2.133965\n",
      "9  1.224745 -0.654654 -0.654654 -0.463423  0.488498\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "std_sca= StandardScaler()\n",
    "x_STD = std_sca.fit_transform(x)\n",
    "print (pd.DataFrame(x_STD))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83efdc9d",
   "metadata": {},
   "source": [
    "Normalization - MinMaxScaler -columnwise process and smallest value in the column will be 0 and highest value 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "129b8a9e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     0    1    2         3         4\n",
      "0  1.0  0.0  0.0  0.739130  0.685714\n",
      "1  0.0  0.0  1.0  0.000000  0.000000\n",
      "2  0.0  1.0  0.0  0.559783  0.171429\n",
      "3  0.0  0.0  1.0  0.521739  0.407143\n",
      "4  0.0  1.0  0.0  0.559783  0.457143\n",
      "5  1.0  0.0  0.0  0.347826  0.285714\n",
      "6  0.0  0.0  1.0  0.521739  0.114286\n",
      "7  1.0  0.0  0.0  0.913043  0.407143\n",
      "8  0.0  1.0  0.0  1.000000  1.000000\n",
      "9  1.0  0.0  0.0  0.434783  0.542857\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "Nm_x= MinMaxScaler()\n",
    "x_NOR = Nm_x.fit_transform(x)\n",
    "print (pd.DataFrame(x_NOR))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "13c48644",
   "metadata": {},
   "source": [
    "Normalization - highest value in row is 1 and lowest value in row is 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "cee14d0f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          0         1         2         3    4\n",
      "0  0.000014  0.000000  0.000000  0.000611  1.0\n",
      "1  0.000000  0.000000  0.000021  0.000562  1.0\n",
      "2  0.000000  0.000019  0.000000  0.000738  1.0\n",
      "3  0.000000  0.000000  0.000016  0.000627  1.0\n",
      "4  0.000000  0.000016  0.000000  0.000623  1.0\n",
      "5  0.000017  0.000000  0.000000  0.000603  1.0\n",
      "6  0.000000  0.000000  0.000019  0.000750  1.0\n",
      "7  0.000016  0.000000  0.000000  0.000771  1.0\n",
      "8  0.000000  0.000012  0.000000  0.000602  1.0\n",
      "9  0.000015  0.000000  0.000000  0.000552  1.0\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import Normalizer\n",
    "Nm_x= Normalizer()\n",
    "x_NOR = Nm_x.fit_transform(x)\n",
    "print (pd.DataFrame(x_NOR))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7bcfa85f",
   "metadata": {},
   "source": [
    "Model Creation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "256d9a22",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          0         1         2         3    4\n",
      "0  0.000000  0.000000  0.000021  0.000562  1.0\n",
      "1  0.000000  0.000012  0.000000  0.000602  1.0\n",
      "2  0.000000  0.000000  0.000019  0.000750  1.0\n",
      "3  0.000000  0.000016  0.000000  0.000623  1.0\n",
      "4  0.000000  0.000019  0.000000  0.000738  1.0\n",
      "5  0.000016  0.000000  0.000000  0.000771  1.0\n",
      "6  0.000014  0.000000  0.000000  0.000611  1.0\n",
      "7  0.000015  0.000000  0.000000  0.000552  1.0 [1 0 0 1 0 1 0 1]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "x_train,x_test, y_train,y_test = train_test_split (x_NOR,y,test_size = 0.2)\n",
    "print (pd.DataFrame(x_train), y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8bf5f656",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}