Statistics_Exam.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1><center>Statistics final exam DSBA</center></h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split \n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn import metrics\n",
    "from scipy import stats\n",
    "from scipy.stats import chisquare\n",
    "from scipy.stats import chi2_contingency\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt  \n",
    "import seaborn as seabornInstance\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('sxof_data.csv', sep=\";\") # open raw data file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3><center>Let us look at the data</center></h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>region</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>income</th>\n",
       "      <th>isMarried</th>\n",
       "      <th>nchildren</th>\n",
       "      <th>religious</th>\n",
       "      <th>education</th>\n",
       "      <th>occupation</th>\n",
       "      <th>rate_marriage</th>\n",
       "      <th>naffairs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>37</td>\n",
       "      <td>563680,0625</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>18</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>27</td>\n",
       "      <td>270797,5938</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>32</td>\n",
       "      <td>365333,3438</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>57</td>\n",
       "      <td>563680,0625</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>253485,5938</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>596</th>\n",
       "      <td>597</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>148468,375</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>597</th>\n",
       "      <td>598</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>32</td>\n",
       "      <td>122240</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>598</th>\n",
       "      <td>599</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>32</td>\n",
       "      <td>276105,2188</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599</th>\n",
       "      <td>600</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>340067,1875</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>18</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>600</th>\n",
       "      <td>601</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>32</td>\n",
       "      <td>332525,3438</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>14</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>601 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      id  region  sex  age       income  isMarried  nchildren  religious  \\\n",
       "0      1       3    1   37  563680,0625          1          0          3   \n",
       "1      2       1    0   27  270797,5938          0          0          4   \n",
       "2      3       3    0   32  365333,3438          0          1          1   \n",
       "3      4       1    1   57  563680,0625          1          1          5   \n",
       "4      5       2    1   22  253485,5938          0          0          2   \n",
       "..   ...     ...  ...  ...          ...        ...        ...        ...   \n",
       "596  597       2    1   22   148468,375          1          1          1   \n",
       "597  598       2    0   32       122240          0          1          2   \n",
       "598  599       1    1   32  276105,2188          1          1          2   \n",
       "599  600       1    1   22  340067,1875          1          1          3   \n",
       "600  601       2    0   32  332525,3438          0          1          3   \n",
       "\n",
       "     education  occupation  rate_marriage  naffairs  \n",
       "0           18           5              4         0  \n",
       "1           14           1              1         0  \n",
       "2           12           1              1         0  \n",
       "3           18           1              5         0  \n",
       "4           17           3              1         0  \n",
       "..         ...         ...            ...       ...  \n",
       "596         12           5              5         1  \n",
       "597         18           1              1         7  \n",
       "598         17           5              5         2  \n",
       "599         18           4              2         2  \n",
       "600         14           4              1         1  \n",
       "\n",
       "[601 rows x 12 columns]"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Here we created new variable to store data and checked if there any null values. Fortunatelly, there are not any of them. </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().values.any()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3><center>Preprocess data to work with it in future</center></h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"income\"] = df[\"income\"].str.replace(',', '.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"income\"] = pd.to_numeric(df[\"income\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1><center>Problem 1</center></h1>\n",
    "<h3><center>a) You are proposed to study the problem set - done</center>\n",
    "<center>b) Test the null hypothesis that income has normal distribution</center>\n",
    "<center>c) Test the null hypothesis that income has log-normal distribution</center></h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "income = df[\"income\"].to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "k2, p = stats.normaltest(income)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set the significance level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.001\n"
     ]
    }
   ],
   "source": [
    "alpha = 1e-3\n",
    "print(alpha)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The null hypothesis can be rejected\n"
     ]
    }
   ],
   "source": [
    "if p < alpha:  # null hypothesis: x comes from a normal distribution\n",
    "    print(\"The null hypothesis can be rejected\")\n",
    "else:\n",
    "    print(\"The null hypothesis cannot be rejected\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3><center>Thus, there is a strong evidence to reject the null hypothesis and make conclusion that distribution is not normal.</center></h3>\n",
    "\n",
    "For item с) I will use Kolmogorov-Smirnov test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = stats.kstest(income, \"lognorm\", stats.lognorm.fit(income))[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The null hypothesis can be rejected\n"
     ]
    }
   ],
   "source": [
    "if p < alpha:  # null hypothesis: x comes from a normal distribution\n",
    "    print(\"The null hypothesis can be rejected\")\n",
    "else:\n",
    "    print(\"The null hypothesis cannot be rejected\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3><center>Thus, there is a strong evidence to reject the null hypothesis and make conclusion that distribution is not log-normal.</center></h3>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<center><h1>Problem 2</h1>\n",
    "<h3>a) Does the proportion of female households vary among various regions?</h3></center>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Moscow - 50.25%\n",
      "Tver - 45.88%\n",
      "Pskov - 46.57%\n"
     ]
    }
   ],
   "source": [
    "print(\"Moscow - \" + \"{:.2f}\".format(100 * (len(df.loc[(df[\"sex\"] == 1) & (df[\"region\"] == 1)])\n",
    "                                    / len(df.loc[df[\"region\"] == 1]))) + \"%\")\n",
    "\n",
    "print(\"Tver - \" + \"{:.2f}\".format(100 * (len(df.loc[(df[\"sex\"] == 1) & (df[\"region\"] == 2)])\n",
    "                                    / len(df.loc[df[\"region\"] == 2]))) + \"%\")\n",
    "\n",
    "print(\"Pskov - \" + \"{:.2f}\".format(100 * (len(df.loc[(df[\"sex\"] == 1) & (df[\"region\"] == 3)])\n",
    "                                    / len(df.loc[df[\"region\"] == 3]))) + \"%\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3><center>\n",
    "    \n",
    "    H0 - means of female households between regions are equal\n",
    "    \n",
    "    H1 - at least two means of female households between regions differ\n",
    "</center></h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "reg1_f = len(df.loc[(df[\"sex\"] == 1) & (df[\"region\"] == 1)])\n",
    "reg2_f = len(df.loc[(df[\"sex\"] == 1) & (df[\"region\"] == 2)])\n",
    "reg3_f = len(df.loc[(df[\"sex\"] == 1) & (df[\"region\"] == 3)])\n",
    "reg1_m = len(df.loc[(df[\"sex\"] == 0) & (df[\"region\"] == 1)])\n",
    "reg2_m = len(df.loc[(df[\"sex\"] == 0) & (df[\"region\"] == 2)])\n",
    "reg3_m = len(df.loc[(df[\"sex\"] == 0) & (df[\"region\"] == 3)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[102 101]\n",
      " [ 89 105]\n",
      " [ 95 109]]\n"
     ]
    }
   ],
   "source": [
    "obs = np.array([[reg1_f, reg1_m], [reg2_f, reg2_m], [reg3_f, reg3_m]])\n",
    "print(obs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "g, p, dof, expctd = chi2_contingency(obs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6414553979577649 > 0.05, then we cannot reject H0 hypothesis. There is a strong evidence that proportions of female households do not significantly differ among these regions.\n"
     ]
    }
   ],
   "source": [
    "print(str(p) + \" > 0.05, then we cannot reject H0 hypothesis.\\\n",
    " There is a strong evidence that proportions of female households do not significantly differ among these regions.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<center><h3>b) Is number of children associated with regions?</h3></center>\n",
    "\n",
    "<h3><center>\n",
    "    \n",
    "    H0 - no, there is not an association\n",
    "    \n",
    "    H1 - yes, there is\n",
    "</center></h3>\n",
    "\n",
    "Let's look at the contingency table:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>region</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nchildren</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>59</td>\n",
       "      <td>59</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>144</td>\n",
       "      <td>135</td>\n",
       "      <td>151</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "region       1    2    3\n",
       "nchildren               \n",
       "0           59   59   53\n",
       "1          144  135  151"
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table = pd.crosstab(df[\"nchildren\"], df[\"region\"])\n",
    "table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = stats.chi2_contingency(table)[1] #doing test for association\n",
    "alpha = 0.05"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The null hypothesis cannot be rejected\n"
     ]
    }
   ],
   "source": [
    "if p < alpha:\n",
    "    print(\"The null hypothesis can be rejected\")\n",
    "else:\n",
    "    print(\"The null hypothesis cannot be rejected\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Therefore, we make a conclusion, that there is not association between regions and number of children."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<center><h3>c) Are religious preferences associated with regions?</h3></center>\n",
    "\n",
    "<h3><center>\n",
    "    \n",
    "    H0 - no, there is not association\n",
    "    \n",
    "    H1 - yes, there is\n",
    "</center></h3>\n",
    "\n",
    "Let's look at the contingency table:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>region</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>religious</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13</td>\n",
       "      <td>15</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>53</td>\n",
       "      <td>56</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>47</td>\n",
       "      <td>36</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>69</td>\n",
       "      <td>62</td>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>21</td>\n",
       "      <td>25</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "region      1   2   3\n",
       "religious            \n",
       "1          13  15  20\n",
       "2          53  56  55\n",
       "3          47  36  46\n",
       "4          69  62  59\n",
       "5          21  25  24"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table = pd.crosstab(df[\"religious\"], df[\"region\"])\n",
    "table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The null hypothesis cannot be rejected\n"
     ]
    }
   ],
   "source": [
    "p = stats.chi2_contingency(table)[1] #doing test for association\n",
    "alpha = 0.05\n",
    "if p < alpha:\n",
    "    print(\"The null hypothesis can be rejected\")\n",
    "else:\n",
    "    print(\"The null hypothesis cannot be rejected\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Therefore, we make a conclusion, that there is not association between regions and religion."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<center><h3>d) Is occupation associated with regions?</h3></center>\n",
    "\n",
    "<h3><center>\n",
    "    \n",
    "    H0 - no, there is not an association\n",
    "    \n",
    "    H1 - yes, there is\n",
    "</center></h3>\n",
    "\n",
    "Let's look at the contingency table:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>region</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>occupation</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>44</td>\n",
       "      <td>28</td>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>34</td>\n",
       "      <td>52</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>46</td>\n",
       "      <td>39</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>33</td>\n",
       "      <td>35</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>46</td>\n",
       "      <td>40</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "region       1   2   3\n",
       "occupation            \n",
       "1           44  28  41\n",
       "2           34  52  56\n",
       "3           46  39  32\n",
       "4           33  35  29\n",
       "5           46  40  46"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table = pd.crosstab(df[\"occupation\"], df[\"region\"])\n",
    "table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The null hypothesis cannot be rejected\n"
     ]
    }
   ],
   "source": [
    "p = stats.chi2_contingency(table)[1] #doing test for association\n",
    "alpha = 0.05\n",
    "if p < alpha:\n",
    "    print(\"The null hypothesis can be rejected\")\n",
    "else:\n",
    "    print(\"The null hypothesis cannot be rejected\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There is evedince that there is not association between regions and occupation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<center><h3>h) Please test any hypothesis you are interested in</h3></center>\n",
    "\n",
    "<h3><center>\n",
    "    \n",
    "    H0 - no, there is not an association between sex and education\n",
    "    \n",
    "    H1 - yes, there is an association between sex and education\n",
    "</center></h3>\n",
    "\n",
    "Let's look at the contingency table:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>education</th>\n",
       "      <th>9</th>\n",
       "      <th>12</th>\n",
       "      <th>14</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>20</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sex</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>36</td>\n",
       "      <td>106</td>\n",
       "      <td>70</td>\n",
       "      <td>52</td>\n",
       "      <td>45</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>48</td>\n",
       "      <td>45</td>\n",
       "      <td>37</td>\n",
       "      <td>67</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "education  9   12   14  16  17  18  20\n",
       "sex                                   \n",
       "0           4  36  106  70  52  45   2\n",
       "1           3   8   48  45  37  67  78"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table = pd.crosstab(df[\"sex\"], df[\"education\"])\n",
    "table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The null hypothesis can be rejected\n"
     ]
    }
   ],
   "source": [
    "p = stats.chi2_contingency(table)[1] #doing test for association\n",
    "alpha = 0.05\n",
    "if p < alpha:\n",
    "    print(\"The null hypothesis can be rejected\")\n",
    "else:\n",
    "    print(\"The null hypothesis cannot be rejected\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We make conclusion that there is an association between sex and length of education"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<center><h3>f) Is the significant difference between the income of men and women? Please redo this\n",
    "analysis separately by regions. </h3></center>\n",
    "\n",
    "<h3><center>\n",
    "    \n",
    "    H0 - average income is the same for men and women\n",
    "    \n",
    "    H1 - no, average income differes for men and women\n",
    "</center></h3>\n",
    "\n",
    "Let's make assumption that variances are equal in population"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [],
   "source": [
    "income_male = df.loc[df['sex'] == 0].income.tolist()\n",
    "income_female = df.loc[df['sex'] == 1].income.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = stats.ttest_ind(income_male, income_female)[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The null hypothesis cannot be rejected\n"
     ]
    }
   ],
   "source": [
    "alpha = 0.05\n",
    "if p < alpha:\n",
    "    print(\"The null hypothesis can be rejected\")\n",
    "else:\n",
    "    print(\"The null hypothesis cannot be rejected\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Therefore there is an evidence that difference between the income of men and women is not significant, in general."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's redo this for different regions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The null hypothesis cannot be rejected\n"
     ]
    }
   ],
   "source": [
    "income_male = df.loc[(df['sex'] == 0) & (df['region'] == 1)].income.tolist()\n",
    "income_female = df.loc[(df['sex'] == 1) & (df['region'] == 1)].income.tolist()\n",
    "p = stats.ttest_ind(income_male, income_female)[1]\n",
    "alpha = 0.05\n",
    "if p < alpha:\n",
    "    print(\"The null hypothesis can be rejected\")\n",
    "else:\n",
    "    print(\"The null hypothesis cannot be rejected\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The null hypothesis cannot be rejected\n"
     ]
    }
   ],
   "source": [
    "income_male = df.loc[(df['sex'] == 0) & (df['region'] == 2)].income.tolist()\n",
    "income_female = df.loc[(df['sex'] == 1) & (df['region'] == 2)].income.tolist()\n",
    "\n",
    "p = stats.ttest_ind(income_male, income_female)[1]\n",
    "alpha = 0.05\n",
    "if p < alpha:\n",
    "    print(\"The null hypothesis can be rejected\")\n",
    "else:\n",
    "    print(\"The null hypothesis cannot be rejected\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The null hypothesis cannot be rejected\n"
     ]
    }
   ],
   "source": [
    "income_male = df.loc[(df['sex'] == 0) & (df['region'] == 3)].income.tolist()\n",
    "income_female = df.loc[(df['sex'] == 1) & (df['region'] == 3)].income.tolist()\n",
    "\n",
    "p = stats.ttest_ind(income_male, income_female)[1]\n",
    "alpha = 0.05\n",
    "if p < alpha:\n",
    "    print(\"The null hypothesis can be rejected\")\n",
    "else:\n",
    "    print(\"The null hypothesis cannot be rejected\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We make a conclusion that there is not significant difference between income of men and women among all given regions."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<center><h3>g) Is the significant difference between the ln(income) of men and women? Please redo this\n",
    "analysis separately by regions. </h3></center>\n",
    "\n",
    "<h3><center>\n",
    "    \n",
    "    H0 - average income is the same for men and women\n",
    "    \n",
    "    H1 - no, average income differes for men and women\n",
    "</center></h3>\n",
    "\n",
    "Let's make assumption that variances are equal in population"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [],
   "source": [
    "income_male = df.loc[df['sex'] == 0].income.to_numpy()\n",
    "income_female = df.loc[df['sex'] == 1].income.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [],
   "source": [
    "income_male = np.log(income_male)\n",
    "income_female = np.log(income_female)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The null hypothesis can be rejected\n"
     ]
    }
   ],
   "source": [
    "p = stats.ttest_ind(income_male, income_female)[1]\n",
    "alpha = 0.05\n",
    "if p < alpha:\n",
    "    print(\"The null hypothesis can be rejected\")\n",
    "else:\n",
    "    print(\"The null hypothesis cannot be rejected\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There is evidence that there is a significant difference between ln(income) of mew and women. However, probably it is not very impornant for real life since people do not earn ln(income)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<center><h3>Please estimate a linear regression model you are interested in. </h3></center>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAAEGCAYAAAAqmOHQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dfXRcdb3v8fe3aWpTLKRIQZoWW6G2PESIxFKpcgCFVK9I5MBR5Fy6uBx7DyA+nlxb5VxEOYd66vFZWRcXrAICglgDHPFEnnwqAk0JEAoUCmJJWqFSikVSaNPv/WN+UybpTDKTmcne2fvzWmtWZr7ZD7/dwHz372H/fubuiIiIRG1c1AUQEREBJSQREYkJJSQREYkFJSQREYkFJSQREYmF8VEXIO72228/nzlzZtTFEBEZU9asWfMXd59ayj5KSMOYOXMmnZ2dURdDRGRMMbM/lbqPmuxERCQWlJBERCQWlJBERCQW1IckIjKMHTt20NPTw/bt26MuSuxMnDiR6dOnU1tbW/axlJBERIbR09PD5MmTmTlzJmYWdXFiw9158cUX6enpYdasWWUfTwlJJALtXb0s71jHxq19TKuvo61lDq1NDVEXSwrYvn27klEeZsZb3vIWNm/eXJHjKSGJjLL2rl6Wruymb0c/AL1b+1i6shtASSnGlIzyq+S/iwY1iIyy5R3rdiejrL4d/SzvWBdRiUTiQQlJZJRt3NpXUlzk2GOPjboIo0JNdiKjbFp9Hb15ks+0+roISiPVUOk+wnvvvbeCpYsv1ZBERllbyxzqamsGxOpqa2hrmRNRiaSSsn2EvVv7cN7oI2zv6h3xMd/85jcD8Otf/5rjjz+e008/nblz53LWWWeRXfV79erVHHvssRx55JHMmzePbdu2sX37ds455xwaGxtpamrinnvuAWDFihW0trZyyimnMGvWLL7//e/zzW9+k6amJubPn8+WLVsAePrpp1m4cCFHH30073vf+3jiiSfK+8cZhhKSyChrbWrgstMaaaivw4CG+jouO61RAxoSotp9hF1dXXz729/mscce45lnnmHVqlW8/vrrfOxjH+M73/kODz/8MHfeeSd1dXX84Ac/AKC7u5sbbriBRYsW7X6W6tFHH+X666/ngQce4Mtf/jKTJk2iq6uL97znPVxzzTUALF68mO9973usWbOGb3zjG5x//vkVuYZC1GQnEoHWpgYloISqdh/hvHnzmD59OgBHHXUUzz77LPvssw8HHngg7373uwHYe++9Afj973/PhRdeCMDcuXN529vexpNPPgnACSecwOTJk5k8eTL77LMPp5xyCgCNjY088sgjvPLKK9x7772cccYZu8/92muvVeQaClFCEhGpoGr3Eb7pTW/a/b6mpoadO3fi7nmHX2eb84Y7zrhx43Z/HjduHDt37mTXrl3U19fz0EMPVaTcxVCTnYhIBUXRRzh37lw2btzI6tWrAdi2bRs7d+7kuOOO47rrrgPgySefZMOGDcyZU1w59t57b2bNmsVPf/pTIJPcHn744epcQKCEJCJSQVH0EU6YMIEbb7yRCy+8kCOPPJKTTjqJ7du3c/7559Pf309jYyMf+9jHWLFixYCa0XCuu+46rrzySo488kgOP/xwbrnllqpdA4ANVaUTaG5udi3QJ5Jujz/+OIceemjUxYitfP8+ZrbG3ZtLOY5qSCIiEgtKSCIiEgtKSCIiRVD3Rn6V/HdRQhIRGcbEiRN58cUXlZQGya6HNHHixIocT88hiYgMY/r06fT09FRs3Z8kya4YWwlKSCIiw6itra3IiqgytKo12ZnZDDO7x8weN7O1ZvaZEN/XzO4ws6fCzykhbmb2XTNbb2aPmNm7co61KGz/lJktyokfbWbdYZ/vWnhUeSTnEBGRaFWzD2kn8AV3PxSYD1xgZocBS4C73H02cFf4DPBBYHZ4LQYuh0xyAS4GjgHmARdnE0zYZnHOfgtDvKRziIhI9KqWkNx9k7s/GN5vAx4HGoBTgavDZlcDreH9qcA1nnEfUG9mBwItwB3uvsXdXwLuABaG3+3t7n/wTE/jNYOOVco5REQkYqMyys7MZgJNwP3AAe6+CTJJC9g/bNYAPJezW0+IDRXvyRNnBOcYXN7FZtZpZp3qxBQRGR1VT0hm9mbgZ8Bn3f2vQ22aJ+YjiA9ZnGL2cfcr3L3Z3ZunTp06zCFFRKQSqpqQzKyWTDK6zt1XhvDz2Way8POFEO8BZuTsPh3YOEx8ep74SM4hIiIRq+YoOwOuBB5392/m/OpWIDtSbhFwS0787DASbj7wcmhu6wBONrMpYTDDyUBH+N02M5sfznX2oGOVcg4REYlYNZ9DWgD8T6DbzLIrPH0JWAbcZGbnAhuA7HKEtwMfAtYDrwLnALj7FjP7GrA6bPdVd98S3p8HrADqgF+GF6WeQ0REoqflJ4ah5SdEREqn5SdERGTMUkISEZFYUEISEZFYUEISEZFY0GzfIpJ47V29LO9Yx8atfUyrr6OtZQ6tTXtM0iIRU0ISkURr7+pl6cpu+nb0A9C7tY+lK7sBlJRiRk12IpJoyzvW7U5GWX07+lnesS6iEkkhSkgikmgbt/aVFJfoKCGJSKJNq68rKS7RUUISkURra5lDXW3NgFhdbQ1tLXMiKpEUokENIpJo2YELGmUXf0pIIpJ4rU0NSkBjgJrsREQkFpSQREQkFpSQREQkFpSQREQkFpSQREQkFpSQREQkFpSQREQkFpSQREQkFpSQREQkFpSQREQkFpSQREQkFpSQREQkFpSQREQkFpSQREQkFpSQREQkFpSQREQkFpSQREQkFpSQREQkFpSQREQkFpSQREQkFpSQREQkFpSQREQkFpSQREQkFsZHXQAREUmO9q5elnesY8JbDzm61H2rVkMys6vM7AUzezQn9hUz6zWzh8LrQzm/W2pm681snZm15MQXhth6M1uSE59lZveb2VNmdqOZTQjxN4XP68PvZw53DhERKV97Vy9LV3bTu7VvRPtXs8luBbAwT/xb7n5UeN0OYGaHAR8HDg/7/NDMasysBvgB8EHgMODMsC3A18OxZgMvAeeG+LnAS+5+CPCtsF3Bc1T4mkVEUmt5xzr6dvSPeP+qJSR3/y2wpcjNTwV+4u6vufsfgfXAvPBa7+7PuPvrwE+AU83MgBOBm8P+VwOtOce6Ory/GXh/2L7QOUREpAI2jrBmlBXFoIZPmdkjoUlvSog1AM/lbNMTYoXibwG2uvvOQfEBxwq/fzlsX+hYezCzxWbWaWadmzdvHtlVioikzLT6urL2H+2EdDlwMHAUsAn4zxC3PNv6COIjOdaeQfcr3L3Z3ZunTp2abxMRERmkrWUOdbUj7wkZ1VF27v589r2Z/Qj4r/CxB5iRs+l0YGN4ny/+F6DezMaHWlDu9tlj9ZjZeGAfMk2HQ51DRETK1NqUaXRa3rGOTSPYf1RrSGZ2YM7HjwLZEXi3Ah8PI+RmAbOBB4DVwOwwom4CmUEJt7q7A/cAp4f9FwG35BxrUXh/OnB32L7QOURkFLR39bJg2d3MWvILFiy7m/au3qiLJFXQ2tTAqiUn8vqf168pdd+q1ZDM7AbgeGA/M+sBLgaON7OjyDSVPQv8bwB3X2tmNwGPATuBC9y9PxznU0AHUANc5e5rwym+CPzEzC4FuoArQ/xK4FozW0+mZvTx4c4hItWVHQ6cHYHVu7WPpSu7gTfuqkUsU3mQQpqbm72zszPqYoiMaQuW3Z332ZSG+jpWLTkxghJJtZnZGndvLmUfTR0kIlVXaDhwucOEJVmUkESk6goNBy53mLAkixKSiFRdvuHAdbU1tLXMiahEEkeaXFVEqi53OPDGrX1Mq6+jrWWOBjTIAEpIIjIqWpsalIBkSGqyExGRWFBCEhGRWFBCEhGRWFBCEhGRWFBCEhGRWFBCEhGRWFBCEhGRWFBCEhGRWFBCEhGRWFBCEhGRWChq6iAzewdwOXCAux9hZu8EPuLul1a1dCKSGO1dvZrLToZUbA3pR8BSYAeAuz9CWIlVRGQ42RVje7f24byxYqyWMZdcxU6uOsndHzCz3NjOKpRHUkp3z8m2vGPd7uXLs/p29LO8Y53+zrJbsQnpL2Z2MOAAZnY6sKlqpZJUyd49Z7+wsnfPQGK/rNKWgLVirBSj2Ca7C4D/B8w1s17gs8B5VSuVpMpQd89JlMbmK60YK8UoKiG5+zPu/gFgKjDX3d/r7s9WtWSSGmm7e05bAgatGCvFKXaUXT1wNjATGJ/tS3L3T1etZJIa0+rr6M2TfJJ695y2BAxaMVaKU2yT3e1kklE3sCbnJVK2tN09q/lKJL9iBzVMdPfPV7Ukklppu3tua5kzYBAHJDsBQzoHrqTVRe3d3HD/c0x46yFHl7pvsQnpWjP7JPBfwGvZoLtvKfWEIvm0NjWk5ospbQkYNOw7LS5q7+bH920Y8f7FJqTXgeXAlwlDv8PPt4/4zCIplqYEDNH3m6VtmH1Ubrj/ubL2LzYhfR44xN3/UtbZRCSVohy4oubC0dPvPvxGQyh2UMNa4NWyziQiqRXlwJU0DrMfq4qtIfUDD5nZPQzsQ9KwbxEZVpT9ZvlqZkPFKynbwd/vTo0ZZx4zg0tbG6t+3rGq2ITUHl4iUgFp7NOIqt+sxixvU1LNwLk5K25wB3+/++7PSU1KDQWaZotV7EwNVwM38MbzR9eHmIiUKI1TB0WpUL9Guf0dwynUwV9ux3+ctbXMobZm5Im+qIRkZscDTwE/AH4IPGlmx434rCKDtHf1smDZ3cxa8gsWLLs70V/O6tMYXQ0FBk4UildKVIkwcmVcXrGDGv4TONnd/87djwNagG+N/LQib0hbjSHqIdBpE9WAikJNgtVuKozS8o517Ng18oxUbEKqdffdt2/u/iRQO+KziuRIW40hrVMHRVULbm1q4LLTGmmor8PI1IwuO62x6v1ZZx4zo6R4EpR7U1XsoIZOM7sSuDZ8PgvNZScVkrYawwlzp+Z9mv2EuVMjKM3oiPpZoCgGVGQHLqRplF2h582KVWxCOo/MmkifBgz4LZm+JJGypW2273ue2FxSPAnSOnXQpa2NiU5Ag+Wbp7EUxTbZjQe+4+6nuftHge8CNcPsI1KUtpY51I4b2K5eO84SO9lo2mqEkM5rTqPc5tGRKDYh3QXknqEOuHOoHczsKjN7wcwezYnta2Z3mNlT4eeUEDcz+66ZrTezR8zsXTn7LArbP2Vmi3LiR5tZd9jnuxYWaRrJOSQGBvfzJrffN5V9SGm8ZkjX6NGs1qYGVi05kdf/vL7kbp1iE9JEd38l+yG8nzTMPiuAhYNiS4C73H02mSS3JMQ/CMwOr8XA5ZBJLsDFwDHAPODibIIJ2yzO2W/hSM4h0VvesY4d/QNH5uzo98QOakjb+k+QvlowpG/0aCUUm5D+NqjWcjQwZF3b3X8LDF6e4lQg+0Dt1UBrTvwaz7gPqDezA8kML7/D3be4+0vAHcDC8Lu93f0P7u7ANYOOVco5JGJpa85pbWrg749u2D38t8aMvz86+bN/7xrmc9KkbfRoJRSbkD4L/NTMfmdmvwNuBD41gvMd4O6bAMLP/UO8Ach9fLknxIaK9+SJj+QcezCzxWbWaWadmzcnt6M5LtLWnNPe1cvP1vTufkCy352frelN9J3zJbetpX/Q8yn9u5xLblsbUYmqL203WpVQ7NRBq4G5ZEbbnQ8c6u6VHPadr8fARxAfyTn2DLpf4e7N7t48dWpyh+LGRdqasNJ45/zSqztKiidB2m60KqHYGhLAu4F3Ak3AmWZ29gjO93y2mSz8fCHEe4Dcp8WmAxuHiU/PEx/JOSRiUT24GJUoZ5+W0VPoubIkP28GbwzkqNoS5mZ2LXAw8BCZpSggU7u4psTz3QosApaFn7fkxD9lZj8hM4DhZXffZGYdwL/nDGQ4GVjq7lvMbJuZzQfuB84GvjeSc5RYfqmStK2gmjb1dbVs7duzNlRfl9wJX9L4vNngB6BLVeyDsc3AYWEAQVHM7AbgeGA/M+shM1puGXCTmZ0LbADOCJvfDnwIWE9mIcBzAELi+RqwOmz3VXfPDpQ4j8xIvjrgl+FFqecQker7ykcO5/M3PjRgIMO4EE+qNPYh5WuOLkWxCelR4K1A0TUKdz+zwK/en2dbJzMTRL7jXAVclSfeCRyRJ/5iqecQkeqrqTF25QzvryljmYKxYNKEGv72+p5fzpMmJHdOgXKTbbF9SPsBj5lZh5ndmn2VdWYRSY20PWsG5E1GQ8WToNwBG8XWkL5S1llEZLcFB+/LqqcHP6KXiSdVGpuv0qjcueyKSkju/psRHV1E9vDYpm0lxZOgflJt3iHe9ZNGZ1BDGpeMj0L233R5x7ri+3dyDJmQzGwb+Z/VMTLdMnuP4JwiqZbGZ3IKDYcajcVTo176Im2yI2Ztaelz2Q2ZkNx98siLJVI83cEm28t5hnwPFa+ktC59MRaV8mCsSFWkbRLKQs/eJPmZnChnLYjqQeRCSzCMdGmGNFBCksilbSqdr3zk8LwzXyf5mZwoZy3ITmJbbLxS0jYlViUUO8pOpGrSNgIrt+M3LU2UUc5a0F+go6pQvFLS+HculxKSRC5tS5hD+qZKivKmo8Ysb/Kpdg0J0vd3Lpea7CRyatpIvkLDu0dj2HdUNSRI54qx5VANSSKXxqaNtI0qfGV7/tF0heKVFFUNqb2rl7abH949Q0Xv1j7abn4Y0HDzQlRDEhllaRtVCLCjwPKwheKVFFUN6ZLb1uadLinJixKWSwlJIpe2L+i0jSpMqzQ+AF0uJSSJXNq+oNM2qlCkWEpIErm0fUGncWnr2gLfNIXiSVCoi2oUBveNWQn+z0HGirR9QadxVOGE8fnXACoUr6RC3//VzgtRzt83VikhSeTS9gXd2tTAZac10lBfh5GZSuay0xoTPfIqyrWBzpp/UEnxSknjFFHlUkKSyKXxC7rzT1v488vbceDPL2+n8097ro8klXFpa+Mea00tOHhfLm1trOp5d/TnH0JYKC56DkliIk1PtF/U3s2P79uw+3O/++7P1f6STKP2rl4e3PDygNiDG16mvau3qv/NpXHF2HKphiQyym64/7mS4lKetI3iHMtUQxIZZVFOZZNGUY3irKsdR1+eJ3/rkjy0kDdmIZnw1kOOLnVfJSSJhYvau7nh/ufod6fGjDOPmaHmK6mIqJZPn1hbkzchTayt/sjCqAxenbdUSkgSOfWpSDVFNfx6a4EZGQrFkyBf82gpkl13lDHh+vs3lBQXKUVUy6dPmpC/JlQongTlrsKrhCSR21XgTrVQXMaeqFZthegevH61wGi6QvEkKPfvqYQkIlU3/+1TSopXUlvLnD2+6MaFeDUVup9K8n1WuQNzlJAkcoVGHSV9NFKaPLZpW0nxSur80xYGDy3YFeLVFGWtMCqqIcmYd9lp78x7B3vZae+MojhSBVEuxRDVc19nHjOjpHgSlFtD0ig7iVzaVow18jfbJPe+OVpRPff1x82vlBRPgkKr8xZLCUliIU1TB6Wxb6G+rpateUa1JXmi0VVP528SLBRPAvUhiYwxaexbOHza5JLiMjaVe4OhGpLEQna6kTQ02aVx6qB7C9QKCsVlbCrnoVhQQpIYGDzdSO/WPpau7AZIbFJKmyibKdM6p1wUXttZ3tIa+otI5DQbs1STRnGOHUpIErmoZmOWdGhtauAT8w/a3UdXY8Yn5h+k2ncMKSFJ5KKa2kXSob2rl+vv37C7j67fnevv30B7V2/EJUuecoflRJKQzOxZM+s2s4fMrDPE9jWzO8zsqfBzSoibmX3XzNab2SNm9q6c4ywK2z9lZoty4keH468P+9pQ55BonTB3aklxkVJ8aeUje8yLuMszcamscvsEo6whneDuR7l7c/i8BLjL3WcDd4XPAB8EZofXYuByyCQX4GLgGGAecHFOgrk8bJvdb+Ew55AI/eKRTSXFRUrxap4BDUPFK6XQKP4Ej+5P1NRBpwJXh/dXA6058Ws84z6g3swOBFqAO9x9i7u/BNwBLAy/29vd/+DuDlwz6Fj5ziERinJaGZFqOfbt+5YUT4Kx+mCsA78yszVmtjjEDnD3TQDh5/4h3gDkTjrVE2JDxXvyxIc6xwBmttjMOs2sc/PmzSO8RBFJsygnlI3KXmWu9RTVc0gL3H2jme0P3GFmTwyxbb46oI8gXjR3vwK4AqC5uTm5TyvGhOZ2kyRKY82/3LWeIqkhufvG8PMF4Odk+oCeD81thJ8vhM17gNzpcacDG4eJT88TZ4hzSITSOLebSBKNuUENZraXmU3OvgdOBh4FbgWyI+UWAbeE97cCZ4fRdvOBl0NzWwdwsplNCYMZTgY6wu+2mdn8MLru7EHHyncOidCUSfnnvyoUF5FkiqLJ7gDg52Ek9njgenf/bzNbDdxkZucCG4Azwva3Ax8C1gOvAucAuPsWM/sasDps91V3z06MdR6wAqgDfhleAMsKnEMiVKgfNMFTu4lIHqOekNz9GeDIPPEXgffniTtwQYFjXQVclSfeCRxR7DkkWi/nWZZgqLiIJFOchn1LSmmmBhEBJSSJgbaWOdSOGzimrnac0dYyJ6ISiUgUlJAkFnYMmttl8GcRST4lJInc0gJzihWKi0gyKSFJ5PItnjZUXMaeQg856+FnyaWEJCJVd8j+e5UUT4I0JuExufzEWNLd+zILlt2ttVNEyrD+hb+VFJexaczN1DAW9W7to+3mh5WUREYojdNDpfGay6WEVKQd/c4lt62NuhiJlMamDRHZkxJSCZI8S2+Uzpp/UElxEUkmJSSJ3B83v1JSXESSSQlJIrfq6S0lxUUknmrKbGdXQiqBlkMQESmsv8wRG0pIRaqtMS4+5fCoiyEiklhRLWE+pjTU19HWMofWpoaoiyIiklhKSMNobNiHVUtOjLoYIiKJpyY7ERGJBSUkERGJBSUkERGJBSUkiZymDhIRUEKSGNAklCICSkgiIhITSkjD0HpIIiKjQwmpCFoPSUSk+pSQiqT1kEREqksJqQRaD0lEpHqUkEREJBaUkEpQX6flJ0REqkUJqQQfPvLAqIsgIpJYSkgluOeJzVEXQUQksZSQSrBxa1/URUikBQfvW1JcRJJJCakE+6gPqSqu++R7mL3/XgNis/ffi+s++Z6ISiQiUVBCKoFpts+qaO/qpeel7QNiPS9t14PIIimjhFSCrXoOqSqWd6yjb0f/gFjfjn6Wd6yLqESSJFE1CRe6gU3yje2USeW1IikhlWBafV3URUikQn1z6rOTSrjuk+/ZI/ksOHjfqjcJn3XMQSXFk+DiUw6ntmbkGVcJqUh1tTW0tcyJuhiJVCjR6wZAKuWM5oNoqK/DgIb6Os5orn5SuLS1kX+cfxA1oUpUY8Y/zj+IS1sbq37uqLQ2NbD89CNpGOH/u+MrXJ5Eaqivo61lDq1NDVEXJZHaWuawdGX3gGa7JN8A1JjR73uu9lST4LacvSbU8LfX+/PGq629q3fAf1+9W/tYurIboOr/T1/a2pjoBJRPa1MDrU0N2NL1a0rdN5U1JDNbaGbrzGy9mS0ZatvGhn1YteREJaMqam1q4LLTGgfcwV52WmNi/83zJaOh4knwbx9tpGbcwIRbM874t49W/8tafZRjR+pqSGZWA/wAOAnoAVab2a3u/li0JUu37F1VGjTU19Gbp39spM0cY0H2b7u8Yx0bt/YxbRRbHdRHOXakLiEB84D17v4MgJn9BDgVUEKSUZG2JsqsqG46phW4AVAfZfykscmuAXgu53NPiO1mZovNrNPMOjdv1nRBUllpa6KMWlvLHOpqB/ZVpeEGYCxKYw0pX8/xgMZ7d78CuAKgubk5uQ37Epk0NVFGLcrmQilNGhNSDzAj5/N0YGNEZRGRUaAbgLEhjU12q4HZZjbLzCYAHwdujbhMIiKpl7oakrvvNLNPAR1ADXCVu6+NuFgiIqmXuoQE4O63A7dHXQ4REXlDGpvsREQkhpSQREQkFswTPF1JJZjZZuBPEZ1+P+AvEZ07Kmm75rRdL+ia02KOu08uZYdU9iGVwt2nRnVuM+t09+aozh+FtF1z2q4XdM1pYWadpe6jJjsREYkFJSQREYkFJaR4uyLqAkQgbdectusFXXNalHzNGtQgIiKxoBqSiIjEghKSiIjEghJSDJjZVWb2gpk9mhPb18zuMLOnws8pUZax0gpc83Ize8LMHjGzn5tZfZRlrLR815zzu38xMzez/aIoW7UUumYzu9DM1pnZWjP7j6jKVw0F/ts+yszuM7OHwlpr86IsYyWZ2Qwzu8fMHg9/z8+EeMnfYUpI8bACWDgotgS4y91nA3eFz0mygj2v+Q7gCHd/J/AksHS0C1VlK9jzmjGzGcBJwIbRLtAoWMGgazazE8is0vxOdz8c+EYE5aqmFez5d/4P4BJ3Pwr4v+FzUuwEvuDuhwLzgQvM7DBG8B2mhBQD7v5bYMug8KnA1eH91UDrqBaqyvJds7v/yt13ho/3kVmrKjEK/J0BvgX8HwYtFJkEBa75PGCZu78Wtnlh1AtWRQWu2YG9w/t9SNAabO6+yd0fDO+3AY+TWYW75O8wJaT4OsDdN0HmDw7sH3F5Rtv/An4ZdSGqzcw+AvS6+8NRl2UUvQN4n5ndb2a/MbN3R12gUfBZYLmZPUemRpi02j8AZjYTaALuZwTfYUpIEjtm9mUyzQDXRV2WajKzScCXyTThpMl4YAqZ5p024CYzs2iLVHXnAZ9z9xnA54ArIy5PxZnZm4GfAZ9197+O5BhKSPH1vJkdCBB+JqpZoxAzWwR8GDjLk/+Q3MHALOBhM3uWTBPlg2b21khLVX09wErPeADYRWby0SRbBKwM738KJGZQA4CZ1ZJJRte5e/Y6S/4OU0KKr1vJ/EdM+HlLhGUZFWa2EPgi8BF3fzXq8lSbu3e7+/7uPtPdZ5L5on6Xu/854qJVWztwIoCZvQOYQPJnwt4I/F14fyLwVIRlqahQu70SeNzdv5nzq5K/wzRTQwyY2Q3A8WTuEp8HLibzP+1NwEFkRl+d4e75OsTHpALXvBR4E/Bi2Ow+d//nSApYBfmu2d2vzPn9s0Czuyfmy7nA3/la4HmtPKAAAAQbSURBVCrgKOB14F/c/e6oylhpBa55HfAdMs2V24Hz3X1NVGWsJDN7L/A7oJtMbRfgS2T6kUr6DlNCEhGRWFCTnYiIxIISkoiIxIISkoiIxIISkoiIxIISkoiIxIISkqSeZfzezD6YE/sHM/vviMt0U5j5/NMjPEZPJWdMD7M3/3PO5xlmdmOlji+iYd8igJkdQeYJ+iagBngIWOjuT5dxzPE5k8WWuu904DfufnAZ5+8hM3v61pEeY9DxDgFuDjNWi1ScakgigLs/CtxGZqaIi4Fr3P1pM1tkZg+EdWx+aGbjAMzsirCuzVoz2z0XXaiV/KuZrQI+amafM7PHzOxhM/vx4POaWZ2ZXW1m3Wb2oJkdF371K2BaOO+xg/Y5wMxWhvM/YGbzQ3xqWHfmQTO7HLAQP8TMHsrZf4mZXRTev8PM7g7le9DMZprZ3iH2YKihfTjsugyYE8q0LPe4ha7DzP7JzG42s46wLs5l5f6tJMHcXS+99HIH2IvME/XdZGaMOILMjBnjw++vAD4R3u8bfo4n85T6YeFzD/D5nGNuAiaE9/V5zvlF4Efh/eHAn8hMpXMI8FCBct4IzA/vZwKPhvc/BL4U3p9KZsmD+sHHIrMuzUXh/RrglPB+IjAJqAUmh9j+wFPh/eDj7P48xHX8E5lpciYDdcBzwLSo/9Z6xfM1vsT8JZJY7v630Cfyiru/ZmYfAN4NdIbJqLNfqABnmtm5ZBLSNOAw4LHwu9x+lbXAj83sFjLJbbD3AsvD+dea2UYyX/SvD1HUD5CpqWQ/TzGzOuA44EPhWLeY2bahrtcyK3ju5+63hX22h/gE4OthSphdwAwbfiXbQtcBcKdn1snBzJ4gM5VMYtYDkspRQhIZaBdvzMdlwFXu/q+5G5jZbOAzwDx33xqa4ibmbPK3nPctZCbVPBW4yMyOcPf+3MONoIwWzj0gaYUEla9TeCcDm+cnhlih7c8ms4jcu9x9Z+iLmphnu8FlKuS1nPf96HtHClAfkkhhdwL/kK0dmNlbzOwgMit/bgP+GqbVb8m3s5nVANM9M3FoGzCVTJNYrt8CZ4XtDwUOBNYXUa4Lcs6THWSQe6xTyDSTAfyZTH/UFDObCPwPAHd/CfhL2BYzm2iZNZr2AV4IyegkMqt/Eq45e8zBRnIdIgMoIYkU4O7dwCXAnWb2CJmBBgcAD5JpnnsU+BGwqsAhxgPXh30fBL6ebbrK8T2gzsy6ySxIePbgmk8eFwALwoCDx4BPhvjFwAfM7EEys033huvYDvw7sJrMkgCP5RzrLOALoYy/J5M0rwWONbNO4AzCUgnu/jyZ5stuM1tWgesQGUDDvkVEJBZUQxIRkVhQQhIRkVhQQhIRkVhQQhIRkVhQQhIRkVhQQhIRkVhQQhIRkVj4/9baDa95xW+hAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df.plot(x='education', y='income', style='o')   \n",
    "plt.xlabel('Years of education')  \n",
    "plt.ylabel('Income')  \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df[\"education\"].values.reshape(-1,1)\n",
    "y = df[\"income\"].values.reshape(-1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None, normalize=False)"
      ]
     },
     "execution_count": 228,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)\n",
    "regressor = LinearRegression()  \n",
    "regressor.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = regressor.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ4AAAEGCAYAAABVSfMhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3df3RcZ33n8fdXPxybghWHOGmIrdoLZiGJyA+L4FOybMpAsHCNzZ7ShbIbHzbFWxoWAmwhLZRA6NkN0ALNtmRJmmx+8Ls0saI4ajCCQOkhJFIaZ/Kj1CYJjoibODUoAYIjyd/94z4jRvKMPCPd+8zMnc/rnDkz85079z5XY893nuc+P8zdERERiaWj0QUQEZH2osQjIiJRKfGIiEhUSjwiIhKVEo+IiETV1egCNLvjjz/e16xZ0+hiiIi0lLGxsSfdfWWl15R4jmLNmjWMjo42uhgiIi3FzH5U7TU1tYmISFRKPCIiEpUSj4iIRKXEIyIiUSnxiIhIVOrVJtJAxWKRkZERJiYm6OnpoVAo0NfX1+hiiWRKiUekQYrFIkNDQ0xOTgIwMTHB0NAQgJKP5Jqa2kQaZGRkZCbplExOTjIyMtKgEonEocQj0iATExN1xUXyQolHpEF6enrqiovkhRKPSIMUCgW6u7tnxbq7uykUCg0qkUgc6lwg0iClDgTq1SbtRolHpIH6+vqUaKTtqKlNRESiUuIREZGolHhERCQqJR4REYlKiUdERKJS4hERkaiUeEREJColHhERiUqJR0REolLiERGRqDJLPGa22sy+ZWYPmtn9ZvbuED/OzHaZ2Z5wvyLEzcwuN7O9ZnavmZ1Vtq9tYfs9ZratLL7ezIrhPZebmS30GCIiEkeWNZ4p4H3u/lJgA3ChmZ0CXAyMuPs6YCQ8BxgA1oXbduAKSJIIcAnwCuBs4JJSIgnbbC9738YQr+sYIiIST2aJx933u/vd4fHTwIPAycAW4Lqw2XXA1vB4C3C9J+4AjjWzk4DXAbvc/aC7/wTYBWwMry139++5uwPXz9lXPccQEZFIolzjMbM1wJnA94ET3X0/JMkJOCFsdjLwaNnbxkNsvvh4hTgLOMbc8m43s1EzGz1w4EA9pyoiIkeReeIxs+cCfwdc5O5PzbdphZgvID5vcWp5j7tf6e797t6/cuXKo+xSRETqkWniMbNukqTzBXe/MYQfLzVvhfsnQnwcWF329lXAY0eJr6oQX8gxREQkkix7tRlwNfCgu3+q7KWbgVLPtG3AYFn8/NDzbAMwEZrJbgPOM7MVoVPBecBt4bWnzWxDONb5c/ZVzzFERCSSLFcgfSXwX4Gimd0TYn8CXAZ81cwuAPYBbwqv3Qq8HtgL/AJ4G4C7HzSzjwF3he0udfeD4fE7gGuBZcBwuFHvMUREJB5LOoRJNf39/T46OtroYoiItBQzG3P3/kqvaeYCERGJSolHRESiUuIREZGolHhERCQqJR4REYlKiUdERKJS4hERkaiUeEREJColHhERiUqJR0REolLiERGRqJR4REQkKiUeERGJSolHRESiUuIREZGolHhERCQqJR4REYlKiUdERKJS4hERkaiUeEREJColHhERiUqJR0REolLiERGRqJR4REQkKiUeERGJSolHRESiUuIREZGolHhERCQqJR4REYlKiUdERKJS4hERkaiUeEREJColHhERiUqJR0REolLiERGRqLoaXQAREWkuxWKRkZERJiYm6OnpoVAo0NfXl9r+M6vxmNk1ZvaEmd1XFvuImf3YzO4Jt9eXvfbHZrbXzH5gZq8ri28Msb1mdnFZfK2Zfd/M9pjZV8xsSYgfE57vDa+vOdoxREQkUSwWGRoaYmJiAoCJiQmGhoYoFoupHSPLprZrgY0V4p929zPC7VYAMzsFeDNwanjPZ82s08w6gb8GBoBTgLeEbQE+Hva1DvgJcEGIXwD8xN1fBHw6bFf1GCmfs4hISxsZGWFycnJWbHJykpGRkdSOkVnicffvAAdr3HwL8GV3P+TuDwN7gbPDba+7P+TuzwJfBraYmQGvBr4W3n8dsLVsX9eFx18DCmH7ascQEZGgVNOpNb4Qjehc8E4zuzc0xa0IsZOBR8u2GQ+xavHnAz9196k58Vn7Cq9PhO2r7UtERIKenp664gsRO/FcAbwQOAPYD/xFiFuFbX0B8YXs6whmtt3MRs1s9MCBA5U2ERHJpUKhQHd396xYd3c3hUIhtWNETTzu/ri7T7v7YeAqftXUNQ6sLtt0FfDYPPEngWPNrGtOfNa+wus9JE1+1fZVqZxXunu/u/evXLlyIacqItKS+vr62Lx580wNp6enh82bN6faqy1qd2ozO8nd94enbwRKPd5uBr5oZp8CXgCsA+4kqaWsM7O1wI9JOgf8nru7mX0L+B2S6z7bgMGyfW0Dvhde/2bYvtoxRCQDWXfJlez09fVl+llllnjM7EvAucDxZjYOXAKca2ZnkDRxPQL8dwB3v9/Mvgo8AEwBF7r7dNjPO4HbgE7gGne/PxziA8CXzezPgH8Crg7xq4EbzGwvSU3nzUc7hoikq9Qlt9Q7qtQlF1DyEcy94mUOCfr7+310dLTRxRBpKZ/5zGcq9oLq6enhoosuakCJJDYzG3P3/kqvacocEUldjC650rqUeEQkdTG65ErrUuIRkdTF6JIrrUuThIpI6kodCNSrTSpR4hGRTGTdJVdal5raREQkKiUeERGJSolHRESiUuIREZGolHhERCQqJR4REYlKiUdERKJS4hERkaiUeEREJColHhERiaqmxGNmLzazETO7Lzx/mZl9KNuiiYhIHtU6V9tVwB8BnwNw93vN7IvAn2VVMBFpbVr6WqqpNfE8x93vNLPy2FQG5RGRHNDS1zKfWhPPk2b2QsABzOx3gP2ZlUqkTeS1VjAyMjKTdEomJycZGRnJxfnJ4tSaeC4ErgReYmY/Bh4G/ktmpRJpA3muFWjpa5lPTZ0L3P0hd38NsBJ4ibuf4+6PZFoykZybr1bQ6rT0tcynphqPmR0LnA+sAbpK13rc/V2ZlUwk5/JcKygUCrNqc6Clr+VXam1quxW4AygCh7Mrjkj76OnpqZhk8lAr0NLXMp9aE89Sd39vpiURaTOqFUizyrrTS62J5wYzeztwC3CoFHT3g6mVRKTN5LlWkOeOE3lXLBbZsWMHhw8njVsTExPs2LEDSO+zqzXxPAt8EvggoUt1uP93qZRCpE319fXl8otY3alb1/Dw8EzSKTl8+DDDw8PRE897gRe5+5OpHFVEci3PHSfy7plnnqkrvhC1ThJ6P/CL1I4qIrmm7tQyn1oTzzRwj5l9zswuL92yLJiItK5CoUB3d/esmDpOtIZly5bVFV+IWpvadoSbiMhR5bnjRN4NDAwwODjI9PT0TKyzs5OBgYHUjlFT4nH368xsCfDiEPqBu0/O9x4RaW957TiRdzF+NNQ6c8G5wHXAI4ABq81sm7t/J7WSiIhIW6i1qe0vgPPc/QeQLAwHfAlYn1XBREryOoOzSDOKMQar1s4F3aWkA+Du/wJ0z7O9SCpK/wlK3XBL/wmKxWKDSyaSTzEmr6018Yya2dVmdm64XQWMpVYKkSryPIOzSDOKMQar1sTzDpKxPO8C3g08APxBaqUQqUIDEUXiijEGq9bE0wX8pbv/J3d/I3A50JlaKUSq0EBEkbhijMGqNfGMAOWjh5YB35jvDWZ2jZk9YWb3lcWOM7NdZrYn3K8IcQuDUvea2b1mdlbZe7aF7feY2bay+HozK4b3XG5hkaCFHEOaV6FQoKNj9j/Tjo4ODUQUyUhfXx+bN2+e+XHX09PD5s2bGzI79VJ3/1npibv/zMyec5T3XAv8FXB9WexiYMTdLzOzi8PzDwADwLpwewVwBfAKMzsOuAToJ5mUdMzMbnb3n4RttpOsE3QrsBEYrvcYNZ6/NFBp4cFqz0UkXVmPwao18fzczM5y97shqW0A884Y5+7fMbM1c8JbgHPD4+uA20mSwhbgend34A4zO9bMTgrb7iotv2Bmu4CNZnY7sNzdvxfi1wNbSRJPXcdw9/01/g2kAUZGRmaNoAaYnp7WLMctYOfOnYyNjeHumBnr169n06ZNjS6WNIFaE89FwN+a2WPh+UnAf17A8U4sfdG7+34zOyHETwYeLdtuPMTmi49XiC/kGEckHjPbTlKbore3t85TlDSpc0Fr2rlzJ6OjozPP3X3muZKP1HSNx93vAl5C0rvtD4GXunua3akrtZ34AuILOcaRQfcr3b3f3ftXrlx5lN1KltS5oDWNjVX+eqgWl/ZSa40H4OXAmvCeM80Md79+/rcc4fFS81ZoSnsixMeB1WXbrQIeC/Fz58RvD/FVFbZfyDGkieV9eei8zsqQtGjXHpf2UlONx8xuAP4cOIckAb2c5IJ/vW4GSj3TtgGDZfHzQ8+zDcBEaC67DTjPzFaE3mnnAbeF1542sw2hN9v5c/ZVzzGkicXoYdMoeZ6VoVoHEHUMEai9xtMPnOJ1/Fwxsy+R1FaON7Nxkt5plwFfNbMLgH3Am8LmtwKvB/aSLDj3NgB3P2hmHwPuCttdWupoQNLsdy1J1+7hcKPeY0jzy+ssx3leHnr9+vWzrvGUx/MgrzXVWGpNPPcBv06FC/HVuPtbqrx0RBtJSGgXVtnPNcA1FeKjwGkV4v9W7zFEGiHPHSdKHQjy2KutWCzOWq9mYmKCwcGkYUXJpza1Jp7jgQfM7E7gUCno7m/IpFQibaCnp6dikslLx4lNmzblItHMNTw8XLGL//DwsBJPjWpNPB/JshAi7SjvHSfy6plnKg9hrBaXI9W6Aum3sy6ISLvR8tDSruZNPGb2NJXHuhjJZZPlmZRKpE3kteNEnnV3dx/RKaQUl9rMm3jc/XmxCiIi0gq6uroqJp6urnqGRbY3/aVEJBN57XKsazyLp8QjIqkrDY4t1QxKg2Mhuy7HsRJd3nsjxlDrejwiIjWLvWR5zFkgYiyUlndKPCKSutiDY2MmujxP4xSLmtpEJHWxm6NiJzr1Rlwc1XhEJHXr1q2rK75YWj6jtSjxiEjq9uzZU1d8sXTdpbUo8YhI6hrR9LVq1apZsVWrVqk5rEnpGo+IpC4sFFkxnoWdO3fy8MMPz4o9/PDD7Ny5M5OJSvM6RikW1XhEJHWxVyCNudR2nhfwi0WJR0RSF/tif8xEF3uMUh4p8YhI6mL3aou51HaeF/CLRYlHRFIXu1dbtSW1s1hqW123F0+JR0RSF7tWsGnTJtauXTsrtnbt2kw6Fqjr9uIp8YhI6mLXCorFIuPj47Ni4+PjmVzw15Q5i6fu1CKSutjLes93wT+LhKApcxZHiUeansZMtJ6+vj727dvH2NgY7o6Zcfrpp2f2uemCf2tR4pGm1oh1XWLKa1ItFovs3r17pjuzu7N79256e3u1Ro7oGo80tzyPmcjzQMTYn5su+LcWJR5panluQslzUm3EXG264N861NQmTS3PTSh5TqqN+Nx0wT89WTcBq8YjTS3PTSjLli2rK95KCoUCnZ2ds2KdnZ25+NzyLkYTsBKPNLU8N6FMTU3VFW81c+dJy2qCUElXjCZgNbVJ08trE8rc/9xHi7eSkZERDh8+PCt2+PDhzMbVQH57CMYWowlYiUdEUhf7+lWxWGRwcJDp6emZ4wwODgL56HYfU4y1lJR4RBpk2bJlPPPMMxXjrS5254Lh4eGZpFMyPT3N8PBwJoknz7WrGEtMKPFI08vrf/JTTz2V0dHRivFWVygUuPHGGyvGs1Apgc8XX4xischNN90080U8MTHBTTfdBOSjdhXjR4M6F0hTy/Mgy9hLB8S0b9++uuKtZGhoqGLHidKMGq2uUCjQ0TE7NXR0dKT6o0GJR5qaBlm2pphLUUPcrul57hRSMvd6TtoL6inxSFPL85dznsfxxFyKGmBgYKCuuFQ3MjJS8XpZmj/2lHikqeV5tce8j+OJrdKA1SzEXGa7EWL82FPikaaW5xHweW6ymTvbxNHiixXjV3pJzGW2GyHGj72GJB4ze8TMimZ2j5mNhthxZrbLzPaE+xUhbmZ2uZntNbN7zeyssv1sC9vvMbNtZfH1Yf97w3ttvmNIc9MI+NZz+umn1xVfrJhNsps2baK/v3+mhmNm9Pf3Z7LMdiPEmKaqkTWe33L3M9y9Pzy/GBhx93XASHgOMACsC7ftwBWQJBHgEuAVwNnAJWWJ5Iqwbel9G49yDGlS842Ab3V5brKJ3WMv9t+yt7eX5cuXA7B8+XJ6e3szOU4jxJimqpnG8WwBzg2PrwNuBz4Q4td78jP3DjM71sxOCtvucveDAGa2C9hoZrcDy939eyF+PbAVGJ7nGNKk8ty5IPYF+Jhif24x/5bFYpEdO3bM/CCamJhgx44dQD7G8UD201Q1qsbjwNfNbMzMtofYie6+HyDcnxDiJwOPlr13PMTmi49XiM93jFnMbLuZjZrZ6IEDBxZ4ipKGPNcK8tyrLfbnFrMTyvDwcMVa+PDwcOrHyqtGJZ5XuvtZJM1oF5rZq+bZttK/VF9AvGbufqW797t7/8qVK+t5q6Qsz7WCPIv9ucVcPiPmLAl51ZCmNnd/LNw/YWY3kVyjedzMTnL3/aEp7Ymw+Tiwuuztq4DHQvzcOfHbQ3xVhe2Z5xjSpPK8EFyev8Biz0NXahbK49RKjZC7heDM7NfM7Hmlx8B5wH3AzUCpZ9o2YDA8vhk4P/Ru2wBMhGay24DzzGxF6FRwHnBbeO1pM9sQerOdP2dflY4hTUoLwbWmuV2bjxZvJXn+3CC/C8GdCHzXzHYDdwI73f3vgcuA15rZHuC14TnArcBDwF7gKuAPAUKngo8Bd4XbpaWOBsA7gL8J7/khSccC5jmGNCktBNeann322briixVzTr+BgYGKY8vyMktCLheCc/eHgCM687v7vwFH/IwNvdkurLKva4BrKsRHgdNqPYY0Ny0EJ0cz35dl2v928t6sp4XgRMjvsgiSnjx3u48txvU5JR5paqUmlNKv2VITCrT+mIk8LwQX25IlSyo24y1ZsiT1Y7XDOJ6saa42aWp5XhYhz9cKYo/jiXlNKe/jeGL0tlSNJyuHDsHb3w433HDka93d9d+6uhb2vjRu8x27I9vfLnluQunr62Pfvn2MjY3h7pgZZ555Zi5+Ned5/FWeu8FDnCEMSjxZueuuykkHYHIyuclRXTLfix/5SKRS1GnJEjjmGFi6NLkvf1x2P3HoEL9x4AAnd3Qw1dXFdFcX07t28fjQECeuXl3xPbXsd+Y+o2UBahG7GdHMKia1PMxwEVuhUJjVvA3pD2GwPPwCyVJ/f7+Pjo7W/8apKfjwh+Fzn/tVopmchByMYxBpWXN/FNSSzOfEvn3HHRwyY6qzk6muLp5avpxDS5fSccwxvG379tpaETJuKVisnTt3zqqJr1+/vu7Zt81srGwS6NmvKfHMb8GJp11MT89OrCnfvnnbbaw4eJDjn3ySrqkpuqam6JyepmtqiuVLliRNmr/8ZZLoRSQ1P+3p4ZoLLuDp5cvp6Ohg69atdTUDz5d41NQmi9PZmdyWLs1k98+ceir/UCHx52H9k49+9KNVX7vkknkbGWszOfmrxDzf/UJfm2ebXxw8OPMDoXPOhXhpDcdOTPDSBx7gzg0bZjpPpHX9UYlHmlopuSy22t+WSs06z31u9EP/1Sc+UfUaz/vf//7Uj5d5Ei9z6aWXVr2e9OEPf7i2nWTVUjA1teh97HvoITqnpxlftYp/Omtm3U31apP20tvby549e5iYmMjdolt5NTAwMGusC0BHR0cuuoqn0mMv45aCxfh/8yTxtCjxSFMrFosMDg7OTC45MTHB4GAyt2seuh3nVZ6nlcnzjOkQp0dic3etkLY3PDx8xIzG09PTuRmsl2f79u3jqaeeAuCpp55i3759DS5ROgqFAh1zeqV1dHTkYsZ0gFNPPbWu+EKoxiNNLe+D9fJq586dlPcGdfeZ57o+19z27NlTV3whVOMRkdSNjY3VFW8leZ8yJ8ZsIUo8IpI6TZnTuqpdq0rzGpYSj4i0vNiTkubZcccdV1d8IZR4pKnF+PUlra+rq/Ll6mpxqe6RRx6pK74QSjzS1AqFAt3d3bNiaU9YKK1Pq7mmJ0YzqX4OSFPL83gQSU/ex9bEFGOmb9V4RKTlVasBZ1EzXrt2bV3xVrNmzZq64guhxCNNrbTMcOnXbGmZ4WKx2OCSSTOpNjg1i0GrZ555Zl3xVnPw4MG64guhxCNN7ZZbbqk4ZuKWW25pUImkFrF7mVVbuiSLJU2qLbueh+XYQeN4RHj22WfriktzyPM4njwvxx6LEo+IpC7P42qqTZaZ1bLeeaTEIyKpy3ONZ6rKarfV4nIkJR4RSV2eazwaM7R4SjwiDZLnJps813hk8TSANCPFYlGDHmVeeW6yyfOAzhgDLBupo6PjiJ6kpXhqx0htTzKjWCwyNDQ0a+zJ0NCQxp4sQJ5rBXluslm3bl1d8VaS99pcpaQzX3whlHgyMDIycsSXx+TkZG76+cc0MDBAZ2fnrFhnZycDAwMNKpHU4v77768rvlhz5/M7WnwxNHHt4inxZED9/NPT19fHli1bZv5T9/T0sGXLllw0W+b5AnzsNWtizk6d59ocxPl3qWs8Gchz+3Yj9PX15SLRzJX3JpuYYia6GEtDN1KMf5eq8WRAU/lLLdRkk56Yf8u8t2hoBdIW1dfXx+bNm2c1D23evDmXv9pl4fQDJT0x/5Z5/8EQ42+ppraM5LV5SNLT19fHvn37GBsbw90xM04//XT9u1mAmOs2FQoFhoaGZnUgytMPhhh/SyUekQYpFovs3r17pu3c3dm9eze9vb1KPgsQ68deOyxOmPXfsi0Tj5ltBP4S6AT+xt0va3CRpA3N1+2+1b/Eli1bVvHCfh7GX4FaNBar7a7xmFkn8NfAAHAK8BYzO6WxpZJ2lOeL1AMDA0eMdO/o6ND4KwHaMPEAZwN73f0hd38W+DKwpcFlkjaU54vUfX19bN26dVYHm61bt6qWIEB7NrWdDDxa9nwceEWDyiJtrB0uUivRSCXtmHgqDb+dNTLKzLYD2wF6e3tjlEnaUDtcpBappB0Tzziwuuz5KuCx8g3c/UrgSoD+/n4NI5fMqFYg7agdr/HcBawzs7VmtgR4M3Bzg8skItI22q7G4+5TZvZO4DaS7tTXuHs2U+aKiMgR2i7xALj7rcCtjS6HiEg7asemNhERaSAlHhERicq09sf8zOwA8KNGl6NGxwNPNroQGcrz+encWlOezw0Wd36/4e4rK72gxJMjZjbq7v2NLkdW8nx+OrfWlOdzg+zOT01tIiISlRKPiIhEpcSTL1c2ugAZy/P56dxaU57PDTI6P13jERGRqFTjERGRqJR4REQkKiWeFmVm15jZE2Z2X1nsODPbZWZ7wv2KRpZxoaqc2yfN7J/N7F4zu8nMjm1kGRej0vmVvfY/zczN7PhGlG2xqp2bmf0PM/uBmd1vZp9oVPkWo8q/yzPM7A4zu8fMRs3s7EaWcaHMbLWZfcvMHgyf0btDPJPvFCWe1nUtsHFO7GJgxN3XASPheSu6liPPbRdwmru/DPgX4I9jFypF13Lk+WFmq4HXAvtiFyhF1zLn3Mzst0hW+X2Zu58K/HkDypWGaznyc/sE8FF3PwP4cHjeiqaA97n7S4ENwIVmdgoZfaco8bQod/8OcHBOeAtwXXh8HbA1aqFSUunc3P3r7j4Vnt5Bso5SS6ry2QF8Gng/cxYmbCVVzu0dwGXufihs80T0gqWgyrk5sDw87mHO2l6twt33u/vd4fHTwIMkqzVn8p2ixJMvJ7r7fkj+IQEnNLg8WflvwHCjC5EmM3sD8GN3393osmTgxcB/MLPvm9m3zezljS5Qii4CPmlmj5LU5Fq5Jg6Ama0BzgS+T0bfKUo80lLM7IMkzQJfaHRZ0mJmzwE+SNJUk0ddwAqSJpw/Ar5qZpWWoG9F7wDe4+6rgfcAVze4PItiZs8F/g64yN2fyuo4Sjz58riZnQQQ7luySaMaM9sG/DbwVs/XALQXAmuB3Wb2CEkz4t1m9usNLVV6xoEbPXEncJhk8sk82AbcGB7/LdCSnQsAzKybJOl8wd1L55TJd4oST77cTPIfgXA/2MCypMrMNgIfAN7g7r9odHnS5O5Fdz/B3de4+xqSL+qz3P1fG1y0tOwAXg1gZi8GlpCfGZ0fA/5jePxqYE8Dy7JgoQZ6NfCgu3+q7KVsvlPcXbcWvAFfAvYDkyRfVBcAzyfpebIn3B/X6HKmeG57gUeBe8Lt/za6nGme35zXHwGOb3Q5U/zslgCfB+4D7gZe3ehypnhu5wBjwG6SayLrG13OBZ7bOSQdJe4t+z/2+qy+UzRljoiIRKWmNhERiUqJR0REolLiERGRqJR4REQkKiUeERGJSolH2p4lvmtmA2Wx3zWzv29wmb4aZuN+1wL3MZ7mLN5hpuI/KHu+2sy+ktb+pX2oO7UIYGankYw8PxPoJBnHsNHdf7iIfXb5ryY2rfe9q4Bvu/sLF3H8cZIZvX+60H3M2d+LgK95MhOzyIKpxiMCuPt9wBDJ7AiXANe7+w/NbJuZ3RnWW/msmXUAmNmVYf2V+81sZo61UMv4UzP7R+CNZvYeM3vAzHab2efnHtfMlpnZdWZWNLO7zexV4aWvAy8Ix/3NOe850cxuDMe/08w2hPjKsGbK3WZ2BWAh/iIzu6fs/Reb2YfC4xeb2TdD+e42szVmtjzE7g41rt8Ob70M+PehTJeV77faeZjZ75vZ18zstrCmy/9e7GclOdDoEbO66dYsN+DXgB8AReAY4DSS6V66wutXAr8XHh8X7ruAfwBOCc/HgfeW7XM/sCQ8PrbCMT8AXBUenwr8iGSk/4uAe6qU8yvAhvB4DXBfePxZ4E/C4y0kI9GPnbsvkjVVPhQejwGbw+OlwHOAbuB5IXYCsCc8nrufmefznMfvk4x6fx6wjGT2iRc0+rPWrbG3rjrzlEhuufvPwzWLn7n7ITN7DfByYDRMplz64gR4i5ldQJJ4XgCcAjwQXiu/7nE/8HkzGyRJYnOdA3wyHP9+M3uM5Av92XmK+hqSmkfp+QozW3Cv56wAAAHPSURBVAa8imSaE9x90Myenu98w2qSx7v7UHjPL0N8CfBxMzuHZELP1Xb0FVGrnQfANzxZ4wUz+2eglxZdt0bSocQjMtvhcIOkqeoad//T8g3MbB3wbuBsd/9paEJbWrbJz8sev45kEsktwIfM7DR3ny7f3QLKaOHYs5JTSESVLtpOMbtZfWmIVdv+fJJFzc5y96lwrWhphe3mlqmaQ2WPp9H3TtvTNR6R6r4B/G7p176ZPd/MeklWnHwaeCpMFf+6Sm82s05glbt/k2QdmpUkTVnlvgO8NWz/UuAkkglRj1auC8uOU7rYX76vzSTNWwD/SnK9aIWZLQU2Abj7T4Anw7aY2VJL1gbqAZ4ISee1JCtREs65tM+5FnIe0qaUeESqcPci8FHgG2Z2L8kF/xNJZlh+gGS25auAf6yyiy7gi+G9dwMfLzU5lfk/wDIzK5Isbnf+3JpMBRcCrwwX/h8A3h7ilwCvMbO7gXOBH4fz+CXwv4C7SKa5f6BsX28F3hfK+F2S5HgD8JtmNgq8iTDVv7s/TtLsWDSzy1I4D2lT6k4tIiJRqcYjIiJRKfGIiEhUSjwiIhKVEo+IiESlxCMiIlEp8YiISFRKPCIiEtX/B7FHhp43EguvAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(X_test, y_test,  color='gray')\n",
    "plt.xlabel('Years of education')  \n",
    "plt.ylabel('Income') \n",
    "plt.plot(X_test, y_pred, color='red', linewidth=2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see that the line has downward trend. Probably such trend is because there are less people, who study for more years and most people still earn average salary. It also maybe because people, who sudy less, have more time to work and get promotions faster."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<center><h1>Problem 3</h1>\n",
    "<h3>a) Test the claim that on average the income at Moscow is greater than at Tver.</h3></center>\n",
    "\n",
    "<h3><center>\n",
    "    \n",
    "    H0 - average income is the same for Moscow and Tver\n",
    "    \n",
    "    H1 - average income is bigger for Moscow, than for Tver\n",
    "</center></h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [],
   "source": [
    "income_mos = df.loc[df['region'] == 1].income.tolist()\n",
    "income_tver = df.loc[df['region'] == 2].income.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.8502292975903151\n"
     ]
    }
   ],
   "source": [
    "t, p = stats.ttest_ind(income_mos, income_tver)\n",
    "print(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.198\n"
     ]
    }
   ],
   "source": [
    "p = np.round(stats.t.cdf(t, df=600), 3)\n",
    "alpha = 0.05\n",
    "print(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The null hypothesis cannot be rejected\n"
     ]
    }
   ],
   "source": [
    "if p < alpha:\n",
    "    print(\"The null hypothesis can be rejected\")\n",
    "else:\n",
    "    print(\"The null hypothesis cannot be rejected\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Due to the results of the test, we can make conclusion that there is not difference for average income of Moscow and Tver."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3><center> b) You noticed that a number of Python developers is 4 times greater than a number of data\n",
    "analysts. Please test this belief. </center></h3>\n",
    "\n",
    "<h3><center>\n",
    "    \n",
    "    H0 - observed == expected number of workers\n",
    "    \n",
    "    H1 - observed != expected number of workers\n",
    "</center></h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_analysts = len(df.loc[df[\"occupation\"] == 2])\n",
    "python_worker = len(df.loc[df[\"occupation\"] == 3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [],
   "source": [
    "obs = [data_analysts, python_worker]\n",
    "exp = [data_analysts, python_worker * 4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = stats.chisquare(obs, exp)[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The null hypothesis can be rejected\n"
     ]
    }
   ],
   "source": [
    "alpha = 0.05\n",
    "if p < alpha:\n",
    "    print(\"The null hypothesis can be rejected\")\n",
    "else:\n",
    "    print(\"The null hypothesis cannot be rejected\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Therefore, we make a conclusion, that there will not be a 4 times more python workers, than data analysts in general population."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3><center> e) Please divide the income into 4 intervals of the same length. Is the distribution of income\n",
    "similar within various occupations areas. </center></h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"income_rank\"] = pd.qcut(df.income, [0, .25, .5, .75, 1.], labels=[1, 2, 3, 4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "table = pd.crosstab(df[\"income_rank\"], df[\"occupation\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's look at the contingency table:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>occupation</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>income_rank</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>30</td>\n",
       "      <td>39</td>\n",
       "      <td>27</td>\n",
       "      <td>31</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>34</td>\n",
       "      <td>31</td>\n",
       "      <td>32</td>\n",
       "      <td>27</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>29</td>\n",
       "      <td>37</td>\n",
       "      <td>26</td>\n",
       "      <td>22</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>35</td>\n",
       "      <td>32</td>\n",
       "      <td>17</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "occupation    1   2   3   4   5\n",
       "income_rank                    \n",
       "1            30  39  27  31  24\n",
       "2            34  31  32  27  26\n",
       "3            29  37  26  22  36\n",
       "4            20  35  32  17  46"
      ]
     },
     "execution_count": 252,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3><center>\n",
    "    \n",
    "    H0 - distribution of income is similar in all groups\n",
    "    \n",
    "    H1 - distribution is different at least in two groups\n",
    "</center></h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = stats.chi2_contingency(table)[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The null hypothesis can be rejected\n"
     ]
    }
   ],
   "source": [
    "alpha = 0.1\n",
    "if p < alpha:\n",
    "    print(\"The null hypothesis can be rejected\")\n",
    "else:\n",
    "    print(\"The null hypothesis cannot be rejected\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The null hypothesis cannot be rejected\n"
     ]
    }
   ],
   "source": [
    "alpha = 0.05\n",
    "if p < alpha:\n",
    "    print(\"The null hypothesis can be rejected\")\n",
    "else:\n",
    "    print(\"The null hypothesis cannot be rejected\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We see that at alpha = 0.1 we can reject H0 hypothesis and concude that distribution of income differs between groups. At alpha = 0.05, we cannot reject the H0 hypothesis and conclude that there is similar distribution of income in all groups. For alpha=0.1 probability of making type1 error is error, so I'll stick to 0.05 hypothesis and will conclude that distribution of salaries might be similar in all groups."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3><center> Now, that is the end of my work, thank you for reading this notebook. </center></h3>"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}