logistic_regression.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "HW06.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "cGcncB6Qkk1z"
      },
      "source": [
        "# Part II: Fit logistic regression iris using binary cross entropy cost function\n",
        "from sklearn import datasets\n",
        "import numpy as np \n",
        "from sklearn.model_selection import train_test_split\n",
        "import numpy as np \n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "class Preprocessing(): \n",
        "\n",
        "  def __init__(self, train_ratio, test_ratio, val_ratio):\n",
        "    ''' Class constructor for Preprocessing. This class holds methods to process\n",
        "        raw input data into training/testing/validation sets for downstream\n",
        "        logistic regression. '''\n",
        "    self.train_ratio =  train_ratio\n",
        "    self.test_ratio = test_ratio\n",
        "    self.val_ratio = val_ratio\n",
        "\n",
        "  \n",
        "  def import_data(self):   \n",
        "    ''' Import iris dataset from SKL '''\n",
        "    iris = datasets.load_iris()  # import iris dataset\n",
        "\n",
        "    features = iris[\"data\"][:, :] # separate features  \n",
        "    labels = (iris[\"target\"] == 2).astype(np.int) # 1 if Iris-Virginica, else 0\n",
        "    \n",
        "    return features, labels\n",
        "\n",
        "\n",
        "  def create_train_set(self, features, labels): \n",
        "    ''' Stratify X, y data into training and testing sets. Want test datasets \n",
        "        to have 10% size of original datasets. '''\n",
        "    # call train_test_split function on original dataset to create train/test\n",
        "    X_train, X_test, y_train, y_test = train_test_split(features, labels, \n",
        "                                                        test_size=self.test_ratio, \n",
        "                                                        random_state=42)\n",
        "    return X_train, X_test, y_train, y_test\n",
        "\n",
        "\n",
        "  def create_validation_set(self, X_train, y_train): \n",
        "    ''' Create validation sets from X_train, y_train input datasets. Validation\n",
        "        set will compose 10% of training sets'''\n",
        "    # call internal variables to definte validation set ratio \n",
        "    val_set_ratio = self.val_ratio / (self.train_ratio + self.test_ratio)\n",
        "    # call train_test_split function on training set to create validation sets\n",
        "    X_train_new, X_val, y_train_new, y_val = train_test_split(X_train, y_train, \n",
        "                                                      test_size=val_set_ratio, \n",
        "                                                      random_state=42)\n",
        "    return X_train_new, X_val, y_train_new, y_val\n",
        "\n"
      ],
      "execution_count": 176,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8D_Q_dyAkrRN"
      },
      "source": [
        "class Logistic_Regression: \n",
        "\n",
        "  def sigmoid(x):\n",
        "    ''' activation function - sigmoid '''\n",
        "    return 1/(1 + np.exp(-x))\n",
        "\n",
        "  \n",
        "  def entropy(y, prediction):\n",
        "    ''' loss function: binary cross entropy ''' \n",
        "    loss = -np.mean(y*(np.log(prediction)) - (1-y)*np.log(1-prediction))\n",
        "    return loss\n",
        "\n",
        "  \n",
        "  def mean_square_error(y, prediction): \n",
        "    ''' loss function: mean square error '''\n",
        "    a = y.shape[0]  # store array shape \n",
        "    y = y.reshape(a,1)\n",
        "    loss = np.square(np.subtract(y, prediction)).mean()\n",
        "    return loss\n",
        "\n",
        "\n",
        "  def gradient_entropy(X, y, prediction):\n",
        "    ''' input training array, labels, and predicted value into gradient of \n",
        "        entropy loss function for gradient descent '''\n",
        "    errors = prediction - y  # store prediction error \n",
        "    dw = (1 / len(X)) * np.dot(X.T, errors)  # gradient wrt weights\n",
        "    db = (1 / len(X)) * np.sum((errors))  # gradient wrt bias \n",
        "    \n",
        "    return dw, db\n",
        "\n",
        "  \n",
        "  def gradient_mse(X, y, prediction): \n",
        "\n",
        "    term1 = prediction - y  # short hand for first subtraction term \n",
        "    term2 = 1 - prediction  # short hand for second subtraction term \n",
        "\n",
        "    # update weight gradient \n",
        "    temp1 = np.dot(term1.T, prediction)  # mtx multiply first two terms \n",
        "    temp2 = np.dot(term2.T, X)  # mtx multiply last two terms \n",
        "    dw = (-2 / len(X) ) * np.dot(temp1, temp2)  # update gradient \n",
        "\n",
        "    # update bias gradient \n",
        "    temp3 = np.dot(term1.T, prediction) \n",
        "    temp4 = np.dot(temp3, term2.T) \n",
        "    db = (-2 / len(X) + np.sum(temp4))\n",
        "\n",
        "    return dw.T, db\n",
        "\n",
        "\n",
        "  def train_entropy(X_train, y_train, X_val, y_val, max_iter, learning_rate):\n",
        "    ''' training function for logistic regression. Intakes training datasets \n",
        "        (X and y), validation datasets (X and y), number of training epochs \n",
        "        (maximum number of training iterations), and learning rate (step size \n",
        "        for gradient descent). Outputs 3 arrays containing final weights, bias,\n",
        "        and losses from each epoch.'''\n",
        "    tolerance = 0.02\n",
        "    m, n = X_train.shape  # store [num training instances, num features]\n",
        "  \n",
        "    w = np.zeros((n,1))  # initialize weights to 0 \n",
        "    b = 0  # initialize bias to 0 \n",
        "    \n",
        "    y_train = y_train.reshape(m,1)  # define y second dimension \n",
        "    losses = []  # initialize empty list to hold loss outputs\n",
        "\n",
        "    for i in range(max_iter):  # limit training by number of epochs \n",
        "      # calculate weighted sum of inputs\n",
        "      weighted_sum = np.dot(X_train,w) + b  \n",
        "      # pass weighted sum into activation function to return prediction\n",
        "      prediction = Logistic_Regression.sigmoid(weighted_sum)  \n",
        "\n",
        "      # calculate gradients based on prediction\n",
        "      dw, db = Logistic_Regression.gradient_entropy(X_train, y_train, \n",
        "                                                    prediction)  \n",
        "      # Update weights and bias \n",
        "      w = w - (learning_rate * dw)\n",
        "      b = b - (learning_rate* db)\n",
        "\n",
        "      ## repeat weighted sum, prediction calculation using validation set \n",
        "      validation_sum = np.dot(X_val,w) + b\n",
        "      validation_prediction = Logistic_Regression.sigmoid(validation_sum)\n",
        "      # calculate loss using validation set and append to list \n",
        "      loss = Logistic_Regression.entropy(y_val, validation_prediction)\n",
        "      losses.append(loss)\n",
        "\n",
        "      if i > 10:  # minimum number of training iterations \n",
        "        term_cond_1 = losses[i]-losses[i-1] # check if loss increased \n",
        "        term_cond_2 = losses[i-1]-losses[i-2]  # check if loss increased again\n",
        "        if term_cond_1 < tolerance and term_cond_2 < tolerance:  # early\n",
        "            break\n",
        "    \n",
        "    return w, b, losses\n",
        "\n",
        "  def train_mse(X_train, y_train, X_val, y_val, max_iter, learning_rate):\n",
        "    ''' training function for logistic regression. Intakes training datasets \n",
        "        (X and y), validation datasets (X and y), number of training epochs \n",
        "        (maximum number of training iterations), and learning rate (step size \n",
        "        for gradient descent). Outputs 3 arrays containing final weights, bias,\n",
        "        and losses from each epoch.'''\n",
        "    tolerance = 0.02\n",
        "    m, n = X_train.shape  # store [num training instances, num features]\n",
        "  \n",
        "    w = np.zeros((n,1))  # initialize weights to 0 \n",
        "    b = 0  # initialize bias to 0 \n",
        "    \n",
        "    y_train = y_train.reshape(m,1)  # define y second dimension \n",
        "    losses = []  # initialize empty list to hold loss outputs\n",
        "\n",
        "    for i in range(max_iter):  # limit training by number of epochs \n",
        "      # calculate weighted sum of inputs\n",
        "      weighted_sum = np.dot(X_train,w) + b  \n",
        "      # pass weighted sum into activation function to return prediction\n",
        "      prediction = Logistic_Regression.sigmoid(weighted_sum)  \n",
        "\n",
        "      # calculate gradients based on prediction\n",
        "      dw, db = Logistic_Regression.gradient_mse(X_train, y_train, prediction)  \n",
        "      # Update weights and bias \n",
        "      w = w - (learning_rate * dw)\n",
        "      b = b - (learning_rate* db)\n",
        "\n",
        "      ## repeat weighted sum, prediction calculation using validation set \n",
        "      validation_sum = np.dot(X_val,w) + b\n",
        "      validation_prediction = Logistic_Regression.sigmoid(validation_sum)\n",
        "      # calculate loss using validation set and append to list \n",
        "      loss = Logistic_Regression.mean_square_error(y_val, validation_prediction)\n",
        "      losses.append(loss)\n",
        "\n",
        "      if i > 10:  # minimum number of training iterations \n",
        "        term_cond_1 = losses[i]-losses[i-1] # check if loss increased \n",
        "        term_cond_2 = losses[i-1]-losses[i-2]  # check if loss increased again\n",
        "        if term_cond_1 < tolerance and term_cond_2 < tolerance:  # early\n",
        "            break\n",
        "    \n",
        "    return w, b, losses\n",
        "\n",
        "  \n",
        "  def predicted_output(X, weights, bias): \n",
        "    ''' Only run this function after completed training. Accepts input dataset\n",
        "        of instances, trained weights and bias. Outputs predictions between \n",
        "        [0,1] and labels [0 or 1]'''\n",
        "    # calculate weighted sum of inputs\n",
        "    weighted_sum = np.dot(X,weights) + bias  \n",
        "    # pass weighted sum into activation function to return prediction\n",
        "    predictions = Logistic_Regression.sigmoid(weighted_sum)  \n",
        "\n",
        "    labels = np.zeros((len(X),1))  # initialize array to hold predicted labels  \n",
        "    for i in range(len(predictions)):  # iter across predicted values \n",
        "      if predictions[i] > 0.5:  # check for predictions >0.5\n",
        "        labels[i] = 1  # set label ==1 for given predictions \n",
        "\n",
        "    return predictions, labels\n",
        "\n",
        "\n",
        "  def plot_loss_epochs(losses): \n",
        "    ''' Plot training loss outputs against number of epochs '''\n",
        "    x = np.arange(0, len(losses))  # create array [0,epochs] to set x axis bounds\n",
        "    y = losses  # store losses as y for plotting purposes \n",
        "    plt.plot(x, y, color = 'k')\n",
        "    plt.xlabel('Number of Training Epochs')\n",
        "    plt.ylabel('Validation Set Loss')\n",
        "    plt.title('Training Loss Over Multiple Epochs')\n",
        "    plt.xlim(0,len(losses))\n",
        "    return  # null return \n",
        "\n",
        "  \n",
        "  def accuracy(predictions, labels): \n",
        "    a = labels.shape[0]\n",
        "    labels = labels.reshape(a,1)\n",
        "\n",
        "    errors = np.absolute(predictions - labels)  # find elt-wise differences\n",
        "    sum_errors = np.sum(errors)  # only wrong predictions will have +1 value\n",
        "    avg = sum_errors / len(errors)  # divide sum by total number of predictions \n",
        "    percent_correct = 1 - avg\n",
        "    return percent_correct\n"
      ],
      "execution_count": 185,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z-7ZuQBDkmcF"
      },
      "source": [
        "# instantiate Preprocessing class instance  \n",
        "Processor = Preprocessing(train_ratio=0.8, test_ratio=0.1, val_ratio=0.1)\n",
        "# import Iris data set and store into X and y matrices \n",
        "X, y = Processor.import_data()   \n",
        "# split X, y input datasets into training and testing sets (90:10)\n",
        "X_train_set, X_test, y_train_set, y_test = Processor.create_train_set(X,y)  \n",
        "# split training dataset into training and validation sets (90:10)\n",
        "X_train, X_val, y_train, y_val = Processor.create_validation_set(X_train_set, \n",
        "                                                                 y_train_set)\n",
        "# train logistic regressor \n",
        "trained_weights, trained_bias, loss = Logistic_Regression.train_entropy(X_train, \n",
        "                                                                y_train, X_val, \n",
        "                                                                y_val, 50, 0.1)\n",
        "# predict labels for test data using trained regressor \n",
        "test_predictions, predicted_labels = Logistic_Regression.predicted_output(X_test,trained_weights, trained_bias)\n",
        "\n"
      ],
      "execution_count": 186,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V6iutY371Crf",
        "outputId": "64cd7999-837e-4ca7-9ebe-e1cf9ce72258"
      },
      "source": [
        "## Question 1: Split data 80/10/10 train/validation/test and report numbers\n",
        "\n",
        "print('Number of training samples:', len(X_train))\n",
        "print('Number of validation samples:', len(X_val))\n",
        "print('Number of testing samples:', len(X_test))"
      ],
      "execution_count": 187,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of training samples: 119\n",
            "Number of validation samples: 16\n",
            "Number of testing samples: 15\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 349
        },
        "id": "9lCmvAdL5u_T",
        "outputId": "4d5111c9-4fd6-4a1a-ea6a-53ffcd14c184"
      },
      "source": [
        "## Question 2: Visualize learning curve for loss and epochs during training\n",
        "Logistic_Regression.plot_loss_epochs(loss) \n",
        "\n",
        "''' Note: Since early termination condition does not initiate until at least \n",
        "          10 epochs of training have occurred, an increase in loss prior to \n",
        "          10th iteration will not result in early termination. I did this purely\n",
        "          for the purpose of showing > 3-4 epochs on the graph. Without the \n",
        "          minimum iterations condition, training terminates after 4 epochs. '''"
      ],
      "execution_count": 188,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "' Note: Since early termination condition does not initiate until at least \\n          10 epochs of training have occurred, an increase in loss prior to \\n          10th iteration will not result in early termination. I did this purely\\n          for the purpose of showing > 3-4 epochs on the graph. Without the \\n          minimum iterations condition, training terminates after 4 epochs. '"
            ]
          },
          "metadata": {},
          "execution_count": 188
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAEWCAYAAACqitpwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5xU9bnH8c8XdukgVUA6OwsRlCLFCgF7jAp2vRrRaLhRczXRWBI0dkPUxBSTGMUoFhI1UcP1WiG7ikoUEKSodKR3CAhLWfa5f5yzZFi3zO6e2Rl2nvfrdV4zc+Z3fuc5wzLP/Mo5R2aGc845F6U6qQ7AOedc7ePJxTnnXOQ8uTjnnIucJxfnnHOR8+TinHMucp5cnHPORc6Ti6s0SW9IGhV1WVezJF0h6f1y3h8iaX6CdQ2TtDK66JJLUr6kq1MdR23mySVDSPoqbimSVBD3+tLK1GVm3zKz8VGXrYxUfpkpcLOkheHnuFzSzyXVr6H9D5Nkkl4psb5vuD6/ivWapFjxazObYmY9qxluIvt9WtKeEn+jnyZ7vy65PLlkCDNrUrwAy4Gz4tY9X1xOUlbqojxo/BYYDVwONAW+BZwEvBj1jsr599gAHCupVdy6UcCCqGOoIQ/G/42aWd9UB+Sqx5NLhituAUi6VdJa4ClJLSS9JmmDpC3h845x2+zvUijuWpH0cFh2qaRvVbFsN0nvSdouaZKk30t6rgrHdHi4362S5kk6O+69MyR9Fu5jlaQfh+tbh8e5VdJmSVMkfe3/h6Rc4FrgUjObamaFZjYPOA84XdKJko6WtFZS3bjtzpE0O3xeR9JtkhZL2iTpRUktw/e6hi2IqyQtB/5ZxmHuAV4FLg63qwtcBMT/UCiuKytuXandQZLeC59+GrYcLirZOpS0TNJPws9vi6SnJDUo49/gMEl/D/+Glkq6vozjKFfcMYyWtFrSmuJ/s/D9+pJ+Hb63OnxeP+79EZJmSdoWft6nx1XfRdIH4d/C25Jah9s0kPRc+G+zVdI0SW2rEn8m8+TiANoBLYEuBL/I6wBPha87AwXAo+VsfzQwH2gNPAg8KUlVKDsB+BhoBdwFfKeyByIpG/hf4G3gUOB/gOclFXfvPAn8t5k1BY7gP1/eNwErgTZAW+CnQGnXRjoJWGlmH8evNLMVwL+AU8zsI2AHcGJckf8Kj48wppHAN4HDgC3A70vs55vA4cBp5RzuMwStJ8Jyc4HV5ZQvk5kNDZ/2DVsOL5RR9NJwXzlAD+D2kgXCpPy/wKdAB4LP7IeSyjuWigwHcoFTgVslnRyuHwMcA/QD+gKDi2OSNJjgM7oZaA4MBZbF1flfwJUEfyf1gOKkNQo4BOhE8Lf4fYL/A64SPLk4gCLgTjPbbWYFZrbJzP5uZjvNbDtwP8GXXVm+NLMnzGwfMB5oT/AFnXBZSZ2BQcDPzGyPmb0PTKzCsRwDNAHGhvX8E3gNuCR8fy/QS1IzM9tiZp/ErW8PdDGzveF4Q2nJpTWwpox9rwnfB/hL8T4lNQXOCNdB8GU1xsxWmtlugkR6vg7sArvLzHaYWZlfamb2IdAyTJyXE3yRJtujZrbCzDYT/F1cUkqZQUAbM7sn/DdYAjxB2Moqw4/DVkLxUnKc7u7w85hD8MOneL+XAveY2Xoz2wDczX9+lFwF/NnM3jGzIjNbZWZfxNX5lJktCD/jFwkSFAR/C62AmJntM7MZZrYtkQ/H/YcnFwewwcx2Fb+Q1EjSnyR9KWkb8B7QPL6bp4S1xU/MbGf4tEklyx4GbI5bB7CiksdBWM8KMyuKW/clwS9oCLqvzgC+lPSupGPD9Q8Bi4C3JS2RdFsZ9W8kSEKlaR++D0Er5dywi+Zc4BMz+zJ8rwvwSvEXKfA5sI8DE3Kix/4s8AOCX/avVFA2CvFxfUnweZfUBTgsPlkQtATL61p62Myaxy0lZxiWtd/DwtelvdcJWFzOPtfGPd/Jf/5mnwXeAv4adrU9GLaIXSV4cnHw9e6fm4CewNFm1oygOwGgrK6uKKwh+BXeKG5dpyrUsxroVGK8pDOwCsDMppnZCIKukFcJB+HNbLuZ3WRm3YGzgRslnVRK/f8M6x8cv1JSJ4JW0+Swvs8Ivui+xYFdYhB8UX6rxJdpAzNbFVcm0cuVP0swBvR6icQMQdccQPxn2i7BessS/2/SmdK74VYAS0scX1MzOyMJ+11NkMxKe28FQfddpYQt17vNrBdwHHAm/+l+dAny5OJK05Sgj3lrONB8Z7J3GP6qnw7cJale2KI4q6LtwsHX/QvBmM1O4BZJ2ZKGhfX8Naz3UkmHmNleYBtBlyCSzpQUC8d//k3QkigquT8zWwA8RjCOc4ykupJ6A38HJpnZpLjiE4AbCJLzS3HrHwPul9Ql3HcbSSMS/7QOiGcpQZflmFLe20CQVC8L4/wu5X/ZrgO6V7DL6yR1DP8uxgCljc18DGxXMEmkYbjvIyQNSuSYynBH2KLuTTBOUrzfvwC3h59ha+BnQPEkkCeBKyWdpGASRQdJ36hoR5KGSzoybKlvI+gm+9rfgiufJxdXml8DDQm6eP4FvFlD+70UOBbYBNxH8AWyu5zyHQiSYPzSiSCZfIsg/j8Al8f1tX8HWBZ2930/3CcEg8WTgK+AqcAfzCyvjP3+ABhH8CX2FcHnk0/Q5RbvLwRf/P80s41x639DMJ70tqTtBJ/x0eUcZ7nM7H0zK2sg/3sEA9qbgN7Ah+VUdRcwPuzKurCMMhMIJkssIehyuq+UePYR/NrvBywl+HcYRzBIXpZbdOB5LhtLvP8uQbflZIIutLfD9fcR/CiZDcwBPimOKZx0cSXwCMEPhnc5sJVTlnbA3wgSy+fhds8msJ2LI79ZmEtXkl4AvjCzpLecXMUkLQOuLtE6S/Y+uxIkqGwzK6yp/brq85aLSxuSBknKCbswTgdGEIyLOOcOMn42tksn7YCXCaaBrgSuMbOZqQ3JOVcV3i3mnHMuct4t5pxzLnIZ1S3WunVr69q1a6rDcM65g8qMGTM2mlmbymyTUcmla9euTJ8+PdVhOOfcQUXSlxWXOpB3iznnnIucJxfnnHOR8+TinHMucp5cnHPORc6Ti3POuch5cnHOORc5Ty7OOecil1HJ5d///neqQ3DOuYyQUclly5YtqQ7BOecyQsqSi6SWkt6RtDB8bFFGuVFhmYWSSt5XG0kTJc1NZJ+7d5d33ynnnHNRSWXL5TZgspnlEtxd7raSBeJusXs0MBi4Mz4JSTqX4E6ACfHk4pxzNSOVyWUEMD58Ph4YWUqZ04B3zGyzmW0B3gFOB5DUBLiRUm6zWpa9e/eyY8eOagXtnHOuYqlMLm3NbE34fC3QtpQyHYAVca9XhusA7gV+CeyszE4XL15cyTCdc85VVlKTi6RJkuaWsoyIL2fBHcsSvmuZpH5Ajpm9kkDZ0ZKmS5oOsGjRosoehnPOuUpK6iX3zezkst6TtE5SezNbI6k9sL6UYquAYXGvOwL5wLHAQEnLCI7hUEn5ZjasxPaY2ePA4+E+zZOLc84lXyq7xSYCxbO/RgH/KKXMW8CpklqEA/mnAm+Z2R/N7DAz6wqcACwoLbGUlJWV5S0X55yrAalMLmOBUyQtBE4OXyNpoKRxAGa2mWBsZVq43BOuq5L69euzcOHCagfunHOufAqGOzJD69atrWHDhqxYsaLiws455wCQNMPMBlZmm4w6Q79+/fqsXLmSgoKCVIfinHO1WsYlF4AlS5akOBLnnKvdMiq5NGjQAPDpyM45l2wZlVyKWy6eXJxzLrkyKrnUrVuXVq1aeXJxzrkky6jkAhCLxXw6snPOJVlGJhdvuTjnXHJlZHJZvny5X37fOeeSKCOTi5mxdOnSVIfinHO1VkYmF/AZY845l0wZl1xyc3MBTy7OOZdMGZdcWrZsSfPmzT25OOdcEmVccpHkM8accy7JMi65gJ/r4pxzyZaxyWXZsmXs2bMn1aE451ytlLHJpaioiC+//DLVoTjnXK2UsckFfMaYc84liycX55xzkcvI5HLooYfStGlTTy7OOZckGZlcfDqyc84lV0YmF/DpyM45l0wZnVyWLl1KYWFhqkNxzrlaJ6OTS2FhIcuXL091KM45V+tkdHIBnzHmnHPJ4MnFk4tzzkUuY5NL+/btadSokScX55xLgpQkF0ktJb0jaWH42KKMcqPCMgsljYpbX0/S45IWSPpC0nlViMGnIzvnXJKkquVyGzDZzHKByeHrA0hqCdwJHA0MBu6MS0JjgPVm1gPoBbxblSA8uTjnXHKkKrmMAMaHz8cDI0spcxrwjpltNrMtwDvA6eF73wV+DmBmRWa2sSpBxGIxFi9ezL59+6qyuXPOuTKkKrm0NbM14fO1QNtSynQAVsS9Xgl0kNQ8fH2vpE8kvSSptO0BkDRa0nRJ0zds2HDAe7FYjD179rBy5cqqH4lzzrmvSVpykTRJ0txSlhHx5czMAKtE1VlAR+BDMzsKmAo8XFZhM3vczAaa2cA2bdoc8J7PGHPOueRIWnIxs5PN7IhSln8A6yS1Bwgf15dSxSqgU9zrjuG6TcBO4OVw/UvAUVWJ0ZOLc84lR6q6xSYCxbO/RgH/KKXMW8CpklqEA/mnAm+FLZ3/BYaF5U4CPqtKEB06dKBBgwaeXJxzLmKpSi5jgVMkLQRODl8jaaCkcQBmthm4F5gWLveE6wBuBe6SNBv4DnBTVYKoU6cOOTk5nlyccy5iWanYqZltImhxlFw/Hbg67vWfgT+XUu5LYGgUsfh0ZOeci17GnqFfrDi5FBUVpToU55yrNTy5xGLs2rWL1atXpzoU55yrNTy5+Iwx55yLnCcXTy7OORe5jE8unTp1Ijs725OLc85FKOOTS926denevbsnF+eci1DGJxeA3NxcTy7OORchTy78ZzpycPK/c8656vLkQpBcduzYwdq1a1MdinPO1QoVJhdJOZLqh8+HSbo+7rL3tYLPGHPOuWgl0nL5O7BPUgx4nOBKxROSGlUN8+TinHPRSiS5FJlZIXAO8Dszuxlon9ywalaXLl3Iysry5OKccxFJJLnslXQJwaXxXwvXZScvpJqXlZVF165dPbk451xEEkkuVwLHAveb2VJJ3YBnkxtWzfPpyM45F50KL7lvZp8B1wOEN+1qama/SHZgNS0Wi/HBBx9gZkhKdTjOOXdQS2S2WL6kZpJaAp8AT0j6VfJDq1mxWIxt27axYcOGVIfinHMHvUS6xQ4xs23AucAzZnY0wd0jaxWfMeacc9FJJLlkSWoPXMh/BvRrHU8uzjkXnUSSyz3AW8BiM5smqTuwMLlh1byuXbtSp04dTy7OOReBRAb0XwJeinu9BDgvmUGlQr169ejSpYsnF+eci0AiA/odJb0iaX24/F1Sx5oIrqYVX8DSOedc9STSLfYUMBE4LFz+N1xX6/i5Ls45F41EkksbM3vKzArD5WmgTZLjSolYLMaWLVvYtGlTqkNxzrmDWiLJZZOkyyTVDZfLgFr57eszxpxzLhqJJJfvEkxDXgusAc4HrkhiTCnjycU556KRyGyxL4Gz49dJehj4cbKCSpVu3bohyZOLc85VU1XvRHlhdXcsqaWkdyQtDB9blFFuVFhmoaRRcesvkTRH0mxJb0pqXd2YGjRoQKdOnTy5OOdcNVU1uURxZcfbgMlmlgtMDl8fuJPgemZ3AkcDg4E7JbWQlAX8BhhuZn2A2cAPIojJpyM751wEykwuYcuitKUV0SSXEcD48Pl4YGQpZU4D3jGzzWa2BXgHOD3cv4DGCi5h3AxYHUFMnlyccy4C5Y25zACM0hPJngj23dbM1oTP1wJtSynTAVgR93ol0MHM9kq6BpgD7CC4HM11pe1E0mhgNEDnzp0rDCo3N5eNGzeydetWmjdvnuixOOeci1Nmy8XMuplZ9/Cx5NI9kcolTZI0t5RlRIl9GUEiS4ikbOAaoD/BiZ2zgZ+UcRyPm9lAMxvYpk3Fp+f4jDHnnKu+qo65JMTMTjazI0pZ/gGsC6+2TPi4vpQqVgGd4l53DNf1C+tfHCamF4HjoojZk4tzzlVfUpNLBSYCxbO/RgH/KKXMW8Cp4SB+C+DUcN0qoJek4qbIKcDnUQTVvXvQKPPk4pxzVVfheS5JNBZ4UdJVwJeE05slDQS+b2ZXm9lmSfcC08Jt7jGzzWG5u4H3JO0Nt78iiqAaNWpEhw4dPLk451w1KOhVKqeA9KyZfaeidQeDgQMH2vTp0yssN2zYMAoLC3n//fdrICrnnEtvkmaY2cDKbJNIt1jvEjupCwyozE4ONj4d2Tnnqqe881x+Imk70EfSNknbw9frKX18pNbIzc1l3bp1bN++PdWhOOfcQam8qcg/N7OmwENm1szMmoZLKzMrddpvbeEzxpxz6WbHjh1Mnz6d8ePHM2vWrFSHU6FEBvTHhJfZ72Zm90rqBLQ3s4+THFvKxCeX/v37pzga51wm2blzJ59//jnz5s07YFm2bNn+MnfccQf9+vVLXZAJSCS5/B4oAk4E7gW+CtcNSmJcKZWTkwN4y8U5lzw7d+7kiy++KDWJFE+0ys7OpmfPnhx99NF897vfpXfv3vTq1Wv/D+B0lkhyOdrMjpI0E8DMtkiql+S4UqpJkya0a9fOk4tzrtoKCgpKTSJLly49IIn06NGDQYMGccUVV9C7d2969+5NTk4O2dnZKT6CqkkkuewNZ4gZQHjiYlFSo0oDB9uMsSVLluy/H41zrubt2rWr1CSyZMmS/UkkKyuLHj16MGDAAC6//PL9SSQWix20SaQsiSSX3wKvAIdKup/gTpS3JzWqNBCLxXj77bdTHUZCFi9eTI8ePfj5z3/OLbfckupwnKvVzIyVK1cya9YsZs6cyaxZs5gzZw5LliyhqCj43V2cRPr3789ll122P4nk5ubWuiRSlkTuRPm8pBnASeGqkWYWyaVW0lksFuPpp59mx44dNG7cONXhlGvSpEkUFRXxwAMPcPXVV9OyZctUh+RcrVBYWMiCBQv2J5Hix02bNgEgidzcXPr27ct//dd/HZBE6tWr1aMHFSozuUhqBOw1s71m9oUkA84ADiei63ils9zcXCBoFfTp0yfF0ZQvPz+fZs2asW3bNsaOHcuDDz6Y6pCcO+js3LmT2bNnH5BEZs+eza5duwCoX78+Rx55JOeccw79+/enX79+9OnThyZNmqQ48vRUXsvlTeAqYKGkGDAVeB44U9KgTDrXJZ2Ti5mRl5fHmWeeSVZWFr/73e+4/vrr6dixY6pDcy5tbdiw4YAkMnPmTBYsWLC/W6tFixb069ePa6+9ln79+tG/f3969uyZMV1aUSgvubQws4Xh81HAX8zsf8KZYjMo4/4ptcXBMh35iy++YN26dQwfPpxTTjmFv/71r9x1112MGzcu1aE5l3JmxtKlS7/WrbVq1ar9ZTp37kz//v256KKL9rdIOnfu7JNjqqm85BJ/RcsTgYcAzGyPpFo/W+yQQw6hTZs2aZ9c8vPzgeBim126dOG6667jN7/5DTfeeCO9evVKbXDO1ZB9+/axYsUKlixZwuLFi5k7dy6zZs1i1qxZbNu2DYC6dety+OGHM3z48P1JpF+/fj5GmSTlJZfZkh4muHdKDHgbQFLG3Pv3YJiOnJeXR8eOHfe3tH7605/y5JNPMmbMGF555ZUUR+dcdLZv374/eSxZsuSA58uWLaOwsHB/2UaNGtG3b18uu+yy/d1avXv3pmHDhik8gsxSXnL5HnAD0BU41cx2hut7AQ8nOa60EIvF9rcM0pGZkZ+fz+mnn76/Cd+6dWtuvvlm7rjjDqZOncqxxx6b4iidS0xRURGrVq36WuIofr5x48YDyrds2ZLu3btz1FFHccEFF9C9e3e6d+9OTk4OHTt2pG7duik6EgflJBczKyC4oVfJ9R8CHyYzqHQRi8V49tlnKSgoSMtfPPPmzWPDhg0MHz78gPU/+tGPePTRR7n11lt59913ve/YpY0dO3awdOnSUlsfS5cuZc+ePfvL1q1bly5dutC9e3fOPfdccnJy9ieQ7t2707x5xnSiHJRSeSfKtFc8Y2zp0qVpOX4RP94Sr3HjxvzsZz/juuuu4/XXX+fb3/52zQfnMt7y5cuZMmUKU6ZMYc6cOSxevJh169YdUKZZs2bk5ORw5JFHMmLEiP0tj+7du9OpUyefnXUQq/BOlLVJoneiLDZt2jQGDx7Mq6++yogRI5IYWdWcd955zJgx44CrpRbbu3cvvXr1omHDhsycOdO7CFxSmRmff/75/mQyZcoUli9fDgSTY/r3709OTs4BrY+cnBxatGjhLeuDQFXuROktl3Kk831dioqKyM/P5+yzzy71/ezsbO677z4uvvhiJkyYwHe+c9DdldqlscLCQmbOnMl7773HlClTeP/99/eftd6uXTuGDBnCj3/8Y4YOHcoRRxzhP24yUIXJRVIP4GagS3x5MzsxiXGlhRYtWtCyZcu0TC5z585l8+bNX+sSi3fBBRfw4IMPcscdd3DhhRdSv379mgvQ1So7d+7ko48+2t8qmTp1Kjt27ACCH2Fnn302Q4YMYciQIeTk5HhrxCXUcnkJeAx4AtiX3HDST7pOR87LywP42mB+vDp16jB27FhOPfVUHnvsMW644YaaCs8d5LZs2cL777+/P5nMmDGDvXv3Iok+ffpw5ZVX7k8m7du3T3W4Lg1VOOYS9rUNqKF4kqqyYy4Al156KR9++CFLly5NUlRVM3LkyP2DpBU5+eST+fTTT1m8eDHNmjWrgejcwWbVqlUHjJfMmTMHgHr16jFo0KD9ieS4447zWVoZKFljLv8r6VqCy+7vLl5pZpsrGd9BKRaL8de//pXdu3enTbdSUVER7733Huecc05C5ceOHcugQYN4+OGHueeee5IcnUtn+/bto7CwkGXLlh2QTIp/PDVp0oTjjjuOCy+8kCFDhjB48OC0nIbv0l8iyWVU+Hhz3DoDukcfTvqJxWIUFRWxbNkyevbsmepwAPj000/ZsmVLuV1i8QYOHMiFF17Ir371K6677jratm2b5AhdooqKinj99dfJz89nz549FBYWsnfv3ko/Jlq2ZE9FmzZtGDJkCNdffz1Dhgyhb9++ZGX5PB9XfYncz6VbTQSSroovvb9w4cK0SS7F4y3lDeaXdN999/Hyyy9z77338uijjyYpsuQwMxYtWkQsFqs1A8U7d+7k2Wef5ZFHHmH+/PnUr1+fhg0bkp2dTVZW1gGPpa2rV68ejRs3/tr68raJf2zTpg0nnHACPXv2rDWfqUsvicwWywauAYaGq/KBP5nZ3iTGlTbScTpyfn4+sVisUpfVz83N5eqrr+ZPf/oTP/zhD/cf18Hg9ttv54EHHqBHjx6MHj2aUaNG0bp161SHVSVr167l97//PX/84x/ZtGkTAwYMYMKECZx//vl+wqCrXcys3AUYB4wnuDLyicBTwLiKtqugzpbAO8DC8LFFGeXeBLYCr5VY3w34CFgEvADUS2S/AwYMsMoqKiqyQw45xK677rpKb5sMhYWFdsghh9j3vve9Sm+7evVqa9SokV188cVJiCw5JkyYYICdeeaZdtxxxxlg9erVs4svvtj++c9/WlFRUapDTMjs2bPtyiuvtHr16pkkGzFihL377rsHTfwuswHTrbLf8xUWgE8TWVepncKDwG3h89uAX5RR7iTgrFKSy4vAxeHzx4BrEtlvVZKLmdmAAQPstNNOq9K2UZs2bZoBNmHChCptP2bMGANsxowZEUcWvenTp1uDBg3shBNOsN27d5uZ2Zw5c+z666+35s2bG2C5ubn20EMP2fr161Mc7dcVFRXZm2++aaeccooB1rBhQ7v22mttwYIFqQ7NuUpJVnL5BMiJe90d+KSyOypR53ygffi8PTC/nLLD4pMLIGAjkBW+PhZ4K5H9VjW5XHTRRZaTk1OlbaP24IMPGmCrV6+u0vZbt261Vq1a2amnnhpxZNFas2aNdejQwTp37mzr1q372vs7d+60Z555xk444QQDLDs72y666CKbPHmy7du3LwUR/0dBQYGNGzfOevfubYC1b9/eHnjgAdu0aVNK43KuqpKVXE4ClhOMtbwLLAOGV3ZHJercGvdc8a9LKVsyubQGFsW97gTMLWf70cB0YHrnzp2r9MGOGTPG6tata3v27KnS9lE644wzrGfPntWq45e//KUBNmnSpIiiitauXbvsmGOOsUaNGtnMmTMrLD937ly74YYbrEWLFgZYLBazX/ziF6UmpWRav3693X333XbooYcaYH369LHx48fvb3U5d7BKSnIJ6qU+0Cdc6ie4zSRgbinLiJLJBNhSTj3VSi7xS1VbLk899ZQBtnDhwiptH5W9e/da06ZN7fvf/3616ikoKLBOnTrZwIED067Pv6ioyK644goD7G9/+1ultt25c6c9++yzNmTIkP2tmQsuuMAmTZqU1NbMZ599ZqNHj7YGDRoYYGeccYZNnjw57T5b56oq0uQCnBg+nlvaUtkdlaj7oOoWmzJligH2xhtvVGn7qPzrX/8ywF544YVq11WcMF988cUIIovOI488YoDdeeed1arns88+sx/96EfWsmVLAywnJ8fGjh1ra9eujSTOoqIimzRpkp1xxhkGWIMGDWz06NH22WefRVK/c+kk6uRyd/j4VCnLnyu7oxJ1P8SBA/oPllP2gOQSrnuJAwf0r01kv1VNLmvXrjXAfvvb31Zp+6iMHTvWgEi+IAsLC613796Wm5ubFt19ZmZvvfWW1alTx84555zIWhoFBQX2/PPP29ChQw2wrKwsO//88+3tt9+u0j52795t48ePt759+xpghx56qN19991pOaHAuagka8ylWyLrKrVTaAVMJpiKPAloGa4fSNw0Z2AKsAEoAFYCp4XruwMfE0xFfinRrrqqJpeioiJr0qSJXX/99VXaPiqnnXaa9erVK7L6Jk6caIA99thjkdVZVfPnz7fmzZvbkUceadu3b0/KPj7//HO78cYbrVWrVgZYt27d7IEHHrA1a9ZUuO2mTZvsgQcesPbt2xtgvXv3tieffNIKCgqSEqtz6SRZyeVrM8OAGZXdUTosVU0uZmb9+vWzM844o8rbV9iW+mAAABkSSURBVNeePXuscePGkZ5vU1RUZMcff7y1a9fOvvrqq8jqraytW7daz549rXXr1rZ06dKk76+goMAmTJhgw4YN29+aOe+88+ytt976WmtmwYIFdu2111qjRo0MsFNOOcXefPNNH09xGSXqbrFvAOcBi0uMt1wBzKvsjtJhqU5yOf/8861Hjx5V3r66PvzwQwPspZdeirTe999/3wC7//77I603UYWFhfatb33LsrKyLD8/v8b3/8UXX9hNN910QGvm/vvvtzfeeMNGjBhhkqxevXp25ZVX2uzZs2s8PufSQdTJZUQ4vrKpxHjLb4HjKrujdFiqk1xuu+02y87Otr1791a5juq4//77DbANGzZEXvdZZ51lzZo1s40bN0Zed0VuvvnmtOia27Vrl/3lL3+x4cOHG8GFWa1Vq1Z2++23J9Rt5lxtlqxusWMrW2m6LtVJLuPGjTPAlixZUuU6quPkk0+2I488Mil1z5kzx+rUqWM33XRTUuovyzPPPGOAXXvttTW634rMnz/f/v73v9uOHTtSHYpzaaEqyaUOFZsp6TpJf5D05+Ilge1qlVRewHL37t188MEHlboKcmUcccQRXH755Tz66KMsX748Kfso6eOPP+Z73/sew4YN49e//nWN7DNRPXr04Nxzz6VRo0apDsW5g1YiyeVZoB1wGsEZ+h2B7ckMKh0VJ5eFCxfW+L6nTZtGQUFBwvdvqYq7774bgDvvvDNp+yi2evVqRo4cSfv27XnppZf8asDO1UKJJJeYmd0B7DCz8cC3gaOTG1b6Oeyww2jYsGFKWi55eXlI4pvf/GbS9tG5c2euu+46nnnmGebNm5e0/RQUFDBy5Ei2bdvGxIkTD9pL5zvnypdIcim+b8tWSUcAhwCHJi+k9CSJWCyWsuTSt29fWrZsmdT9/PSnP6VJkyb89Kc/TUr9Zsbo0aOZNm0azz33HEceeWRS9uOcS71EksvjkloAdwATgc8ILpmfcVKRXHbt2sXUqVOTNt4Sr1WrVtx6661MnDiRDz74IPL6H374YZ577jnuvfdeRo4cGXn9zrn0UWFyMbNxZrbFzN41s+5mdqiZPVYTwaWbWCzG4sWL2bdvX43t86OPPmLXrl1JHW+Jd8MNN9CuXTtuvfXW4tmCkXj99de59dZbueCCCxgzZkxk9Trn0lOZtzmWdGN5G5rZr6IPJ73FYjH27NnDqlWr6Ny5c43ss3i8ZejQoRUXjkDjxo258847ueaaa3jttdc466yzql3n559/ziWXXEK/fv146qmn/J7tzmWA8louTcNlIHAN0CFcvg8clfzQ0k8qpiPn5+fTv39/mjdvXmP7vOqqq8jNzeUnP/lJtVtpW7Zs4eyzz6ZBgwa8+uqrNG7cOKIonXPprMzkYmZ3m9ndBFOPjzKzm8zsJmAAUDM/29NMTSeXgoICpk6dWmNdYsWys7O5//77mTdvHs8991yV6yksLOSiiy7iyy+/5OWXX66x1p5zLvUSGdBvC+yJe70nXJdxOnbsSP369WvsXJepU6eyZ8+eGk8uAOeffz4DBw7kZz/7Gbt27apSHbfccgvvvPMOf/zjHzn++OMjjtA5l84SSS7PAB9LukvSXcBHwNPJDCpd1alTh5ycnBprueTn51OnTh1OOOGEGtlfPEmMHTuW5cuX84c//KHS2z/11FM88sgjXH/99Vx11VVJiNA5l86UyIwgSUcBQ8KX75nZzKRGlSQDBw606dOnV6uOESNGsGTJEubMmRNRVGUbMmQIu3fv5uOPP076vspy6qmnMmPGDJYsWcIhhxyS0DYffvghw4cPZ+jQobzxxhtkZZU5b8Q5dxCQNMPMBlZmmzJbLpKahY8tgWUEl4F5FvgyXJeRiqcjFxUVJXU/O3fu5KOPPkpJl1i8sWPHsnnzZh566KGEyq9YsYJzzz2XTp068cILL3hicS5DldctNiF8nAFMj1uKX2ekWCxGQUEBa9asSep+PvjgA/bu3VsjJ0+W56ijjuLiiy/mkUceqfCYd+7cyciRI9m5cycTJ05M+hUFnHPpq7zZYmeGj93CkyeLl25m1r3mQkwvNTVjLD8/n7p166ZkvKWke++9lz179nDPPfeUWcbM+O53v8vMmTOZMGECvXr1qsEInXPpprxusaPKW2oyyHRSU8klLy+PQYMG0bRp06TuJxGxWIzRo0fzxBNPlDlTbuzYsbzwwgs88MADnHnmmTUcoXMu3ZTXIf7Lct4z4MSIYzkodOrUiezs7KROR/7qq6+YNm0aN998c9L2UVl33HEHTz/9NLfffjsvvPDCAe9NnDiRMWPGcMkll3DrrbemKELnXDopM7mYWWpHktNUVlYW3bt3T2rL5YMPPqCwsDDl4y3x2rVrx4033sh9993HLbfcwoABAwCYN28el156KUcddRRPPvmkX9rFOQckdp4Lko6QdKGky4uXZAeWzpJ9deS8vDyys7PT7sTDm2++mVatWnHbbbcBsGnTJs4++2yaNGnCq6++SsOGDVMcoXMuXVSYXCTdCfwuXIYTXG7/7CTHldaKk0uUVw2Ol5eXx+DBg9PuOlzNmjXj9ttvZ9KkSbzxxhtccMEFrFy5kldeeYWOHTumOjznXBpJpOVyPnASsNbMrgT6EtwwLGPFYjF27NjBunXrIq9727ZtzJgxI626xOJdc801dOnShXPOOYe8vDwef/xxjjnmmFSH5ZxLM4kklwIzKwIKwxMr1wOdkhtWekvmjLH333+fffv2pfzkybLUr1+fe++9l927d3PjjTcyatSoVIfknEtDiZw+PV1Sc+AJghMovwKmJjWqNBefXKI+DyUvL4969epx7LHHRlpvlC677DL69Onjtyl2zpWpvPNcfi/peDO71sy2hnefPAUYFXaPVZmklpLekbQwfGxRRrk3JW2V9FqJ9c9Lmi9prqQ/S8quTjyV1aVLF+rWrZuU6ch5eXkcffTRNGrUKPK6oyKJvn37UqdOQvNBnHMZqLxvhwXAw5KWSXpQUn8zW2ZmsyPY723AZDPLBSaHr0vzEPCdUtY/D3wDOBJoCFwdQUwJy87OpmvXrpF3i23dupWZM2embZeYc84lqrzLv/zGzI4FvglsAv4s6QtJd0rqUc39jgDGh8/HAyPLiGEysL2U9a9bCPiY4IZmNSo3Nzfy5DJlyhSKioo8uTjnDnoV9muY2Zdm9gsz6w9cQpAIPq/mftuaWfFVENdSxZuPhd1h3wHeLKfMaEnTJU3fsGFDVXZTqmRMR87Ly6N+/fo++8o5d9BL5DyXLElnSXoeeAOYD5ybwHaTwjGRksuI+HJh66Oq39B/ILi/zJSyCpjZ42Y20MwGtmnTpoq7+bpYLMa2bdvYuHFjZHXm5+dz7LHH0qBBg8jqdM65VChztpikUwhaKmcQdD39FRhtZjsSqdjMTi6n7nWS2pvZGkntCaY3V0p4cmcb4L8ru20U4meMRZG0Nm/ezKxZs7jrrruqXZdzzqVaeS2XnwAfAoeb2dlmNiHRxJKAiUDxCRKjgH9UZmNJVwOnAZeE5+DUuKjPdXnvvfcwMx9vcc7VCuUN6J9oZuPMbEsS9jsWOEXSQuDk8DWSBkoaV1xI0hTgJeAkSSslnRa+9RjBOM1USbMk/SwJMZara9eu1KlTJ7Lkkp+fT4MGDRg8eHAk9TnnXCql5B60ZraJ4JIyJddPJ25asZkNKWP7lN87t379+nTu3Dmyc13y8vI4/vjjqV+/fiT1OedcKvlZcNUQ1XTkjRs3Mnv2bO8Sc87VGp5cqiGqS++/++67AGl7sUrnnKssTy7VEIvF2LJlC5s3b65WPfn5+TRq1IhBgwZFFJlzzqWWJ5dqiGrGWF5eHieccAL16tWLIiznnEs5Ty7VEEVyWb9+PfPmzfMuMedcreLJpRq6d++OpGoll+LxFh/Md87VJp5cqqFBgwZ07NixWtOR8/LyaNKkCQMGDIgwMuecSy1PLtVU3RljeXl5DBkyhOzsGr0ljXPOJZUnl2qqzrkua9eu5YsvvvDxFudcrePJpZpisRgbN25k69atld42Pz8f8PEW51zt48mlmopnjC1evLjS2+bl5dGsWTP69+8fdVjOOZdSnlyqqTrTkfPz8xkyZAhZWSm/VJpzzkXKk0s1de/eHah8clm9ejULFizwLjHnXK3kyaWaGjduzGGHHVbp5JKXlwf4eItzrnby5BKBWCxW6XNd8vLyaN68OX379k1SVM45lzqeXCJQlXNd8vPzGTp0KHXr1k1SVM45lzqeXCKQm5vLunXr2L59e0LlV6xYweLFi71LzDlXa3lyiUBlpyMXj7f4yZPOudrKk0sEKjsdOT8/n5YtW9KnT59khuWccynjySUCOTk5QOLJJS8vj29+85vUqeMfv3OudvJvtwg0bdqUtm3bJpRcli1bxrJly3y8xTlXq3lyiUii05GLryfm4y3OudrMk0tEEp2OnJeXR+vWrendu3cNROWcc6nhySUiubm5rF69mh07dpRZxszIy8tj2LBhPt7inKvV/BsuIsUzxpYsWVJmmSVLlrBixQrvEnPO1XqeXCKSyHRkv3+Lcy5TpCS5SGop6R1JC8PHFmWUe1PSVkmvlfH+byV9ldxoE5PIdOS8vDwOPfRQDj/88JoKyznnUiJVLZfbgMlmlgtMDl+X5iHgO6W9IWkgUGpSSoXmzZvTunXrMpNL/HiLpBqOzjnnalaqkssIYHz4fDwwsrRCZjYZ+NoFuyTVJUg8tyQrwKoob8bYokWLWL16tXeJOecyQqqSS1szWxM+Xwu0reT2PwAmxtVRJkmjJU2XNH3Dhg2VjbNSyjvXxe/f4pzLJElLLpImSZpbyjIivpyZGWCVqPcw4ALgd4mUN7PHzWygmQ1s06ZNpY6hsmKxGCtWrKCgoOBr7+Xl5dGuXTt69OiR1Biccy4dJO3m7WZ2clnvSVonqb2ZrZHUHlhfiar7AzFgUTh20UjSIjOLVS/i6svNzQVg6dKl9OrVa/96MyM/P5/hw4f7eItzLiOkqltsIjAqfD4K+EeiG5rZ/5lZOzPramZdgZ3pkFig7OnI8+fPZ+3atd4l5pzLGKlKLmOBUyQtBE4OXyNpoKRxxYUkTQFeAk6StFLSaSmJNkFlJRcfb3HOZZqkdYuVx8w2ASeVsn46cHXc6yEJ1NUk2uiqrmXLlrRo0eJrySU/P58OHTrsPxfGOedqOz9DP2IlpyP7eItzLhN5colYyenIn332GevXr/cuMedcRvHkErFYLMby5cvZvXs38J/xFr9YpXMuk3hyiVgsFqOoqIhly5YBwXhL586d6datW2oDc865GuTJJWLF57osWrSIoqIiH29xzmUkTy4Ri5+OPHfuXDZt2uRdYs65jJOSqci1WevWrWnWrBmLFi3a31rxwXznXKbx5BIxSfunI69cuZJu3brRpUuXVIflnHM1ypNLEsRiMaZNm8bWrVsZObLUuwk451yt5mMuSRCLxVi6dClbtmzxLjHnXEby5JIExYP64OMtzrnM5MklCYqnI8diMTp27JjiaJxzruZ5ckmC4paLT0F2zmUqH9BPgrZt23LPPfdw7rnnpjoU55xLCU8uSSCJO+64I9VhOOdcyni3mHPOuch5cnHOORc5Ty7OOeci58nFOedc5Dy5OOeci5wnF+ecc5Hz5OKccy5ynlycc85FTmaW6hhqjKTtwPxUx5FErYGNqQ4iSWrzsYEf38Guth9fTzNrWpkNMu0M/flmNjDVQSSLpOm19fhq87GBH9/BLhOOr7LbeLeYc865yHlycc45F7lMSy6PpzqAJKvNx1ebjw38+A52fnwlZNSAvnPOuZqRaS0X55xzNcCTi3POuchlRHKRdLqk+ZIWSbot1fFESVInSXmSPpM0T9INqY4pGSTVlTRT0mupjiVqkppL+pukLyR9LunYVMcUJUk/Cv8250r6i6QGqY6pOiT9WdJ6SXPj1rWU9I6kheFji1TGWFVlHNtD4d/mbEmvSGqeSF21PrlIqgv8HvgW0Au4RFKv1EYVqULgJjPrBRwDXFfLjq/YDcDnqQ4iSX4DvGlm3wD6UouOU1IH4HpgoJkdAdQFLk5tVNX2NHB6iXW3AZPNLBeYHL4+GD3N14/tHeAIM+sDLAB+kkhFtT65AIOBRWa2xMz2AH8FRqQ4psiY2Roz+yR8vp3gi6lDaqOKlqSOwLeBcamOJWqSDgGGAk8CmNkeM9ua2qgilwU0lJQFNAJWpzieajGz94DNJVaPAMaHz8cDI2s0qIiUdmxm9raZFYYv/wV0TKSuTEguHYAVca9XUsu+fItJ6gr0Bz5KbSSR+zVwC1CU6kCSoBuwAXgq7PYbJ6lxqoOKipmtAh4GlgNrgH+b2dupjSop2prZmvD5WqBtKoNJou8CbyRSMBOSS0aQ1AT4O/BDM9uW6niiIulMYL2ZzUh1LEmSBRwF/NHM+gM7OHi7VL4mHHsYQZBEDwMaS7ostVEllwXnd9S6czwkjSHohn8+kfKZkFxWAZ3iXncM19UakrIJEsvzZvZyquOJ2PHA2ZKWEXRpnijpudSGFKmVwEozK25t/o0g2dQWJwNLzWyDme0FXgaOS3FMybBOUnuA8HF9iuOJlKQrgDOBSy3BkyMzIblMA3IldZNUj2AwcWKKY4qMJBH0139uZr9KdTxRM7OfmFlHM+tK8G/3TzOrNb98zWwtsEJSz3DVScBnKQwpasuBYyQ1Cv9WT6IWTViIMxEYFT4fBfwjhbFEStLpBN3SZ5vZzkS3q/XJJRyI+gHwFsEf9YtmNi+1UUXqeOA7BL/oZ4XLGakOylXK/wDPS5oN9AMeSHE8kQlbZH8DPgHmEHznHNSXSpH0F2Aq0FPSSklXAWOBUyQtJGitjU1ljFVVxrE9CjQF3gm/Xx5LqC6//Itzzrmo1fqWi3POuZrnycU551zkPLk455yLnCcX55xzkfPk4pxzLnKeXFxKSTJJv4x7/WNJd0VU99OSzo+irgr2c0F4NeO8uHVHxk0N3yxpafh8UoJ1nl3RFbwlHSbpb9WNP6zrCkkb4mKeFeUFUCXdJenHUdXn0l9WqgNwGW83cK6kn5vZxlQHU0xSVtzF+ipyFfA9M3u/eIWZzSE4ZwVJTwOvmdkBiaC8fZjZRCo42dfMVgNRJs8XzOwHEdbnMpi3XFyqFRKcVPejkm+UbHlI+ip8HCbpXUn/kLRE0lhJl0r6WNIcSTlx1ZwsabqkBeF1yorvDfOQpGnhPSr+O67eKZImUspZ8pIuCeufK+kX4bqfAScAT0p6qKKDlZQv6deSpgM3SDpL0kfhRSsnSWoblrtC0qNxn8NvJX0YHu/54fquCu+7EZZ/WdKbCu4p8mDcPq8Kj/9jSU8U15uI8DN5T9L/Kbgn0mOS6pT1eYTrT5f0iaRPJU2Oq65XePxLJF0flm0c1v1pWM9Ficbm0pu3XFw6+D0wO/4LMQF9gcMJLg++BBhnZoMV3Cztf4AfhuW6Etx2IQfIkxQDLie4Ou8gSfWBDyQVX6n3KIJ7VyyN35mkw4BfAAOALcDbkkaa2T2STgR+bGbTE4y9npkNDOttARxjZibpaoLLbNxUyjbtCZLYNwhaNKV1h/UjuCr2bmC+pN8B+4A7wuPaDvwT+LSMuC6SdELc6+Kblg0muBfSl8CbBC3NDynl8wA+AJ4AhprZUkkt4+r7BjCc4Gzv+ZL+SHDvkNVm9u3w8zikjNjcQcaTi0s5M9sm6RmCm0oVJLjZtOJLnEtaDBQnhzkEX2DFXjSzImChpCUEX3CnAn3iWkWHALnAHuDjkoklNAjIN7MN4T6fJ7gPy6sJxhvvhbjnHYEXFFzssB5Q2r4BXg2P47Pi1k0pJpvZv8P4PgO6AK2Bd81sc7j+JaBHWXGV7BaTBMFnsiR8/ReCJLeX0j+PfcB7xZ9h8X5D/2dmu4HdktYTXJZ+DvDLsOXzmplNKSM2d5DxbjGXLn5NMHYRfy+TQsK/0bArpl7ce7vjnhfFvS7iwB9NJa9vZICA/zGzfuHSLe4eIzuqdRSJid/H74BHzexI4L+Bsm4BHH+8SqDMPqL78VjaZ1gVX4vPzBYQtKrmAPeF3YyuFvDk4tJC+Av3RYIEU2wZQbcLwNlAdhWqvkBSnXAcpjswn+AiptcouFUBknqo4ht0fQx8U1JrBbfOvgR4twrxlHQI/7kFxKjyClbRNIK4Wyi4E+R5VahjsIKritcBLgLep+zP41/AUEndILi3fHkVh92NO83sOeAhatftBjKad4u5dPJLgitYF3sC+IekTwn6+qvSqlhO8EXYDPi+me2SNI5gLOYTBf0+G6jgtrRmtkbB1OA8gpbD/5lZFJdVvwt4SdIWgvGQbhHUuZ+ZrZL0AMFnsBn4Avh3GcVLjrlcGz5OI7gybozg+F8xs6KyPg9Jo4GXw2S0HjilnBCPBB6SVETQ1XZNFQ7TpSG/KrJztZykJmb2VdhyeQX4s5m9kuC2wwgmK5yZzBhd7ePdYs7VfndJmgXMJZgwUJVJCM5VirdcnHPORc5bLs455yLnycU551zkPLk455yLnCcX55xzkfPk4pxzLnL/D8/rfSPb85w6AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TwOnoQhB6XMA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1ebe6a48-faa9-4387-cf53-1040cbbc4ce9"
      },
      "source": [
        "## Question 3: Accuracy on training, validation, and testing dataset \n",
        "\n",
        "# predict labels for test/valid/train data using trained regressor \n",
        "predictions_test, labels_test = Logistic_Regression.predicted_output(X_test,trained_weights, trained_bias)\n",
        "predictions_valid, labels_valid = Logistic_Regression.predicted_output(X_val,trained_weights, trained_bias)\n",
        "predictions_train, labels_train = Logistic_Regression.predicted_output(X_train,trained_weights, trained_bias)\n",
        "# calculate accuracies for train/valid/test datasets \n",
        "accuracy_train = Logistic_Regression.accuracy(labels_test, y_test)\n",
        "accuracy_valid = Logistic_Regression.accuracy(labels_valid, y_val)\n",
        "accuracy_test = Logistic_Regression.accuracy(labels_train, y_train)\n",
        "\n",
        "print('Training accuracy:', accuracy_train)\n",
        "print('Validation accuracy:', accuracy_valid)\n",
        "print('Testing accuracy:', accuracy_test)"
      ],
      "execution_count": 196,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training accuracy: 0.8666666666666667\n",
            "Validation accuracy: 1.0\n",
            "Testing accuracy: 0.9495798319327731\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6WXk67LXlQX9",
        "outputId": "45916507-6d01-4787-c124-cb3e645507fa"
      },
      "source": [
        "## Question 4: Comparison to SKL logistic regressor \n",
        "\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "\n",
        "# instantiate Preprocessing class instance (repeated from above)\n",
        "Processor = Preprocessing(train_ratio=0.8, test_ratio=0.1, val_ratio=0.1)\n",
        "X, y = Processor.import_data()   \n",
        "X_train_set, X_test, y_train_set, y_test = Processor.create_train_set(X,y)  \n",
        "X_train, X_val, y_train, y_val = Processor.create_validation_set(X_train_set, \n",
        "                                                                 y_train_set)\n",
        "\n",
        "clf = LogisticRegression()  # instantiate log reg instance\n",
        "clf.fit(X_train, y_train)  # fit to training data\n",
        "\n",
        "# predict labels for train/validation/test data sets \n",
        "predictions_train = clf.predict(X_train)\n",
        "predictions_val = clf.predict(X_val)  \n",
        "predictions_test = clf.predict(X_test)\n",
        "# accuracy scores for train/validation/test data sets \n",
        "accuracy_train = clf.score(X_train, y_train)\n",
        "accuracy_valid = clf.score(X_val, y_val)\n",
        "accuracy_test = clf.score(X_test, y_test)\n",
        "\n",
        "print('SKL training accuracy:', accuracy_train)\n",
        "print('SKL validation accuracy:', accuracy_valid)\n",
        "print('SKL testing accuracy:', accuracy_test)"
      ],
      "execution_count": 197,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "SKL training accuracy: 0.9747899159663865\n",
            "SKL validation accuracy: 1.0\n",
            "SKL testing accuracy: 1.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T1GuyEtCqJGY"
      },
      "source": [
        "# instantiate Preprocessing class instance  \n",
        "Processor = Preprocessing(train_ratio=0.8, test_ratio=0.1, val_ratio=0.1)\n",
        "# import Iris data set and store into X and y matrices \n",
        "X, y = Processor.import_data()   \n",
        "# split X, y input datasets into training and testing sets (90:10)\n",
        "X_train_set, X_test, y_train_set, y_test = Processor.create_train_set(X,y)  \n",
        "# split training dataset into training and validation sets (90:10)\n",
        "X_train, X_val, y_train, y_val = Processor.create_validation_set(X_train_set, \n",
        "                                                                 y_train_set)\n",
        "# train logistic regressor using MSE\n",
        "trained_weights2, trained_bias2, loss2 = Logistic_Regression.train_mse(X_train, \n",
        "                                                                y_train, X_val, \n",
        "                                                                y_val, 50, 0.1)\n"
      ],
      "execution_count": 198,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 349
        },
        "id": "y6sk8Q-tr-g2",
        "outputId": "3b556058-aa3b-4b12-83f2-69fc2728b98f"
      },
      "source": [
        "## Question 5: Visualize learning curve for loss and epochs during training\n",
        "Logistic_Regression.plot_loss_epochs(loss2) \n",
        "\n",
        "''' Note: Since early termination condition does not initiate until at least \n",
        "10 epochs of training have occurred, an increase in loss prior to \n",
        "10th iteration will not result in early termination. I did this purely\n",
        "for the purpose of showing > 3-4 epochs on the graph. Without the \n",
        "minimum iterations condition, training terminates after 4 epochs. '''\n"
      ],
      "execution_count": 199,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "' Note: Since early termination condition does not initiate until at least \\n10 epochs of training have occurred, an increase in loss prior to \\n10th iteration will not result in early termination. I did this purely\\nfor the purpose of showing > 3-4 epochs on the graph. Without the \\nminimum iterations condition, training terminates after 4 epochs. '"
            ]
          },
          "metadata": {},
          "execution_count": 199
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5hU9dn/8feHuiAdAZEOs4nyGEvAyhNjeYzYwCSaSESUoMRGrIkFRcDYgiYkys+ICIqINQqiqCj2YKEYiKLo0ntHqpTd+/fHOUuGdcvMMjNnd/Z+XddcnDn1PrPDuedbzvnKzHDOOecSVS3qAJxzzlUunjicc84lxROHc865pHjicM45lxRPHM4555LiicM551xSPHG4fUh6TdLFqV7XZZakSyR9WMryn0ial+C+TpK0LHXRpZekdyVdGnUc2cwTRxaQtDXuVSBpR9z7C5PZl5mdYWZPpHrdZER5oVLgD5K+CT/HJZLukVQ7Q8c/SZJJeqnI/CPC+e+Wc78mKVb43sw+MLMf7me4iRz3cUm7inxHZ6f7uC69PHFkATOrV/gClgDnxM17qnA9STWii7LS+DvQH+gD1AfOAE4Fnkv1gUr5e6wFjpfUNG7excDXqY4hQ/4c/x01syOiDsjtH08cWazwl7ukmyStAsZIaizpFUlrJW0Mp1vHbbO3mF9Y3SHp/nDdhZLOKOe6HSS9L2mLpLckjZA0rhzndGh43E2SvpDUI27ZmZLmhsdYLunGcP6B4XlukrRB0geSvvfdl5QLXAlcaGYfmdkeM/sC+CXQXdIpko6VtEpS9bjtfi5pTjhdTdLNkuZLWi/pOUlNwmXtw1/+/SQtAd4u4TR3AROAC8LtqgO/BuJ/BBTuq0bcvGKraCS9H07ODn/x/7poqU7SIkm3hJ/fRkljJOWU8Dc4WNI/w+/QQkm/L+E8ShV3Dv0lrZC0svBvFi6vLWl4uGxFOF07bnlPSf+WtDn8vLvH7b6dpH+F34Upkg4Mt8mRNC7822ySNF1Si/LEX5V54sh+BwFNgHYEv6SrAWPC922BHcBDpWx/LDAPOBD4M/CYJJVj3fHAp0BTYDBwUbInIqkmMAmYAjQHBgBPSSqscnkM+J2Z1QcO478X5huAZUAzoAVwK1Dcs3ZOBZaZ2afxM81sKfAxcJqZfQJsA06JW+U34fkRxnQu8FPgYGAjMKLIcX4KHAqcXsrpjiUo9RCu9zmwopT1S2RmJ4aTR4S/+J8tYdULw2N1An4A3FZ0hTDhTgJmA60IPrNrJZV2LmU5GcgFfgbcJOn/wvkDgeOAI4EjgGMKY5J0DMFn9AegEXAisChun78B+hJ8T2oBhQnpYqAh0Ibgu3g5wf8BlwRPHNmvALjDzHaa2Q4zW29m/zSz7Wa2BbiL4EJWksVm9qiZ5QNPAC0JLr4JryupLXA0MMjMdpnZh8DL5TiX44B6wL3hft4GXgF6hct3A50lNTCzjWY2K25+S6Cdme0O6/eLSxwHAitLOPbKcDnA04XHlFQfODOcB8GFaKCZLTOznQRJ8jztWy012My2mVmJFywzmwY0CZNiH4KLZLo9ZGZLzWwDwfeiVzHrHA00M7Oh4d9gAfAoYemoBDeGv+4LX0XbxYaEn8d/CH7UFB73QmComa0xs7XAEP77g6MfMNrM3jSzAjNbbmZfxe1zjJl9HX7GzxEkHwi+C02BmJnlm9lMM9ucyIfj/ssTR/Zba2bfFb6RVFfSI5IWS9oMvA80iq96KWJV4YSZbQ8n6yW57sHAhrh5AEuTPA/C/Sw1s4K4eYsJfvlCUKV0JrBY0nuSjg/nDwPygCmSFki6uYT9ryNIMMVpGS6HoHTxi7Da5BfALDNbHC5rB7xUeJEEvgTy2TfZJnruTwJXE/wif6mMdVMhPq7FBJ93Ue2Ag+MTAUEJrrTqnvvNrFHcq2hPvJKOe3D4vrhlbYD5pRxzVdz0dv77nX0SeAN4Jqz++nNYknVJ8MSR/Yr+sr4B+CFwrJk1ICjiA5RU/ZQKKwl+PdeNm9emHPtZAbQp0j7RFlgOYGbTzawnQfXEBMIGbTPbYmY3mFlHoAdwvaRTi9n/2+H+j4mfKakNQWlnari/uQQXsTPYt5oKgovgGUUulDlmtjxunUQfSf0kQZvL5CJJF4LqMoD4z/SgBPdbkvi/SVuKrxpbCiwscn71zezMNBx3BUGiKm7ZUoIqtaSEJc4hZtYZOAE4m/9WCboEeeKoeuoT1OluChtt70j3AcNf4zOAwZJqhSWBc8raLmzI3PsiaCPZDvxRUk1JJ4X7eSbc74WSGprZbmAzQTUdks6WFAvbW74lKAEUFD2emX0N/IOg3eQ4SdUl/Q/wT+AtM3srbvXxwDUEiff5uPn/AO6S1C48djNJPRP/tPaJZyFBNeLAYpatJUiYvcM4f0vpF9LVQMcyDnmVpNbh92IgUFxbyKfAFgUdLuqExz5M0tGJnFMJbg9Lwv9D0C5ReNyngdvCz/BAYBBQ2KHiMaCvpFMVdEhoJemQsg4k6WRJPwpL2JsJqq6+911wpfPEUfUMB+oQVLt8DLyeoeNeCBwPrAf+RHBx2FnK+q0IElz8qw1BojiDIP7/B/SJq9u+CFgUVsFdHh4TgobXt4CtwEfA/zOzd0o47tXAKIIL1FaCz+ddgmqweE8TXNTfNrN1cfP/RtB+M0XSFoLP+NhSzrNUZvahmZXUKH4ZQePweuB/gGml7Gow8ERYvfSrEtYZT9DxYAFBNdCfioknn+BX+pHAQoK/wyiCBueS/FH73sexrsjy9wiqEqcSVGtNCef/ieAHxxzgP8CswpjCDgx9gb8S/Bh4j31LJyU5CHiBIGl8GW73ZALbuTjygZxcFCQ9C3xlZmkv8biySVoEXFqkVJXuY7YnSD41zWxPpo7r9p+XOFxGSDpaUqewWqE70JOgHcI5V8n4ncQuUw4CXiToCrkMuMLMPos2JOdceXhVlXPOuaR4VZVzzrmkZE1V1YEHHmjt27ePOgznnKtUZs6cuc7MmiWzTdYkjvbt2zNjxoyow3DOuUpF0uKy19qXV1U555xLiicO55xzSfHE4ZxzLimeOJxzziXFE4dzzrmkeOJwzjmXFE8czjnnkpI1iWP58uWsW1f0ac3OOedSLWsSx6pVq5g7d27UYTjnXNbLmsQBkJeXF3UIzjmX9TxxOOecS0rWJI7atWt74nDOuQzwxOGccy4pWZM4cnJyyMvLwwemcs659Epr4pDUXdI8SXmSbi5m+YmSZknaI+m8Isvuk/R5+Pp1WceqXbs2W7ZsYe3atak8Beecc0WkLXFIqg6MAM4AOgO9JHUustoS4BJgfJFtzwJ+DBwJHAvcKKlBacerXbs24A3kzjmXbukscRwD5JnZAjPbBTwD9IxfwcwWmdkcoKDItp2B981sj5ltA+YA3Us7WE5ODgDffPNNisJ3zjlXnHQmjlbA0rj3y8J5iZgNdJdUV9KBwMlAm9I2qFWrFtWrV/cSh3POpVmFHDrWzKZIOhqYBqwFPgLyi64nqT/QH6Bt27a0a9fOE4dzzqVZOkscy9m3lNA6nJcQM7vLzI40s9MAAV8Xs85IM+tqZl2bNWtGLBbzxOGcc2mWzsQxHciV1EFSLeAC4OVENpRUXVLTcPpw4HBgSlnb5ebm8s0333iXXOecS6O0JQ4z2wNcDbwBfAk8Z2ZfSBoqqQeApKMlLQPOBx6R9EW4eU3gA0lzgZFA73B/pYrFYnz77bds2LAhHafknHOONLdxmNlkYHKReYPipqcTVGEV3e47gp5VSYnFYkDQJbdp06bJbu6ccy4BWXPnOOybOJxzzqVHViWODh06IMnv5XDOuTTKqsRRu3Zt2rZt6yUO55xLo6xKHIB3yXXOuTTzxOGccy4pWZc4cnNzWb9+PRs3bow6FOecy0pZlzgKe1bNnz8/4kiccy47ZW3i8Ooq55xLj6xLHB07dgT88erOOZcuWZc46tSpQ+vWrb3E4ZxzaZJ1iQO8Z5VzzqWTJw7nnHNJycrEkZuby5o1a9i8eXPUoTjnXNapkCMA7q/4LrlHHXVUxNE4l3579uxhzZo1rF69mtWrV7Nr165i1yttrJryLoPgcT916tTZ51W3bt290zk5OVSrlpW/U6ukrE4ceXl5njhcpZWfn8+6detYtWoVq1evLvXf9evXV/gBzAqTS3xCKSnRlDa/bt261KtXj3r16lG/fn3q16+/d7p27dpRn2aVkJWJo1OnToDfy+EqnoKCAtavX19iEoifXrt2LQUFBd/bR506dTjooIM46KCDiMVi/O///i8tWrTgoIMOokWLFrRo0YKcnJwSY5CU0mVmxs6dO9mxY8fe1/bt2/d5X9r87du3s2HDBpYvX/69+Tt37izl0/y+mjVr7k0ixSWWROcVTtetW7fUz6SqysrEccABB9CyZUu/l8NFauPGjbz66qtMmjSJr7/+mtWrV7NmzRry8/O/t25OTs7ei3/79u057rjjvpcMCqfr1atXZS5mBQUFfPfdd/skk+3bt7Nt2za2bNnC1q1b2bJlS4nThf+uWrVqn+UlVeUVJYl69erRoEEDGjRoQP369cs1Xb9+ferUqZM1f7esTBzgPatcNBYvXszEiROZOHEi7733Hvn5+bRs2ZIuXbrQtWvX7yWBwn8bNGiQNReVVKpWrRp169albt26Kd3vrl272Lp1a6nJpnB68+bNe98XTq9cuXLv9ObNm4stGRZVvXr1hBPOZZddRoMGDVJ6zqmU1Ynj9ddfjzoMl+XMjNmzZzNx4kQmTJjAv//9bwA6d+7MTTfdxLnnnkuXLl28YbiCqVWrFk2aNKFJkyb7vS8zY8eOHWzevHmfZBI/XdK8jRs3snjx4r3ztm7dipnRu3dvTxxRiMVirFy5km3btnHAAQdEHY7LIrt37+aDDz7YW7JYvHgxkujWrRvDhg2jZ8+e5ObmRh2myxBJe0tFBx100H7tq6CgoFJcs7I2cRT+x50/fz6HH354xNG4ym7r1q288cYbTJgwgVdffZWNGzeSk5PDz372MwYNGsTZZ59N8+bNow7TVXLVqlWjfv36UYdRpqxNHPFdcj1xuPJYtWoVkyZNYuLEibz11lvs3LmTpk2b0qNHD84991xOO+20Cv/L0Ll0yNrE4V1yXXnMmzePCRMmMHHiRD7++GPMjA4dOnDllVfSs2dPunXrRo0aWfvfxrmEZO3/gAYNGtC8eXPvkutKVVBQwCeffLK3cXvevHkAdOnShSFDhnDuuedy2GGHeY8n5+JkbeIA75Lrirdz506mTp3KhAkTmDRpEqtWraJGjRqcdNJJDBgwgB49etCmTZuow3Suwsr6xPH2229HHYarQD777DN++ctfsnDhQurVq8eZZ55Jz549OfPMM2nUqFHU4TlXKWR94hg7diw7duygTp06UYfjIjZ27Fh+97vf0bRpUyZOnMjpp5/uzzZyrhyy+q6kwp5VCxYsiDgSF6Vdu3Zx9dVXc/HFF3Pccccxc+ZMevTo4UnDuXLK6sRReC+Ht3NUXStWrODkk09mxIgR3HDDDbz55pu0aNEi6rCcq9TSmjgkdZc0T1KepJuLWX6ipFmS9kg6r8iyP0v6QtKXkv6ucnRr8S65VdsHH3zAj3/8Y2bPns0zzzzD/fff711pnUuBtCUOSdWBEcAZQGegl6TORVZbAlwCjC+y7QlAN+Bw4DDgaOCnycbQuHFjmjZt6omjijEz/va3v3HKKafQoEEDPvnkE379619HHZZzWSOdP7+OAfLMbAGApGeAnsDcwhXMbFG4rOijJQ3IAWoBAmoCq8sTRCwW83s5qpBt27bRv39/xo8fT48ePRg7diwNGzaMOiznsko6q6paAUvj3i8L55XJzD4C3gFWhq83zOzLoutJ6i9phqQZa9euLXZffi9H1ZGXl8fxxx/P008/zZ/+9CdeeuklTxrOpUGFbByXFAMOBVoTJJtTJP2k6HpmNtLMuppZ12bNmhW7r1gsxpIlS5IeScxVLq+++ipdu3Zl2bJlvPbaawwcONAfZe5cmqTzf9ZyIP7229bhvET8HPjYzLaa2VbgNeD48gQRi8UwMxYuXFiezV0FV1BQwODBgzn77LPp0KEDM2fO5PTTT486LOeyWjoTx3QgV1IHSbWAC4CXE9x2CfBTSTUk1SRoGP9eVVUivEtu9tq4cSPnnHMOQ4YMoU+fPkybNo0OHTpEHZZzWS9ticPM9gBXA28QXPSfM7MvJA2V1ANA0tGSlgHnA49I+iLc/AVgPvAfYDYw28wmlSeO+Meru+wxZ84cunbtyptvvsmIESN4/PHH/ekAzmVIWju1m9lkYHKReYPipqcTVGEV3S4f+F0qYmjSpAmNGjXyxJFFnnrqKS677DIaNWrEu+++ywknnBB1SM5VKVnfeijJu+Rmid27d3PNNdfQu3dvunbtyqxZszxpOBeBMhOHpE6SaofTJ0n6vaRK9RhR75Jb+a1cuZJTTjmFv//971xzzTVMnTp1v8d3ds6VTyIljn8C+WEX2ZEEPaXGl75JxRKLxVi0aBG7du2KOhRXDtOmTaNLly7MnDmT8ePHM3z4cGrWrBl1WM5VWYkkjoKwofvnwINm9gegZXrDSq1YLEZBQQGLFy+OOhSXBDPjoYce4qc//Sl169bl448/plevXlGH5VyVl0ji2C2pF3Ax8Eo4r1L93POeVZXP9u3bufjiixkwYACnn346M2bM4PDDD486LOcciSWOvgQ3391lZgsldQCeTG9YqeX3clQuCxYs4IQTTmDcuHEMGTKEl19+2Ufnc64CKbM7rpnNBX4PIKkxUN/M7kt3YKnUrFkz6tev74mjEnjttde48MILMTNeeeUVzjzzzKhDcs4VkUivqnclNZDUBJgFPCrpL+kPLXUKu+R64qi4CgoK+NOf/sRZZ51FmzZtmDFjhicN5yqoRKqqGprZZuAXwFgzOxb4v/SGlXp+L0fFtWnTJs4991xuv/12fvOb3/DRRx/tHYTLOVfxJJI4akhqCfyK/zaOVzqxWIyFCxeyZ8+eqENxcfLz8znnnHN47bXX+Pvf/86TTz5J3bp1ow7LOVeKRBLHUILnTc03s+mSOgKV7qd7LBZjz549LFmyJOpQXJwHH3yQDz/8kFGjRjFgwADKMUKwcy7DykwcZva8mR1uZleE7xeY2S/TH1pqeZfciicvL49bb72Vs846iz59+kQdjnMuQYk0jreW9JKkNeHrn5K+92DCis4TR8VSUFBAv379qFWrFo888oiXNJyrRBKpqhpDMI7GweFrUjivUmnZsiV169b1xFFBPPzww7z//vv85S9/oVWrhEYUds5VEIkkjmZmNsbM9oSvx4Hix2mtwLxLbsWxcOFCbrrpJk4//XT69u0bdTjOuSQlkjjWS+otqXr46g2sT3dg6eBdcqNnZlx22WVIYuTIkV5F5VwllEji+C1BV9xVwErgPOCSNMaUNrFYjAULFpCfnx91KFXWqFGjmDp1KsOGDaNt27ZRh+OcK4dEelUtNrMeZtbMzJqb2bnANRmILeVisRi7du1i2bJlUYdSJS1dupQbbriBk08+mf79+0cdjnOunMo7AuCvUhpFhnjPquiYGf379yc/P59Ro0ZRrVrWDz7pXNYq7//eSlkx7YkjOk888QSvv/469957Lx07dow6HOfcfijx6bjhQw2LXUQlTRytWrUiJyfHE0eGrVixguuuu46f/OQnXHXVVVGH45zbT6U9Vn0mYBSfJCrlGKzVqlWjU6dOnjgyyMy4/PLL+e6773jssce8isq5LFBi4jCzDpkMJFP8Xo7MGj9+PJMmTeKBBx7YO6CWc65yq3I//woTR0FBQdShZL1Vq1bx+9//nuOPP55rrqmUHfGcc8Wokonju+++Y8WKFVGHktXMjKuuuopt27YxevRoqlevHnVIzrkUqZKJA7xnVbo9//zzvPjiiwwZMoRDDjkk6nCccymUyNNxn0xkXmXhiSP91q5dy1VXXcXRRx/NDTfcEHU4zrkUS6TE8T/xbyRVB7qkJ5z0a9OmDbVq1fLEkUYDBgzg22+/ZfTo0dSoUVrHPedcZVRi4pB0i6QtwOGSNkvaEr5fA0zMWIQpVr16dTp27OiJI01eeuklnn32WQYNGsRhhx0WdTjOuTQoMXGY2T1mVh8YZmYNzKx++GpqZrcksnNJ3SXNk5Qn6eZilp8oaZakPZLOi5t/sqR/x72+k3Ruuc6wGN4lNz3Wr1/PFVdcwVFHHcVNN90UdTjOuTRJpKpqYPhY9dsBJLWRdExZG4VVWiOAM4DOQC9JnYustoTgSbvj42ea2TtmdqSZHQmcAmwHpiQQa0IKE4eZpWqXDrj22mtZv349Y8aMoWbNmlGH45xLk0QSxwjgeOA34fut4byyHAPkhWOU7wKeAXrGr2Bmi8xsDlDaTRXnAa+Z2fYEjpmQWCzGtm3bWLVqVap2WeW98sorjBs3jltvvZUjjjgi6nCcc2mUSOI41syuAr4DMLONQK0EtmsFLI17vyycl6wLgKeLWyCpv6QZkmasXbs24R16z6rU2rRpE7/73e/40Y9+xMCBA6MOxzmXZokkjt1htZMBSGpG6SWElJHUEvgR8EZxy81spJl1NbOuzZolPpqtJ47Uuv7661m9ejVjxoyhVq1EflM45yqzRBLH34GXgOaS7gI+BO5OYLvlQJu4963Decn4FfCSme1OcrtStWvXjho1anjiSIHXX3+dMWPG8Mc//pEuXSptL23nXBLK7GRvZk9JmgmcGs4618y+TGDf04FcSR0IEsYF/LedJFG9gIR6cCWjRo0atG/f3hPHftq8eTOXXXYZhx56KIMGDYo6HOdchpR2H0ddSTUBzOwr4C2Cto1DE9mxme0BriaoZvoSeM7MvpA0VFKP8BhHS1oGnA88IumLuOO3JyixvFeO8ypTbm6uJ4799Ic//IEVK1YwZswYcnJyog7HOZchpZU4Xgf6Ad9IigEfAU8BZ0s6OpF7OcxsMjC5yLxBcdPTCaqwitt2EeVrTE9ILBbjX//6F2aGVCnHpYrU1KlTGTlyJDfeeCPHHnts1OE45zKotDaOxmb2TTh9MfC0mQ0guC/j7LRHlmaxWIzNmzeTTG8sF9i6dSuXXnopP/jBDxg6dGjU4TjnMqy0xBF/d9wpwJsA4T0ZlX4wC+9ZVX633HILixcvZvTo0dSpUyfqcJxzGVZa4pgj6X5J1wExwju3JTXKSGRp5omjfN5//30eeughBgwYQLdu3aIOxzkXgdISx2XAOqA98LO4O7c7A/enOa60a9++PdWqVfPEkYTt27fz29/+lo4dO3L33Yn0yHbOZaPSxhzfAdxbzPxpwLR0BpUJtWrVol27dp44knDbbbcxf/583nnnHQ444ICow3HORaTKjQAYz7vkJm7atGkMHz6cK664gpNOOinqcJxzEarSicMfr56YHTt28Nvf/pa2bdty3333RR2Ocy5iVXp4tlgsxsaNG9mwYQNNmjSJOpwKa/DgwcybN48pU6ZQv379qMNxzkUskTHHfyDpUUlTJL1d+MpEcOlW2LPqm2++KWPNquvTTz/l/vvv59JLL+W0006LOhznXAWQSInjeeAfwKNAfnrDyaz4Lrl+9/P37dy5k759+3LwwQdz//2VviOdcy5FEkkce8zs4bRHEoEOHTogyds5SnDnnXcyd+5cJk+eTMOGDaMOxzlXQSTSOD5J0pWSWkpqUvhKe2QZkJOTQ5s2bTxxFGPWrFnce++9XHzxxZxxxhlRh+Ocq0ASKXFcHP77h7h5BnRMfTiZ5z2rvm/Xrl307duX5s2b89e//jXqcJxzFUwi43F0yEQgUcnNzeWf//xn1GFUKPfffz9z5sxh4sSJNG7cOOpwnHMVTJmJIxyT4wrgxHDWu8AjqR6VLyqxWIx169axadMmGjXKisdw7Zd169Zx77338vOf/5wePXpEHY5zrgJKpI3jYaAL8P/CV5dwXlbwhx3u65577mHbtm3cddddUYfinKugEmnjONrMjoh7/7ak2ekKKNPiE0fXrl0jjiZay5YtY8SIEfTp04dDD01ooEfnXBWUSIkjX1KnwjeSOpJF93N07Bi08XuJA4YOHYqZMXjw4KhDcc5VYImUOP4AvCNpASCgHdA3rVFlUN26dWnVqlWVTxxff/01o0eP5qqrrqJdu3ZRh+Ocq8AS6VU1VVIu8MNw1jwz25nesDLLu+TCoEGDyMnJYeDAgVGH4pyr4EqsqpJ0SvjvL4CzCEYBjAFnhfOyRlVPHJ999hnPPvss1113Hc2bN486HOdcBVdaieOnwNvAOcUsM+DFtEQUgdzcXFavXs2WLVuq5NNfBw4cSOPGjbnhhhuiDsU5VwmUNgLgHeHkUDNbGL9MUlbdFFjYs2r+/PkceeSREUeTWR988AGvvfYa9913n9/H4pxLSCK9qoq7rfqFVAcSpar6eHUz45ZbbqFly5ZcffXVUYfjnKskSixxSDoE+B+gYZE2jQZATroDy6ROnYLexlWtnWPy5Mn861//4uGHH6Zu3bpRh+OcqyRKa+P4IXA20Ih92zm2AJelM6hMq1evHgcddFCVShwFBQUMHDiQTp060a9fv6jDcc5VIqW1cUwEJko63sw+ymBMkahqPaueffZZZs+ezVNPPUXNmjWjDsc5V4kkcgPgZ5KuIqi22ltFZWa/TVtUEYjFYkyZMiXqMDJi9+7dDBo0iMMPP5wLLrgg6nCcc5VMIo3jTwIHAacD7wGtCaqrskpubi4rVqxg27ZtUYeSdmPGjCEvL4+77rqLatUS+Qo459x/JXLViJnZ7cA2M3uC4GbAhAboltRd0jxJeZJuLmb5iZJmSdoj6bwiy9pKmiLpS0lzJbVP5JjlVdizasGCBek8TOR27NjBkCFDOOGEEzjrrLOiDsc5VwklkjgKx93YJOkwoCFQ5u3FkqoDI4AzgM5AL0mdi6y2BLgEGF/MLsYCw8zsUOAYYE0CsZZbVemSO2LECFasWME999yDpKjDcc5VQom0cYyU1Bi4HXgZqAcMSmC7Y4A8M1sAIOkZoCcwt3AFM1sULiuI3zBMMDXM7M1wva0JHG+/VIUuud9++y333Hh6BvUAABZhSURBVHMP3bt358QTTyx7A+ecK0YiDzkcFU6+R3LjjLcClsa9X0aCVVzADwhKOC8CHYC3gJvNbJ/HuUvqD/QHaNu2bRKhfV/Dhg1p1qxZVieOBx54gA0bNvggTc65/VLaDYDXl7ahmf0l9eHsVQP4CXAUQXXWswRVWo8ViWEkMBKga9eutr8HzeYuuWvWrOEvf/kL559/Pj/+8Y+jDsc5V4mV1sZRP3x1JRhzvFX4uhxI5MqzHGgT9751OC8Ry4B/m9kCM9sDTEjwmPslmxPH3XffzXfffcedd94ZdSjOuUquxMRhZkPMbAjBBf/HZnaDmd1AMOZ4IvVC04FcSR0k1QIuIGgjScR0oJGkZuH7U4hrG0mXWCzG0qVL2bFjR7oPlVGLFy/m4Ycf5pJLLuGHP/xh2Rs451wpEulV1QLYFfd+VzivVGFJ4WrgDeBL4Dkz+0LSUEk9ACQdLWkZcD7wiKQvwm3zgRuBqZL+QzDy4KOJn1b55ObmArBw4cIy1qxchgwZgiTuuOOOsld2zrkyJNKraizwqaSXwvfnAo8nsnMzmwxMLjJvUNz0dIISTXHbvgkcnshxUqWwS25eXh6dOxftOVw5ffXVVzzxxBNcc801tGnTpuwNnHOuDIn0qrpL0msEjdUAfc3ss/SGFY1svJfj9ttvp27dutxyyy1Rh+KcyxKl9apqYGabJTUBFoWvwmVNzGxD+sPLrMaNG9OkSZOsaSCfOXMmL7zwAnfccQfNmjUrewPnnEtAaSWO8QSPVZ9JMFRsIYXvk7mno9LIpp5Vt956K02bNuX660vtWe2cc0kp7bHqZ4f/ZtUwsWWJxWJMmzYt6jD227vvvsuUKVN44IEHaNCgQdThOOeySGlVVaXeN2Fms1IfTvRisRjPPPMMO3fupHbt2lGHUy6FQ8K2bt2aK6+8MupwnHNZprSqqgdKWWYE91ZknVgsRkFBAYsWLaq09zxMmjSJjz/+mJEjR5KTk1Wj/DrnKoDSqqpOzmQgFUXhvRx5eXmVMnHk5+czcOBAcnNz6du3b9ThOOeyUCL3cRA+Tr0z+44AODZdQUWpsnfJffrpp/n888955plnqFEjoT+vc84lpcwri6Q7gJMIEsdkgvE1PiS4MTDrNG3alIYNG1bKnlW7du1i0KBBHHnkkZx//vlRh+Ocy1KJ/CQ9DzgC+MzM+kpqAYxLb1jRkVRpu+SOGjWKhQsXMnnyZB8S1jmXNolcXXaYWQGwR1IDgpH4svrZFZUxcWzfvp0777yTn/zkJ3Tv3j3qcJxzWSyRxDFDUiOChwzOBGYBH6U1qojFYjEWLVrE7t27y165gnjwwQdZtWqVDwnrnEu70u7jGAGMN7PCGwH+Iel1oIGZzclIdBGJxWLk5+ezePHivY3lFdmmTZu47777OOuss+jWrVvU4TjnslxpJY6vgfslLZL0Z0lHmdmibE8asO9TciuDYcOGsXHjRh8S1jmXEaUN5PQ3Mzse+CmwHhgt6StJd0j6QcYijED8vRwV3apVqxg+fDi9evXiiCOOiDoc51wVUGYbh5ktNrP7zOwooBfBeBxfpj2yCDVv3px69epVins57rrrLnbu3MnQoUOjDsU5V0WUmTgk1ZB0jqSngNeAecAv0h5ZhCpLl9yFCxfyyCOP0K9fv0rRFuOcyw6lNY6fRlDCOBP4FHgG6G9m2zIUW6RisRhz5lTs5pzBgwdTvXp1Bg0aVPbKzjmXIqWVOG4BpgGHmlkPMxtfVZIGBIlj4cKF7NmzJ+pQivXFF1/w5JNPcvXVV9OqVauow3HOVSGlPeQwK59+m6hYLMbu3btZunQpHTpUvCFJbrvtNurXr8/NN98cdSjOuSrGn0tRgorcJfeTTz5hwoQJ3HjjjTRt2jTqcJxzVYwnjhJU5C65AwcOpFmzZlx77bVRh+Kcq4L8udslaNmyJXXq1KlwiWPq1KlMnTqV4cOHU79+/ajDcc5VQV7iKEFhl9yKdC+HmXHrrbfStm1bLr/88qjDcc5VUV7iKEUsFuOrr76KOoy9JkyYwKeffsro0aMr7XjozrnKz0scpYjFYsyfP5/8/PyoQyE/P5/bbruNQw45hIsuuijqcJxzVZiXOEoRi8XYtWsXy5cvp23btpHGMm7cOObOncsLL7zgQ8I65yLlJY5SVJQuuTt37uSOO+6gS5cu/OIXWf20F+dcJeCJoxQVJXE89NBDLF68mLvvvtsHaXLORS6tiUNSd0nzJOVJ+t4tzpJOlDRL0h5J5xVZli/p3+Hr5XTGWZLWrVtTu3btSBPH3LlzGThwIOeccw6nnXZaZHE451yhtFWWS6oOjABOA5YB0yW9bGZz41ZbAlwC3FjMLnaY2ZHpii8R1apVo1OnTpF1yd21axe9e/emQYMGPProo17acM5VCOlsZT0GyDOzBQCSngF6AnsTh5ktCpcVpDGO/RLl49UHDx7MZ599xoQJE2jRokUkMTjnXFHprKpqBSyNe78snJeoHEkzJH0s6dziVpDUP1xnxtq1a/cn1hIVdsktKMhsbvvwww+577776NevHz179szosZ1zrjQVuXG8nZl1BX4DDJfUqegKZjbSzLqaWddmzZqlJYhYLMaOHTtYuXJlWvZfnC1bttCnTx/at2/PX//614wd1znnEpHOxLEcaBP3vnU4LyFmtjz8dwHwLnBUKoNLVBQ9q6677joWL17M2LFj/XlUzrkKJ52JYzqQK6mDpFrABUBCvaMkNZZUO5w+EOhGXNtIJmU6cUycOJHHHnuMm2++mW7dumXkmM45l4y0JQ4z2wNcDbwBfAk8Z2ZfSBoqqQeApKMlLQPOBx6R9EW4+aHADEmzgXeAe4v0xsqYNm3aULNmzYwkjtWrV3PZZZdx1FFHcccdd6T9eM45Vx5pfXaFmU0GJheZNyhuejpBFVbR7aYBP0pnbImqUaMGHTt2THviMDMuvfRSNm/ezLhx46hVq1Zaj+ecc+XlDz1KQCYerz5q1CheeeUVhg8fTufOndN6LOec2x8VuVdVhVF4L4eZpWX/eXl5XHfddZx66qkMGDAgLcdwzrlU8cSRgFgsxrZt21i9enXK971nzx769OlDzZo1efzxx6lWzf8kzrmKza9SCUhnz6p7772Xjz76iIcffpjWrb/X3OOccxWOJ44EpCtxzJgxgyFDhtCrVy8uuOCClO7bOefSxRNHAtq1a0f16tVTmji2b9/ORRddRIsWLRgxYkTK9uucc+nmvaoSULNmTTp06JDSxHHzzTfz1Vdf8dZbb9G4ceOU7dc559LNSxwJSmWX3ClTpvDggw9y7bXXcuqpp6Zkn845lymeOBKUqi65GzZsoG/fvnTu3Jm77747RdE551zmeOJIUCwWY/Pmzaxbt67c+zAzLr/8ctauXcu4ceOoU6dOCiN0zrnM8MSRoFT0rBo/fjzPP/88Q4YM4aijInnYr3PO7TdPHAna38SxZMkSrrrqKrp168Yf//jHVIbmnHMZ5YkjQe3bt6datWrlShwFBQVccskl5OfnM3bsWKpXr56GCJ1zLjO8O26CateuTdu2bcuVOIYPH84777zDY489RseOHdMQnXPOZY6XOJKQm5ubdOL4/PPPueWWW+jZsyd9+/ZNU2TOOZc5njiSkOy9HDt37qR37940atSIkSNHIimN0TnnXGZ4VVUSYrEYGzduZMOGDTRp0qTM9e+44w5mz57NpEmTaN68eQYidM659PMSRxKS6Vn1wQcf8Oc//5n+/ftz9tlnpzs055zLGE8cSUg0cWzevJk+ffrQsWNHHnjggUyE5pxzGeNVVUno2LEjkspMHNdccw1Llizhww8/pF69ehmKzjnnMsNLHEnIycmhdevWpSaOF198kccff5xbb72V448/PoPROedcZnjiSFLhww6Ls2rVKvr370+XLl0YNGhQhiNzzrnM8MSRpNzc3GK75JoZ/fr1Y9u2bYwbN46aNWtGEJ1zzqWfJ44kxWIx1q1bx6ZNm/aZP3LkSCZPnsywYcM45JBDIorOOefSzxNHkgp7Vs2fP3/vvK+//prrr7+en/3sZ1x55ZVRheaccxnhiSNJRbvk7tmzh4suuojatWszevRoqlXzj9Q5l928O26SCh9SWJg47r77bj799FOeffZZWrVqFWVozjmXEf7zOEkHHHAABx98MHl5eUyfPp2hQ4dy4YUX8qtf/Srq0JxzLiPSmjgkdZc0T1KepJuLWX6ipFmS9kg6r5jlDSQtk/RQOuNMViwWY86cOfTu3ZuDDz6Yhx6qUOE551xapa2qSlJ1YARwGrAMmC7pZTObG7faEuAS4MYSdnMn8H66Yiyv3NxcHnvsMQDefvttGjVqFHFEzjmXOekscRwD5JnZAjPbBTwD9IxfwcwWmdkcoKDoxpK6AC2AKWmMsVwKG8ivv/56Tj755Iijcc65zEpn43grYGnc+2XAsYlsKKka8ADQG/i/UtbrD/QHaNu2bbkDTdavfvUr1q9fz5133pmxYzrnXEVRURvHrwQmm9my0lYys5Fm1tXMujZr1ixDoQU9q4YNG0ZOTk7GjumccxVFOkscy4E2ce9bh/MScTzwE0lXAvWAWpK2mtn3Gtidc85lVjoTx3QgV1IHgoRxAfCbRDY0swsLpyVdAnT1pOGccxVD2qqqzGwPcDXwBvAl8JyZfSFpqKQeAJKOlrQMOB94RNIX6YrHOedcasjMoo4hJbp27WozZsyIOgznnKtUJM00s67JbFNRG8edc85VUJ44nHPOJcUTh3POuaR44nDOOZeUrGkcl7QFmBd1HGl0ILAu6iDSyM+vcsvm88vmcwP4oZnVT2aDbBqPY16yPQMqE0kz/PwqLz+/yiubzw2C80t2G6+qcs45lxRPHM4555KSTYljZNQBpJmfX+Xm51d5ZfO5QTnOL2sax51zzmVGNpU4nHPOZYAnDuecc0nJisQhqbukeZLyJGXV49cltZH0jqS5kr6QdE3UMaWapOqSPpP0StSxpJqkRpJekPSVpC8lHR91TKkk6brwe/m5pKclVerRzSSNlrRG0udx85pIelPSN+G/jaOMcX+UcH7Dwu/nHEkvSWpU1n4qfeKQVB0YAZwBdAZ6SeocbVQptQe4wcw6A8cBV2XZ+QFcQ/Do/Wz0N+B1MzsEOIIsOk9JrYDfE4yXcxhQnWDcncrscaB7kXk3A1PNLBeYGr6vrB7n++f3JnCYmR0OfA3cUtZOKn3iAI4B8sxsgZntAp4BekYcU8qY2UozmxVObyG48LSKNqrUkdQaOAsYFXUsqSapIXAi8BiAme0ys03RRpVyNYA6kmoAdYEVEcezX8zsfWBDkdk9gSfC6SeAczMaVAoVd35mNiUcPwngY4LRWkuVDYmjFbA07v0ysujCGk9Se+Ao4JNoI0mp4cAfgYKoA0mDDsBaYExYFTdK0gFRB5UqZrYcuB9YAqwEvjWzKdFGlRYtzGxlOL0KaBFlMGn2W+C1slbKhsRRJUiqB/wTuNbMNkcdTypIOhtYY2Yzo44lTWoAPwYeNrOjgG1U7mqOfYR1/T0JEuTBwAGSekcbVXpZcP9CVt7DIGkgQdX4U2Wtmw2JYznQJu5963Be1pBUkyBpPGVmL0YdTwp1A3pIWkRQxXiKpHHRhpRSy4BlZlZYQnyBIJFki/8DFprZWjPbDbwInBBxTOmwWlJLgPDfNRHHk3KSLgHOBi60BG7uy4bEMR3IldRBUi2CxrmXI44pZSSJoI78SzP7S9TxpJKZ3WJmrc2sPcHf7W0zy5pfrGa2Clgq6YfhrFOBuRGGlGpLgOMk1Q2/p6eSRY3/cV4GLg6nLwYmRhhLyknqTlBd3MPMtieyTaVPHGGjztXAGwRf2ufM7Itoo0qpbsBFBL/G/x2+zow6KJewAcBTkuYARwJ3RxxPyoQlqReAWcB/CK4nlfrxHJKeBj4CfihpmaR+wL3AaZK+IShl3RtljPujhPN7CKgPvBleX/5R5n78kSPOOeeSUelLHM455zLLE4dzzrmkeOJwzjmXFE8czjnnkuKJwznnXFI8cbi0kmSSHoh7f6OkwSna9+OSzkvFvso4zvnhk23fiZv3o7ju0RskLQyn30pwnz3KepKzpIMlvbC/8Yf7ukTS2riY/53Kh2VKGizpxlTtz1VsNaIOwGW9ncAvJN1jZuuiDqaQpBpxD3YrSz/gMjP7sHCGmf2H4L4MJD0OvGJm+1zkSzuGmb1MGTeqmtkKIJWJ8VkzuzqF+3NVlJc4XLrtIbgp7LqiC4qWGCRtDf89SdJ7kiZKWiDpXkkXSvpU0n8kdYrbzf9JmiHp6/DZV4XjewyTND0cY+B3cfv9QNLLFHMHt6Re4f4/l3RfOG8Q8L/AY5KGlXWykt6VNFzSDOAaSedI+iR8yOFbklqE610i6aG4z+HvkqaF53teOL+9wnETwvVflPS6gnEh/hx3zH7h+X8q6dHC/SYi/Ezel/SqgjFt/iGpWkmfRzi/u6RZkmZLmhq3u87h+S+Q9Ptw3QPCfc8O9/PrRGNzFZeXOFwmjADmxF/sEnAEcCjBI6AXAKPM7BgFA1kNAK4N12tP8Gj9TsA7kmJAH4IntR4tqTbwL0mFT239McHYAwvjDybpYOA+oAuwEZgi6VwzGyrpFOBGM5uRYOy1zKxruN/GwHFmZpIuJXi0ww3FbNOSIEEdQlASKa6K6kiCpyPvBOZJehDIB24Pz2sL8DYwu4S4fi3pf+PeFw4qdQzBWDaLgdcJSojTKObzAP4FPAqcaGYLJTWJ298hwMkEdyHPk/QwwdgPK8zsrPDzaFhCbK4S8cTh0s7MNksaSzDoz44EN5te+ChrSfOBwgv/fwguToWeM7MC4BtJCwguXj8DDo8rzTQEcoFdwKdFk0boaOBdM1sbHvMpgrE0JiQYb7xn46ZbA88qeDheLaC4YwNMCM9jbmGppBhTzezbML65QDvgQOA9M9sQzn8e+EFJcRWtqpIEwWeyIHz/NEEC203xn0c+8H7hZ1h43NCrZrYT2ClpDcHjx/8DPBCWWF4xsw9KiM1VIl5V5TJlOEFbQfx4FHsIv4Nh9UituGU746YL4t4XsO8PnqLPzDFAwAAzOzJ8dYgbJ2Lbfp1FYuKP8SDwkJn9CPgdUNLQqvHnqwTWySd1P/yK+wzL43vxmdnXBKWh/wB/Cqv+XCXnicNlRPjL9DmC5FFoEUFVCEAPoGY5dn2+pGphu0dHYB7BAy+vUPA4eiT9QGUPoPQp8FNJByoYjrgX8F454imqIf99zP/Fpa1YTtMJ4m6sYBS+X5ZjH8coeLp0NeDXwIeU/Hl8DJwoqQME43GXtuOwCnC7mY0DhpFdj5WvsryqymXSAwRPMi70KDBR0myCuvXylAaWEFzkGgCXm9l3kkYRtH3MUlAXs5Yyhvs0s5UKuse+Q/CL/1UzS8XjswcDz0vaSND+0CEF+9zLzJZLupvgM9gAfAV8W8LqRds4rgz/nU7whNQYwfm/ZGYFJX0ekvoDL4aJZg1wWikh/ggYJqmAoPrrinKcpqtg/Om4zlVykuqZ2dawxPESMNrMXkpw25MIGv7PTmeMLrt4VZVzld9gSf8GPidofC9Pg75zCfMSh3POuaR4icM551xSPHE455xLiicO55xzSfHE4ZxzLimeOJxzziXl/wNglSplgv578gAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RrCRwb77uuh7",
        "outputId": "96d0b3d6-7822-40d6-8a48-e3d5b26c3b9e"
      },
      "source": [
        "## Question 6: Accuracy on training, validation, and testing dataset \n",
        "\n",
        "\n",
        "# predict labels for test/valid/train data using trained regressor \n",
        "predictions_test2, labels_test2 = Logistic_Regression.predicted_output(X_test, trained_weights2, trained_bias2)\n",
        "predictions_valid2, labels_valid2 = Logistic_Regression.predicted_output(X_val,trained_weights2, trained_bias2)\n",
        "predictions_train2, labels_train2 = Logistic_Regression.predicted_output(X_train,trained_weights2, trained_bias2)\n",
        "# calculate accuracies for train/valid/test datasets \n",
        "accuracy_train = Logistic_Regression.accuracy(labels_train2, y_train)\n",
        "accuracy_valid = Logistic_Regression.accuracy(labels_valid2, y_val)\n",
        "accuracy_test = Logistic_Regression.accuracy(labels_test2, y_test)\n",
        "\n",
        "print('Training accuracy:', accuracy_train)\n",
        "print('Validation accuracy:', accuracy_valid)\n",
        "print('Testing accuracy:', accuracy_test)"
      ],
      "execution_count": 201,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training accuracy: 0.8571428571428572\n",
            "Validation accuracy: 0.8125\n",
            "Testing accuracy: 0.9333333333333333\n"
          ]
        }
      ]
    }
  ]
}