Assignment1_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy import array\n",
    "from numpy import asarray\n",
    "from numpy import zeros\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.utils import pad_sequences\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Flatten\n",
    "from keras.layers import Embedding\n",
    "from pathlib import Path\n",
    "import os\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from keras.utils import np_utils\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Mac OS, runs faster on CPU. Possibly beacuse of the small batch size and the subsequent non parallelization of the GPU\n",
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"-1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def file_iterator():\n",
    "    data_dir = Path('dependency_treebank')\n",
    "    for data_file in filter(lambda f: os.path.isfile(data_dir/f) and f.endswith('.dp'), os.listdir(data_dir)):\n",
    "        yield data_dir/data_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set = []\n",
    "test_set = []\n",
    "val_set = []\n",
    "\n",
    "train_split = 100\n",
    "val_split = 150\n",
    "\n",
    "file_counter = 0\n",
    "for file in file_iterator():\n",
    "    file_counter += 1\n",
    "    if file_counter <= train_split:\n",
    "        train_set.append(pd.read_csv(file, sep=\"\\t\", names=['token', 'pos'], usecols=[0, 1], engine='python'))\n",
    "    elif file_counter <= val_split:\n",
    "        val_set.append(pd.read_csv(file, sep=\"\\t\", names=['token', 'pos'], usecols=[0, 1], engine='python'))\n",
    "    else:\n",
    "        test_set.append(pd.read_csv(file, sep=\"\\t\", names=['token', 'pos'], usecols=[0, 1], engine='python'))\n",
    "\n",
    "assert len(train_set) == 100\n",
    "assert len(val_set) == 50\n",
    "assert len(test_set) == 49"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_frame = pd.concat(train_set)\n",
    "test_frame = pd.concat(test_set)\n",
    "val_frame = pd.concat(val_set)\n",
    "\n",
    "assert sum([e.shape[0] for e in train_set]) == train_frame.shape[0]\n",
    "assert sum([e.shape[0] for e in test_set]) == test_frame.shape[0]\n",
    "assert sum([e.shape[0] for e in val_set]) == val_frame.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.utils import pad_sequences\n",
    "\n",
    "# Tokenizer.from_pretrained('bert-base-uncased')\n",
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts(train_frame['token'].values)\n",
    "vocab_size = len(tokenizer.word_index) + 1\n",
    "\n",
    "encoded_doc = tokenizer.texts_to_sequences(train_frame['token'].values)\n",
    "\n",
    "# Not necessary since we encode words and not sentences \n",
    "max_length = 1\n",
    "padded_docs = pad_sequences(encoded_doc, maxlen=max_length, padding='post')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 400000 word vectors.\n"
     ]
    }
   ],
   "source": [
    "# load the whole embedding into memory\n",
    "embeddings_index = dict()\n",
    "f = open('glove.6B.100d.txt')\n",
    "for line in f:\n",
    "\tvalues = line.split()\n",
    "\tword = values[0]\n",
    "\tcoefs = asarray(values[1:], dtype='float32')\n",
    "\tembeddings_index[word] = coefs\n",
    "f.close()\n",
    "print('Loaded %s word vectors.' % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encountered 71 OOV tokens\n"
     ]
    }
   ],
   "source": [
    "# create a weight matrix for words in training docs\n",
    "# use 0 embedding as placeholder\n",
    "embedding_matrix = zeros((vocab_size, 100))\n",
    "c = 0\n",
    "for word, i in tokenizer.word_index.items():\n",
    "\tembedding_vector = embeddings_index.get(word)\n",
    "\tif embedding_vector is not None:\n",
    "\t\tembedding_matrix[i] = embedding_vector\n",
    "\telse:\n",
    "\t\tc+=1\n",
    "\t#TODO: Add edgecase OOV\n",
    "\n",
    "print(f\"Encountered {c} OOV tokens\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# One hot encode train_frame['pos'].values\n",
    "encoder = LabelEncoder()\n",
    "encoder.fit(train_frame['pos'].values)\n",
    "encoded_Y = encoder.transform(train_frame['pos'].values)\n",
    "# convert integers to dummy variables (i.e. one hot encoded)\n",
    "labels = np_utils.to_categorical(encoded_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-12-07 18:21:01.852203: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " embedding (Embedding)       (None, 1, 100)            632700    \n",
      "                                                                 \n",
      " bidirectional (Bidirectiona  (None, 128)              84480     \n",
      " l)                                                              \n",
      "                                                                 \n",
      " dense (Dense)               (None, 44)                5676      \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 722,856\n",
      "Trainable params: 90,156\n",
      "Non-trainable params: 632,700\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "from tensorflow import keras\n",
    "# define model\n",
    "model = Sequential()\n",
    "e = Embedding(vocab_size, 100, weights=[embedding_matrix], input_length=1, trainable=False)\n",
    "model.add(e)\n",
    "model.add(keras.layers.Bidirectional(keras.layers.LSTM(units=64), input_shape=(None, 50)))\n",
    "model.add(Dense(44, activation='softmax'))\n",
    "# compile the model\n",
    "model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "# summarize the model\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/25\n",
      "1413/1413 [==============================] - 8s 3ms/step - loss: 1.1226 - accuracy: 0.6992\n",
      "Epoch 2/25\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.6184 - accuracy: 0.8067\n",
      "Epoch 3/25\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.5354 - accuracy: 0.8237\n",
      "Epoch 4/25\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.4903 - accuracy: 0.8338\n",
      "Epoch 5/25\n",
      "1413/1413 [==============================] - 5s 3ms/step - loss: 0.4626 - accuracy: 0.8397\n",
      "Epoch 6/25\n",
      "1413/1413 [==============================] - 5s 3ms/step - loss: 0.4420 - accuracy: 0.8450\n",
      "Epoch 7/25\n",
      "1413/1413 [==============================] - 5s 3ms/step - loss: 0.4283 - accuracy: 0.8466\n",
      "Epoch 8/25\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.4170 - accuracy: 0.8489\n",
      "Epoch 9/25\n",
      "1413/1413 [==============================] - 5s 3ms/step - loss: 0.4084 - accuracy: 0.8498\n",
      "Epoch 10/25\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.4017 - accuracy: 0.8515\n",
      "Epoch 11/25\n",
      "1413/1413 [==============================] - 5s 3ms/step - loss: 0.3948 - accuracy: 0.8516\n",
      "Epoch 12/25\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3906 - accuracy: 0.8527\n",
      "Epoch 13/25\n",
      "1413/1413 [==============================] - 5s 3ms/step - loss: 0.3860 - accuracy: 0.8546\n",
      "Epoch 14/25\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3820 - accuracy: 0.8545\n",
      "Epoch 15/25\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3791 - accuracy: 0.8559\n",
      "Epoch 16/25\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3763 - accuracy: 0.8541\n",
      "Epoch 17/25\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3741 - accuracy: 0.8566\n",
      "Epoch 18/25\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3719 - accuracy: 0.8570\n",
      "Epoch 19/25\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3695 - accuracy: 0.8559\n",
      "Epoch 20/25\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3677 - accuracy: 0.8570\n",
      "Epoch 21/25\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3661 - accuracy: 0.8591\n",
      "Epoch 22/25\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3647 - accuracy: 0.8578\n",
      "Epoch 23/25\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3643 - accuracy: 0.8572\n",
      "Epoch 24/25\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3633 - accuracy: 0.8573\n",
      "Epoch 25/25\n",
      "1413/1413 [==============================] - 5s 3ms/step - loss: 0.3614 - accuracy: 0.8585\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1317c74f0>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from keras.callbacks import CSVLogger\n",
    "\n",
    "csv_logger = CSVLogger('log/log_lstm_64.csv', append=False, separator=';')\n",
    "\n",
    "# fit the model\n",
    "model.fit(padded_docs, labels, epochs=25, callbacks=csv_logger)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1413/1413 [==============================] - 5s 4ms/step - loss: 0.3473 - accuracy: 0.8646\n",
      "Train set accuracy: 86.46 %\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvYAAAF2CAYAAAAFudxlAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABxdklEQVR4nO3deVxU5f4H8M/MMAyggLIIiihmhXpVMAxyKfWGkhpJi7lDdNUyKXVui7iRtnCrG2GmUl5ouWmSpuW9Gr/mElQmSkJWpGCumMnmhg4yDMz5/XGawZFFhm3gzOf9ep3XzDxzzpnv+UZnvj7znOfIBEEQQEREREREnZrc2gEQEREREVHLsbAnIiIiIpIAFvZERERERBLAwp6IiIiISAJY2BMRERERSQALeyIiIiIiCWBhT0REREQkASzsiYiIiIgkgIU9EREREZEEsLAnIiIiIpIAFvbUaW3YsAEymQwhISHWDoWIqNM6fvw4nnjiCdxyyy1wcHCAi4sLRo0ahbVr1+LatWvWDq9JXnzxRchksgaXoqIii/e5ZcsWJCYmtn6w7cjPzw/333+/tcOgdmRn7QCImmvz5s3w8/NDdnY2jh07hltvvdXaIRERdSq7d+/G1KlToVKpEBkZicGDB6Oqqgp79+7Fc889h19//RXvvfeetcNsso0bN6Jr16512rt162bxvrZs2YK8vDwsXry45YERtRMW9tQpnTx5Evv27cOOHTvwxBNPYPPmzYiLi7N2WHVotVp06dLF2mEQEdVx8uRJTJ8+HX379sXXX3+Nnj17mt5buHAhjh07ht27dze4vcFgQFVVFRwcHNoj3CZ55JFH4OHh0e6fW1lZCXt7e8jlHAhB1sW/QOqUNm/ejO7du2Py5Ml45JFHsHnz5jrrXLp0CUuWLIGfnx9UKhV69+6NyMhIlJWVmdaprKzEiy++iNtvvx0ODg7o2bMnHnroIRw/fhwAkJmZCZlMhszMTLN9nzp1CjKZDB988IGp7bHHHkPXrl1x/PhxTJo0Cc7Ozpg1axYA4LvvvsPUqVPRp08fqFQq+Pr6YsmSJfX+zJ2fn49HH30Unp6ecHR0hL+/P5YvXw4AyMjIgEwmw86dO+tst2XLFshkMmRlZVmcTyKyPa+//jquXr2K5ORks6Le6NZbb8WiRYtMr2UyGWJiYrB582b85S9/gUqlQlpaGgDgxx9/xMSJE+Hi4oKuXbvi3nvvxf79+832p9frsXr1atx2221wcHCAu7s7Ro8eDY1GY1qnqKgI0dHR6N27N1QqFXr27IkpU6bg1KlTrXLMxnP6p59+ildeeQW9e/eGg4MD7r33Xhw7dsy03tixY7F7926cPn3aNJzHz8/PbB9bt27FihUr4OPjAycnJ5SXlwMAtm3bhqCgIDg6OsLDwwOzZ8/G2bNnzeIwfl+cOHECYWFh6NKlC3r16oU1a9ZAEAQAgCAI8PPzw5QpU+ocR2VlJVxdXfHEE0+0OCfV1dV46aWX0L9/f6hUKvj5+WHZsmXQ6XRm6x08eBBhYWHw8PCAo6Mj+vXrh8cff9xsna1btyIoKAjOzs5wcXHBkCFDsHbt2hbHSE3HHnvqlDZv3oyHHnoI9vb2mDFjBjZu3IgffvgBd955JwDg6tWruPvuu3HkyBE8/vjjuOOOO1BWVoZdu3bh999/h4eHB2pqanD//fcjPT0d06dPx6JFi3DlyhVoNBrk5eWhf//+FsdVXV2NsLAwjB49Gv/85z/h5OQEQDzRV1RUYMGCBXB3d0d2djbWrVuH33//Hdu2bTNt//PPP+Puu++GUqnE/Pnz4efnh+PHj+M///kPXnnlFYwdOxa+vr7YvHkzHnzwwTo56d+/P0aMGNGCzBKRrfjPf/6DW265BSNHjmzyNl9//TU+/fRTxMTEwMPDA35+fvj1119x9913w8XFBc8//zyUSiXeffddjB07Ft98843pOqgXX3wR8fHxmDt3LoKDg1FeXo6DBw8iNzcX48ePBwA8/PDD+PXXX/H000/Dz88PJSUl0Gg0KCwsNBXWjblw4UKdNjs7uzpDcf7xj39ALpfj2WefxeXLl/H6669j1qxZOHDgAABg+fLluHz5Mn7//Xe89dZbAFBniM9LL70Ee3t7PPvss9DpdLC3t8cHH3yA6Oho3HnnnYiPj0dxcTHWrl2L77//Hj/++KNZHDU1Nbjvvvtw11134fXXX0daWhri4uJQXV2NNWvWQCaTYfbs2Xj99ddx4cIFuLm5mbb9z3/+g/LycsyePfumObmZuXPn4sMPP8QjjzyCv//97zhw4ADi4+Nx5MgRUydSSUkJJkyYAE9PTyxduhTdunXDqVOnsGPHDtN+NBoNZsyYgXvvvRevvfYaAODIkSP4/vvvzf6BSG1MIOpkDh48KAAQNBqNIAiCYDAYhN69ewuLFi0yrbNq1SoBgLBjx4462xsMBkEQBCElJUUAICQkJDS4TkZGhgBAyMjIMHv/5MmTAgDh/fffN7VFRUUJAISlS5fW2V9FRUWdtvj4eEEmkwmnT582td1zzz2Cs7OzWdv18QiCIMTGxgoqlUq4dOmSqa2kpESws7MT4uLi6nwOEdGNLl++LAAQpkyZ0uRtAAhyuVz49ddfzdojIiIEe3t74fjx46a2P/74Q3B2dhbuueceU1tAQIAwefLkBvd/8eJFAYDwxhtvNP1A/hQXFycAqHfx9/c3rWc8pw8cOFDQ6XSm9rVr1woAhF9++cXUNnnyZKFv3751Psu4j1tuucXs3F5VVSX06NFDGDx4sHDt2jVT+3//+18BgLBq1SpTm/H74umnnza1GQwGYfLkyYK9vb1QWloqCIIgFBQUCACEjRs3msXwwAMPCH5+fmbfDfXp27dvozk/dOiQAECYO3euWfuzzz4rABC+/vprQRAEYefOnQIA4YcffmhwX4sWLRJcXFyE6urqRmOitsWhONTpbN68GV5eXhg3bhwA8efhadOmYevWraipqQEAfPbZZwgICKjTq21c37iOh4cHnn766QbXaY4FCxbUaXN0dDQ912q1KCsrw8iRIyEIAn788UcAQGlpKb799ls8/vjj6NOnT4PxREZGQqfTYfv27aa21NRUVFdXt0rvDRFJn3HYiLOzs0XbjRkzBoMGDTK9rqmpwVdffYWIiAjccsstpvaePXti5syZ2Lt3r+mzunXrhl9//RW//fZbvft2dHSEvb09MjMzcfHiRUsPCYB4XtdoNGbL+++/X2e96Oho2Nvbm17ffffdAIATJ040+bOioqLMzu0HDx5ESUkJnnrqKbPrDiZPnowBAwbUe71CTEyM6blxqFNVVRX+97//AQBuv/12hISEmA03vXDhAr788kvMmjWrRd9VALBnzx4AgFqtNmv/+9//DgCmmI2/NPz3v/+FXq+vd1/dunWDVqs1G1pF7Y+FPXUqNTU12Lp1K8aNG4eTJ0/i2LFjOHbsGEJCQlBcXIz09HQA4vRtgwcPbnRfx48fh7+/P+zsWm9Emp2dHXr37l2nvbCwEI899hjc3NzQtWtXeHp6YsyYMQCAy5cvA6j9QrlZ3AMGDMCdd95pdqLfvHkz7rrrLs4MRERN4uLiAgC4cuWKRdv169fP7HVpaSkqKirg7+9fZ92BAwfCYDDgzJkzAIA1a9bg0qVLuP322zFkyBA899xz+Pnnn03rq1QqvPbaa/jyyy/h5eWFe+65B6+//rpFU1Xec889CA0NNVvqG554Y+dJ9+7dAcCif1DcmIvTp08DQL25GDBggOl9I7lcbvaPIUAs5AGYXVMQGRmJ77//3rT9tm3boNfrMWfOnCbH2pDTp09DLpfX+e7w9vZGt27dTJ85ZswYPPzww1i9ejU8PDwwZcoUvP/++2bj8J966incfvvtmDhxInr37o3HH3/cdA0GtR8W9tSpfP311zh37hy2bt2K2267zbQ8+uijAFDvRbQt0VBviPGXgRupVKo6syLU1NRg/Pjx2L17N1544QV8/vnn0Gg0pgtvDQaDxXFFRkbim2++we+//47jx49j//797K0noiZzcXFBr169kJeXZ9F21/dQW+qee+7B8ePHkZKSgsGDB+Nf//oX7rjjDvzrX/8yrbN48WIcPXoU8fHxcHBwwMqVKzFw4EDTL5utRaFQ1Nsu/HnhalO0JBeWmD59OpRKpen77eOPP8bw4cPr/QdEc92s518mk2H79u3IyspCTEwMzp49i8cffxxBQUG4evUqAKBHjx44dOgQdu3ahQceeAAZGRmYOHEioqKiWi1OujkW9tSpbN68GT169MC2bdvqLDNmzMDOnTtx7do19O/f/6ZfWP3790dBQUGDPysCtb04ly5dMmu/seelMb/88guOHj2KN998Ey+88AKmTJmC0NBQ9OrVy2w9Y89NU75op0+fDoVCgU8++QSbN2+GUqnEtGnTmhwTEdH999+P48ePt2gmLU9PTzg5OaGgoKDOe/n5+ZDL5fD19TW1ubm5ITo6Gp988gnOnDmDoUOH4sUXXzTbrn///vj73/+Or776Cnl5eaiqqsKbb77Z7Biby9JhLn379gWAenNRUFBget/IYDDUGfpz9OhRADC7UNjNzQ2TJ0/G5s2bcfr0aXz//fet0ltvjNlgMNQZHlVcXIxLly7Vifmuu+7CK6+8goMHD2Lz5s349ddfsXXrVtP79vb2CA8Px4YNG0w3Pvvoo4/MZhyitsXCnjqNa9euYceOHbj//vvxyCOP1FliYmJw5coV7Nq1Cw8//DB++umneqeFNPbIPPzwwygrK8M777zT4Dp9+/aFQqHAt99+a/b+hg0bmhy3sWfo+p4gQRDqTAHm6emJe+65BykpKSgsLKw3HiMPDw9MnDgRH3/8MTZv3oz77rvPKnM3E1Hn9fzzz6NLly6YO3cuiouL67x//Pjxm05VqFAoMGHCBHzxxRdmw0eKi4uxZcsWjB492jTs5/z582bbdu3aFbfeeqtpOEdFRQUqKyvN1unfvz+cnZ3rTL3YHrp06WIaKtkUw4cPR48ePZCUlGQW75dffokjR45g8uTJdba5/vtHEAS88847UCqVuPfee83WmzNnDg4fPoznnnsOCoUC06dPb8YR1TVp0iQAqHOH3YSEBAAwxXzx4sU630OBgYEAYDrWG//7yuVyDB061Gwdanuc7pI6jV27duHKlSt44IEH6n3/rrvugqenJzZv3owtW7Zg+/btmDp1qunnwgsXLmDXrl1ISkpCQEAAIiMj8dFHH0GtViM7Oxt33303tFot/ve//+Gpp57ClClT4OrqiqlTp2LdunWQyWTo378//vvf/6KkpKTJcQ8YMAD9+/fHs88+i7Nnz8LFxQWfffZZvWM53377bYwePRp33HEH5s+fj379+uHUqVPYvXs3Dh06ZLZuZGQkHnnkEQDitGtERJbo378/tmzZgmnTpmHgwIFmd57dt28ftm3bhscee+ym+3n55Zeh0WgwevRoPPXUU7Czs8O7774LnU6H119/3bTeoEGDMHbsWAQFBcHNzQ0HDx7E9u3bTReQHj16FPfeey8effRRDBo0CHZ2dti5cyeKi4ubXMhu37693jvPjh8/Hl5eXk1LzJ+CgoKQmpoKtVqNO++8E127dkV4eHiD6yuVSrz22muIjo7GmDFjMGPGDNN0l35+fliyZInZ+g4ODkhLS0NUVBRCQkLw5ZdfYvfu3Vi2bBk8PT3N1p08eTLc3d2xbds2TJw4ET169GjycRw7dgwvv/xynfZhw4Zh8uTJiIqKwnvvvYdLly5hzJgxyM7OxocffoiIiAjTJBUffvghNmzYgAcffBD9+/fHlStXsGnTJri4uJj+cTB37lxcuHABf/3rX9G7d2+cPn0a69atQ2BgIAYOHNjkeKmFrDYfD5GFwsPDBQcHB0Gr1Ta4zmOPPSYolUqhrKxMOH/+vBATEyP4+PgI9vb2Qu/evYWoqCihrKzMtH5FRYWwfPlyoV+/foJSqRS8vb2FRx55xGzattLSUuHhhx8WnJychO7duwtPPPGEkJeXV+90l126dKk3rsOHDwuhoaFC165dBQ8PD2HevHnCTz/9VGcfgiAIeXl5woMPPih069ZNcHBwEPz9/YWVK1fW2adOpxO6d+8uuLq6mk2tRkRkiaNHjwrz5s0T/Pz8BHt7e8HZ2VkYNWqUsG7dOqGystK0HgBh4cKF9e4jNzdXCAsLE7p27So4OTkJ48aNE/bt22e2zssvvywEBwcL3bp1ExwdHYUBAwYIr7zyilBVVSUIgiCUlZUJCxcuFAYMGCB06dJFcHV1FUJCQoRPP/30psfQ2HSXuG7KYuNUldu2bTPbvr4pjK9evSrMnDlT6NatmwDANPVlQ/swSk1NFYYNGyaoVCrBzc1NmDVrlvD777+brWP8vjh+/LgwYcIEwcnJSfDy8hLi4uKEmpqaevf71FNPCQCELVu23DQfRn379m0wJ3/7298EQRAEvV4vrF692vQ96OvrK8TGxpr9t8/NzRVmzJgh9OnTR1CpVEKPHj2E+++/Xzh48KBpne3btwsTJkwQevToIdjb2wt9+vQRnnjiCeHcuXNNjpdaTiYIFlwpQkQdRnV1NXr16oXw8HAkJydbOxwiImqixx57DNu3bzddeNoUS5YsQXJyMoqKikw3PyS6EcfYE3VSn3/+OUpLSxEZGWntUIiIqA1VVlbi448/xsMPP8yinhrFMfZEncyBAwfw888/46WXXsKwYcNM8+ETEZG0lJSU4H//+x+2b9+O8+fPY9GiRdYOiTo4FvZEnczGjRvx8ccfIzAw0DQXPhERSc/hw4cxa9Ys9OjRA2+//bZpJhqihnCMPRERERGRBHCMPRERERGRBLCwJyIiIiKSAMmMsTcYDPjjjz/g7Oxs8W2giYg6G0EQcOXKFfTq1QtyuW310fB8T0S2xJLzvWQK+z/++AO+vr7WDoOIqF2dOXMGvXv3tnYY7YrneyKyRU0530umsHd2dgYAnDx5Em5ublaOxnr0ej2++uorTJgwAUql0trhWA3zIGIeRFLMQ3l5OXx9fU3nPlvC871Iin/XzcE8iJgHkRTzYMn5XjKFvfHnWGdnZ7i4uFg5GuvR6/VwcnKCi4uLZP6gm4N5EDEPIinnwRaHovB8L5Ly37UlmAcR8yCSch6acr63rYGZREREREQSxcKeiIiIiEgCWNgTEREREUkAC3siIiIiIglgYU9EREREJAHNKuzXr18PPz8/ODg4ICQkBNnZ2Y2un5iYCH9/fzg6OsLX1xdLlixBZWWl2Tpnz57F7Nmz4e7uDkdHRwwZMgQHDx5sTnhERERERDbH4ukuU1NToVarkZSUhJCQECQmJiIsLAwFBQXo0aNHnfW3bNmCpUuXIiUlBSNHjsTRo0fx2GOPQSaTISEhAQBw8eJFjBo1CuPGjcOXX34JT09P/Pbbb+jevXvLj5CIiIiIyAZYXNgnJCRg3rx5iI6OBgAkJSVh9+7dSElJwdKlS+usv2/fPowaNQozZ84EAPj5+WHGjBk4cOCAaZ3XXnsNvr6+eP/9901t/fr1s/hgiIiIiIhslUVDcaqqqpCTk4PQ0NDaHcjlCA0NRVZWVr3bjBw5Ejk5OabhOidOnMCePXswadIk0zq7du3C8OHDMXXqVPTo0QPDhg3Dpk2bmnM8REREREQ2yaIe+7KyMtTU1MDLy8us3cvLC/n5+fVuM3PmTJSVlWH06NEQBAHV1dV48sknsWzZMtM6J06cwMaNG6FWq7Fs2TL88MMPeOaZZ2Bvb4+oqKh696vT6aDT6Uyvy8vLAYh3HNPr9ZYclqQYj92WcwAwD0bMg0iKeZDSsRARUeuweCiOpTIzM/Hqq69iw4YNCAkJwbFjx7Bo0SK89NJLWLlyJQDAYDBg+PDhePXVVwEAw4YNQ15eHpKSkhos7OPj47F69eo67RkZGXBycmq7A+okNBqNtUPoEJgHkS3mwWAATp1yxU8/eeLXX91hbz8cW7cWwtf3Cvr0uYKePa/Czk6wdpjNVlFRYe0QrK+iAnB3t3YUREQdhkWFvYeHBxQKBYqLi83ai4uL4e3tXe82K1euxJw5czB37lwAwJAhQ6DVajF//nwsX74ccrkcPXv2xKBBg8y2GzhwID777LMGY4mNjYVarTa9Li8vh6+vL8aNGwd3Gz7R6/V6aDQajB8/Hkql0trhWA3zILK1PJw+DaSny5CeLkdGhgxlZTKz9/ftq32uVAq47TZg0CDBtAwcKODWW4GOliqDQTy2I0dkpiUvr9zaYVnfxYuAr6+1oyAi6jAsKuzt7e0RFBSE9PR0REREABB729PT0xETE1PvNhUVFZDLzYfyKxQKAIAgiL1lo0aNQkFBgdk6R48eRd++fRuMRaVSQaVS1WlXKpU2UcDcDPMgYh5EUs3DxYtARgbwv/8BGg1w7Jj5+126AGPHAmPG1ODw4SMwGAYhP1+Ow4eBq1dlOHwYOHzYvPhXKgF/f+AvfwFuuQXw8BA7hW98dHUF5K18J5CaGuDECfwZV+1y5Ahw7dqNa/M2JLh40doREBF1KBYPxVGr1YiKisLw4cMRHByMxMREaLVa0yw5kZGR8PHxQXx8PAAgPDwcCQkJGDZsmGkozsqVKxEeHm4q8JcsWYKRI0fi1VdfxaOPPors7Gy89957eO+991rxUImoLVRXA1otYGcnFtJtRRCAsjLgl1+A9HSxmD94UOzNNlIogJAQIDRUXEJCAHt7QK83YM+e45g0yR9KpRwGA3DmDPDrr+Jy+HDto1YL5OWJS2MUCsDNrW7B7+IixioIYmxNedRqgYICcbnu0iEz9vbAgAHAoEHi4ucHREa2Wno7JdmlS9YOgYioQ7G4sJ82bRpKS0uxatUqFBUVITAwEGlpaaYLagsLC8166FesWAGZTIYVK1bg7Nmz8PT0RHh4OF555RXTOnfeeSd27tyJ2NhYrFmzBv369UNiYiJmzZrVCodIRE1VXAx88QVw8iRw9apYcGq15s9vfH19IertDdx6K3DbbbXLrbeKS9euTYtBrxc/Pz+/dikoEB8vXKi7/oABwPjxYiE/ZozYk34zcjnQt6+4XDdBFwwGoLCwtuA/exY4f178B4XxsaxMPP6aGqC0VFxak6MjMHBgbQFvXPr1E//xZFTOkTj1/0EQEdmwZl08GxMT0+DQm8zMTPMPsLNDXFwc4uLiGt3n/fffj/vvv7854RBJ3uXLYrFrSYHcVEVFwI4dwLZtwLffmveAN2dfRUXA3r113+vZs7bQNxb9PXqIQ0+uL+KPHxeL+/rIZGIxPmpUba98797Nj/dGcrnYE+7nB0ye3PB6Op1Y6N9Y9J8/D1y5IsYplzf9UanEn+P9xc9u7SE+benbb7/FG2+8gZycHJw7dw47d+40DdWsz7lz5/D3v/8dBw8exLFjx/DMM88gMTGxeR/OHnsiIjNtPisOETVddTXw22/Azz+bL4WF4vsKBTBsGDB6dO1yw+yzTXLunHkxL1w3OcyddwIjR4r/gOjaVRxeY1wae11ZKY5x/+232kfjcv68+JnnzomfdzNOTmJPvL+/+GhcbrtN7NG2NpUK6NVLXGydVqtFQEAAHn/8cTz00EM3XV+n08HT0xMrVqzAW2+91aLPlnGMPRGRGRb2RC1gMIgz7ikU4jAJOzuxF7YpSkrqFvCHDzc8xrpbN7GD8uBBcTF2ct56K3D33bWF/m231R/DH38An30mFvN795oX88HBwNSpwCOPiD3GzeHgAAwfLi43unix/oK/tFS8QPXGAt7Hp3P1WtuyiRMnYuLEiU1e38/PD2vXrgUApKSktOzDWdgTEZlhYU9kAUEQC9L0dODrr8UZWc6fN19HLjcv9O3s6r7Wahsem92lCzBkCDB0qPlj9+5iz/3334uF+d694oWkx46Jy/vvi9t7etYW+XfcIcN//3sL3nhDgX37zIv5kJDaYr6RCahaRffu4j8egoPb9nPItrDHnojIHAt7opv44w+xkDcW82fONL6+wSAuN7sxqEwm9rYbi3fj0q9fw73VffqIy4wZ4utLl4CsLOC778RCPztb/AfDzp3iIv4vPsS0/YgRYjH/8MPifog6g4buNG64cMGm78ArxTsqNwfzIGIeRFLMgyXHwsKe6AYXLwKZmbXFfH6++fv29mKBfO+9wF//CgQEiD3h1dW1S01N46+Nc6W3dHrIbt2AiRPFBRCH8eTk1PboHzwowNX1AubO7YZHH1XwXj7UKTV0p/HLJ0/iuz17rBBRx2KLd5auD/MgYh5EUsqDJXcaZ2FPNksQxN74o0drp1PcuxfIzTUfsiKTAUFBtYX86NHixZ0dkUolXvg6ciTw/POAXl+NPXv2YtKkSVAqFdYOj6hZGrrTeHcAk66fr9TG2NqdpRvCPIiYB5EU81BuwfzGLOxJ8i5fri3ejx6tff7bb+JY9/oMHCgW8ffeK965tHv3dg2ZiK7T0J3GZZcuSeaLuyWkemdpSzEPIuZBJKU8WHIcLOypw7p0CThwAPjpJ3GISU1N0xedToEffxyN+fPtUFLS8GcoFLWzstx+uziV5F//ymkMiZrq6tWrOHbsmOn1yZMncejQIbi5uaFPnz6IjY3F2bNn8dFHH5nWOXTokGnb0tJSHDp0CPb29hg0aJBlH86LZ4mIzLCwpw7BONtMVhawb5+4/Pqr+ZAYy8gBuJte9ewpFu7GAt74vF8/cbw7ETXPwYMHMW7cONNr45CZqKgofPDBBzh37hwKjTdi+NOwYcNMz3NycrBlyxb07dsXp06dsuizZTodcO1ax7i5ARFRB8DCnqyiokKci91YxGdliXfuvFH//uINk1xcxN71+hbj9JLXLzJZDUpLf8SjjwZg0CAlnJ3b/xiJbMHYsWMhNPIv8A8++KBOW2PrW+zCBfHGB0RExMKe2o4giOPbCwvFKSLPnAGOHBGL+B9/FGeHuZ5KVXvX0xEjxKU5d1UFAL3egD17zuKOOwLYI08kZefPs7AnIvoTC3tqtmvXxGL9+sL9xtdXrza8fa9etTO4jBwpjm+3t2+/+IlIAi5csHYEREQdBgt7arLqanHYzK5dwH/+I84u0xTu7uLNkHx9xTHtd90lFvK+vuJUkkREzcbCnojIhIU9NerKFeD//k8s5nfvrvsd2rWrWKD7+tYW79c/79274875TkQSwMKeiMiEhT3VceaM2CO/axeQkQFUVdW+5+YGTJ4MPPAAMG6c+Jq97kRkNSzsiYhMWNgTBAE4dEgs5L/4Qryw9Xq33gpMmSIW8yNHAnb8qyGijuL8eWtHQETUYbBEs0FarTjV5IEDwP794iw1RUW178tkYgH/wAPi4u/PXnki6qDYY09EZMLCXuIMBqCgoLaI378fyMsT7856PScnYMIEsZCfPBno0cM68RIRWYSFPRGRCQt7iTl/Hjh4sAeys+X44QcgO1ucS/5GPj7i7DQhIeLj8OG8eSMRdUIs7ImITFjYS4ReD7z0EhAfb4fq6hFm7zk6ioW7sYgPCRFnqyEi6vQ4xp6IyISFvQQcPQrMng388AMAyNCr11Xce68TRoyQ4667gMGDwbuvEpE0sceeiMiEhX0nJgjAu+8Cf/87UFEBdOsGvPNONbp2TcekSZOgVMqtHSIRUdtiYU9EZMLKr5MqLgbCw4EFC8Si/q9/BX75BXj0UcHaoRERtZ9r18SFiIhY2HdGu3YBQ4aId4JVqYCEBECj4bh5IrItgvzPrzD22hMRAWhmYb9+/Xr4+fnBwcEBISEhyM7ObnT9xMRE+Pv7w9HREb6+vliyZAkqKyvrXfcf//gHZDIZFi9e3JzQJO3qVWD+fPFmUaWlYnH/ww/AkiWAnP9EIyJb0727+MjCnogIQDMK+9TUVKjVasTFxSE3NxcBAQEICwtDSUlJvetv2bIFS5cuRVxcHI4cOYLk5GSkpqZi2bJlddb94Ycf8O6772Lo0KGWH4nEHTgADBsGbNok3izq2WfFon7IEGtHRkRkJd26iY8s7ImIADSjsE9ISMC8efMQHR2NQYMGISkpCU5OTkhJSal3/X379mHUqFGYOXMm/Pz8MGHCBMyYMaNOL//Vq1cxa9YsbNq0Cd2NvTCE6mpg9Wpg1Cjg2DFxuE16OvDGG+IwHCIiWyW4uYlPWNgTEQGwsLCvqqpCTk4OQkNDa3cglyM0NBRZWVn1bjNy5Ejk5OSYCvkTJ05gz549mDRpktl6CxcuxOTJk832beuOHQNGjwZefFG8U+yMGcDPPwPjxlk7MiKiDsDYY8+57ImIAFg43WVZWRlqamrg5eVl1u7l5YX8/Px6t5k5cybKysowevRoCIKA6upqPPnkk2ZDcbZu3Yrc3Fz8IE7E3iQ6nQ46nc70ury8HACg1+uh1+stOawO6eOPZXj6aQW0WhlcXQW8/XYNZswQZ7xp7PCMxy6FHLQE8yBiHkRSzIOUjqXZOMaeiMhMm89jn5mZiVdffRUbNmxASEgIjh07hkWLFuGll17CypUrcebMGSxatAgajQYODg5N3m98fDxWr15dpz0jIwNOTk6teQjtSq+XIzl5MNLS+gEABg8uxaJFP8LV9Rr27Gn6fjQaTRtF2LkwDyLmQSSlPFRUVFg7BKsTWNgTEZmxqLD38PCAQqFAcXGxWXtxcTG8vb3r3WblypWYM2cO5s6dCwAYMmQItFot5s+fj+XLlyMnJwclJSW44447TNvU1NTg22+/xTvvvAOdTgeFQlFnv7GxsVCr1abX5eXl8PX1xbhx4+Du7m7JYXUYZ84AM2YokJ0th0wmYMUKA5Yv7wa5vOljb/R6PTQaDcaPHw+lDd9ulnkQMQ8iKebB+CulTWNhT0RkxqLC3t7eHkFBQUhPT0dERAQAwGAwID09HTExMfVuU1FRAfkNczEaC3VBEHDvvffil19+MXs/OjoaAwYMwAsvvFBvUQ8AKpUKqnquHlUqlZ3yizs9HZg+HSgrE7+rNm+WYeJEBYD6j/9mOmseWhvzIGIeRFLKg1SOo0WMhT3H2BMRAWjGUBy1Wo2oqCgMHz4cwcHBSExMhFarRXR0NAAgMjISPj4+iI+PBwCEh4cjISEBw4YNMw3FWblyJcLDw6FQKODs7IzBgwebfUaXLl3g7u5ep12KBAF4/XVg2TLAYBCntPzsM6BfP2tHRkTUsXEoDhGROYsL+2nTpqG0tBSrVq1CUVERAgMDkZaWZrqgtrCw0KyHfsWKFZDJZFixYgXOnj0LT09PhIeH45VXXmm9o+ikLl8GoqOBnTvF1489BmzYADg6WjUsIqLOgYU9EZGZZl08GxMT0+DQm8zMTPMPsLNDXFwc4uLimrz/G/chRXl5wEMPAb/9BtjbA+vWAfPmiTefIiKim+M89kRE5tp8Vhyq65NPgLlzgYoKwNdXHHpz553WjoqIqJPhnWeJiMxYfOdZaj69Hli8GJg5UyzqQ0OB3FwW9UTUeX377bcIDw9Hr169IJPJ8Pnnn990m8zMTNxxxx1QqVS49dZb8cEHHzTvw41DcSoqgMrK5u2DiEhCWNi3k3PnxDvGrl0rvl62DEhLAzw8rBsXEVFLaLVaBAQEYP369U1a/+TJk5g8eTLGjRuHQ4cOYfHixZg7dy7+7//+z/IPd3YGjDOnsdeeiIhDcdrDvn3Aww8DRUWAiwvw0UfAlCnWjoqIqOUmTpyIiRMnNnn9pKQk9OvXD2+++SYAYODAgdi7dy/eeusthIWFWfbhMpnYa19WJhb2vXpZtj0RkcSwsG9jp08DkycDly4BgwcDO3YAt91m7aiIiKwjKysLoaGhZm1hYWFYvHhxg9vodDrodDrTa+PNufR6PQQ3N8jKylBdUgJBr2+TmDsq/Z/Hq7ex474R8yBiHkRSzIMlx8LCvg3p9eJNpy5dEsfRZ2QAXbpYOyoiIuspKioyTY9s5OXlhfLycly7dg2O9cz3Gx8fj9WrV9dpz8jIQJhMBjcAuRoNzl250lZhd2gajcbaIXQIzIOIeRBJKQ8VFRVNXpeFfRtasQLYvx9wdQVSU1nUExE1R2xsLNRqtel1eXk5fH19MW7cOHT7+GOgoAB39OsHYdIkK0bZ/vR6PTQaDcaPH2/TdyJmHkTMg0iKeTD+StkULOzbyJ494h1lASAlhXeSJSICAG9vbxQXF5u1FRcXw8XFpd7eegBQqVRQqVR12pVKJeTu7gAAu8uXAYl8iVtKqVRKpoBpCeZBxDyIpJQHS46Ds+K0gd9/ByIjxecxMeKNqIiICBgxYgTS09PN2jQaDUaMGNG8Hf5Z2HNWHCIiFvatrrpanKf+/Hlg2DDgjTesHRERUdu5evUqDh06hEOHDgEQp7M8dOgQCgsLAYjDaCKNPR0AnnzySZw4cQLPP/888vPzsWHDBnz66adYsmRJ8wIw3n32/PmWHAYRkSSwsG9lq1cD330nTq/86aeAg4O1IyIiajsHDx7EsGHDMGzYMACAWq3GsGHDsGrVKgDAuXPnTEU+APTr1w+7d++GRqNBQEAA3nzzTfzrX/+yfKpLI2Nhzx57IiKOsW9N//sf8Mor4vP33gNuvdW68RARtbWxY8dCEIQG36/vrrJjx47Fjz/+2DoBsLAnIjJhj30rKSoCZs0CBAGYP1+c5pKIiNoYx9gTEZmwsG8FNTViUV9SAgwZAiQmWjsiIiIbwTH2REQmLOxbwauvAl9/Lc5T/+mnQAMzthERUWvjUBwiIhMW9i30zTfAiy+KzzduBAYMsGo4RES2xVjYV1QAlZXWjYWIyMpY2LdAaSkwYwZgMACPPQbMmWPtiIiIbIyrK6BQiM8vXrRuLEREVsbCvpkMBrGQP3cOGDgQeOcda0dERGSDZDKge3fxOcfZE5GNY2HfTG+8Afzf/4nj6T/9VBxfT0REVsBx9kREAFjYN8v33wPLl4vP160DBg+2bjxERDaNhT0REQAW9hY7f14cV2+c4vLxx60dERGRjWNhT0QEgIW9xf72N+DMGeD228VZcGQya0dERGTjeJMqIiIALOwtkpUFfPEFoFQCqamAs7O1IyIiIt6kiohIxMLeAm+9JT7Ong0EBlo1FCIiMuJQHCIiAM0s7NevXw8/Pz84ODggJCQE2dnZja6fmJgIf39/ODo6wtfXF0uWLEHldTcSiY+Px5133glnZ2f06NEDERERKCgoaE5obebUKeCzz8TnS5ZYNRQiIroeC3siIgDNKOxTU1OhVqsRFxeH3NxcBAQEICwsDCUlJfWuv2XLFixduhRxcXE4cuQIkpOTkZqaimXLlpnW+eabb7Bw4ULs378fGo0Ger0eEyZMgFarbf6RtbJ168S568ePB4YMsXY0RERkwjH2REQAADtLN0hISMC8efMQHR0NAEhKSsLu3buRkpKCpUuX1ll/3759GDVqFGbOnAkA8PPzw4wZM3DgwAHTOmlpaWbbfPDBB+jRowdycnJwzz33WBpiqysvBzZtEp+zt56IqIPhGHsiIgAW9thXVVUhJycHoaGhtTuQyxEaGoqsrKx6txk5ciRycnJMw3VOnDiBPXv2YNKkSQ1+zuXLlwEAbsaTtZWlpABXroh3mA0Ls3Y0RERkhkNxiIgAWNhjX1ZWhpqaGnh5eZm1e3l5IT8/v95tZs6cibKyMowePRqCIKC6uhpPPvmk2VCc6xkMBixevBijRo3C4Ebu/KTT6aDT6Uyvy8vLAQB6vR56vd6Sw2pUTQ2wdq0dABmeeaYaNTUCampabfetznjsrZmDzoh5EDEPIinmQUrH0mIs7ImIADRjKI6lMjMz8eqrr2LDhg0ICQnBsWPHsGjRIrz00ktYuXJlnfUXLlyIvLw87N27t9H9xsfHY/Xq1XXaMzIy4OTk1Grx79vXE6dOBcPFRQc3t6+wZ4+h1fbdljQajbVD6BCYBxHzIJJSHioqKqwdQsdhHGOv1QI6HaBSWTceIiIrsaiw9/DwgEKhQHFxsVl7cXExvL29691m5cqVmDNnDubOnQsAGDJkCLRaLebPn4/ly5dDLq8dDRQTE4P//ve/+Pbbb9G7d+9GY4mNjYVarTa9Li8vh6+vL8aNGwd340m+Fbz2muLP2Ozw4IP3tdp+24per4dGo8H48eOhVCqtHY7VMA8i5kEkxTwYf6UkAC4ugFwuznBw4QLQs6e1IyIisgqLCnt7e3sEBQUhPT0dERERAMShM+np6YiJial3m4qKCrPiHQAUCrFYFgTB9Pj0009j586dyMzMRL9+/W4ai0qlgqqeXhmlUtlqX9wHDog3pbK3B55+WgGlUtEq+20PrZmHzox5EDEPIinlQSrH0SrkcqB7d/HiWRb2RGTDLB6Ko1arERUVheHDhyM4OBiJiYnQarWmWXIiIyPh4+OD+Ph4AEB4eDgSEhIwbNgw01CclStXIjw83FTgL1y4EFu2bMEXX3wBZ2dnFBUVAQBcXV3h6OjYWsdqMeMNqWbOBBr4QYKIiDoCN7fawp6IyEZZXNhPmzYNpaWlWLVqFYqKihAYGIi0tDTTBbWFhYVmPfQrVqyATCbDihUrcPbsWXh6eiI8PByvvPKKaZ2NGzcCAMaOHWv2We+//z4ee+yxZhxWy50+DWzfLj7nFJdERB0cL6AlImrexbMxMTENDr3JzMw0/wA7O8TFxSEuLq7B/RmH5HQk69aJM+Lcey8wdKi1oyEiokYZr63iXPZEZMMsvvOsLbhypfaGVNddn0tERB0Ve+yJiFjY1yclRbzb7IABwH0dfyIcIiJiYU9ExML+RuINqcTnixeLky0QEVEHx8KeiIiF/Y2++AI4eVIcrjlnjrWjISLq+NavXw8/Pz84ODggJCQE2dnZDa6r1+uxZs0a9O/fHw4ODggICEBaWlrLg+AYeyIiFvY3SkgQH598EmjFG9gSEUlSamoq1Go14uLikJubi4CAAISFhaGkpKTe9VesWIF3330X69atw+HDh/Hkk0/iwQcfxI8//tiyQNhjT0TEwv562dnA998DSiWwcKG1oyEi6vgSEhIwb948REdHY9CgQUhKSoKTkxNSUlLqXf/f//43li1bhkmTJuGWW27BggULMGnSJLz55pstC4SFPRFR86a7lKrrb0jFGxcSETWuqqoKOTk5iI2NNbXJ5XKEhoYiKyur3m10Oh0cHBzM2hwdHbF3794GP0en00Gn05lel5eXAxCH9ej1egCAzMUFdgCECxdQ/Web1BmPXW8jx9sQ5kHEPIikmAdLjoWF/Z8KC4Ft28TnvCEVEdHNlZWVoaamxnSDQiMvLy/k5+fXu01YWBgSEhJwzz33oH///khPT8eOHTtQU1PT4OfEx8dj9erVddozMjLg9OeYSadz5zAeQE1JCfbs2dP8g+qENBqNtUPoEJgHEfMgklIeKioqmrwuC/s/vfOOOCPOX/8KBARYOxoiImlau3Yt5s2bhwEDBkAmk6F///6Ijo5ucOgOAMTGxkJ93U1FysvL4evri3HjxsHdeNHsxYvAggWwq6zEpNBQwN6+rQ/F6vR6PTQaDcaPHw+lUmntcKyGeRAxDyIp5sH4K2VTsLCHeEOq994Tn/OGVERETePh4QGFQoHi4mKz9uLiYnh7e9e7jaenJz7//HNUVlbi/Pnz6NWrF5YuXYpbbrmlwc9RqVRQqVR12pVKZe0Xt4cHIJMBggDllStAA58vRWZ5sGHMg4h5EEkpD5YcBy+eBfD++8Dly4C/PzBxorWjISLqHOzt7REUFIT09HRTm8FgQHp6OkaMGNHotg4ODvDx8UF1dTU+++wzTJkypWXByOVA9+7ic15AS0Q2yuZ77HlDKiKi5lOr1YiKisLw4cMRHByMxMREaLVaREdHAwAiIyPh4+OD+Ph4AMCBAwdw9uxZBAYG4uzZs3jxxRdhMBjw/PPPtzwYd3exqGdhT0Q2yuYL+127gBMnxJnSIiOtHQ0RUecybdo0lJaWYtWqVSgqKkJgYCDS0tJMF9QWFhZCfl2PSWVlJVasWIETJ06ga9eumDRpEv7973+jW7duLQ/GOOUlb1JFRDbK5gt73pCKiKhlYmJiEBMTU+97mZmZZq/HjBmDw4cPt00gnMueiGycTQ88+eEHYO9e3pCKiEgSWNgTkY2z6cLeeEOq6dOBXr2sGwsREbUQC3sisnE2W9ifO8cbUhERSYpxTnuOsSciG2WzhX1eHlBdDQwcCAwbZu1oiIioxdhjT0Q2zmYLe+N5/4Y7oRMRUWfFwp6IbJzNF/bG7wEiIurkWNgTkY1jYc/CnohIGjjGnohsHAt7FvZERNLAHnsisnEs7FnYExFJg/GEfvUqUFVl3ViIiKyAhT0LeyIiaXB1BWQy8fnFi9aNhYjICppV2K9fvx5+fn5wcHBASEgIsrOzG10/MTER/v7+cHR0hK+vL5YsWYLKysoW7bOlWNgTEUmMQgF07y4+53AcIrJBFhf2qampUKvViIuLQ25uLgICAhAWFoaSkpJ619+yZQuWLl2KuLg4HDlyBMnJyUhNTcWyZcuavc/WwMKeiEiCjCd1XkBLRDbI4sI+ISEB8+bNQ3R0NAYNGoSkpCQ4OTkhJSWl3vX37duHUaNGYebMmfDz88OECRMwY8YMsx55S/fZGljYExFJEC+gJSIbZlFhX1VVhZycHISGhtbuQC5HaGgosrKy6t1m5MiRyMnJMRXyJ06cwJ49ezBp0qRm77OlBIGFPRGRJLGwJyIbZmfJymVlZaipqYHXDbdr9fLyQn5+fr3bzJw5E2VlZRg9ejQEQUB1dTWefPJJ01Cc5uwTAHQ6HXQ6nel1eXk5AECv10Ov1zd6HFotUFWlBAA4O+txk9U7FeOx3ywHUsc8iJgHkRTzIKVjaVXGuexZ2BORDbKosG+OzMxMvPrqq9iwYQNCQkJw7NgxLFq0CC+99BJWrlzZ7P3Gx8dj9erVddozMjLg5OTU6LalpQ4AwmBnV4PMzD2mSRSkRKPRWDuEDoF5EDEPIinloaKiwtohdEwcY09ENsyiwt7DwwMKhQLFxcVm7cXFxfD29q53m5UrV2LOnDmYO3cuAGDIkCHQarWYP38+li9f3qx9AkBsbCzUarXpdXl5OXx9fTFu3Di4G3tsGvDTT8bjkWPy5EmNrtvZ6PV6aDQajB8/Hkql0trhWA3zIGIeRFLMg/FXSroBh+IQkQ2zqLC3t7dHUFAQ0tPTERERAQAwGAxIT09HTExMvdtUVFRALjcfyq9QKAAAgiA0a58AoFKpoFKp6rQrlcqbfnFfvSo+urnJJPMlf6Om5MEWMA8i5kEkpTxI5ThaHQt7IrJhFg/FUavViIqKwvDhwxEcHIzExERotVpER0cDACIjI+Hj44P4+HgAQHh4OBISEjBs2DDTUJyVK1ciPDzcVODfbJ+tzXi+N053TEREEsHCnohsmMWF/bRp01BaWopVq1ahqKgIgYGBSEtLM138WlhYaNZDv2LFCshkMqxYsQJnz56Fp6cnwsPD8corrzR5n62NM+IQEUmUcSgmx9gTkQ1q1sWzMTExDQ6TyczMNP8AOzvExcUhLi6u2ftsbSzsiYgkij32RGTDLL5BlRSwsCcikigW9kRkw1jYExGRdBhP7FeuQFI3KSEiagIW9kREJB3dusF0cxL22hORjWFhT0RE0qFQiMU9wMKeiGwOC3siIpIWjrMnIhvFwp6IiFpk/fr18PPzg4ODA0JCQpCdnd3o+omJifD394ejoyN8fX2xZMkSVFZWtl5ALOyJyEaxsCciomZLTU2FWq1GXFwccnNzERAQgLCwMJSUlNS7/pYtW7B06VLExcXhyJEjSE5ORmpqKpYtW9Z6QXEueyKyUTZX2Ot0gFYrPmdhT0TUMgkJCZg3bx6io6MxaNAgJCUlwcnJCSkpKfWuv2/fPowaNQozZ86En58fJkyYgBkzZty0l98i7LEnIhvVrBtUdWYXL4qPcjng4mLdWIiIOrOqqirk5OQgNjbW1CaXyxEaGoqsrKx6txk5ciQ+/vhjZGdnIzg4GCdOnMCePXswZ86cBj9Hp9NBp9OZXpeXlwMA9Ho99PVMaSnv1g0KADWlpTBIeMpL47HXlwNbwjyImAeRFPNgybHYXGFv7MDp3l0s7omIqHnKyspQU1MDLy8vs3YvLy/k5+fXu83MmTNRVlaG0aNHQxAEVFdX48knn2x0KE58fDxWr15dpz0jIwNOTk512v3Pn8cAAIWHDuHnPXssO6hOSKPRWDuEDoF5EDEPIinloaKiosnr2mxhz2E4RETtLzMzE6+++io2bNiAkJAQHDt2DIsWLcJLL72ElStX1rtNbGws1Gq16XV5eTl8fX0xbtw4uBvH019HfuIEkJqKvs7O6D1pUpsdi7Xp9XpoNBqMHz8eSqXS2uFYDfMgYh5EUsyD8VfKpmBhT0REzeLh4QGFQoHi4mKz9uLiYnh7e9e7zcqVKzFnzhzMnTsXADBkyBBotVrMnz8fy5cvh7yen1JVKhVUKlWddqVSWf8Xt6cnAEB+8SLkEvlib0yDebAxzIOIeRBJKQ+WHIfNDUZhYU9E1Drs7e0RFBSE9PR0U5vBYEB6ejpGjBhR7zYVFRV1ineFQgEAEAShdQLjxbNEZKPYY09ERM2mVqsRFRWF4cOHIzg4GImJidBqtYiOjgYAREZGwsfHB/Hx8QCA8PBwJCQkYNiwYaahOCtXrkR4eLipwG8xFvZEZKNY2BMRUbNNmzYNpaWlWLVqFYqKihAYGIi0tDTTBbWFhYVmPfQrVqyATCbDihUrcPbsWXh6eiI8PByvvPJK6wXFwp6IbBQLeyIiapGYmBjExMTU+15mZqbZazs7O8TFxSEuLq7tAjJeUFteDuj1gETG2RIR3QzH2BMRkbR061b73HjzEiIiG8DCnoiIpEWhqC3uORyHiGwIC3siIpIejrMnIhvEwp6IiKTHOM7+/HnrxkFE1I5Y2BMRkfSwx56IbJBNFfbV1cDly+JzFvZERBLGwp6IbJBNFfaXLtU+v37SBCIikhgW9kRkg2yqsDee311dATubm8GfiMiGcIw9EdmgZhX269evh5+fHxwcHBASEoLs7OwG1x07dixkMlmdZfLkyaZ1rl69ipiYGPTu3RuOjo4YNGgQkpKSmhNaozi+nojIRrDHnohskMWFfWpqKtRqNeLi4pCbm4uAgACEhYWhpKSk3vV37NiBc+fOmZa8vDwoFApMnTrVtI5arUZaWho+/vhjHDlyBIsXL0ZMTAx27drV/COrBwt7IiIbwcKeiGyQxYV9QkIC5s2bh+joaFPPupOTE1JSUupd383NDd7e3qZFo9HAycnJrLDft28foqKiMHbsWPj5+WH+/PkICAho9JeA5jCe37t3b9XdEhFRR8PCnohskEWFfVVVFXJychAaGlq7A7kcoaGhyMrKatI+kpOTMX36dHTp0sXUNnLkSOzatQtnz56FIAjIyMjA0aNHMWHCBEvCuynjncXZY09EJHHGEz3H2BORDbHoEtKysjLU1NTAy8vLrN3Lywv5+fk33T47Oxt5eXlITk42a1+3bh3mz5+P3r17w87ODnK5HJs2bcI999zT4L50Oh10Op3pdXl5OQBAr9dDr9fXu01pqRyAAt261UCvN9w03s7IeOwN5cBWMA8i5kEkxTxI6VjahPHiWfbYE5ENade5YZKTkzFkyBAEBwebta9btw779+/Hrl270LdvX3z77bdYuHAhevXqZfbrwPXi4+OxevXqOu0ZGRlwcnKqd5sffxwMoD8uXDiOPXuOtPh4OjKNRmPtEDoE5kHEPIiklIeKigprh9CxGXvsy8sBvR5QKq0bDxFRO7CosPfw8IBCoUBxcbFZe3FxMby9vRvdVqvVYuvWrVizZo1Z+7Vr17Bs2TLs3LnTNFPO0KFDcejQIfzzn/9ssLCPjY2FWq02vS4vL4evry/GjRsHd2NPzQ0+/VQBALjzzv6YNKlf4wfbSen1emg0GowfPx5KG/4iYx5EzINIinkw/kpJDbj+ZiWXLgGentaKhIio3VhU2Nvb2yMoKAjp6emIiIgAABgMBqSnpyMmJqbRbbdt2wadTofZs2ebtRuHzsjl5sP9FQoFDIaGh8uoVCqoVKo67UqlssEvbuMNqjw9FVAqFY3G29k1lgdbwjyImAeRlPIgleNoM3Z24k1LLl8Wh+OwsCciG2DxUBy1Wo2oqCgMHz4cwcHBSExMhFarRXR0NAAgMjISPj4+iI+PN9suOTkZERERdXrTXVxcMGbMGDz33HNwdHRE37598c033+Cjjz5CQkJCCw6tLk53SURkQ9zdxcKeF9ASkY2wuLCfNm0aSktLsWrVKhQVFSEwMBBpaWmmC2oLCwvr9L4XFBRg7969+Oqrr+rd59atWxEbG4tZs2bhwoUL6Nu3L1555RU8+eSTzTikhrGwJyKyIW5uwIkTvICWiGxGsy6ejYmJaXDoTWZmZp02f39/CILQ4P68vb3x/vvvNycUi7CwJyKyIZzLnohsjMU3qOqsDAYW9kRENoWFPRHZGJsp7K9cEYt7gHeeJSKyCcZrujjGnohshM0U9sYOGycnwMHBurEQEVE7YI89EdkYmyvsOQyHiMhGsLAnIhvDwp6IiKSJhT0R2RgW9kREJE0cY09ENoaFPRERtcj69evh5+cHBwcHhISEIDs7u8F1x44dC5lMVmeZPHly6wfGHnsisjEs7ImIqNlSU1OhVqsRFxeH3NxcBAQEICwsDCUlJfWuv2PHDpw7d8605OXlQaFQYOrUqa0fHAt7IrIxLOyJiKjZEhISMG/ePERHR2PQoEFISkqCk5MTUlJS6l3fzc0N3t7epkWj0cDJyaltC/vLl4Hq6tbfPxFRB8PCnoiImqWqqgo5OTkIDQ01tcnlcoSGhiIrK6tJ+0hOTsb06dPRpUuX1g/w+puWXLzY+vsnIupg7KwdQHthYU9E1LrKyspQU1MDLy8vs3YvLy/k5+ffdPvs7Gzk5eUhOTm50fV0Oh10Op3pdXl5OQBAr9dDr9c3uq2dqytkly9DX1wMdOt205g6E+Ox3ywHUsc8iJgHkRTzYMmxsLAnIiKrSE5OxpAhQxAcHNzoevHx8Vi9enWd9oyMDDg5OTW6baiDA7pcvoys3btx8fjxFsXbUWk0GmuH0CEwDyLmQSSlPFRUVDR5XRb2RETULB4eHlAoFCguLjZrLy4uhre3d6PbarVabN26FWvWrLnp58TGxkKtVptel5eXw9fXF+PGjYO7cUrLBih8fIDiYowcMADCpEk3/azORK/XQ6PRYPz48VAqldYOx2qYBxHzIJJiHoy/UjYFC3siImoWe3t7BAUFIT09HREREQAAg8GA9PR0xMTENLrttm3boNPpMHv27Jt+jkqlgkqlqtOuVCpv/sX9Z+FvV14OSORL/kZNyoMNYB5EzINISnmw5DhsorAXBBb2RERtQa1WIyoqCsOHD0dwcDASExOh1WoRHR0NAIiMjISPjw/i4+PNtktOTkZERMRNe9xbjDepIiIbYhOFfUUFUFUlPmdhT0TUeqZNm4bS0lKsWrUKRUVFCAwMRFpamumC2sLCQsjl5hOwFRQUYO/evfjqq6/aPkDOZU9ENsQmCnvj+dzeHrjJdVZERGShmJiYBofeZGZm1mnz9/eHIAhtHNWfWNgTkQ2xiXnsrx+GI5NZNxYiImpHLOyJyIbYXGFPREQ2hGPsiciG2FRhf/1NCImIyAawx56IbIhNFfbssScisjEs7InIhrCwJyIi6WJhT0Q2hIU9ERFJl3GM/aVLQHW1VUMhImprLOyJiEi6rr+46tIlq4VBRNQemlXYr1+/Hn5+fnBwcEBISAiys7MbXHfs2LGQyWR1lsmTJ5utd+TIETzwwANwdXVFly5dcOedd6KwsLA54dVx8aL4yMKeiMjG2NkBLi7icw7HISKJs7iwT01NhVqtRlxcHHJzcxEQEICwsDCUlJTUu/6OHTtw7tw505KXlweFQoGpU6ea1jl+/DhGjx6NAQMGIDMzEz///DNWrlwJBweH5h/ZddhjT0RkwzjOnohshMV3nk1ISMC8efMQHR0NAEhKSsLu3buRkpKCpUuX1lnf7YZqeuvWrXBycjIr7JcvX45Jkybh9ddfN7X179/f0tAaxMKeiMiGubkBp05xLnsikjyLeuyrqqqQk5OD0NDQ2h3I5QgNDUVWVlaT9pGcnIzp06ejS5cuAACDwYDdu3fj9ttvR1hYGHr06IGQkBB8/vnnloTWKBb2REQ2zHgBLXvsiUjiLOqxLysrQ01NDby8vMzavby8kJ+ff9Pts7OzkZeXh+TkZFNbSUkJrl69in/84x94+eWX8dprryEtLQ0PPfQQMjIyMGbMmHr3pdPpoNPpTK/Ly8sBAHq9Hnq93mzdCxfsAMjg7KzHDW9JjvHYb8yBrWEeRMyDSIp5kNKxtDkOxSEiG2HxUJyWSE5OxpAhQxAcHGxqMxgMAIApU6ZgyZIlAIDAwEDs27cPSUlJDRb28fHxWL16dZ32jIwMODk5mV7r9XJoteEAgIMHv0J+vm1Md6bRaKwdQofAPIiYB5GU8lBRUWHtEDoPFvZEZCMsKuw9PDygUChQXFxs1l5cXAxvb+9Gt9Vqtdi6dSvWrFlTZ592dnYYNGiQWfvAgQOxd+/eBvcXGxsLtVptel1eXg5fX1+MGzcO7safXQEUFYmPcrmARx6ZALnEJ/jU6/XQaDQYP348lEqltcOxGuZBxDyIpJgH46+U1ATGwp5j7IlI4iwq7O3t7REUFIT09HREREQAEHvc09PTERMT0+i227Ztg06nw+zZs+vs884770RBQYFZ+9GjR9G3b98G96dSqaBSqeq0K5VKsy/uK1fEx+7dZVCppPGF3hQ35sFWMQ8i5kEkpTxI5TjaBcfYE5GNsHgojlqtRlRUFIYPH47g4GAkJiZCq9WaZsmJjIyEj48P4uPjzbZLTk5GRESEWW+60XPPPYdp06bhnnvuwbhx45CWlob//Oc/yMzMbN5RXYcXzhIR2TgOxSEiG2FxYT9t2jSUlpZi1apVKCoqQmBgINLS0kwX1BYWFkJ+w3iXgoIC7N27F1999VW9+3zwwQeRlJSE+Ph4PPPMM/D398dnn32G0aNHN+OQzLGwJyKycSzsichGNOvi2ZiYmAaH3tTXy+7v7w9BEBrd5+OPP47HH3+8OeE0ioU9EZGNY2FPRDZC4peSsrAnIrJ5xiGgvHiWiCSOhT0REUmb8Qvg0iWgpsaqoRARtSUW9kREJG3du9c+v3TJamEQEbU1FvZERCRtSiXg7Cw+5zh7IpIwFvZERCR9HGdPRDaAhT0REUmf8Uvg9GnrxkFE1IZY2BMRkfT99a/iY3KydeMgImpDLOyJiEj6Fi4E5HJAowEOH7Z2NEREbULShX11NXD5svichT0RUdtYv349/Pz84ODggJCQEGRnZze6/qVLl7Bw4UL07NkTKpUKt99+O/bs2dO2Qfr5AVOmiM/ffrttP4uIyEokXdhfP6tZt27WioKISLpSU1OhVqsRFxeH3NxcBAQEICwsDCUlJfWuX1VVhfHjx+PUqVPYvn07CgoKsGnTJvj4+LR9sM88Iz5+9BFnxyEiSZJ0YW88b7u6AnZ21o2FiEiKEhISMG/ePERHR2PQoEFISkqCk5MTUlJS6l0/JSUFFy5cwOeff45Ro0bBz88PY8aMQUBAQNsHO2YMMHQocO0ax9oTkSTZRGHPYThERK2vqqoKOTk5CA0NNbXJ5XKEhoYiKyur3m127dqFESNGYOHChfDy8sLgwYPx6quvoqY97ggrkwGLFonP33lHHK9JRCQhku7HZmFPRNR2ysrKUFNTAy8vL7N2Ly8v5Ofn17vNiRMn8PXXX2PWrFnYs2cPjh07hqeeegp6vR5xcXH1bqPT6aDT6Uyvy8vLAQB6vR56vd6yoKdOhd0LL0BWWIjqzz6D8NBDlm3fgRiP3eIcSAzzIGIeRFLMgyXHYhOF/fV3EyciIusxGAzo0aMH3nvvPSgUCgQFBeHs2bN44403Gizs4+PjsXr16jrtGRkZcHJysjiGgWPH4vbt23FpzRp87+Bg8fYdjUajsXYIHQLzIGIeRFLKQ0VFRZPXtYnCnj32REStz8PDAwqFAsXFxWbtxcXF8Pb2rnebnj17QqlUQqFQmNoGDhyIoqIiVFVVwd7evs42sbGxUKvVptfl5eXw9fXFuHHj4G68o6wlAgIgfP45PA4fxqSePYFhwyzfRweg1+uh0Wgwfvx4KJVKa4djNcyDiHkQSTEPxl8pm4KFPRERNYu9vT2CgoKQnp6OiIgIAGKPfHp6OmJiYurdZtSoUdiyZQsMBgPkcvEyr6NHj6Jnz571FvUAoFKpoFKp6rQrlcrmfXH7+QGPPAJs3Qrlxo3A++9bvo8OpNl5kBjmQcQ8iKSUB0uOgxfPEhFRs6nVamzatAkffvghjhw5ggULFkCr1SI6OhoAEBkZidjYWNP6CxYswIULF7Bo0SIcPXoUu3fvxquvvoqFCxe2b+DGi2i3bAEamJqTiKizYY89ERE127Rp01BaWopVq1ahqKgIgYGBSEtLM11QW1hYaOqZBwBfX1/83//9H5YsWYKhQ4fCx8cHixYtwgsvvNC+gd91FxAcDGRnA+++C6xc2b6fT0TUBljYExFRi8TExDQ49CYzM7NO24gRI7B///42jqoJnnkGmD0b2LABeOEFoIGhQEREnQWH4hARkW2aOhXo2RMoKgK2b7d2NERELcbCnoiIbJO9PbBggfh87VrrxkJE1AokXdhfvCg+srAnIqJ6PfGEWOBnZwMdYXgQEVELSLawNxjYY09ERDfRowcwY4b4nL32RNTJSbawv3JFLO4B3nmWiIgaYZz6cvt24OxZ68ZCRNQCzSrs169fDz8/Pzg4OCAkJATZ2dkNrjt27FjIZLI6y+TJk+td/8knn4RMJkNiYmJzQjMx9tY7OQESuGM4ERG1lWHDgLvvBqqrgY0brR0NEVGzWVzYp6amQq1WIy4uDrm5uQgICEBYWBhKGrjBx44dO3Du3DnTkpeXB4VCgalTp9ZZd+fOndi/fz969epl+ZHcgMNwiIioyYy99u++C1RWWjcWIqJmsriwT0hIwLx58xAdHY1BgwYhKSkJTk5OSElJqXd9Nzc3eHt7mxaNRgMnJ6c6hf3Zs2fx9NNPY/Pmza1yC2AW9kRE1GRTpgB9+gBlZeLdaImIOiGLCvuqqirk5OQgNDS0dgdyOUJDQ5GVldWkfSQnJ2P69Ono0qWLqc1gMGDOnDl47rnn8Je//MWSkBrEwp6IiJrMzg5YuFB8vnYtIAjWjYeIqBksuvNsWVkZampqTLcKN/Ly8kJ+fv5Nt8/OzkZeXh6Sk5PN2l977TXY2dnhmWeeaXIsOp0OOp3O9Lq8vBwAoNfrodfrUVoqB6BAt24G6PU1Td5vZ6fX680ebRXzIGIeRFLMg5SOpcOYOxd48UXg55+Bb78FxoyxdkRERBaxqLBvqeTkZAwZMgTBwcGmtpycHKxduxa5ubmQyWRN3ld8fDxWr15dpz0jIwNOTk7Yv/92AAOh1RZiz56fWiP8TkWj0Vg7hA6BeRAxDyIp5aGiosLaIUiPmxsQGSmOs1+7loU9EXU6FhX2Hh4eUCgUKC4uNmsvLi6Gt7d3o9tqtVps3boVa9asMWv/7rvvUFJSgj59+pjaampq8Pe//x2JiYk4depUvfuLjY2FWq02vS4vL4evry/GjRsHd3d3ZGaKo4yGDvXFpEk+lhxmp6bX66HRaDB+/PhWuVahs2IeRMyDSIp5MP5KSa3smWfEwv6LL4BTpwA/P2tHRETUZBYV9vb29ggKCkJ6ejoiIiIAiOPj09PTERMT0+i227Ztg06nw+zZs83a58yZYzZmHwDCwsIwZ84cREdHN7g/lUoFlUpVp12pVEKpVOLSJfG1p6cCSqXi5gcnMcY82DrmQcQ8iKSUB6kcR4czaBAQGgr873/AO+8A//yntSMiImoyi4fiqNVqREVFYfjw4QgODkZiYiK0Wq2pCI+MjISPjw/i4+PNtktOTkZERATc3d3N2t3d3eu0KZVKeHt7w9/f39LwTHjxLBERNcuiRWJh/69/iWPuu3a1dkRERE1icWE/bdo0lJaWYtWqVSgqKkJgYCDS0tJMF9QWFhZCLjefbKegoAB79+7FV1991TpRNwELeyIiapZJk4BbbwWOHQP+/W9gwQJrR0RE1CTNung2JiamwaE3mZmZddr8/f0hWDB1WEPj6i3Bwp6IiJpFLgeeflrsuX/7beCJJ8Q2IqIOTrJnKhb2RETUbI89Bjg7A/n5wJdfWjsaIqImkWRhLwgs7ImIqAVcXMR57QFg3jzg3DnrxkNE1ASSLOwrKoCqKvE5C3siImqW1avFWXLOnQOmTq39YiEi6qAkWdgbe+vt7QEnJ+vGQkREnZSzM/D554CrK/D998CSJdaOiIioUZIu7N3cAAtuZktERGTuttuAjz8Wn2/YALz/vnXjISJqhOQLeyIioha5/35xWA4gTn35ww/WjYeIqAEs7ImIiG5mxQrggQcAnQ546CGgpMTaERER1SHpwr57d+vGQUREEiGXAx99BPj7A7//Djz6KKDXWzsqIiIzki7s2WNPREStxtVVvJjW2Rn45hvgueesHRERkRkW9kRERE01YIDYcw8Aa9cC//63deMhIroOC3siImqR9evXw8/PDw4ODggJCUF2dnaD637wwQeQyWRmi4ODQztG2woiIsQx9wAwfz6Qm2vVcIiIjFjYExFRs6WmpkKtViMuLg65ubkICAhAWFgYShq5uNTFxQXnzp0zLadPn27HiFvJiy8CkyYBlZXixbRlZdaOiIiIhT0RETVfQkIC5s2bh+joaAwaNAhJSUlwcnJCSkpKg9vIZDJ4e3ubFi8vr3aMuJUoFMDmzcCttwKnTwPTpwPV1daOiohsnJ21A2gLLOyJiNpeVVUVcnJyEBsba2qTy+UIDQ1FVlZWg9tdvXoVffv2hcFgwB133IFXX30Vf/nLXxpcX6fTQafTmV6Xl5cDAPR6PfTWnJmmSxfg009hd/fdkKWno+aFF2D4xz/a7eONx27VHHQAzIOIeRBJMQ+WHAsLeyIiapaysjLU1NTU6XH38vJCfn5+vdv4+/sjJSUFQ4cOxeXLl/HPf/4TI0eOxK+//orevXvXu018fDxWG28QdZ2MjAw4OTm1/EBaqOfChQh+/XUoEhLwo0yGs3ff3a6fr9Fo2vXzOirmQcQ8iKSUh4qKiiavy8KeiIjazYgRIzBixAjT65EjR2LgwIF499138dJLL9W7TWxsLNRqtel1eXk5fH19MW7cOLi7u7d5zDc1aRJqDAYo/vlPBG3YgIAZM4ChQ9v8Y/V6PTQaDcaPHw+lUtnmn9dRMQ8i5kEkxTwYf6VsCskV9jodYPyHDQt7IqK24+HhAYVCgeLiYrP24uJieHt7N2kfSqUSw4YNw7FjxxpcR6VSQaVS1btth/ni/sc/gJ9/huyrr6B89FHghx/a7UuoQ+XBipgHEfMgklIeLDkOyV08e+mS+CiXAy4uVg2FiEjS7O3tERQUhPT0dFObwWBAenq6Wa98Y2pqavDLL7+gZ8+ebRVm+1AogE8+Afr1A06cAMaPB06etHZURGRjJFfYX7woPnbvLhb3RETUdtRqNTZt2oQPP/wQR44cwYIFC6DVahEdHQ0AiIyMNLu4ds2aNfjqq69w4sQJ5ObmYvbs2Th9+jTmzp1rrUNoPW5u4p1p3d3Fue2DgoA9e6wdFRHZEMkNxbl0SQaAw3CIiNrDtGnTUFpailWrVqGoqAiBgYFIS0szXVBbWFgI+XW9LBcvXsS8efNQVFSE7t27IygoCPv27cOgQYOsdQita+hQsaifOhXIzgYmTwZWrgTi4sRefSKiNiS5wt7YY8/CnoiofcTExCAmJqbe9zIzM81ev/XWW3jrrbfaISor6tMH+PZbQK0GNmwAXnoJ2L8f2LIF8PCwdnREJGGSG6xiHGPPwp6IiKxGpQLWrwc+/hhwcgI0GuCOO4ADB6wdGRFJmOQK+4sXORSHiIg6iFmzxGL+9tuBM2eAu+8We/EFwdqREZEESbCwFx9Z2BMRUYcweLA4/eXDDwN6PbBwITBnDqDVWjsyIpKYZhX269evh5+fHxwcHBASEoLs7OwG1x07dixkMlmdZfLkyQDEGwm88MILGDJkCLp06YJevXohMjISf/zxR7MOiENxiIiow3FxAbZtA958U7yIdvNmICQEKCiwdmREJCEWF/apqalQq9WIi4tDbm4uAgICEBYWhpKSknrX37FjB86dO2da8vLyoFAoMHXqVADibXJzc3OxcuVK5ObmYseOHSgoKMADDzzQrAPiUBwiIuqQZDLxgtqMDMDbG/j1V+DOO4HPPrN2ZEQkERYX9gkJCZg3bx6io6MxaNAgJCUlwcnJCSkpKfWu7+bmBm9vb9Oi0Wjg5ORkKuxdXV2h0Wjw6KOPwt/fH3fddRfeeecd5OTkoLCw0OIDYo89ERF1aHffDfz4I3DPPcCVK8AjjwDPPisO0yEiagGLCvuqqirk5OQgNDS0dgdyOUJDQ5GVldWkfSQnJ2P69Ono0qVLg+tcvnwZMpkM3bp1syQ8ABxjT0REnYC3N5CeDjz3nPj6zTeBUaM4aw4RtYhF89iXlZWhpqbGdOMRIy8vL+Tn5990++zsbOTl5SE5ObnBdSorK/HCCy9gxowZcHFxaXA9nU4HnU5nel1eXg6gtrB3camGXm97sw7o/+zx0dt4zw/zIGIeRFLMg5SOxWbZ2QGvvw6MGAE89ph4ge1ddwGzZwP/+Afg42PtCImok2nXG1QlJydjyJAhCA4Orvd9vV6PRx99FIIgYOPGjY3uKz4+HqtXr67TXlpaAwD4+edMnD9vuzMOaDQaa4fQITAPIuZBJKU8VFRUWDsEai0PPigW9MuXAx98IM59v2MH8MIL4hAdJydrR0hEnYRFhb2HhwcUCgWKi4vN2ouLi+Ht7d3otlqtFlu3bsWaNWvqfd9Y1J8+fRpff/11o731ABAbGwu1Wm16XV5eDl9fX1y7pgQAPPjgGJu8wZ9er4dGo8H48eOhVCqtHY7VMA8i5kEkxTwYf6UkiejZE0hJEafCXLQI+P57IC4O+Ne/xF79adPEi2+JiBphUWFvb2+PoKAgpKenIyIiAgBgMBiQnp7e4O3EjbZt2wadTofZs2fXec9Y1P/222/IyMiAu7v7TWNRqVRQqVQNvu/pqYRdu/4e0bEolUrJFDAtwTyImAeRlPIgleOgGwQFAd99B3z6KfD880BhITBjBrBuHZCYKM6iQ0TUAItnxVGr1di0aRM+/PBDHDlyBAsWLIBWq0V0dDQAIDIyErGxsXW2S05ORkRERJ2iXa/X45FHHsHBgwexefNm1NTUoKioCEVFRaiqqmrWQbm6wqaLeiIi6sRkMrGHPj8feOklcSjOvn1AcLA4Fr+Z93khIumzuLCfNm0a/vnPf2LVqlUIDAzEoUOHkJaWZrqgtrCwEOfOnTPbpqCgAHv37sXf/va3Ovs7e/Ysdu3ahd9//x2BgYHo2bOnadm3b1+zDooz4hARUafn6AisWAEcPSreqRYAPvwQuP124JVXgGvXrBsfEXU4zerXjomJaXDoTWZmZp02f39/CEL9M9T4+fk1+F5zsbAnIiLJ8PEBPvoIiIkBFi8GsrKAFStgt2kTfB56CAgNBTg0i4jQjB77zoCFPRERSU5wsHhR7ebNQO/ekJ0+jeFvvQU7Pz9gyRLgl1+sHSERWZkkC/vu3a0dARERURuQyYCZM4H8fNTExaGye3fIysrEC2uHDhUvrt2wofamLkRkUyRZ2LPHnoiIJK1LFxiWL8dX//oXqj//HHj4YXE4zsGD4pSZPXuK/wDQaACDwdrRElE7YWFPRETUSQkKBYRJk4Dt24GzZ4G33gKGDAF0OuCTT4AJE4B+/cQ58U+etHa4RNTGWNgTERFJgaeneHHtTz+JPfdPPQV06ybOhb9mDXDLLcBf/wr8+9/A5cvWjpaI2gALeyIiIimRycQbXa1fL855v2ULMH682J6RAURGiv8ICAsDNm4Ue/qJSBJY2BMREUmVo6N459qvvhKH4qxeDQwcCOj1YttTTwG9ewMhIUB8PHDkiLUjJqIWYGFPRERkC/r2BVatAg4fFu9q+9prwIgR4nvZ2cCyZcCgQcCAAcDSpcD+/bzwlqiTYWFPRERka/z9geefB/btE4frvPsuMHEiYG8PFBTUFv0+PsATTwBffglotdaOmohugoU9ERGRLevZE5g/H9izBygtBbZuBaZPB5ydgaIi4L33gEmTxAtxQ0KAZ58Fdu0CLlywduREdAM7awfQFniDKiIiomZwcQGmTRMXnQ7IzAQ+/xzYvRs4c0YcspOdDbz5prj+X/4C3H03cM894mPv3taMnsjmSa6wd3QU4OBg7SiIiIg6OZVKnDknLAwQBOD0aeC772qX/Hzg11/FJSlJ3MbPTyzwjcX+7beLs/EQUbuQ3FAc9tYTEbWv9evXw8/PDw4ODggJCUF2dnaTttu6dStkMhkiIiLaNkBqOZlMLNrnzBGH5hw5ApSUADt2AEuWAMOHA3I5cOqUOE/+/PniRbjduwNjxgCLFgHvvw8cOgRUVVn5YIikS3I99t26WTsCIiLbkZqaCrVajaSkJISEhCAxMRFhYWEoKChAjx49Gtzu1KlTePbZZ3H33Xe3Y7TUqjw9gQcfFBcAuHIFyMqq7dE/cEC8Eda334qLkVIpzr4TGGi+8AucqMUkV9h37y5YOwQiIpuRkJCAefPmITo6GgCQlJSE3bt3IyUlBUuXLq13m5qaGsyaNQurV6/Gd999h0uXLrVjxNRmnJ2BCRPEBRDnys/PF3vpf/xRfDx0CLh4Ubw77k8/AR9+WLu9n19tkT9smPjo68uhPEQWkFxhz3/wExG1j6qqKuTk5CA2NtbUJpfLERoaiqysrAa3W7NmDXr06IG//e1v+O677276OTqdDjqdzvS6vLwcAKDX66HX61twBJ2b8dg7dA4GDBCX6dPF14IAnDkD2U8/QXbokPj488+QnTolDuM5dUq8WPdPgrs7hIAACIGB4mNAgDhVp0JhWqdT5KEdMA8iKebBkmORXGHPMfZERO2jrKwMNTU18PLyMmv38vJCfn5+vdvs3bsXycnJOHToUJM/Jz4+HqtXr67TnpGRAScnJ4tiliKNRmPtECynUABBQeICQHn1KlxOnYLriRNwPXkSridPwvnMGcjPn4fs66+Br782bVptb49yPz9cvuUWXO7XD5f79YOib9/OmYc2wDyIpJSHioqKJq8rucK+WzcOxSEi6oiuXLmCOXPmYNOmTfDw8GjydrGxsVCr1abX5eXl8PX1xbhx4+Du7t4WoXYKer0eGo0G48ePh1KptHY4ra6mshI1hw/X9uwfOgTZzz/DTquF29GjcDt61Gx9wcMDQp8+gK9v7aOvL9C3r/jo6Sle4CtRUv97aCop5sH4K2VTSK6wZ489EVH78PDwgEKhQHFxsVl7cXExvL2966x//PhxnDp1CuHh4aY2g8EAALCzs0NBQQH69+9fZzuVSgWVSlWnXalUSuaLuyUkmwelUrwhVkhIbVtNDXDsWO2Y/R9/hPDjj5CVlkJWVgZZWRmQm1v//lQqccx+nz61S79+tYuPj9kQn85Ksn8PFpJSHiw5DgkW9uyxJyJqD/b29ggKCkJ6erppykqDwYD09HTExMTUWX/AgAH45ZdfzNpWrFiBK1euYO3atfD19W2PsKkzUyjEMfb+/qZx+9VVVdBs24bx/v5QnjsHFBbWXf74Q7zh1rFj4lIfpbK22L/lFvOiv18/wMODF/JShye5wp4XzxIRtR+1Wo2oqCgMHz4cwcHBSExMhFarNc2SExkZCR8fH8THx8PBwQGDBw82277bnyftG9uJmkwmg75rVyAgQJxPvz56vVjcX1/snzoFnDwpLqdPi+scPy4u9enaVZy5p08f8Q67xsXXt/Z5165tdZRETSK5wp5DcYiI2s+0adNQWlqKVatWoaioCIGBgUhLSzNdUFtYWAi5hMc1UyehVAJ9+4pLfWpqgLNnawt943LihPj4xx/A1atAXp64NMTVtf6i38tLHOPv4SE+Ojuz95/ahOQKe148S0TUvmJiYuodegMAmZmZjW77wQcftH5ARJZSKGrH3Y8ZU/f9ykqxV//kSeD3382XM2fEx/Jy8YZcly8Dv/7a+OfZ29cW+dcX/De+9vAQFzc38R8nRDchucKePfZERETUqhwcasf2N6S8XOz1r6/wLykBysqA0lKgogKoqhJ/Bfjjj6bH4OpaW+gbF3d303NZt25w/+038RcCT09xbHLXrpKeCYjqalZhv379erzxxhsoKipCQEAA1q1bh+Dg4HrXHTt2LL755ps67ZMmTcLu3bsBAIIgIC4uDps2bcKlS5cwatQobNy4EbfddpvFsbGwJyIionbn4iIuAwc2vl5FRW2Rb3y88XlpKXD+vNh2/rx4Yy/jrwENXANgB2A0ACxfXtsol4v/IHB1FQv9hpYb3ze+dnGRxExBtsTiwj41NRVqtRpJSUkICQlBYmIiwsLCUFBQgB49etRZf8eOHaiqqjK9Pn/+PAICAjB16lRT2+uvv463334bH374Ifr164eVK1ciLCwMhw8fhoODg0Xx8V4lRERE1GE5OdUO+2mKmhrg0iWxyDcuxqL/usVQWoqKwkJ0qamB7OJF8VcBgwG4eFFcmsvFpf7Cv3t38Xn37rXLja+7dOG1BO3M4sI+ISEB8+bNM814kJSUhN27dyMlJQVLly6ts76bm5vZ661bt8LJyclU2AuCgMTERKxYsQJTpkwBAHz00Ufw8vLC559/junG21A3Ef9+iIiISDIUCnHIjbt7o0OBavR6pO/Zg0mTJonznldWiv8guNly8WLtrwHGtsuXgWvXxB2Xl4vLmTOWx25nZ17su7iIFw47O4vDhIzP61uuf79rV/E+BCzybsqiwr6qqgo5OTmIjY01tcnlcoSGhiIrK6tJ+0hOTsb06dPRpUsXAMDJkydRVFSE0NBQ0zqurq4ICQlBVlaWxYU9ERERkc1zcAC8vcWlOXS6+gt+4z8Grv+HgfG18fnFi+L0odXVtb8qtJRcLhb4XbqYP97QJnd0xO1//AF5fr74jwInJ/PF0bFum5OTeEGzBP7hYFFhX1ZWhpqaGtM0ZkZeXl7Iz8+/6fbZ2dnIy8tDcnKyqa2oqMi0jxv3aXyvPjqdDjqdzvTaeLtdvV4PvV5/84ORKOOx23IOAObBiHkQSTEPUjoWIuqAVCqgRw9xsZQgiD3+1xf6Fy8CV65YvlRWivs0GGp/PWiEAsBAAPjkE8tilsnEX0euX+zs6rbd+L69vZgrS5YuXWp/vTBem2F8bvyVwq5589u066w4ycnJGDJkSIMX2loiPj4eq1evrtOekZEBJw60h0ajsXYIHQLzIGIeRFLKQ0VFhbVDICKqn0xW2xPu49OyfdXUAFqteB+BGx/raau5cgVnDh9GH09PyCsrxYuVG1tqasTPEQTxF4bq6pYff2twcqot9v8c5dIUFhX2Hh4eUCgUKC4uNmsvLi6G901+6tFqtdi6dSvWrFlj1m7crri4GD179jTbZ2BgYIP7i42NhVqtNr0uLy+Hr68vxo0bB3d396YekuTo9XpoNBqMHz9eHGNno5gHEfMgkmIeym/Sa0VEJAkKRW2vdhMY9Hr8tGcPfCZNgrwp53u93rzIt2SprhYvUtbpmr5UVor/CLlyRfz1wfhofG4cjWKM6Yaa+2YsKuzt7e0RFBSE9PR0REREiAk0GJCent7gzUmMtm3bBp1Oh9mzZ5u19+vXD97e3khPTzcV8uXl5Thw4AAWLFjQ4P5UKhVUKlWddqVSKZkv7pZgHkTMg4h5EEkpD1I5DiIiq1Iqa6cE7Qh0utphSMZiv6gIuG42ycZYPBRHrVYjKioKw4cPR3BwMBITE6HVak2z5ERGRsLHxwfx8fFm2yUnJyMiIqJOb7pMJsPixYvx8ssv47bbbjNNd9mrVy/TPx6IiIiIiCTPOA7fw6O2zYJfaC0u7KdNm4bS0lKsWrUKRUVFCAwMRFpamuni18LCQshvuMtZQUEB9u7di6+++qrefT7//PPQarWYP38+Ll26hNGjRyMtLc3iOeyJiIiIiGxVsy6ejYmJaXDoTWZmZp02f39/CILQ4P5kMhnWrFlTZ/w9ERERERE1jfzmqxARERERUUfHwp6IiIiISAJY2BMRERERSQALeyIiIiIiCWBhT0REREQkASzsiYiIiIgkgIU9EREREZEEsLAnIiIiIpKAZt2gqiMy3gDrypUrUCqVVo7GevR6PSoqKlBeXs48MA/Mw5+kmIfyP28x3tjN/6SK53uRFP+um4N5EDEPIinmwZLzvWQK+/PnzwMA+vXrZ+VIiIjaz5UrV+Dq6mrtMNoVz/dEZIuacr6XTGHv5uYGACgsLLS5L7nrlZeXw9fXF2fOnIGLi4u1w7Ea5kHEPIikmAdBEHDlyhX06tXL2qG0O57vRVL8u24O5kHEPIikmAdLzveSKezlcvFyAVdXV8n8h2wJFxcX5gHMgxHzIJJaHmy1qOX53pzU/q6bi3kQMQ8iqeWhqed7XjxLRERERCQBLOyJiIiIiCRAMoW9SqVCXFwcVCqVtUOxKuZBxDyImAcR8yAt/O8pYh5EzIOIeRDZeh5kgi3OlUZEREREJDGS6bEnIiIiIrJlLOyJiIiIiCSAhT0RERERkQSwsCciIiIikgBJFPbr16+Hn58fHBwcEBISguzsbGuH1K5efPFFyGQys2XAgAHWDqvNffvttwgPD0evXr0gk8nw+eefm70vCAJWrVqFnj17wtHREaGhofjtt9+sE2wbu1kuHnvssTp/I/fdd591gm0j8fHxuPPOO+Hs7IwePXogIiICBQUFZutUVlZi4cKFcHd3R9euXfHwww+juLjYShFTc/Gcz3O+LZ/zeb4X8Zxfv05f2KempkKtViMuLg65ubkICAhAWFgYSkpKrB1au/rLX/6Cc+fOmZa9e/daO6Q2p9VqERAQgPXr19f7/uuvv463334bSUlJOHDgALp06YKwsDBUVla2c6Rt72a5AID77rvP7G/kk08+accI294333yDhQsXYv/+/dBoNNDr9ZgwYQK0Wq1pnSVLluA///kPtm3bhm+++QZ//PEHHnroIStGTZbiOV/Ec35dtnLO5/lexHN+A4ROLjg4WFi4cKHpdU1NjdCrVy8hPj7eilG1r7i4OCEgIMDaYVgVAGHnzp2m1waDQfD29hbeeOMNU9ulS5cElUolfPLJJ1aIsP3cmAtBEISoqChhypQpVonHWkpKSgQAwjfffCMIgvjfX6lUCtu2bTOtc+TIEQGAkJWVZa0wyUI85/OcLwg85xvxfF+L53xRp+6xr6qqQk5ODkJDQ01tcrkcoaGhyMrKsmJk7e+3335Dr169cMstt2DWrFkoLCy0dkhWdfLkSRQVFZn9bbi6uiIkJMTm/jaMMjMz0aNHD/j7+2PBggU4f/68tUNqU5cvXwYAuLm5AQBycnKg1+vN/iYGDBiAPn362OzfRGfDc34tnvPN8ZxvztbO9wDP+UadurAvKytDTU0NvLy8zNq9vLxQVFRkpajaX0hICD744AOkpaVh48aNOHnyJO6++25cuXLF2qFZjfG/v63/bRjdd999+Oijj5Ceno7XXnsN33zzDSZOnIiamhprh9YmDAYDFi9ejFGjRmHw4MEAxL8Je3t7dOvWzWxdW/2b6Ix4zhfxnF8Xz/m1bO18D/Ccfz07awdALTdx4kTT86FDhyIkJAR9+/bFp59+ir/97W9WjIw6iunTp5ueDxkyBEOHDkX//v2RmZmJe++914qRtY2FCxciLy/PJsYdk+3hOZ8aY2vne4Dn/Ot16h57Dw8PKBSKOlc4FxcXw9vb20pRWV+3bt1w++2349ixY9YOxWqM//35t1G/W265BR4eHpL8G4mJicF///tfZGRkoHfv3qZ2b29vVFVV4dKlS2br82+i8+A5v3485/Oc3xgpn+8BnvNv1KkLe3t7ewQFBSE9Pd3UZjAYkJ6ejhEjRlgxMuu6evUqjh8/jp49e1o7FKvp168fvL29zf42ysvLceDAAZv+2zD6/fffcf78eUn9jQiCgJiYGOzcuRNff/01+vXrZ/Z+UFAQlEql2d9EQUEBCgsL+TfRSfCcXz+e83nOb4wUz/cAz/kN6fRDcdRqNaKiojB8+HAEBwcjMTERWq0W0dHR1g6t3Tz77LMIDw9H37598ccffyAuLg4KhQIzZsywdmht6urVq2Y9ECdPnsShQ4fg5uaGPn36YPHixXj55Zdx2223oV+/fli5ciV69eqFiIgI6wXdRhrLhZubG1avXo2HH34Y3t7eOH78OJ5//nnceuutCAsLs2LUrWvhwoXYsmULvvjiCzg7O5vGULq6usLR0RGurq7429/+BrVaDTc3N7i4uODpp5/GiBEjcNddd1k5emoqnvN5zjey1XM+z/cinvMbYO1peVrDunXrhD59+gj29vZCcHCwsH//fmuH1K6mTZsm9OzZU7C3txd8fHyEadOmCceOHbN2WG0uIyNDAFBniYqKEgRBnP5s5cqVgpeXl6BSqYR7771XKCgosG7QbaSxXFRUVAgTJkwQPD09BaVSKfTt21eYN2+eUFRUZO2wW1V9xw9AeP/9903rXLt2TXjqqaeE7t27C05OTsKDDz4onDt3znpBU7PwnM9zvi2f83m+F/GcXz+ZIAhC2//zgYiIiIiI2lKnHmNPREREREQiFvZERERERBLAwp6IiIiISAJY2BMRERERSQALeyIiIiIiCWBhT0REREQkASzsiYiIiIgkgIU9EREREZEEsLAnIiIiIpIAFvZERERERBLAwp6IiIiISAJY2BMRERERScD/A4xtuKHWdp5bAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 900x400 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import my_plot\n",
    "\n",
    "# evaluate the model\n",
    "loss, accuracy = model.evaluate(padded_docs, labels)\n",
    "\n",
    "print('Train set accuracy: {a:.2f} %'.format(a=accuracy*100))\n",
    "\n",
    "my_plot.plot_log_results('log_lstm_64.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " embedding_2 (Embedding)     (None, 1, 100)            632700    \n",
      "                                                                 \n",
      " bidirectional_2 (Bidirectio  (None, 512)              549888    \n",
      " nal)                                                            \n",
      "                                                                 \n",
      " dense_2 (Dense)             (None, 44)                22572     \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 1,205,160\n",
      "Trainable params: 572,460\n",
      "Non-trainable params: 632,700\n",
      "_________________________________________________________________\n",
      "None\n",
      "Epoch 1/50\n",
      "1413/1413 [==============================] - 5s 2ms/step - loss: 0.8852 - accuracy: 0.7417\n",
      "Epoch 2/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.5811 - accuracy: 0.8107\n",
      "Epoch 3/50\n",
      "1413/1413 [==============================] - 4s 2ms/step - loss: 0.5130 - accuracy: 0.8256\n",
      "Epoch 4/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.4749 - accuracy: 0.8345\n",
      "Epoch 5/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.4498 - accuracy: 0.8389\n",
      "Epoch 6/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.4338 - accuracy: 0.8425\n",
      "Epoch 7/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.4200 - accuracy: 0.8453\n",
      "Epoch 8/50\n",
      "1413/1413 [==============================] - 4s 2ms/step - loss: 0.4093 - accuracy: 0.8469\n",
      "Epoch 9/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.4015 - accuracy: 0.8488\n",
      "Epoch 10/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3946 - accuracy: 0.8494\n",
      "Epoch 11/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3883 - accuracy: 0.8506\n",
      "Epoch 12/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3844 - accuracy: 0.8524\n",
      "Epoch 13/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3806 - accuracy: 0.8523\n",
      "Epoch 14/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3780 - accuracy: 0.8534\n",
      "Epoch 15/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3751 - accuracy: 0.8526\n",
      "Epoch 16/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3730 - accuracy: 0.8540\n",
      "Epoch 17/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3711 - accuracy: 0.8549\n",
      "Epoch 18/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3693 - accuracy: 0.8546\n",
      "Epoch 19/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3678 - accuracy: 0.8552\n",
      "Epoch 20/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3661 - accuracy: 0.8551\n",
      "Epoch 21/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3643 - accuracy: 0.8562\n",
      "Epoch 22/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3642 - accuracy: 0.8565\n",
      "Epoch 23/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3629 - accuracy: 0.8553\n",
      "Epoch 24/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3622 - accuracy: 0.8562\n",
      "Epoch 25/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3612 - accuracy: 0.8560\n",
      "Epoch 26/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3602 - accuracy: 0.8575\n",
      "Epoch 27/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3601 - accuracy: 0.8566\n",
      "Epoch 28/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3581 - accuracy: 0.8569\n",
      "Epoch 29/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3590 - accuracy: 0.8579\n",
      "Epoch 30/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3575 - accuracy: 0.8570\n",
      "Epoch 31/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3578 - accuracy: 0.8564\n",
      "Epoch 32/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3562 - accuracy: 0.8576\n",
      "Epoch 33/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3567 - accuracy: 0.8573\n",
      "Epoch 34/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3561 - accuracy: 0.8576\n",
      "Epoch 35/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3547 - accuracy: 0.8584\n",
      "Epoch 36/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3547 - accuracy: 0.8574\n",
      "Epoch 37/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3546 - accuracy: 0.8569\n",
      "Epoch 38/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3536 - accuracy: 0.8584\n",
      "Epoch 39/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3539 - accuracy: 0.8565\n",
      "Epoch 40/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3530 - accuracy: 0.8580\n",
      "Epoch 41/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3535 - accuracy: 0.8574\n",
      "Epoch 42/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3530 - accuracy: 0.8584\n",
      "Epoch 43/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3518 - accuracy: 0.8577\n",
      "Epoch 44/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3522 - accuracy: 0.8573\n",
      "Epoch 45/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3514 - accuracy: 0.8575\n",
      "Epoch 46/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3512 - accuracy: 0.8571\n",
      "Epoch 47/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3508 - accuracy: 0.8580\n",
      "Epoch 48/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3506 - accuracy: 0.8579\n",
      "Epoch 49/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3502 - accuracy: 0.8573\n",
      "Epoch 50/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3498 - accuracy: 0.8592\n",
      "1413/1413 [==============================] - 2s 932us/step - loss: 0.3373 - accuracy: 0.8560\n",
      "Accuracy: 85.595453\n"
     ]
    }
   ],
   "source": [
    "# Using GRU instead of LSTM\n",
    "# define model\n",
    "model = Sequential()\n",
    "e = Embedding(vocab_size, 100, weights=[embedding_matrix], input_length=1, trainable=False)\n",
    "model.add(e)\n",
    "# A\n",
    "model.add(keras.layers.Bidirectional(keras.layers.GRU(units=128), input_shape=(None, 50)))\n",
    "model.add(Dense(44, activation='softmax'))\n",
    "# compile the model\n",
    "model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "# summarize the model\n",
    "print(model.summary())\n",
    "\n",
    "# fit the model\n",
    "model.fit(padded_docs, labels, epochs=50)\n",
    "# evaluate the model\n",
    "loss, accuracy = model.evaluate(padded_docs, labels)\n",
    "print('Accuracy: %f' % (accuracy*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_9\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " embedding_9 (Embedding)     (None, 1, 100)            632700    \n",
      "                                                                 \n",
      " bidirectional_10 (Bidirecti  (None, 1, 256)           234496    \n",
      " onal)                                                           \n",
      "                                                                 \n",
      " lstm_14 (LSTM)              (None, 128)               197120    \n",
      "                                                                 \n",
      " dense_3 (Dense)             (None, 44)                5676      \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 1,069,992\n",
      "Trainable params: 437,292\n",
      "Non-trainable params: 632,700\n",
      "_________________________________________________________________\n",
      "None\n",
      "Epoch 1/50\n",
      "1413/1413 [==============================] - 6s 3ms/step - loss: 0.9629 - accuracy: 0.7225\n",
      "Epoch 2/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.5385 - accuracy: 0.8202\n",
      "Epoch 3/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.4713 - accuracy: 0.8359\n",
      "Epoch 4/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.4409 - accuracy: 0.8414\n",
      "Epoch 5/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.4207 - accuracy: 0.8448\n",
      "Epoch 6/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.4071 - accuracy: 0.8488\n",
      "Epoch 7/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3957 - accuracy: 0.8506\n",
      "Epoch 8/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3881 - accuracy: 0.8528\n",
      "Epoch 9/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3813 - accuracy: 0.8537\n",
      "Epoch 10/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3767 - accuracy: 0.8542\n",
      "Epoch 11/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3726 - accuracy: 0.8547\n",
      "Epoch 12/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3690 - accuracy: 0.8553\n",
      "Epoch 13/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3657 - accuracy: 0.8581\n",
      "Epoch 14/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3638 - accuracy: 0.8568\n",
      "Epoch 15/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3614 - accuracy: 0.8585\n",
      "Epoch 16/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3586 - accuracy: 0.8585\n",
      "Epoch 17/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3569 - accuracy: 0.8585\n",
      "Epoch 18/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3555 - accuracy: 0.8583\n",
      "Epoch 19/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3542 - accuracy: 0.8577\n",
      "Epoch 20/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3527 - accuracy: 0.8590\n",
      "Epoch 21/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3512 - accuracy: 0.8591\n",
      "Epoch 22/50\n",
      "1413/1413 [==============================] - 5s 3ms/step - loss: 0.3509 - accuracy: 0.8589\n",
      "Epoch 23/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3492 - accuracy: 0.8580\n",
      "Epoch 24/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3486 - accuracy: 0.8596\n",
      "Epoch 25/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3477 - accuracy: 0.8592\n",
      "Epoch 26/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3466 - accuracy: 0.8593\n",
      "Epoch 27/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3457 - accuracy: 0.8604\n",
      "Epoch 28/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3458 - accuracy: 0.8590\n",
      "Epoch 29/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3444 - accuracy: 0.8599\n",
      "Epoch 30/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3439 - accuracy: 0.8605\n",
      "Epoch 31/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3435 - accuracy: 0.8611\n",
      "Epoch 32/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3432 - accuracy: 0.8592\n",
      "Epoch 33/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3423 - accuracy: 0.8610\n",
      "Epoch 34/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3420 - accuracy: 0.8601\n",
      "Epoch 35/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3420 - accuracy: 0.8603\n",
      "Epoch 36/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3412 - accuracy: 0.8606\n",
      "Epoch 37/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3407 - accuracy: 0.8606\n",
      "Epoch 38/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3407 - accuracy: 0.8586\n",
      "Epoch 39/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3402 - accuracy: 0.8619\n",
      "Epoch 40/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3401 - accuracy: 0.8610\n",
      "Epoch 41/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3403 - accuracy: 0.8608\n",
      "Epoch 42/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3396 - accuracy: 0.8613\n",
      "Epoch 43/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3394 - accuracy: 0.8604\n",
      "Epoch 44/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3394 - accuracy: 0.8605\n",
      "Epoch 45/50\n",
      "1413/1413 [==============================] - 6s 4ms/step - loss: 0.3386 - accuracy: 0.8616\n",
      "Epoch 46/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3386 - accuracy: 0.8606\n",
      "Epoch 47/50\n",
      "1413/1413 [==============================] - 5s 3ms/step - loss: 0.3385 - accuracy: 0.8621\n",
      "Epoch 48/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3391 - accuracy: 0.8623\n",
      "Epoch 49/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3385 - accuracy: 0.8621\n",
      "Epoch 50/50\n",
      "1413/1413 [==============================] - 4s 3ms/step - loss: 0.3380 - accuracy: 0.8617\n",
      "1413/1413 [==============================] - 2s 1ms/step - loss: 0.3317 - accuracy: 0.8650\n",
      "Accuracy: 86.504722\n"
     ]
    }
   ],
   "source": [
    "# Use additional LSTM layer\n",
    "# define model\n",
    "model = Sequential()\n",
    "e = Embedding(vocab_size, 100, weights=[embedding_matrix], input_length=1, trainable=False)\n",
    "model.add(e)\n",
    "# A\n",
    "model.add(keras.layers.Bidirectional(keras.layers.LSTM(units=128, return_sequences=True), input_shape=(None, 50)))\n",
    "model.add(keras.layers.LSTM(128))\n",
    "model.add(Dense(44, activation='softmax'))\n",
    "# compile the model\n",
    "model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "# summarize the model\n",
    "print(model.summary())\n",
    "\n",
    "# fit the model\n",
    "model.fit(padded_docs, labels, epochs=50)\n",
    "# evaluate the model\n",
    "loss, accuracy = model.evaluate(padded_docs, labels)\n",
    "print('Accuracy: %f' % (accuracy*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_13\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " embedding_13 (Embedding)    (None, 1, 100)            632700    \n",
      "                                                                 \n",
      " bidirectional_13 (Bidirecti  (None, 256)              234496    \n",
      " onal)                                                           \n",
      "                                                                 \n",
      " dense_23 (Dense)            (None, 128)               32896     \n",
      "                                                                 \n",
      " dense_24 (Dense)            (None, 44)                5676      \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 905,768\n",
      "Trainable params: 273,068\n",
      "Non-trainable params: 632,700\n",
      "_________________________________________________________________\n",
      "None\n",
      "Epoch 1/50\n",
      "1413/1413 [==============================] - 4s 2ms/step - loss: 0.8624 - accuracy: 0.7429\n",
      "Epoch 2/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.5419 - accuracy: 0.8184\n",
      "Epoch 3/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.4848 - accuracy: 0.8306\n",
      "Epoch 4/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.4530 - accuracy: 0.8382\n",
      "Epoch 5/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.4346 - accuracy: 0.8423\n",
      "Epoch 6/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.4210 - accuracy: 0.8444\n",
      "Epoch 7/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.4092 - accuracy: 0.8481\n",
      "Epoch 8/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.4019 - accuracy: 0.8497\n",
      "Epoch 9/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3953 - accuracy: 0.8491\n",
      "Epoch 10/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3922 - accuracy: 0.8502\n",
      "Epoch 11/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3874 - accuracy: 0.8511\n",
      "Epoch 12/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3836 - accuracy: 0.8533\n",
      "Epoch 13/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3810 - accuracy: 0.8539\n",
      "Epoch 14/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3807 - accuracy: 0.8537\n",
      "Epoch 15/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3769 - accuracy: 0.8545\n",
      "Epoch 16/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3766 - accuracy: 0.8543\n",
      "Epoch 17/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3742 - accuracy: 0.8547\n",
      "Epoch 18/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3729 - accuracy: 0.8556\n",
      "Epoch 19/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3723 - accuracy: 0.8561\n",
      "Epoch 20/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3697 - accuracy: 0.8555\n",
      "Epoch 21/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3694 - accuracy: 0.8565\n",
      "Epoch 22/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3689 - accuracy: 0.8557\n",
      "Epoch 23/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3677 - accuracy: 0.8556\n",
      "Epoch 24/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3673 - accuracy: 0.8564\n",
      "Epoch 25/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3657 - accuracy: 0.8576\n",
      "Epoch 26/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3656 - accuracy: 0.8572\n",
      "Epoch 27/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3662 - accuracy: 0.8557\n",
      "Epoch 28/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3643 - accuracy: 0.8556\n",
      "Epoch 29/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3645 - accuracy: 0.8546\n",
      "Epoch 30/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3639 - accuracy: 0.8569\n",
      "Epoch 31/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3627 - accuracy: 0.8563\n",
      "Epoch 32/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3622 - accuracy: 0.8567\n",
      "Epoch 33/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3626 - accuracy: 0.8558\n",
      "Epoch 34/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3602 - accuracy: 0.8564\n",
      "Epoch 35/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3608 - accuracy: 0.8558\n",
      "Epoch 36/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3600 - accuracy: 0.8575\n",
      "Epoch 37/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3595 - accuracy: 0.8568\n",
      "Epoch 38/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3597 - accuracy: 0.8577\n",
      "Epoch 39/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3585 - accuracy: 0.8571\n",
      "Epoch 40/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3591 - accuracy: 0.8563\n",
      "Epoch 41/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3574 - accuracy: 0.8570\n",
      "Epoch 42/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3571 - accuracy: 0.8595\n",
      "Epoch 43/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3578 - accuracy: 0.8573\n",
      "Epoch 44/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3576 - accuracy: 0.8581\n",
      "Epoch 45/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3565 - accuracy: 0.8579\n",
      "Epoch 46/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3569 - accuracy: 0.8573\n",
      "Epoch 47/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3558 - accuracy: 0.8579\n",
      "Epoch 48/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3563 - accuracy: 0.8571\n",
      "Epoch 49/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3549 - accuracy: 0.8577\n",
      "Epoch 50/50\n",
      "1413/1413 [==============================] - 3s 2ms/step - loss: 0.3553 - accuracy: 0.8577\n",
      "1413/1413 [==============================] - 2s 731us/step - loss: 0.3399 - accuracy: 0.8555\n",
      "Accuracy: 85.548991\n"
     ]
    }
   ],
   "source": [
    "# Use additional Dense layer\n",
    "from tensorflow import keras\n",
    "# define model\n",
    "model = Sequential()\n",
    "e = Embedding(vocab_size, 100, weights=[embedding_matrix], input_length=1, trainable=False)\n",
    "model.add(e)\n",
    "model.add(keras.layers.Bidirectional(keras.layers.LSTM(units=128), input_shape=(None, 50)))\n",
    "model.add(Dense(128))\n",
    "model.add(Dense(44, activation='softmax'))\n",
    "# compile the model\n",
    "model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "# summarize the model\n",
    "print(model.summary())\n",
    "\n",
    "# fit the model\n",
    "model.fit(padded_docs, labels, epochs=50)\n",
    "# evaluate the model\n",
    "loss, accuracy = model.evaluate(padded_docs, labels)\n",
    "print('Accuracy: %f' % (accuracy*100))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.9 64-bit ('nlp-env')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.9"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "83996dc0b81395b17a2f017c39a7462422633737611de1f2a39e1edfb0be2356"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}