04_Explicit_Feedback_deep_learning_with_Tensorflow_Rendered.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Explicit Feedback Neural Recommender Systems\n",
    "\n",
    "Goals:\n",
    "- Understand recommendation system \n",
    "- Build different models architectures using Tensorflow\n",
    "- Retrieve Embeddings and visualize them\n",
    "- Add metadata information as input to the model\n",
    "\n",
    "\n",
    "This notebook is inspired by Oliver Grisel Notebook who used Keras\n",
    "https://github.com/ogrisel for building the moels. We will be using Basic Tensorflow APIs instead. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "import tensorflow as tf\n",
    "from tensorflow.contrib import layers\n",
    "from tensorflow.python.estimator.inputs import numpy_io\n",
    "from tensorflow.contrib.learn import *\n",
    "%matplotlib inline "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensorflow Version : 1.5.0\n"
     ]
    }
   ],
   "source": [
    "print('Tensorflow Version : {0}'.format(tf.__version__))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ratings file\n",
    "\n",
    "Each line contains a rated movie: \n",
    "- a user\n",
    "- an item\n",
    "- a rating from 1 to 5 stars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Base Path for MovieLens dataset\n",
    "ML_100K_PATH = os.path.join('processed','ml-100k','ml-100k')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>196</td>\n",
       "      <td>242</td>\n",
       "      <td>3</td>\n",
       "      <td>881250949</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>186</td>\n",
       "      <td>302</td>\n",
       "      <td>3</td>\n",
       "      <td>891717742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>22</td>\n",
       "      <td>377</td>\n",
       "      <td>1</td>\n",
       "      <td>878887116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>244</td>\n",
       "      <td>51</td>\n",
       "      <td>2</td>\n",
       "      <td>880606923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>166</td>\n",
       "      <td>346</td>\n",
       "      <td>1</td>\n",
       "      <td>886397596</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id  rating  timestamp\n",
       "0      196      242       3  881250949\n",
       "1      186      302       3  891717742\n",
       "2       22      377       1  878887116\n",
       "3      244       51       2  880606923\n",
       "4      166      346       1  886397596"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_raw_ratings = pd.read_csv(os.path.join(ML_100K_PATH, 'u.data'), sep='\\t',\n",
    "                      names=[\"user_id\", \"item_id\", \"rating\", \"timestamp\"])\n",
    "df_raw_ratings.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Item metadata file\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>title</th>\n",
       "      <th>release_date</th>\n",
       "      <th>video_release_date</th>\n",
       "      <th>imdb_url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Toy%20Story%2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>GoldenEye (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?GoldenEye%20(...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Four Rooms (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Four%20Rooms%...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Get Shorty (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Get%20Shorty%...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Copycat (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Copycat%20(1995)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   item_id              title release_date  video_release_date  \\\n",
       "0        1   Toy Story (1995)  01-Jan-1995                 NaN   \n",
       "1        2   GoldenEye (1995)  01-Jan-1995                 NaN   \n",
       "2        3  Four Rooms (1995)  01-Jan-1995                 NaN   \n",
       "3        4  Get Shorty (1995)  01-Jan-1995                 NaN   \n",
       "4        5     Copycat (1995)  01-Jan-1995                 NaN   \n",
       "\n",
       "                                            imdb_url  \n",
       "0  http://us.imdb.com/M/title-exact?Toy%20Story%2...  \n",
       "1  http://us.imdb.com/M/title-exact?GoldenEye%20(...  \n",
       "2  http://us.imdb.com/M/title-exact?Four%20Rooms%...  \n",
       "3  http://us.imdb.com/M/title-exact?Get%20Shorty%...  \n",
       "4  http://us.imdb.com/M/title-exact?Copycat%20(1995)  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m_cols = ['item_id', 'title', 'release_date', 'video_release_date', 'imdb_url']\n",
    "# Loading only 5 columns\n",
    "df_items = pd.read_csv(os.path.join(ML_100K_PATH, 'u.item'), sep='|',\n",
    "                    names=m_cols, usecols=range(5), encoding='latin-1')\n",
    "df_items.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_release_year(x):\n",
    "    splits = str(x).split('-')\n",
    "    if(len(splits) == 3):\n",
    "        return int(splits[2])\n",
    "    else:\n",
    "        return 1920\n",
    "    \n",
    "df_items['release_year'] = df_items['release_date'].map(lambda x : get_release_year(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>title</th>\n",
       "      <th>release_date</th>\n",
       "      <th>video_release_date</th>\n",
       "      <th>imdb_url</th>\n",
       "      <th>release_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Toy%20Story%2...</td>\n",
       "      <td>1995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>GoldenEye (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?GoldenEye%20(...</td>\n",
       "      <td>1995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Four Rooms (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Four%20Rooms%...</td>\n",
       "      <td>1995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Get Shorty (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Get%20Shorty%...</td>\n",
       "      <td>1995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Copycat (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Copycat%20(1995)</td>\n",
       "      <td>1995</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   item_id              title release_date  video_release_date  \\\n",
       "0        1   Toy Story (1995)  01-Jan-1995                 NaN   \n",
       "1        2   GoldenEye (1995)  01-Jan-1995                 NaN   \n",
       "2        3  Four Rooms (1995)  01-Jan-1995                 NaN   \n",
       "3        4  Get Shorty (1995)  01-Jan-1995                 NaN   \n",
       "4        5     Copycat (1995)  01-Jan-1995                 NaN   \n",
       "\n",
       "                                            imdb_url  release_year  \n",
       "0  http://us.imdb.com/M/title-exact?Toy%20Story%2...          1995  \n",
       "1  http://us.imdb.com/M/title-exact?GoldenEye%20(...          1995  \n",
       "2  http://us.imdb.com/M/title-exact?Four%20Rooms%...          1995  \n",
       "3  http://us.imdb.com/M/title-exact?Get%20Shorty%...          1995  \n",
       "4  http://us.imdb.com/M/title-exact?Copycat%20(1995)          1995  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_items.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Merge Rating with Item Metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_all_ratings = pd.merge(df_items, df_raw_ratings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>title</th>\n",
       "      <th>release_date</th>\n",
       "      <th>video_release_date</th>\n",
       "      <th>imdb_url</th>\n",
       "      <th>release_year</th>\n",
       "      <th>user_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Toy%20Story%2...</td>\n",
       "      <td>1995</td>\n",
       "      <td>308</td>\n",
       "      <td>4</td>\n",
       "      <td>887736532</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Toy%20Story%2...</td>\n",
       "      <td>1995</td>\n",
       "      <td>287</td>\n",
       "      <td>5</td>\n",
       "      <td>875334088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Toy%20Story%2...</td>\n",
       "      <td>1995</td>\n",
       "      <td>148</td>\n",
       "      <td>4</td>\n",
       "      <td>877019411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Toy%20Story%2...</td>\n",
       "      <td>1995</td>\n",
       "      <td>280</td>\n",
       "      <td>4</td>\n",
       "      <td>891700426</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Toy%20Story%2...</td>\n",
       "      <td>1995</td>\n",
       "      <td>66</td>\n",
       "      <td>3</td>\n",
       "      <td>883601324</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   item_id             title release_date  video_release_date  \\\n",
       "0        1  Toy Story (1995)  01-Jan-1995                 NaN   \n",
       "1        1  Toy Story (1995)  01-Jan-1995                 NaN   \n",
       "2        1  Toy Story (1995)  01-Jan-1995                 NaN   \n",
       "3        1  Toy Story (1995)  01-Jan-1995                 NaN   \n",
       "4        1  Toy Story (1995)  01-Jan-1995                 NaN   \n",
       "\n",
       "                                            imdb_url  release_year  user_id  \\\n",
       "0  http://us.imdb.com/M/title-exact?Toy%20Story%2...          1995      308   \n",
       "1  http://us.imdb.com/M/title-exact?Toy%20Story%2...          1995      287   \n",
       "2  http://us.imdb.com/M/title-exact?Toy%20Story%2...          1995      148   \n",
       "3  http://us.imdb.com/M/title-exact?Toy%20Story%2...          1995      280   \n",
       "4  http://us.imdb.com/M/title-exact?Toy%20Story%2...          1995       66   \n",
       "\n",
       "   rating  timestamp  \n",
       "0       4  887736532  \n",
       "1       5  875334088  \n",
       "2       4  877019411  \n",
       "3       4  891700426  \n",
       "4       3  883601324  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all_ratings.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preprocessing\n",
    "\n",
    "To understand well the distribution of the data, the following statistics are computed:\n",
    "- the number of users\n",
    "- the number of items\n",
    "- the rating distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "943"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of users\n",
    "max_user_id = df_all_ratings['user_id'].max()\n",
    "max_user_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1682"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of items\n",
    "max_item_id = df_all_ratings['item_id'].max()\n",
    "max_item_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEKCAYAAAD5MJl4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAFStJREFUeJzt3X+s3XWd5/Hnawoo648F5Uq6bd2SmcYVzUzVa+2GZFfRQIHJlsnoBpKVrmHTGQO7mnV3LfMP4w8SzGZ0Q6IkuHQou46V+CM0Uqc2iDHuyo8LVKCi4S5W6bSxdQsIYxYD894/zqfL2X5ue8/9IecyfT6Sb873vL+f7/e+z0ng1e/3+znnpKqQJGnY74y7AUnS0mM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqXPKuBuYr7POOqtWr1497jYk6WXl/vvv/2VVTcw27mUbDqtXr2ZqamrcbUjSy0qSn40ybtbLSklemeTeJD9MsjfJJ1r9liQ/TbKnLWtbPUluSDKd5KEkbx861qYkj7Vl01D9HUkebvvckCRzf8mSpMUyypnDc8D5VfVsklOB7yf5Vtv2H6vqq8eMvwhY05Z3ATcC70ryOuBaYBIo4P4kO6rqyTZmM3A3sBPYAHwLSdJYzHrmUAPPtqentuVEX+W6Ebi17Xc3cEaS5cCFwO6qOtICYTewoW17bVX9oAZfEXsrcOkCXpMkaYFGmq2UZFmSPcAhBv+Dv6dtuq5dOvpckle02grgiaHd97faier7Z6hLksZkpHCoqheqai2wEliX5K3ANcA/Ad4JvA74eBs+0/2Cmke9k2RzkqkkU4cPHx6ldUnSPMzpcw5V9RTwXWBDVR1sl46eA/4SWNeG7QdWDe22EjgwS33lDPWZ/v5NVTVZVZMTE7POxJIkzdMos5UmkpzR1k8H3gf8uN0roM0suhR4pO2yA7iizVpaDzxdVQeBXcAFSc5MciZwAbCrbXsmyfp2rCuA2xf3ZUqS5mKU2UrLgW1JljEIk9uq6ptJvpNkgsFloT3An7bxO4GLgWng18CHAKrqSJJPAfe1cZ+sqiNt/cPALcDpDGYpOVNJksYoL9ffkJ6cnCw/BCdJc5Pk/qqanG3cy/YT0pJ+u1ZvuWPcLQCw7/pLxt3CSckv3pMkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVJn1nBI8sok9yb5YZK9ST7R6uckuSfJY0m+kuS0Vn9Fez7dtq8eOtY1rf6TJBcO1Te02nSSLYv/MiVJczHKmcNzwPlV9QfAWmBDkvXAZ4DPVdUa4Engyjb+SuDJqvo94HNtHEnOBS4D3gJsAL6QZFmSZcDngYuAc4HL21hJ0pjMGg418Gx7empbCjgf+GqrbwMubesb23Pa9vcmSatvr6rnquqnwDSwri3TVfV4Vf0G2N7GSpLGZKR7Du1f+HuAQ8Bu4H8BT1XV823IfmBFW18BPAHQtj8NvH64fsw+x6tLksZkpHCoqheqai2wksG/9N8807D2mONsm2u9k2RzkqkkU4cPH569cUnSvMxptlJVPQV8F1gPnJHklLZpJXCgre8HVgG07f8QODJcP2af49Vn+vs3VdVkVU1OTEzMpXVJ0hyMMltpIskZbf104H3Ao8BdwPvbsE3A7W19R3tO2/6dqqpWv6zNZjoHWAPcC9wHrGmzn05jcNN6x2K8OEnS/Jwy+xCWA9varKLfAW6rqm8m+RGwPcmngQeBm9v4m4H/lmSawRnDZQBVtTfJbcCPgOeBq6rqBYAkVwO7gGXA1qrau2ivUJI0Z7OGQ1U9BLxthvrjDO4/HFv/P8AHjnOs64DrZqjvBHaO0K8k6SXgJ6QlSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUGeVbWaWTxuotd4y7BQD2XX/JuFvQSc4zB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSZ9ZwSLIqyV1JHk2yN8lHWv3Pk/xNkj1tuXhon2uSTCf5SZILh+obWm06yZah+jlJ7knyWJKvJDltsV+oJGl0o5w5PA98rKreDKwHrkpybtv2uapa25adAG3bZcBbgA3AF5IsS7IM+DxwEXAucPnQcT7TjrUGeBK4cpFenyRpHmYNh6o6WFUPtPVngEeBFSfYZSOwvaqeq6qfAtPAurZMV9XjVfUbYDuwMUmA84Gvtv23AZfO9wVJkhZuTvcckqwG3gbc00pXJ3koydYkZ7baCuCJod32t9rx6q8Hnqqq54+pz/T3NyeZSjJ1+PDhubQuSZqDkcMhyauBrwEfrapfATcCvwusBQ4Cf3F06Ay71zzqfbHqpqqarKrJiYmJUVuXJM3RSF/ZneRUBsHwpar6OkBV/WJo+xeBb7an+4FVQ7uvBA609ZnqvwTOSHJKO3sYHi9JGoNRZisFuBl4tKo+O1RfPjTsj4BH2voO4LIkr0hyDrAGuBe4D1jTZiadxuCm9Y6qKuAu4P1t/03A7Qt7WZKkhRjlzOE84IPAw0n2tNqfMZhttJbBJaB9wJ8AVNXeJLcBP2Iw0+mqqnoBIMnVwC5gGbC1qva2430c2J7k08CDDMJIkjQms4ZDVX2fme8L7DzBPtcB181Q3znTflX1OIPZTJKkJcBPSEuSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOqP8TKgkndRWb7lj3C0AsO/6S16yv+WZgySpYzhIkjqGgySpM2s4JFmV5K4kjybZm+Qjrf66JLuTPNYez2z1JLkhyXSSh5K8fehYm9r4x5JsGqq/I8nDbZ8bkuS38WIlSaMZ5czheeBjVfVmYD1wVZJzgS3AnVW1BrizPQe4CFjTls3AjTAIE+Ba4F3AOuDao4HSxmwe2m/Dwl+aJGm+Zg2HqjpYVQ+09WeAR4EVwEZgWxu2Dbi0rW8Ebq2Bu4EzkiwHLgR2V9WRqnoS2A1saNteW1U/qKoCbh06liRpDOZ0zyHJauBtwD3A2VV1EAYBAryhDVsBPDG02/5WO1F9/wx1SdKYjBwOSV4NfA34aFX96kRDZ6jVPOoz9bA5yVSSqcOHD8/WsiRpnkYKhySnMgiGL1XV11v5F+2SEO3xUKvvB1YN7b4SODBLfeUM9U5V3VRVk1U1OTExMUrrkqR5GGW2UoCbgUer6rNDm3YAR2ccbQJuH6pf0WYtrQeebpeddgEXJDmz3Yi+ANjVtj2TZH37W1cMHUuSNAajfH3GecAHgYeT7Gm1PwOuB25LciXwc+ADbdtO4GJgGvg18CGAqjqS5FPAfW3cJ6vqSFv/MHALcDrwrbZIksZk1nCoqu8z830BgPfOML6Aq45zrK3A1hnqU8BbZ+tFkvTS8BPSkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqTOrOGQZGuSQ0keGar9eZK/SbKnLRcPbbsmyXSSnyS5cKi+odWmk2wZqp+T5J4kjyX5SpLTFvMFSpLmbpQzh1uADTPUP1dVa9uyEyDJucBlwFvaPl9IsizJMuDzwEXAucDlbSzAZ9qx1gBPAlcu5AVJkhZu1nCoqu8BR0Y83kZge1U9V1U/BaaBdW2ZrqrHq+o3wHZgY5IA5wNfbftvAy6d42uQJC2yhdxzuDrJQ+2y05mttgJ4YmjM/lY7Xv31wFNV9fwxdUnSGM03HG4EfhdYCxwE/qLVM8PYmkd9Rkk2J5lKMnX48OG5dSxJGtm8wqGqflFVL1TV3wFfZHDZCAb/8l81NHQlcOAE9V8CZyQ55Zj68f7uTVU1WVWTExMT82ldkjSCeYVDkuVDT/8IODqTaQdwWZJXJDkHWAPcC9wHrGkzk05jcNN6R1UVcBfw/rb/JuD2+fQkSVo8p8w2IMmXgXcDZyXZD1wLvDvJWgaXgPYBfwJQVXuT3Ab8CHgeuKqqXmjHuRrYBSwDtlbV3vYnPg5sT/Jp4EHg5kV7dZKkeZk1HKrq8hnKx/0feFVdB1w3Q30nsHOG+uO8eFlKkrQE+AlpSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVJn1l+C099/q7fcMe4WANh3/SXjbkFS45mDJKljOEiSOoaDJKljOEiSOrOGQ5KtSQ4leWSo9roku5M81h7PbPUkuSHJdJKHkrx9aJ9NbfxjSTYN1d+R5OG2zw1JstgvUpI0N6OcOdwCbDimtgW4s6rWAHe25wAXAWvashm4EQZhAlwLvAtYB1x7NFDamM1D+x37tyRJL7FZw6GqvgccOaa8EdjW1rcBlw7Vb62Bu4EzkiwHLgR2V9WRqnoS2A1saNteW1U/qKoCbh06liRpTOZ7z+HsqjoI0B7f0OorgCeGxu1vtRPV989Qn1GSzUmmkkwdPnx4nq1Lkmaz2DekZ7pfUPOoz6iqbqqqyaqanJiYmGeLkqTZzDccftEuCdEeD7X6fmDV0LiVwIFZ6itnqEuSxmi+4bADODrjaBNw+1D9ijZraT3wdLvstAu4IMmZ7Ub0BcCutu2ZJOvbLKUrho4lSRqTWb9bKcmXgXcDZyXZz2DW0fXAbUmuBH4OfKAN3wlcDEwDvwY+BFBVR5J8CrivjftkVR29yf1hBjOiTge+1RZJ0hjNGg5VdflxNr13hrEFXHWc42wFts5QnwLeOlsfkqSXjp+QliR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUmfWnwn9+2r1ljvG3QIA+66/ZNwtSFLHMwdJUsdwkCR1FhQOSfYleTjJniRTrfa6JLuTPNYez2z1JLkhyXSSh5K8feg4m9r4x5JsWthLkiQt1GKcObynqtZW1WR7vgW4s6rWAHe25wAXAWvashm4EQZhAlwLvAtYB1x7NFAkSePx27istBHY1ta3AZcO1W+tgbuBM5IsBy4EdlfVkap6EtgNbPgt9CVJGtFCw6GAbye5P8nmVju7qg4CtMc3tPoK4Imhffe32vHqkqQxWehU1vOq6kCSNwC7k/z4BGMzQ61OUO8PMAigzQBvfOMb59qrJGlECzpzqKoD7fEQ8A0G9wx+0S4X0R4PteH7gVVDu68EDpygPtPfu6mqJqtqcmJiYiGtS5JOYN7hkORVSV5zdB24AHgE2AEcnXG0Cbi9re8ArmizltYDT7fLTruAC5Kc2W5EX9BqkqQxWchlpbOBbyQ5epy/qqq/TnIfcFuSK4GfAx9o43cCFwPTwK+BDwFU1ZEknwLua+M+WVVHFtCXJGmB5h0OVfU48Acz1P838N4Z6gVcdZxjbQW2zrcXSdLi8hPSkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqTOkgmHJBuS/CTJdJIt4+5Hkk5mSyIckiwDPg9cBJwLXJ7k3PF2JUknryURDsA6YLqqHq+q3wDbgY1j7kmSTlpLJRxWAE8MPd/fapKkMUhVjbsHknwAuLCq/k17/kFgXVX922PGbQY2t6dvAn7ykjbaOwv45Zh7WCp8L17ke/Ei34sXLZX34h9X1cRsg055KToZwX5g1dDzlcCBYwdV1U3ATS9VU7NJMlVVk+PuYynwvXiR78WLfC9e9HJ7L5bKZaX7gDVJzklyGnAZsGPMPUnSSWtJnDlU1fNJrgZ2AcuArVW1d8xtSdJJa0mEA0BV7QR2jruPOVoyl7iWAN+LF/levMj34kUvq/diSdyQliQtLUvlnoMkaQkxHOYhydYkh5I8Mu5exi3JqiR3JXk0yd4kHxl3T+OS5JVJ7k3yw/ZefGLcPY1TkmVJHkzyzXH3Mm5J9iV5OMmeJFPj7mcUXlaahyT/DHgWuLWq3jrufsYpyXJgeVU9kOQ1wP3ApVX1ozG39pJLEuBVVfVsklOB7wMfqaq7x9zaWCT598Ak8Nqq+sNx9zNOSfYBk1W1FD7nMBLPHOahqr4HHBl3H0tBVR2sqgfa+jPAo5ykn26vgWfb01PbclL+6yvJSuAS4L+OuxfNj+GgRZNkNfA24J7xdjI+7VLKHuAQsLuqTtb34r8A/wn4u3E3skQU8O0k97dveljyDActiiSvBr4GfLSqfjXufsalql6oqrUMPuW/LslJd9kxyR8Ch6rq/nH3soScV1VvZ/DN01e1S9NLmuGgBWvX178GfKmqvj7ufpaCqnoK+C6wYcytjMN5wL9o19m3A+cn+e/jbWm8qupAezwEfIPBN1EvaYaDFqTdhL0ZeLSqPjvufsYpyUSSM9r66cD7gB+Pt6uXXlVdU1Urq2o1g6/C+U5V/asxtzU2SV7VJmuQ5FXABcCSn+loOMxDki8DPwDelGR/kivH3dMYnQd8kMG/Dve05eJxNzUmy4G7kjzE4PvCdlfVST+NU5wNfD/JD4F7gTuq6q/H3NOsnMoqSep45iBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgO0gIl+WiSfzD0fOfRzztIL1dOZZVG0D7sl6rqvivo5fiNm9JsPHOQjiPJ6vY7FV8AHgBuTjI1/FsNSf4d8I8YfPjtrlbbl+Ssof2/2Pb5dvvkNEnemeShJD9I8p/9bRAtNYaDdGJvYvC7HW8DPlZVk8DvA/88ye9X1Q3AAeA9VfWeGfZfA3y+qt4CPAX8cav/JfCnVfVPgRd+669CmiPDQTqxnw39WM+/TPIA8CDwFuDcEfb/aVXtaev3A6vb/YjXVNX/bPW/WtSOpUVwyrgbkJa4vwVIcg7wH4B3VtWTSW4BXjnC/s8Nrb8AnA5ksZuUFptnDtJoXssgKJ5OcjaD7+U/6hngNaMeqKqeBJ5Jsr6VLlu0LqVF4pmDNIKq+mGSB4G9wOPA/xjafBPwrSQHj3PfYSZXAl9M8rcMfvfh6cXsV1oop7JKY5Dk1Ud/bzrJFmB5VX1kzG1J/49nDtJ4XJLkGgb/Df4M+NfjbUf6/3nmIEnqeENaktQxHCRJHcNBktQxHCRJHcNBktQxHCRJnf8LUYFS9K/jqDIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1020c1a58>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_all_ratings.groupby('rating')['rating'].count().plot(kind='bar', rot=0);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    100000.000000\n",
       "mean          3.529860\n",
       "std           1.125674\n",
       "min           1.000000\n",
       "25%           3.000000\n",
       "50%           4.000000\n",
       "75%           4.000000\n",
       "max           5.000000\n",
       "Name: rating, dtype: float64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ratings\n",
    "df_all_ratings['rating'].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add Popularity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "popularity = df_all_ratings.groupby('item_id').size().reset_index(name='popularity')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Enrich the ratings data with the popularity as an additional metadata."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>title</th>\n",
       "      <th>release_date</th>\n",
       "      <th>video_release_date</th>\n",
       "      <th>imdb_url</th>\n",
       "      <th>release_year</th>\n",
       "      <th>user_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>popularity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Toy%20Story%2...</td>\n",
       "      <td>1995</td>\n",
       "      <td>308</td>\n",
       "      <td>4</td>\n",
       "      <td>887736532</td>\n",
       "      <td>452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Toy%20Story%2...</td>\n",
       "      <td>1995</td>\n",
       "      <td>287</td>\n",
       "      <td>5</td>\n",
       "      <td>875334088</td>\n",
       "      <td>452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Toy%20Story%2...</td>\n",
       "      <td>1995</td>\n",
       "      <td>148</td>\n",
       "      <td>4</td>\n",
       "      <td>877019411</td>\n",
       "      <td>452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Toy%20Story%2...</td>\n",
       "      <td>1995</td>\n",
       "      <td>280</td>\n",
       "      <td>4</td>\n",
       "      <td>891700426</td>\n",
       "      <td>452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Toy%20Story%2...</td>\n",
       "      <td>1995</td>\n",
       "      <td>66</td>\n",
       "      <td>3</td>\n",
       "      <td>883601324</td>\n",
       "      <td>452</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   item_id             title release_date  video_release_date  \\\n",
       "0        1  Toy Story (1995)  01-Jan-1995                 NaN   \n",
       "1        1  Toy Story (1995)  01-Jan-1995                 NaN   \n",
       "2        1  Toy Story (1995)  01-Jan-1995                 NaN   \n",
       "3        1  Toy Story (1995)  01-Jan-1995                 NaN   \n",
       "4        1  Toy Story (1995)  01-Jan-1995                 NaN   \n",
       "\n",
       "                                            imdb_url  release_year  user_id  \\\n",
       "0  http://us.imdb.com/M/title-exact?Toy%20Story%2...          1995      308   \n",
       "1  http://us.imdb.com/M/title-exact?Toy%20Story%2...          1995      287   \n",
       "2  http://us.imdb.com/M/title-exact?Toy%20Story%2...          1995      148   \n",
       "3  http://us.imdb.com/M/title-exact?Toy%20Story%2...          1995      280   \n",
       "4  http://us.imdb.com/M/title-exact?Toy%20Story%2...          1995       66   \n",
       "\n",
       "   rating  timestamp  popularity  \n",
       "0       4  887736532         452  \n",
       "1       5  875334088         452  \n",
       "2       4  877019411         452  \n",
       "3       4  891700426         452  \n",
       "4       3  883601324         452  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all_ratings = pd.merge(df_all_ratings, popularity)\n",
    "df_all_ratings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>title</th>\n",
       "      <th>release_date</th>\n",
       "      <th>video_release_date</th>\n",
       "      <th>imdb_url</th>\n",
       "      <th>release_year</th>\n",
       "      <th>user_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>popularity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6402</th>\n",
       "      <td>50</td>\n",
       "      <td>Star Wars (1977)</td>\n",
       "      <td>01-Jan-1977</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Star%20Wars%2...</td>\n",
       "      <td>1977</td>\n",
       "      <td>290</td>\n",
       "      <td>5</td>\n",
       "      <td>880473582</td>\n",
       "      <td>583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6403</th>\n",
       "      <td>50</td>\n",
       "      <td>Star Wars (1977)</td>\n",
       "      <td>01-Jan-1977</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Star%20Wars%2...</td>\n",
       "      <td>1977</td>\n",
       "      <td>79</td>\n",
       "      <td>4</td>\n",
       "      <td>891271545</td>\n",
       "      <td>583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6404</th>\n",
       "      <td>50</td>\n",
       "      <td>Star Wars (1977)</td>\n",
       "      <td>01-Jan-1977</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Star%20Wars%2...</td>\n",
       "      <td>1977</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>888552084</td>\n",
       "      <td>583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6405</th>\n",
       "      <td>50</td>\n",
       "      <td>Star Wars (1977)</td>\n",
       "      <td>01-Jan-1977</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Star%20Wars%2...</td>\n",
       "      <td>1977</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>879362124</td>\n",
       "      <td>583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6406</th>\n",
       "      <td>50</td>\n",
       "      <td>Star Wars (1977)</td>\n",
       "      <td>01-Jan-1977</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Star%20Wars%2...</td>\n",
       "      <td>1977</td>\n",
       "      <td>274</td>\n",
       "      <td>5</td>\n",
       "      <td>878944679</td>\n",
       "      <td>583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6407</th>\n",
       "      <td>50</td>\n",
       "      <td>Star Wars (1977)</td>\n",
       "      <td>01-Jan-1977</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Star%20Wars%2...</td>\n",
       "      <td>1977</td>\n",
       "      <td>227</td>\n",
       "      <td>4</td>\n",
       "      <td>879035347</td>\n",
       "      <td>583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6408</th>\n",
       "      <td>50</td>\n",
       "      <td>Star Wars (1977)</td>\n",
       "      <td>01-Jan-1977</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Star%20Wars%2...</td>\n",
       "      <td>1977</td>\n",
       "      <td>99</td>\n",
       "      <td>5</td>\n",
       "      <td>885679998</td>\n",
       "      <td>583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6409</th>\n",
       "      <td>50</td>\n",
       "      <td>Star Wars (1977)</td>\n",
       "      <td>01-Jan-1977</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Star%20Wars%2...</td>\n",
       "      <td>1977</td>\n",
       "      <td>305</td>\n",
       "      <td>5</td>\n",
       "      <td>886321799</td>\n",
       "      <td>583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6410</th>\n",
       "      <td>50</td>\n",
       "      <td>Star Wars (1977)</td>\n",
       "      <td>01-Jan-1977</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Star%20Wars%2...</td>\n",
       "      <td>1977</td>\n",
       "      <td>108</td>\n",
       "      <td>4</td>\n",
       "      <td>879879739</td>\n",
       "      <td>583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6411</th>\n",
       "      <td>50</td>\n",
       "      <td>Star Wars (1977)</td>\n",
       "      <td>01-Jan-1977</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Star%20Wars%2...</td>\n",
       "      <td>1977</td>\n",
       "      <td>63</td>\n",
       "      <td>4</td>\n",
       "      <td>875747292</td>\n",
       "      <td>583</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      item_id             title release_date  video_release_date  \\\n",
       "6402       50  Star Wars (1977)  01-Jan-1977                 NaN   \n",
       "6403       50  Star Wars (1977)  01-Jan-1977                 NaN   \n",
       "6404       50  Star Wars (1977)  01-Jan-1977                 NaN   \n",
       "6405       50  Star Wars (1977)  01-Jan-1977                 NaN   \n",
       "6406       50  Star Wars (1977)  01-Jan-1977                 NaN   \n",
       "6407       50  Star Wars (1977)  01-Jan-1977                 NaN   \n",
       "6408       50  Star Wars (1977)  01-Jan-1977                 NaN   \n",
       "6409       50  Star Wars (1977)  01-Jan-1977                 NaN   \n",
       "6410       50  Star Wars (1977)  01-Jan-1977                 NaN   \n",
       "6411       50  Star Wars (1977)  01-Jan-1977                 NaN   \n",
       "\n",
       "                                               imdb_url  release_year  \\\n",
       "6402  http://us.imdb.com/M/title-exact?Star%20Wars%2...          1977   \n",
       "6403  http://us.imdb.com/M/title-exact?Star%20Wars%2...          1977   \n",
       "6404  http://us.imdb.com/M/title-exact?Star%20Wars%2...          1977   \n",
       "6405  http://us.imdb.com/M/title-exact?Star%20Wars%2...          1977   \n",
       "6406  http://us.imdb.com/M/title-exact?Star%20Wars%2...          1977   \n",
       "6407  http://us.imdb.com/M/title-exact?Star%20Wars%2...          1977   \n",
       "6408  http://us.imdb.com/M/title-exact?Star%20Wars%2...          1977   \n",
       "6409  http://us.imdb.com/M/title-exact?Star%20Wars%2...          1977   \n",
       "6410  http://us.imdb.com/M/title-exact?Star%20Wars%2...          1977   \n",
       "6411  http://us.imdb.com/M/title-exact?Star%20Wars%2...          1977   \n",
       "\n",
       "      user_id  rating  timestamp  popularity  \n",
       "6402      290       5  880473582         583  \n",
       "6403       79       4  891271545         583  \n",
       "6404        2       5  888552084         583  \n",
       "6405        8       5  879362124         583  \n",
       "6406      274       5  878944679         583  \n",
       "6407      227       4  879035347         583  \n",
       "6408       99       5  885679998         583  \n",
       "6409      305       5  886321799         583  \n",
       "6410      108       4  879879739         583  \n",
       "6411       63       4  875747292         583  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all_ratings.nlargest(10, 'popularity')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Later in the analysis we will assume that this popularity does not come from the ratings themselves but from an external metadata, e.g. box office numbers in the month after the release in movie theaters."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train Test Validation Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total rating rows count: 100000 \n",
      "Total training rows count: 80000 \n",
      "Total validation rows count: 16000 \n",
      "Total test rows count: 20000 \n"
     ]
    }
   ],
   "source": [
    "# Split All ratings into train_val and test\n",
    "ratings_train_val, ratings_test = train_test_split(df_all_ratings, test_size=0.2, random_state=0)\n",
    "# Split train_val into training and validation set\n",
    "ratings_train, ratings_val = train_test_split(ratings_train_val, test_size=0.2, random_state=0)\n",
    "\n",
    "print('Total rating rows count: {0} '.format(len(df_all_ratings)))\n",
    "print('Total training rows count: {0} '.format(len(ratings_train_val)))\n",
    "print('Total validation rows count: {0} '.format(len(ratings_val)))\n",
    "print('Total test rows count: {0} '.format(len(ratings_test)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 64000 entries, 39110 to 59493\n",
      "Data columns (total 10 columns):\n",
      "item_id               64000 non-null int64\n",
      "title                 64000 non-null object\n",
      "release_date          63995 non-null object\n",
      "video_release_date    0 non-null float64\n",
      "imdb_url              63992 non-null object\n",
      "release_year          64000 non-null int64\n",
      "user_id               64000 non-null int64\n",
      "rating                64000 non-null int64\n",
      "timestamp             64000 non-null int64\n",
      "popularity            64000 non-null int64\n",
      "dtypes: float64(1), int64(6), object(3)\n",
      "memory usage: 5.4+ MB\n"
     ]
    }
   ],
   "source": [
    "ratings_train.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Explicit feedback: supervised ratings prediction\n",
    "\n",
    "For each pair of (user, item) try to predict the rating the user would give to the item.\n",
    "\n",
    "This is the classical setup for building recommender systems from offline data with explicit supervision signal. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predictive ratings  as a regression problem\n",
    "\n",
    "The following code implements the following architecture:\n",
    "\n",
    "<img src=\"images/rec_archi_1.svg\" style=\"width: 600px;\" />"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Matrix Factorization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_size = 30 # embedding size\n",
    "reg_param = 0.01 # regularization parameter lambda\n",
    "learning_rate = 0.01 # learning rate \n",
    "\n",
    "\n",
    "# create tensorflow graph\n",
    "g = tf.Graph()\n",
    "with g.as_default():\n",
    "    # setting up random seed\n",
    "    tf.set_random_seed(1234)\n",
    "    \n",
    "    # placeholders\n",
    "    users = tf.placeholder(shape=[None], dtype=tf.int64)\n",
    "    items = tf.placeholder(shape=[None], dtype=tf.int64)\n",
    "    ratings = tf.placeholder(shape=[None], dtype=tf.float32)\n",
    "    \n",
    "    # variables\n",
    "    with tf.variable_scope(\"embedding\"):\n",
    "        user_weight = tf.get_variable(\"user_w\"\n",
    "                                      , shape=[max_user_id + 1, embedding_size]\n",
    "                                      , dtype=tf.float32\n",
    "                                      , initializer=layers.xavier_initializer())\n",
    "\n",
    "        item_weight = tf.get_variable(\"item_w\"\n",
    "                                       , shape=[max_item_id + 1, embedding_size]\n",
    "                                       , dtype=tf.float32\n",
    "                                       , initializer=layers.xavier_initializer())\n",
    "    # prediction\n",
    "    with tf.name_scope(\"inference\"):\n",
    "        user_embedding = tf.nn.embedding_lookup(user_weight, users)\n",
    "        item_embedding = tf.nn.embedding_lookup(item_weight, items)\n",
    "        pred = tf.reduce_sum(tf.multiply(user_embedding, item_embedding), 1) \n",
    "        \n",
    "    # loss \n",
    "    with tf.name_scope(\"loss\"):\n",
    "        reg_loss = tf.contrib.layers.apply_regularization(layers.l2_regularizer(scale=reg_param),\n",
    "                                               weights_list=[user_weight, item_weight])\n",
    "        loss = tf.nn.l2_loss(pred - ratings) + reg_loss\n",
    "        train_ops = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)\n",
    "        rmse = tf.sqrt(tf.reduce_mean(tf.pow(pred - ratings, 2)))\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def train_model():\n",
    "    # Training \n",
    "    epochs = 1000 # number of iterations \n",
    "    losses_train = []\n",
    "    losses_val = []\n",
    "\n",
    "\n",
    "\n",
    "    with tf.Session(graph=g) as sess:\n",
    "        # initializer\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "\n",
    "\n",
    "        train_input_dict = {  users: ratings_train['user_id']\n",
    "                            , items: ratings_train['item_id']\n",
    "                            , ratings: ratings_train['rating']}\n",
    "        val_input_dict =  {  users: ratings_val['user_id']\n",
    "                            , items: ratings_val['item_id']\n",
    "                            , ratings: ratings_val['rating']}\n",
    "\n",
    "        test_input_dict =  {  users: ratings_test['user_id']\n",
    "                            , items: ratings_test['item_id']\n",
    "                            , ratings: ratings_test['rating']}\n",
    "\n",
    "        def check_overfit(validation_loss):\n",
    "            n = len(validation_loss)\n",
    "            if n < 5:\n",
    "                return False\n",
    "            count = 0 \n",
    "            for i in range(n-4, n):\n",
    "                if validation_loss[i] < validation_loss[i-1]:\n",
    "                    count += 1\n",
    "                if count >=2:\n",
    "                    return False\n",
    "            return True\n",
    "\n",
    "        for i in range(epochs):\n",
    "            # run the training operation\n",
    "            sess.run([train_ops], feed_dict=train_input_dict)\n",
    "\n",
    "            # show intermediate results \n",
    "            if i % 5 == 0:\n",
    "                loss_train = sess.run(loss, feed_dict=train_input_dict)\n",
    "                loss_val = sess.run(loss, feed_dict=val_input_dict)\n",
    "                losses_train.append(loss_train)\n",
    "                losses_val.append(loss_val)\n",
    "\n",
    "\n",
    "                # check early stopping \n",
    "                if(check_overfit(losses_val)):\n",
    "                    print('overfit !')\n",
    "                    break\n",
    "\n",
    "                print(\"iteration : {0} train loss: {1:.3f} , valid loss {2:.3f}\".format(i,loss_train, loss_val))\n",
    "\n",
    "        # calculate RMSE on the test dataset\n",
    "        print('RMSE on test dataset : {0:.4f}'.format(sess.run(rmse, feed_dict=test_input_dict)))\n",
    "\n",
    "        plt.plot(losses_train, label='train')\n",
    "        plt.plot(losses_val, label='validation')\n",
    "        #plt.ylim(0, 50000)\n",
    "        plt.legend(loc='best')\n",
    "        plt.title('Loss');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration : 0 train loss: 438954.844 , valid loss 109274.789\n",
      "iteration : 5 train loss: 430119.469 , valid loss 107681.461\n",
      "iteration : 10 train loss: 401145.969 , valid loss 100981.586\n",
      "iteration : 15 train loss: 345095.969 , valid loss 87444.492\n",
      "iteration : 20 train loss: 262872.094 , valid loss 67228.461\n",
      "iteration : 25 train loss: 166455.266 , valid loss 43223.102\n",
      "iteration : 30 train loss: 81420.000 , valid loss 21761.330\n",
      "iteration : 35 train loss: 38146.625 , valid loss 10548.836\n",
      "iteration : 40 train loss: 37472.977 , valid loss 10132.517\n",
      "iteration : 45 train loss: 39926.707 , valid loss 10855.712\n",
      "iteration : 50 train loss: 31300.855 , valid loss 9038.124\n",
      "iteration : 55 train loss: 25830.330 , valid loss 8010.942\n",
      "iteration : 60 train loss: 25812.863 , valid loss 8221.631\n",
      "iteration : 65 train loss: 25626.330 , valid loss 8261.295\n",
      "iteration : 70 train loss: 24081.344 , valid loss 7883.961\n",
      "iteration : 75 train loss: 22998.752 , valid loss 7608.090\n",
      "iteration : 80 train loss: 22692.918 , valid loss 7565.453\n",
      "iteration : 85 train loss: 22296.045 , valid loss 7557.205\n",
      "iteration : 90 train loss: 21739.768 , valid loss 7546.205\n",
      "iteration : 95 train loss: 21318.223 , valid loss 7574.492\n",
      "iteration : 100 train loss: 20961.021 , valid loss 7605.824\n",
      "overfit !\n",
      "RMSE on test dataset : 0.9642\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEICAYAAACj2qi6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8FfW9//HX55yThSQsSQgQkmBQFlllCYt1KbggoFe0IsXbRa2V1tpW29tebe/tz3p/bX9drFpvrb22WrXXjWpVei9IXVCpKwGVfVNRQiCENYSQ7eT7++NMQsAsh5BkTnLez8fjPObMd2bOfHKMeTPfmfmOOecQERGJRsDvAkREpOtQaIiISNQUGiIiEjWFhoiIRE2hISIiUVNoiIhI1BQaIiISNYWGSBuZ2TYzu8DvOkQ6k0JDRESiptAQaWdmdr2ZbTWzfWa2yMwGeu1mZneZ2W4zO2hmq81stLdstpmtN7NDZrbDzL7n708h0jSFhkg7MrPzgP8HzAOygY+BJ7zFM4BzgWFAH+DzwF5v2QPA15xzPYHRwMudWLZI1EJ+FyDSzXwBeNA5twrAzH4A7DezfKAG6AmcDrzjnNvQaLsaYKSZve+c2w/s79SqRaKkIw2R9jWQyNEFAM65ciJHEznOuZeB3wL3AiVmdr+Z9fJWvQKYDXxsZq+a2ZmdXLdIVBQaIu2rGDilfsbMUoFMYAeAc+4e59xEYBSRbqrve+0rnHNzgH7As8DCTq5bJCoKDZGTk2BmyfUvIn/srzWzcWaWBPwMeNs5t83MJpnZFDNLAA4DlUDYzBLN7Atm1ts5VwOUAWHffiKRFig0RE7OYuBIo9c5wI+Ap4GdwGnAfG/dXsAfiJyv+JhIt9Ud3rIvAdvMrAz4OvDFTqpf5ISYHsIkIiLR0pGGiIhETaEhIiJRU2iIiEjUFBoiIhK1bndHeN++fV1+fr7fZYiIdCkrV67c45zLam29bhca+fn5FBYW+l2GiEiXYmYft76WuqdEROQEKDRERCRqCg0REYlatzunISLdR01NDUVFRVRWVvpdSreRnJxMbm4uCQkJbdpeoSEiMauoqIiePXuSn5+PmfldTpfnnGPv3r0UFRUxePDgNn2GuqdEJGZVVlaSmZmpwGgnZkZmZuZJHbkpNEQkpikw2tfJfp/qnvI8824R5VVhpg3LIi8jxe9yRERikkLD87f3d/Lyxt0ADO6bymeHZXHusL5MPTWTlER9TSLx6MCBAzz22GN84xvfOKHtZs+ezWOPPUafPn06qDL/dLvnaRQUFLi23BHunOPDPYd5bXMpr24u5a0P91JZU0diMMCkweleiGQxvH9PHS6LdJINGzYwYsQI3/a/bds2LrnkEtauXXtMezgcJhgM+lTVyWvqezWzlc65gta21T+hPWbGaVlpnJaVxrVnDaayJkzhtv28unk3r23ew88Wb+RnizfSv1cS5w7N4rPDszh7SF/6pCT6XbqIdJBbb72VDz74gHHjxpGQkEBaWhrZ2dm89957rF+/nssuu4zt27dTWVnJTTfdxIIFC4CjwxmVl5cza9Yszj77bN544w1ycnJ47rnn6NGjh88/WdvpSCNKOw8eYfnmPby6uZTlW0opq6wlYHBGXh+mD+/HV84eTFqSMlikPTX+F/Htf1vH+uKydv38kQN7cds/jWp2eeMjjVdeeYWLL76YtWvXNlyuum/fPjIyMjhy5AiTJk3i1VdfJTMz85jQGDJkCIWFhYwbN4558+Zx6aWX8sUv+vs0Xx1pdILs3j2YNymPeZPyqA3X8X7RQV7bXMprW0q568XNvLC+hIeunURmWpLfpYpIB5k8efIx9zfcc889PPPMMwBs376dLVu2kJmZecw2gwcPZty4cQBMnDiRbdu2dVq9HUGh0QahYICJp6Qz8ZR0vnPhMF7eWMI3Hl3Flb9/k0eum0xuuq6+EmlvLR0RdJbU1NSG96+88govvvgib775JikpKUybNq3J+x+Sko7+QzIYDHLkyJFOqbWj6D6NdnDe6f357+umsKe8irn3vcnmkkN+lyQi7aBnz54cOtT0/88HDx4kPT2dlJQUNm7cyFtvvdXJ1flDodFOCvIzWPj1M6lzjit//yYrP97vd0kicpIyMzM566yzGD16NN///vePWTZz5kxqa2sZO3YsP/rRj5g6dapPVXYunQhvZ9v3VfClB96mpKyK331xAtOH9/OtFpGuzu9LbrurkzkRriONdpaXkcJTN3yG0/qlcv3DhTz77g6/SxIRaTcKjQ7QNy2Jx6+fyqT8DG5+8j3+9PpHfpckItIuFBodpGdyAn+6dhIzRw3g9r+t546lm+huXYEiEn8UGh0oOSHIvV+YwFWT8/jtsq3827NrCdcpOESk69J9Gh0sGDB+dvkYMlOT+O2yrew/XM3d88eRFOq649aISPzSkUYnMDO+d9Fw/s8lI1mydhfX/mkF5VW1fpclInLCog4NMwua2btm9j/e/GAze9vMtpjZk2aW6LUnefNbveX5jT7jB177JjO7qFH7TK9tq5nd2qi9yX10VV85ezB3f34c73y0j6vuf4s95VV+lyQi7SwtLQ2A4uJi5s6d2+Q606ZNo7VbA+6++24qKioa5mfPns2BAwfar9A2OpEjjZuADY3mfwHc5ZwbCuwHrvParwP2O+eGAHd562FmI4H5wChgJvA7L4iCwL3ALGAkcJW3bkv76LIuG5/DH64uYMvuQ1z5+zfZvq+i9Y1EpMsZOHAgTz31VJu3Pz40Fi9eHBPP54gqNMwsF7gY+KM3b8B5QP038jBwmfd+jjePt/x8b/05wBPOuSrn3EfAVmCy99rqnPvQOVcNPAHMaWUfXdr04f149KtT2Xe4mq88tIKacJ3fJYlIM2655RZ+97vfNcz/+Mc/5vbbb+f8889nwoQJjBkzhueee+5T223bto3Ro0cDcOTIEebPn8/YsWP5/Oc/f8z4UzfccAMFBQWMGjWK2267DYgMhFhcXMz06dOZPn06EBlufc+ePQDceeedjB49mtGjR3P33Xc37G/EiBFcf/31jBo1ihkzZnTIOFfRngi/G/hXoKc3nwkccM7Vd8wXATne+xxgO4BzrtbMDnrr5wCNB2dpvM3249qntLKPY5jZAmABwKBBg6L8kfw18ZR07rjyDK5/pJDH3/mEL5+Z73dJIrFtya2wa037fuaAMTDr5y2uMn/+fG6++eaGp/ctXLiQ559/nu985zv06tWLPXv2MHXqVC699NJmH9B23333kZKSwurVq1m9ejUTJkxoWPbTn/6UjIwMwuEw559/PqtXr+bb3/42d955J8uWLaNv377HfNbKlSv505/+xNtvv41zjilTpvDZz36W9PR0tmzZwuOPP84f/vAH5s2bx9NPP93uw7C3eqRhZpcAu51zKxs3N7Gqa2VZe7V/utG5+51zBc65gqysrKZWiUkXjOjHmadmctcLmzl4pMbvckSkCePHj2f37t0UFxfz/vvvk56eTnZ2Nj/84Q8ZO3YsF1xwATt27KCkpKTZz3jttdca/niPHTuWsWPHNixbuHAhEyZMYPz48axbt47169e3WM8//vEPLr/8clJTU0lLS+Nzn/scy5cvBzpnGPZojjTOAi41s9lAMtCLyJFHHzMLeUcCuUCxt34RkAcUmVkI6A3sa9Rer/E2TbXvaWEf3YKZ8e+XjOCS//wHv315C/928cjWNxKJV60cEXSkuXPn8tRTT7Fr1y7mz5/Po48+SmlpKStXriQhIYH8/Pwmh0VvrKmjkI8++og77riDFStWkJ6ezjXXXNPq57R0k3BnDMPe6pGGc+4Hzrlc51w+kRPZLzvnvgAsA+ovDbgaqO/UW+TN4y1/2UV+ykXAfO/qqsHAUOAdYAUw1LtSKtHbxyJvm+b20W2MGtibKyfm8tAb2/h472G/yxGRJsyfP58nnniCp556irlz53Lw4EH69etHQkICy5Yt4+OPP25x+3PPPZdHH30UgLVr17J69WoAysrKSE1NpXfv3pSUlLBkyZKGbZoblv3cc8/l2WefpaKigsOHD/PMM89wzjnntONP27KTuU/jFuC7ZraVyPmHB7z2B4BMr/27wK0Azrl1wEJgPfA8cKNzLuwdRXwTWErk6qyF3rot7aNb+d6M4SQEA/x8yUa/SxGRJowaNYpDhw6Rk5NDdnY2X/jCFygsLKSgoIBHH32U008/vcXtb7jhBsrLyxk7diy//OUvmTx5MgBnnHEG48ePZ9SoUXzlK1/hrLPOathmwYIFzJo1q+FEeL0JEyZwzTXXMHnyZKZMmcJXv/pVxo8f3/4/dDM0NHqM+M+XtvDrFzbz5IKpTDk1s/UNROKAhkbvGBoavRu4/txTGdg7mZ/87wbqND6ViMQohUaMSE4I8q8zT2fNjoM8o2dwiEiMUmjEkEvPGMgZub351dJNVFRrbCoRaPlqITlxJ/t9KjRiSCBg/OiSkewqq+T+1z70uxwR3yUnJ7N3714FRztxzrF3716Sk5Pb/BkaGj3GFORncPGYbP7r1Q+ZP2kQA3q3/T+uSFeXm5tLUVERpaWlfpfSbSQnJ5Obm9vm7RUaMejWWafzwvoSfrV0E7+ed4bf5Yj4JiEhgcGDB/tdhjSi7qkYlJeRwrVn5/P0qiLWFB30uxwRkQYKjRh14/QhZKYm8pP/Xa/+XBGJGQqNGNUrOYHvXDiMtz/ax9J1zQ+EJiLSmRQaMWz+pDyG9U/j/y3ZQHWtnrkhIv5TaMSwUDDAv108ko/3VvDIm9v8LkdERKER6z47LItpw7P4zUtb2He42u9yRCTOKTS6gH+bPYKK6jC/eXGz36WISJxTaHQBQ/v35KrJefz325+wdXe53+WISBxTaHQR37lgGCkJQX62eIPfpYhIHFNodBGZaUl887whvLxxN8u3aEgFEfGHQqMLueasfPIyevDT/91AWM/cEBEfKDS6kKRQkB/MGsHGXYdYWLjd73JEJA4pNLqYWaMHMCk/nV//fROHKmv8LkdE4oxCo4sxM344ewR7yqv56yo94U9EOpdCowsaPyid0Tm91EUlIp1OodFFzSvIY11xGWt3aOh0Eek8Co0uas4ZOSSGAvxFRxsi0okUGl1U75QEZo4awLPvFVNZE/a7HBGJEwqNLuzzk/I4eKSGv6/X8zZEpHMoNLqwM0/NJDe9BwtXqItKRDqHQqMLCwSMKyfm8foHe9i+r8LvckQkDig0uri5BbkAPLWyyOdKRCQeKDS6uJw+PTh7SF+eWllEncajEpEOptDoBuYV5LHjwBFe/2CP36WISDen0OgGZozqT5+UBBYWqotKRDqWQqMbSAoFuWxcDkvX7eJAhZ4jLiIdR6HRTVxZkEt1bR3PvVfsdyki0o0pNLqJUQN7MzqnF0/qng0R6UAKjW5kXkEe63dqEEMR6TgKjW6kfhBDDZkuIh1FodGNNAxi+O4ODWIoIh2i1dAws2Qze8fM3jezdWZ2u9c+2MzeNrMtZvakmSV67Une/FZveX6jz/qB177JzC5q1D7Ta9tqZrc2am9yH9K8z0/Ko6yylqXrdvldioh0Q9EcaVQB5znnzgDGATPNbCrwC+Au59xQYD9wnbf+dcB+59wQ4C5vPcxsJDAfGAXMBH5nZkEzCwL3ArOAkcBV3rq0sA9pRv0ghn/RPRsi0gFaDQ0XUe7NJngvB5wHPOW1Pwxc5r2f483jLT/fzMxrf8I5V+Wc+wjYCkz2Xludcx8656qBJ4A53jbN7UOaUT+I4T+2ahBDEWl/UZ3T8I4I3gN2Ay8AHwAHnHO13ipFQI73PgfYDuAtPwhkNm4/bpvm2jNb2Mfx9S0ws0IzKywtLY3mR+rW5hbkYqZBDEWk/UUVGs65sHNuHJBL5MhgRFOreVNrZll7tTdV3/3OuQLnXEFWVlZTq8SVxoMYhjWIoYi0oxO6eso5dwB4BZgK9DGzkLcoF6i/FbkIyAPwlvcG9jVuP26b5tr3tLAPaUX9IIZvaBBDEWlH0Vw9lWVmfbz3PYALgA3AMmCut9rVwHPe+0XePN7yl51zzmuf711dNRgYCrwDrACGeldKJRI5Wb7I26a5fUgr6gcx1B3iItKeQq2vQjbwsHeVUwBY6Jz7HzNbDzxhZj8B3gUe8NZ/APizmW0lcoQxH8A5t87MFgLrgVrgRudcGMDMvgksBYLAg865dd5n3dLMPqQV9YMYPvb2JxyoqKZPiq5WFpGTZ5F/0HcfBQUFrrCw0O8yYsL64jJm37Oc2y8dxdWfyfe7HBGJYWa20jlX0Np6uiO8Gxs5sJcGMRSRdqXQ6OY0iKGItCeFRjenQQxFpD0pNLo5DWIoIu1JoREHNIihiLQXhUYc0CCGItJeFBpxQIMYikh7UWjECQ1iKCLtQaERJzSIoYi0B4VGHNEghiJyshQacaR+EMOFOiEuIm2k0IgjSaEgF4/J5sX1JRyp1j0bInLiFBpx5uIx2RypCfPq5t1+lyIiXZBCI85MHpxBRmoii9foRj8ROXEKjTgTCga4aFR/XtpQomFFROSEKTTi0KzR2RyuDrN8i66iEpETo9CIQ2eelknvHgksWbPT71JEpItRaMShhGCAGSP788KGEqpq1UUlItFTaMSp2WOyOVRZyxtb9/pdioh0IQqNOPWZIZn0TA6xWF1UInICFBpxKikU5MIR/fn7+hJqwnV+lyMiXYRCI47NGpPNwSM1vPmBuqhEJDoKjTh2ztC+pCYGWbJWXVQiEh2FRhxLTghy/oj+LF1XQq26qEQkCgqNODdr9AD2Ha7mnY/2+V2KiHQBCo04N214P3okBFmsLioRiYJCI871SAwy/fQsnl9boif6iUirFBrCrNHZ7CmvonCbuqhEpGUKDWH66f1ICgVYslbDpYtIyxQaQlpSiM8Oy2LJ2p3UqYtKRFqg0BAgMhZVSVkV727f73cpIhLDFBoCwHkj+pEYDOiJfiLSIoWGANArOYFzhvZlyZqdOKcuKhFpmkJDGswak03xwUreLzrodykiEqMUGtLgwhH9CQVMT/QTkWYpNKRB75QEzhrSl8Vr1UUlIk1TaMgxZo8ZwPZ9R1hXXOZ3KSISg1oNDTPLM7NlZrbBzNaZ2U1ee4aZvWBmW7xputduZnaPmW01s9VmNqHRZ13trb/FzK5u1D7RzNZ429xjZtbSPqTjXDhyAMGA6Yl+ItKkaI40aoF/cc6NAKYCN5rZSOBW4CXn3FDgJW8eYBYw1HstAO6DSAAAtwFTgMnAbY1C4D5v3frtZnrtze1DOkhGaiJnnprJYl1FJSJNaDU0nHM7nXOrvPeHgA1ADjAHeNhb7WHgMu/9HOARF/EW0MfMsoGLgBecc/ucc/uBF4CZ3rJezrk3XeSv1CPHfVZT+5AONGvMALbtrWDjrkN+lyIiMeaEzmmYWT4wHngb6O+c2wmRYAH6eavlANsbbVbktbXUXtREOy3s4/i6FphZoZkVlpaWnsiPJE2YMXIAAUNXUYnIp0QdGmaWBjwN3Oyca+ksqTXR5trQHjXn3P3OuQLnXEFWVtaJbCpNyOqZxOTBGSzWAIYicpyoQsPMEogExqPOub96zSVe1xLedLfXXgTkNdo8FyhupT23ifaW9iEdbPaYbLbuLmdLibqoROSoaK6eMuABYINz7s5GixYB9VdAXQ0816j9y95VVFOBg17X0lJghpmleyfAZwBLvWWHzGyqt68vH/dZTe1DOthFowZghsaiEpFjRHOkcRbwJeA8M3vPe80Gfg5caGZbgAu9eYDFwIfAVuAPwDcAnHP7gP8LrPBe/+G1AdwA/NHb5gNgidfe3D6kg/XvlUzBKeks0WNgRaSRUGsrOOf+QdPnHQDOb2J9B9zYzGc9CDzYRHshMLqJ9r1N7UM6x6zR2fzH/6zng9JyTstK87scEYkBuiNcmjVz9AAAntcJcRHxKDSkWQP79GD8oD66O1xEGig0pEWzR2ezrriMj/ce9rsUEYkBCg1pUX0X1RJ1UYkICg1pRV5GCmNze+vucBEBFBoShVmjs3m/6CBF+yv8LkVEfKbQkFbN0lVUIuJRaEir8vumMjK7l66iEhGFhkRn9pgBrPrkADsPHvG7FBHxkUJDonLx2IEALHqvuJU1RaQ7U2hIVAb3TWX8oD48vapIT/QTiWMKDYna3Im5bC4pZ82Og36XIiI+UWhI1C4ZO5DEUICnVxa1vrKIdEsKDYla7x4JXDiyP4veL6a6ts7vckTEBwoNOSFzJ+Syv6KGlzfqIYoi8UihISfknKF9yeqZxFPqohKJSwoNOSGhYIDLx+fwyqbd7C2v8rscEelkCg05YVdMyKW2zvGc7tkQiTsKDTlhwwf0ZHROL55epS4qkXij0JA2mTshl3XFZWzYWeZ3KSLSiRQa0iaXjsshIWi6Z0Mkzig0pE0yUhOZPrwfz75XTG1Y92yIxAuFhrTZFRNz2VNexWtbSv0uRUQ6iUJD2mz68H5kpCbqng2ROKLQkDZLDAW49IyBvLh+Nwcqqv0uR0Q6gUJDTsrciblUh+v422o91U8kHig05KSMGtiL4f176ioqkTih0JCTYmbMnZjLe9sPsHV3ud/liEgHU2jISZszfiDBgOkOcZE4oNCQk9avZzLnDu3LM6t2EK7To2BFujOFhrSLKybmsquskjc+2ON3KSLSgRQa0i4uGNGfXskhnRAX6eYUGtIukhOC/NMZA3l+3S4OVdb4XY6IdBCFhrSbKybmUllTx+I1umdDpLsK+V1AzFi/CMqKISkNEtO8ac9G8z0j01Ci35XGrPF5fTi1bypPr9zB5ycN8rscEekACo16Kx+CD15qfb1AwqcDJbk39B8FORMhtwB6DezwcmORmXHFxFx+tXQTH+89zCmZqX6XJCLtrNXQMLMHgUuA3c650V5bBvAkkA9sA+Y55/abmQG/AWYDFcA1zrlV3jZXA//ufexPnHMPe+0TgYeAHsBi4CbnnGtuHyf9EzfnqsehqhyqD3nT8qPz1YebWVYOVYfg0C748BWo8/rye2ZHAiRnAuQUwMDxkNyrw0qPJZePz+GOv2/i6VU7+O6Fw/wuR0TamTnX8nX1ZnYuUA480ig0fgnsc8793MxuBdKdc7eY2WzgW0RCYwrwG+fcFC8ACoECwAErgYle0LwD3AS8RSQ07nHOLWluH639QAUFBa6wsLAt38XJqamEkrVQVAg7VkZe+z7wFhr0HeYdiUyMTPuN6rZdXV/849ts23uY174/nUDA/C5HRKJgZiudcwWtrdfqkYZz7jUzyz+ueQ4wzXv/MPAKcIvX/oiLJNFbZtbHzLK9dV9wzu3zinsBmGlmrwC9nHNveu2PAJcBS1rYR2xKSI50TeU2+s4r9kHxKtixKhIiW/4O7z8WWRZMguyxcNp5MPGabtWldcXEHL7z5Pu8s20fU0/N9LscEWlHbT2n0d85txPAObfTzPp57TnA9kbrFXltLbUXNdHe0j66jpQMGHJB5AXgHBzcfvRopKgQXv0lLP81jLgUpnwd8iaDde1/nV80agCpiWt5emWRQkOkm2nvE+FN/bVzbWg/sZ2aLQAWAAwaFMNX7ZhBn0GR1+jPRdr2fQQr/gir/gzr/grZZ8Dkr8HoKyJHL11QSmKIi8dm87+rd3L7nFGkJOp6C5Huoq33aZR43U54091eexGQ12i9XKC4lfbcJtpb2senOOfud84VOOcKsrKy2vgj+SRjMFz0U/juerj4Tqitgue+AXeNhJf+Aw7u8LvCNrliQi6Hq8M8v3aX36WISDtqa2gsAq723l8NPNeo/csWMRU46HUxLQVmmFm6maUDM4Cl3rJDZjbVu/Lqy8d9VlP76J6S0mDSdfCNt+DLz0HeFFh+J9w9Bv5yDXz8ZqR7q4uYlJ9BXkYPjXwr0s1Ec8nt40ROSPc1syLgNuDnwEIzuw74BLjSW30xkSunthK55PZaAOfcPjP7v8AKb73/qD8pDtzA0Utul3gvWthH92YGp06LvOq7rt79M6x7BgaMhSlfg9FzY77rKhAwPjc+l3te3kLxgSMM7NPD75JEpB20esltV+PbJbcdqfowrH4S3r4fSjdASmbkiqvPfBt69PG7umZ9sreCc3+1jO9fNJwbpw/xuxwRaUG0l9xq7KmuIDEVCr4C33gTvrwIBp0Z6bq6d0pk+JMYNSgzhcmDM3hqZRHd7R8nIvFKodGVmMGpn4X5j8KCZZCWBQu/BE98Acpic5DAuRNy+WjPYVZ9csDvUkSkHSg0uqqB4+H6ZXDB7bD1Rbh3MhQ+CHV1fld2jFljBpCcEGDhiu2trywiMU+h0ZUFE+Dsm+GGN2DgOPif78BDF0PpZr8ra9AzOYF5BXksXLmdN7bqqX4iXZ1CozvIPC1yrmPOvbB7Pfz+rMid5rXVflcGwK2zTufUvqnc/OR77C2v8rscETkJCo3uwgzGfxG+uQJOvwSW/RT+61zYvqL1bTtYSmKI3/7zBA4cqeFf/vI+dXU6KS7SVSk0upu0fnDln+CqJ6GqDB64EBb/a2QIdx+NyO7Fv188glc2lfLg6x/5WouItJ1Co7saPhNufBsmXw/v3A/3ToXNS30t6UtTT2HGyP784vmNrCk66GstItI2Co3uLKknzP4VXPf3yDAlj82Dp74C5aW+lGNm/HLuWPqmJfGtx1dRXlXrSx0i0nYKjXiQNxm+thym/RA2/A3uOxM+fNWXUvqkJPKb+eP5ZF8FP3p2rS81iEjbKTTiRSgRpt0CX3sNemTAI3MiV1j5cF/H5MEZfPv8oTzz7g6eXqkBDUW6EoVGvOk3Aq5/GcZcGbnC6tEr4HDn3z/xrfOGMnlwBj96bi0flpZ3+v5FpG0UGvEoKQ0+dz9ccjdsex1+fw588lanlhAMGL+ZP47EUIBvPf4uVbXhTt2/iLSNQiNemUHBtfDVFyCUBH+aDa/f06nP7Mju3YNfzT2DdcVl/GLJpk7br4i0nUIj3mWfAV97FU6/GF74ETzxz3Bkf6ft/sKR/bnmM/k8+PpHvLyxpNP2KyJto9AQSO4N8x6BmT+HLX+P3Em+Y1Wn7f7WWaczIrsX3/vLakrKKjttvyJy4hQaEmEGU2+Aa5+PdFE9eBG884dO6a5HprFHAAALK0lEQVRKTgjy238ez5HqMDc/8R5hDTMiErMUGnKsvEmRy3JPnQaLvxe5GbAThiA5LSuN2+eM4s0P9/K7ZVs7fH8i0jYKDfm0lIzI2FXn3wbrn4X7p8Gujr8R78qJuVx6xkDufmkLhdv2tb6BiHQ6hYY0LRCAc74LV/8Nqsrhj+fDqj93aHeVmfHTy0eT06cHNz3xHgcrajpsXyLSNgoNaVn+2fD15ZA3BRZ9E/5ydYfeDNgzOYF7rhpPSVkltzy9Ws8WF4kxIb8LkC4grR986Rl4/Tew7Gfw8RuRGwNHXNIhuxuX14d/nTmcny3eyKNvf8IXp57Sbp/tnKO8qpb9h2vYV1HN/sPV7Dtczf6KyGvf4RqOVNcyflA6Zw/ty6l9UzGzdtu/SFdn3e1fcgUFBa6wsNDvMrqvknXwzNdg1xoYOx9m/QJ69Gn33dTVOa55aAVvfrCHUzJTCQWMxFCAUMAIBQMkBgOEgkZCMECCNw0FAiSGjFAgQEIwQFVt2AuC6oaQOFBRTU246d/5YMBIT0kkFDB2eZf+5vTpwTlD+3LO0CzOGpJJn5TEdv9ZRWKBma10zhW0up5CQ05YbTUsvwNeuwPS+sOc/4QhF7T7bvaWV3HXi5vZf7iG6nAdteE6asKOmnAdNeE6autcw/zxy2rCjsRQgPSUBDJSE0lPSYxMUxPJSPGmqQkN7X1SEumVHGo4qvhkbwXLt5ayfPMeXv9gD4cqazGDsbl9ONcLkfGD+pAQVA+vdA8KDel4O1bBszdA6UaYeA3M+EnkGR7dTG24jveLDrJ8Syn/2LKHd7cfIFznSE0McuZpmZwzNItzhvZlsLqypAtTaEjnqKmMjJb7xn9Cnzy47L7IyfNurKyyhjc/2MvyLaUs37KHj/dWAJGurEn56Qwf0IvhA9IY1r8nOX16KEikS1BoSOf65C145uuw/yOYcgOc/38gMcXvqjpF466s1UUHKD54dCiUtKQQw/qnMXxAT4b178nw/j0ZPqAnmWlJPlYs8mkKDel81YfhxR9HnkmeOQQu+33kDvM4U1ZZw5aSQ2zaVc6mXWVsKjnEpl2H2N/ovpO+aYkM6x8JktMH9OTUrDQSQ82fH2npWMUMAt7RTMCsYT4yjWwdsMh9MAEDI7IsGDBSEoOkJIZa3LfEB4WG+OfDV+C5b0LZDjjrZph2a2T49TjmnKO0vIrNu8rZVHKIzbsOsbHkEFtKDlFR7f+zRBKDAVKSgqQmhkhNCpKaFDr6PjFEalKIlKQgaYkhUpJCpCQGSQoFSE4IkpwQIDkUJCnhuLZG87pgIPYpNMRflWWw9Ifw7p+h3yi4+A4YdGbkn8XSoK7OsePAET7cc5i6ZgZqdDT//6hz3guoc86bd8fM13n/jx+dj6wTrnNUVIc5XFXL4YZpLYeraqmoDlNeVUtFlTetruVwVZjqcNseDxwMGMmhAEkJQQLeEU8wYJH3gciRUdCOHgFF1oksi7QbwUBknUAgsk4wECDYaP1gwAgEjFDDepFpMBhpCwUil2mHvHVCwQDBgJEQjHxWQtBbJ2CEgpHPCwUCBANE9lU/9WoItdDWUE993Y3fm2Hez3VM7d7RoF8UGhIbNi+FRd+C8hJIz488ZnbMPMga5ndl0gbVtXVUVNdypCZMZU0dVbWRaWVNmMqaMFW1kfdVNXVU1nrTmjCVtUfXr3ORsKxzjnCdF2DOHdfuzbvG8960DsLe+/pX/bKwc9TVOWrrItP69WrrHOGwo6aujrB3qXYsChwXgvXhFwo0CsHA0TA9PpB+/8WJ5GW07VxitKGhO8KlYw27CL61EtYvgjV/geW/htd+FXn405grYfQV0Gug31VKlBJDARJDibT/7Zydy3khVX+/z7GBcjRYGgdTU0FVH061TbQ1DrGGaaMwPGa5F4THr3tMCDbs/9jtw+Gj64WCHX+koiMN6VyHdsHav8KahVD8LmAw+JzI0ceIf+qQu8tFpHXqnpLYt2dr5OhjzULY9yEEk2DYjEiADJ0BCcl+VygSNxQa0nU4F7m7fM1fYO3TcHg3JPWGkf8EIy+H1EywIARCEAh674ON3oeOm/faXB2Ea7xXtfdq9L6utpn2MCSmQXKvyB3uSb28970iny3SDSk0pGsK18JHr0YCZMPfoLrc74qOlZB6NECaCpXEVAgle68kSOgRmdbPh3o0vSzoDYTo6ryXA9yx8w3v6xotc8c94+S4/59bWgZggcirceA2TAOfDmYLRp61It2OToRL1xQMwZDzI69L7orcaV5bGTkqqAuDC0emDe9rj5uvb6uN/NELJnqvhEbThGPbAwnHrmOByI2KVQcjj7qtLItMq8q892VH31eWwcGio+01FX5/g53DgkcD55jgsePaj1/PjrvsutH7VttP4iRvi5eyRvu5JxDITf5jvIm2T613kv+I/9IzkHHqyX1GK2I+NMxsJvAbIAj80Tn3c59Lks6S0ANOm+53FSfGOaitigRdbRXUHjk6X1PZfHu4CrDj/rh6U47/Q9xovn5ZYy39gWy8rOEmj+NC14Whru64+drj2upafx2znrcfV3fs/o/OtNx+Uj0iLWzb4uc6PhUon/pumwu945Y1u04T653MvRqhHm3fNtpddPgeToKZBYF7gQuBImCFmS1yzq33tzKRZphFTuDrJL50U7HeOTkZ2Oqc+9A5Vw08AczxuSYRkbgV66GRA2xvNF/ktR3DzBaYWaGZFZaWlnZacSIi8SbWQ6Opzr1PdUI65+53zhU45wqysrI6oSwRkfgU66FRBOQ1ms8Fin2qRUQk7sV6aKwAhprZYDNLBOYDi3yuSUQkbsX01VPOuVoz+yawlMgltw8659b5XJaISNyK6dAAcM4tBhb7XYeIiMR+95SIiMSQbjf2lJmVAh+3cfO+wJ52LKc70nfUMn0/LdP30zq/vqNTnHOtXn7a7ULjZJhZYTQDdsUzfUct0/fTMn0/rYv170jdUyIiEjWFhoiIRE2hcaz7/S6gC9B31DJ9Py3T99O6mP6OdE5DRESipiMNERGJmkJDRESiptDwmNlMM9tkZlvN7Fa/64k1ZrbNzNaY2XtmpoewA2b2oJntNrO1jdoyzOwFM9viTdP9rNFPzXw/PzazHd7v0XtmNtvPGv1kZnlmtszMNpjZOjO7yWuP6d8hhQbHPCFwFjASuMrMRvpbVUya7pwbF8vXkHeyh4CZx7XdCrzknBsKvOTNx6uH+PT3A3CX93s0zhsmKF7VAv/inBsBTAVu9P7uxPTvkEIjQk8IlBPmnHsN2Hdc8xzgYe/9w8BlnVpUDGnm+xGPc26nc26V9/4QsIHIQ+Zi+ndIoRER1RMC45wD/m5mK81sgd/FxLD+zrmdEPmjAPTzuZ5Y9E0zW+11X8VU14tfzCwfGA+8TYz/Dik0IqJ6QmCcO8s5N4FIF96NZnau3wVJl3QfcBowDtgJ/NrfcvxnZmnA08DNzrkyv+tpjUIjQk8IbIVzrtib7gaeIdKlJ59WYmbZAN50t8/1xBTnXIlzLuycqwP+QJz/HplZApHAeNQ591evOaZ/hxQaEXpCYAvMLNXMeta/B2YAa1veKm4tAq723l8NPOdjLTGn/o+h53Li+PfIzAx4ANjgnLuz0aKY/h3SHeEe79K/uzn6hMCf+lxSzDCzU4kcXUDkwV2P6fsBM3scmEZkKOsS4DbgWWAhMAj4BLjSOReXJ4Ob+X6mEemacsA24Gv1/ffxxszOBpYDa4A6r/mHRM5rxOzvkEJDRESipu4pERGJmkJDRESiptAQEZGoKTRERCRqCg0REYmaQkNERKKm0BARkaj9f4lWYm6Or15+AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x102015320>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_model()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Matrix Factorization with Biases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_size = 30 # embedding size\n",
    "reg_param = 0.01 # regularization parameter lambda\n",
    "learning_rate = 0.01 # learning rate \n",
    "\n",
    "\n",
    "# create tensorflow graph\n",
    "g = tf.Graph()\n",
    "with g.as_default():\n",
    "    \n",
    "    tf.set_random_seed(1234)\n",
    "    \n",
    "    # placeholders\n",
    "    users = tf.placeholder(shape=[None], dtype=tf.int64)\n",
    "    items = tf.placeholder(shape=[None], dtype=tf.int64)\n",
    "    ratings = tf.placeholder(shape=[None], dtype=tf.float32)\n",
    "    \n",
    "    # variables\n",
    "    with tf.variable_scope(\"embedding\"):\n",
    "        user_weight = tf.get_variable(\"user_w\"\n",
    "                                      , shape=[max_user_id + 1, embedding_size]\n",
    "                                      , dtype=tf.float32\n",
    "                                      , initializer=layers.xavier_initializer())\n",
    "\n",
    "        item_weight = tf.get_variable(\"item_w\"\n",
    "                                       , shape=[max_item_id + 1, embedding_size]\n",
    "                                       , dtype=tf.float32\n",
    "                                       , initializer=layers.xavier_initializer())\n",
    "        \n",
    "        user_bias = tf.get_variable(\"user_b\"\n",
    "                                , shape=[max_user_id + 1]\n",
    "                                , dtype=tf.float32\n",
    "                                , initializer=tf.zeros_initializer)\n",
    "        \n",
    "        item_bias = tf.get_variable(\"item_b\"\n",
    "                                 , shape=[max_item_id + 1]\n",
    "                                 , dtype=tf.float32\n",
    "                                 , initializer=tf.zeros_initializer)\n",
    "        \n",
    "    # prediction\n",
    "    with tf.name_scope(\"inference\"):\n",
    "        user_embedding = tf.nn.embedding_lookup(user_weight, users)\n",
    "        item_embedding = tf.nn.embedding_lookup(item_weight, items)\n",
    "        user_b = tf.nn.embedding_lookup(user_bias, users)\n",
    "        item_b = tf.nn.embedding_lookup(item_bias, items)\n",
    "        pred = tf.reduce_sum(tf.multiply(user_embedding, item_embedding), 1) + user_b + item_b\n",
    "        \n",
    "    # loss \n",
    "    with tf.name_scope(\"loss\"):\n",
    "        reg_loss = tf.contrib.layers.apply_regularization(layers.l2_regularizer(scale=reg_param),\n",
    "                                               weights_list=[user_weight, item_weight])\n",
    "        loss = tf.nn.l2_loss(pred - ratings) + reg_loss\n",
    "        train_ops = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)\n",
    "        rmse = tf.sqrt(tf.reduce_mean(tf.pow(pred - ratings, 2)))\n",
    "\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration : 0 train loss: 434451.062 , valid loss 108152.547\n",
      "iteration : 5 train loss: 403790.500 , valid loss 101102.758\n",
      "iteration : 10 train loss: 355479.844 , valid loss 89540.172\n",
      "iteration : 15 train loss: 285449.938 , valid loss 72448.680\n",
      "iteration : 20 train loss: 198575.859 , valid loss 50982.094\n",
      "iteration : 25 train loss: 111163.250 , valid loss 29129.332\n",
      "iteration : 30 train loss: 50093.434 , valid loss 13611.050\n",
      "iteration : 35 train loss: 33727.246 , valid loss 9234.554\n",
      "iteration : 40 train loss: 38895.691 , valid loss 10511.686\n",
      "iteration : 45 train loss: 34300.242 , valid loss 9630.757\n",
      "iteration : 50 train loss: 26736.699 , valid loss 8091.473\n",
      "iteration : 55 train loss: 25132.494 , valid loss 7959.958\n",
      "iteration : 60 train loss: 25422.391 , valid loss 8179.475\n",
      "iteration : 65 train loss: 24283.762 , valid loss 7953.400\n",
      "iteration : 70 train loss: 22859.615 , valid loss 7620.199\n",
      "iteration : 75 train loss: 22283.582 , valid loss 7516.657\n",
      "iteration : 80 train loss: 21930.398 , valid loss 7519.730\n",
      "iteration : 85 train loss: 21348.547 , valid loss 7511.425\n",
      "iteration : 90 train loss: 20815.717 , valid loss 7532.250\n",
      "overfit !\n",
      "RMSE on test dataset : 0.9624\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEICAYAAACj2qi6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8FfW9//HXJycnCQlbCIiBEHYXQNYQsbigWAS1otYqaK1WK3Vp1fZeb217e21729vW61Xrtdi6VvvzihQX0LrUBZfWBcK+qYQ97GtYEkKW7++PmYQDZjmE5MxJzvv5eMzjzPnOzJnPGZK8mfnOYs45REREopEUdAEiItJyKDRERCRqCg0REYmaQkNERKKm0BARkagpNEREJGoKDRERiZpCQ6SRzGytmZ0fdB0isaTQEBGRqCk0RJqYmd1kZoVmtsvMZplZN7/dzOwBM9tmZsVmttjMBvnTLjSz5Wa2z8w2mtm/BvstRGqn0BBpQmZ2HvAb4EogG1gHTPMnjwPOBk4COgJXATv9aU8A33XOtQMGAe/GsGyRqCUHXYBIK3MN8KRzbj6Amf0Y2G1mvYByoB1wCjDHObciYrlyYICZLXLO7QZ2x7RqkShpT0OkaXXD27sAwDm3H29vortz7l3gYeAPwFYze9TM2vuzfh24EFhnZu+b2RkxrlskKgoNkaa1CehZ/cbMMoAsYCOAc+4h59wIYCDeYaq7/Pa5zrmJwAnAy8D0GNctEhWFhsjxCZtZWvWA98f+22Y21MxSgf8CPnXOrTWzkWZ2upmFgQPAQaDSzFLM7Boz6+CcKwf2ApWBfSOReig0RI7Pa0BpxHAW8DPgBWAz0BeY5M/bHngMr79iHd5hq/v8adcCa81sL3Az8M0Y1S9yTEwPYRIRkWhpT0NERKKm0BARkagpNEREJGoKDRERiVqruyK8c+fOrlevXkGXISLSosybN2+Hc65LQ/O1utDo1asXBQUFQZchItKimNm6hufS4SkRETkGCg0REYmaQkNERKLW6vo0RKT1KC8vp6ioiIMHDwZdSquRlpZGTk4O4XC4UcsrNEQkbhUVFdGuXTt69eqFmQVdTovnnGPnzp0UFRXRu3fvRn2GDk+JSNw6ePAgWVlZCowmYmZkZWUd156bQkNE4poCo2kd7/ZUaPjeXLaFF+cXBV2GiEhcU2jgHeebNmc9P5y+iKnvFaLbxYsIwJ49e5g6deoxL3fhhReyZ8+eZqgoeAoNvN21P147gq8N6ca9b3zOz2YupbJKwSGS6OoKjcrK+h+s+Nprr9GxY8fmKitQOnvKl5oc4vdXDaVbhzT+9MFqtu4t46FJw2iTEgq6NBEJyN13382qVasYOnQo4XCYtm3bkp2dzcKFC1m+fDmXXnopGzZs4ODBg9xxxx1MmTIFOHw7o/379zNhwgTOPPNMPvroI7p3787MmTNp06ZNwN+s8RQaEZKSjB9feCrZHdL4xavLufrxT3jiupF0ykgJujSRhPeLV5axfNPeJv3MAd3ac8/XBtY5/be//S1Lly5l4cKFvPfee1x00UUsXbq05nTVJ598kk6dOlFaWsrIkSP5+te/TlZW1hGfsXLlSp577jkee+wxrrzySl544QW++c2W+zRfHZ6qxfWjezP16uEs27SXKx75iPU7S4IuSUTiQH5+/hHXNzz00EMMGTKEUaNGsWHDBlauXPmlZXr37s3QoUMBGDFiBGvXro1Vuc1Cexp1mHBaNl3apXLj0wVc/sg/eer6fE7L6RB0WSIJq749gljJyMioGX/vvfd4++23+fjjj0lPT2fMmDG1Xv+QmppaMx4KhSgtLY1Jrc1Fexr1yOvViRduOYPU5BBXPfoxsz/fFnRJIhJD7dq1Y9++fbVOKy4uJjMzk/T0dD777DM++eSTGFcXDIVGA/qd0I6Xbv0KvbIy+M7TBUyfuyHokkQkRrKyshg9ejSDBg3irrvuOmLa+PHjqaioYPDgwfzsZz9j1KhRAVUZW9barknIy8tzzfEQpn0Hy7n12fl8uHIHPzj/JG4f209Xqoo0sxUrVnDqqacGXUarU9t2NbN5zrm8hpbVnkaU2qWFefL6kVw+vDsPvP0FP3lpCRWVVUGXJSISU+oIPwbhUBL/840hdOvQhodnF7J1bxkPXz2M9BRtRhFJDNrTOEZmxr9ecDK/vmwQ732+jUmPfsKO/WVBlyUiEhMKjUa65vSePHptHl9s3cflUz9izY4DQZckItLsog4NMwuZ2QIze9V/39vMPjWzlWb2vJml+O2p/vtCf3qviM/4sd/+uZldENE+3m8rNLO7I9prXUe8OH9AV567aRT7yyr4+iMfsWD97qBLEhFpVseyp3EHsCLi/e+AB5xz/YHdwI1++43AbudcP+ABfz7MbAAwCRgIjAem+kEUAv4ATAAGAJP9eetbR9wYlpvJC7d8hbapyXz7z3PZUqzHUopI6xVVaJhZDnAR8Lj/3oDzgBn+LE8Dl/rjE/33+NPH+vNPBKY558qcc2uAQiDfHwqdc6udc4eAacDEBtYRV3p3zuCpb4+krLyKf/nrQqp0h1yRhNW2bVsANm3axBVXXFHrPGPGjKGhSwMefPBBSkoO38IoXm63Hu2exoPAvwHV55hmAXuccxX++yKguz/eHdgA4E8v9uevaT9qmbra61vHEcxsipkVmFnB9u3bo/xKTatvl7bc87UB/LNwJ499uDqQGkQkfnTr1o0ZM2Y0PGMdjg6NeLndeoOhYWYXA9ucc/Mim2uZ1TUwranav9zo3KPOuTznXF6XLl1qmyUmrhrZgwmDTuS/3/ycJUXFgdUhIk3nRz/60RHP1Pj5z3/OL37xC8aOHcvw4cM57bTTmDlz5peWW7t2LYMGDQKgtLSUSZMmMXjwYK666qoj7j91yy23kJeXx8CBA7nnnnsA70aImzZt4txzz+Xcc88FvNut79ixA4D777+fQYMGMWjQIB588MGa9Z166qncdNNNDBw4kHHjxjXLfa6iucBgNHCJmV0IpAHt8fY8OppZsr8nkANs8ucvAnoARWaWDHQAdkW0V4tcprb2HfWsIy6ZGb+5/DQWbtjD7dMW8Or3zyQjVddwiDSJ1++GLUua9jNPPA0m/LbeWSZNmsSdd97JrbfeCsD06dN54403+MEPfkD79u3ZsWMHo0aN4pJLLqnzLhGPPPII6enpLF68mMWLFzN8+PCaab/+9a/p1KkTlZWVjB07lsWLF3P77bdz//33M3v2bDp37nzEZ82bN4+nnnqKTz/9FOccp59+Oueccw6ZmZkxuQ17g3sazrkfO+dynHO98Dqy33XOXQPMBqoP2F0HVEftLP89/vR3nXevklnAJP/sqt5Af2AOMBfo758pleKvY5a/TF3riFsd01O4/8qhrN15gF++sjzockTkOA0bNoxt27axadMmFi1aRGZmJtnZ2fzkJz9h8ODBnH/++WzcuJGtW7fW+RkffPBBzR/vwYMHM3jw4Jpp06dPZ/jw4QwbNoxly5axfHn9fzf+8Y9/cNlll5GRkUHbtm25/PLL+fDDD4HY3Ib9eP4b/CNgmpn9ClgAPOG3PwH8xcwK8fYwJgE455aZ2XRgOVAB3OacqwQws+8BbwIh4Enn3LIG1hHXzuibxS3n9GXqe6sYc3IXJpyWHXRJIi1fA3sEzemKK65gxowZbNmyhUmTJvHss8+yfft25s2bRzgcplevXrXeFj1SbXsha9as4b777mPu3LlkZmZy/fXXN/g59d0vMBa3YT+mi/ucc+855y72x1c75/Kdc/2cc99wzpX57Qf99/386asjlv+1c66vc+5k59zrEe2vOedO8qf9OqK91nW0BD/46kkMyenA3S8uYdOeln3/fJFEN2nSJKZNm8aMGTO44oorKC4u5oQTTiAcDjN79mzWrVtX7/Jnn302zz77LABLly5l8eLFAOzdu5eMjAw6dOjA1q1bef31mj+Ldd6W/eyzz+bll1+mpKSEAwcO8NJLL3HWWWc14betn64IbybhUBK/nzSM8soqfvD8Qip1Gq5IizVw4ED27dtH9+7dyc7O5pprrqGgoIC8vDyeffZZTjnllHqXv+WWW9i/fz+DBw/m3nvvJT8/H4AhQ4YwbNgwBg4cyA033MDo0aNrlpkyZQoTJkyo6QivNnz4cK6//nry8/M5/fTT+c53vsOwYcOa/kvXQbdGb2Z/LdjAXTMWc9cFJ3Pbuf2CLkekRdGt0ZuHbo0ex64YkcNFg7N54K0vWLgh+AtzRESOh0KjmZkZ/3XpaXRtn8Yd0xawv6yi4YVEROKUQiMGOqSHeeCqoWzYVcLPZy1reAERqdHaDqEH7Xi3p0IjRvJ7d+K2c/sxY14RryyK62sUReJGWloaO3fuVHA0EeccO3fuJC0trdGfocuVY+j2sf35R+EOfvLSEobldiQnMz3okkTiWk5ODkVFRQR1T7nWKC0tjZycnEYvr7OnYmz9zhIufOhDBmS357kpowgl1X7bARGRWNLZU3EqNyudX04cyJy1u5g6uzDockREjolCIwCXDevOxKHdePCdlcxbp6f9iUjLodAIgJnxn5cOIrtDGnc+v4B9B8uDLklEJCoKjYC0Twvz+0lD2bi7lHtm6jRcEWkZFBoBGtGzE7eP7c+LCzYyc+HGoMsREWmQQiNg3zu3H3k9M/n3l5ayYVdJwwuIiARIoRGw5FASD1zlPTTlrhmLdBGTiMQ1hUYc6NEpnbvGn8wnq3fx8aqdQZcjIlInhUacuDKvB13bp/LgOyuDLkVEpE4KjTiRFg5xyzl9mbNmF5+s1t6GiMQnhUYcmZSfS5d2qfz+be1tiEh8UmjEkbRwiJvP6cvHq3cyZ82uoMsREfkShUacuTo/l85tU3lIfRsiEocUGnGmTUqI757dh38U7mDeOu1tiEh8UWjEoWtG5ZKVkcJD7+guuCISXxQacSg9JZmbzu7D+19sZ+GGPUGXIyJSQ6ERp64d1ZPM9LD6NkQkrig04lRGajLfOasP7362jcVF2tsQkfig0Ihj3zqjJx3ahNW3ISJxQ6ERx9qlhbnxzN68vWIrSzcWB12OiIhCI95dP7oX7dKS+d931bchIsFTaMS59mlhbhjdmzeXbWXF5r1BlyMiCU6h0QLcMLo37VK1tyEiwVNotAAd0sNcP7oXry3Zwudb9gVdjogkMIVGC3Hjmb3JSAlpb0NEAqXQaCE6pqdw3Vd68bclm1m5VXsbIhIMhUYL8p2z+tAmHOLh2bpuQ0SCodBoQTplpHDtGT15ZdEmVm3fH3Q5IpKAFBotzE1n9SE1OcQf3tXehojEXoOhYWZpZjbHzBaZ2TIz+4Xf3tvMPjWzlWb2vJml+O2p/vtCf3qviM/6sd/+uZldENE+3m8rNLO7I9prXUci69w2lW+OyuXlhRtZs+NA0OWISIKJZk+jDDjPOTcEGAqMN7NRwO+AB5xz/YHdwI3+/DcCu51z/YAH/PkwswHAJGAgMB6YamYhMwsBfwAmAAOAyf681LOOhDbl7L6EQ0n8QX0bIhJjDYaG81QfQA/7gwPOA2b47U8Dl/rjE/33+NPHmpn57dOcc2XOuTVAIZDvD4XOudXOuUPANGCiv0xd60hoXdqlcs3pPXlpwUbW7dTehojETlR9Gv4ewUJgG/AWsArY45yr8GcpArr7492BDQD+9GIgK7L9qGXqas+qZx1H1zfFzArMrGD79u3RfKUW7+Zz+hBKMqbOXhV0KSKSQKIKDedcpXNuKJCDt2dwam2z+a9Wx7Smaq+tvkedc3nOubwuXbrUNkurc0L7NK7Oz+WF+UVs2FUSdDkikiCO6ewp59we4D1gFNDRzJL9STnAJn+8COgB4E/vAOyKbD9qmbrad9SzDgG+e04fksyY+p72NkQkNqI5e6qLmXX0x9sA5wMrgNnAFf5s1wEz/fFZ/nv86e8655zfPsk/u6o30B+YA8wF+vtnSqXgdZbP8pepax0CZHdow1UjezBj3gY27ikNuhwRSQDR7GlkA7PNbDHeH/i3nHOvAj8CfmhmhXj9D0/48z8BZPntPwTuBnDOLQOmA8uBN4Db/MNeFcD3gDfxwmi6Py/1rEN8N4/pC8Aj7+lMKhFpfub9h771yMvLcwUFBUGXEVM/eWkJMwqKeP/fxpDdoU3Q5YhIC2Rm85xzeQ3NpyvCW4FbzulLlXP8UX0bItLMFBqtQI9O6Vw2rDvTC4rYe7A86HJEpBVTaLQS157Rk9LySmYu1AlmItJ8FBqtxGndOzCwW3v+79P1tLZ+KhGJHwqNVsLMmJyfy4rNe1lcVBx0OSLSSik0WpGJQ7vRJhziuTnrgy5FRFophUYr0i4tzCVDujFr0Sb2qUNcRJqBQqOVmXx6LiWHKpm1SB3iItL0FBqtzJCcDpya3V6HqESkWSg0Whkz4+r8HizduJcl6hAXkSam0GiFJg7rTlo4if/T3oaINDGFRivUPi3MxYO7MWvhRvaXVTS8gIhIlBQardTk/FwOHKrkFXWIi0gTUmi0UsNzO3Jy13bqEBeRJqXQaKW8K8R7sLiomKUb1SEuIk1DodGKXTYsh9TkJO1tiEiTUWi0Yh3Sw1w0OJuZCzdxQB3iItIEFBqt3NX5uewvq+BvizcHXYqItAIKjVZuRM9M+p/QVtdsiEiTUGi0ctW3TF+4YQ/LN+0NuhwRaeEUGgng8uHdSUlOYtpc7W2IyPFRaCSAjukpXHRaNi/N30jpocqgyxGRFkyhkSAm5+eyr6yCVxfrCnERaTyFRoIY2SuTvl0ydM2GiBwXhUaCqO4Qn79+D59tUYe4iDSOQiOBfH14DimhJKbN2RB0KSLSQik0EkhmRgoTTjuRF+cXqUNcRBpFoZFgJufnsvdgBa8t0RXiInLsFBoJ5vTenejTWR3iItI4Co0EU90hXrBuN19s3Rd0OSLSwig0EtDXR3gd4trbEJFjpdBIQJ0yUhg3sCsvzt/IwXJ1iItI9BQaCerq/FyKS8t5fak6xEUkegqNBDWqTxa9stJ57lNdsyEi0VNoJKikJGNSfi5z1u6icJs6xEUkOgqNBHbFiBzCIeM5XSEuIlFSaCSwzm1TGTfgRF6YX6QOcRGJikIjwU3Oz2VPSTlvLtsSdCki0gI0GBpm1sPMZpvZCjNbZmZ3+O2dzOwtM1vpv2b67WZmD5lZoZktNrPhEZ91nT//SjO7LqJ9hJkt8Zd5yMysvnVI0/lK3yxyO6Xzf5/qmg0RaVg0exoVwL84504FRgG3mdkA4G7gHedcf+Ad/z3ABKC/P0wBHgEvAIB7gNOBfOCeiBB4xJ+3ernxfntd65Am4nWI9+DTNbtYtX1/0OWISJxrMDScc5udc/P98X3ACqA7MBF42p/taeBSf3wi8IzzfAJ0NLNs4ALgLefcLufcbuAtYLw/rb1z7mPnnAOeOeqzaluHNKErRuSQnGRM0xXiItKAY+rTMLNewDDgU6Crc24zeMECnODP1h2IPB2nyG+rr72olnbqWcfRdU0xswIzK9i+ffuxfCUBTmiXxlcHdGXGvCLKKtQhLiJ1izo0zKwt8AJwp3Ouvke/WS1trhHtUXPOPeqcy3PO5XXp0uVYFhXfpPxcdpeU8/dlW4MuRUTiWFShYWZhvMB41jn3ot+81T+0hP+6zW8vAnpELJ4DbGqgPaeW9vrWIU3srH6d6d6xDdPm6hCViNQtmrOnDHgCWOGcuz9i0iyg+gyo64CZEe3f8s+iGgUU+4eW3gTGmVmm3wE+DnjTn7bPzEb56/rWUZ9V2zqkiSUlGZNG9uCfhTtZt/NA0OWISJyKZk9jNHAtcJ6ZLfSHC4HfAl81s5XAV/33AK8Bq4FC4DHgVgDn3C7gP4G5/vBLvw3gFuBxf5lVwOt+e13rkGbwjbweJBlMm6srxEWkduadsNR65OXluYKCgqDLaLG+8/RcFm4o5uMfn0c4pGs/RRKFmc1zzuU1NJ/+KsgRJufnsmN/Ge+sUIe4iHyZQkOOcM5JXTixfZpuYigitVJoyBGSQ0lcmZfDByu3U7S7JOhyRCTOKDTkS64c6Z0ZPV0d4iJyFIWGfElOZjpn9+/C9IIiKiqrgi5HROKIQkNqNTm/B1v2HuT9L3RbFhE5TKEhtRp7alc6t03lOd3EUEQiKDSkVuFQEt/Iy+Hdz7axpfhg0OWISJxQaEidJo3sQZWDvxaoQ1xEPAoNqVPPrAxG98ti2twNVFW1rjsHiEjjKDSkXpNG5rJxTykfFu4IuhQRiQMKDanXuIFdyUwP66l+IgIoNKQBqckhrhiRw1vLt7J9X1nQ5YhIwBQa0qCrRuZSUeWYMa+o4ZlFpFVTaEiD+p3QlvxenXh+7npa2630ReTYKDQkKpNP78HanSV8vHpn0KWISIAUGhKVCYOyaZ+WrFumiyQ4hYZEJS0c4vLhOby5dAu7DhwKuhwRCYhCQ6I2Kb8HhyqreHG+OsRFEpVCQ6J2yontGZbbkefmqENcJFEpNOSYTB6Zy6rtByhYtzvoUkQkAAoNOSYXD8mmbWqybpkukqAUGnJM0lOSmTi0G39bvJnikvKgyxGRGFNoyDGbnJ9LWUUVLy/cGHQpIhJjCg05ZoO6d+C07h3UIS6SgBQa0iiT8nvw2ZZ9LNywJ+hSRCSGFBrSKJcM6UabcIhpukJcJKEoNKRR2qWFuWRIN15ZvIl9B9UhLpIoFBrSaJPye1ByqJJZizYFXYqIxIhCQxptaI+OnHJiOx2iEkkgCg1pNDNjcn4uSzYWs3RjcdDliEgMKDTkuFw6tDupyUm6QlwkQSg05Lh0SA9z0WnZzFy4iZJDFUGXIyLNTKEhx23y6bnsL6vg1cWbgy5FRJqZQkOOW17PTPqd0JZnP1mnK8RFWjmFRrUVr8Li6bBjJVRVBV1Ni2Jm3DC6N4uKinlj6ZagyxGRZpQcdAFxY86fYM0H3nhKO+g2FLoNOzxk9gKzQEuMZ1fm5fD0R2v5zeufcd6pJ5CaHAq6JBFpBg3uaZjZk2a2zcyWRrR1MrO3zGyl/5rpt5uZPWRmhWa22MyGRyxznT//SjO7LqJ9hJkt8Zd5yMz7y1zXOprNN1+Cm/8JlzwMg6+EQwfg0z/CjG/DQ0Phd73gmUvh7V/AileguAh0KKZGciiJf7/4VNbvKuHP/1wbdDki0kysoWPQZnY2sB94xjk3yG+7F9jlnPutmd0NZDrnfmRmFwLfBy4ETgd+75w73cw6AQVAHuCAecAI59xuM5sD3AF8ArwGPOSce72udTT0hfLy8lxBQUFjtsWXVRyCbcth0wJ/mA/bVkCVf5ZQRpcj90Zy8iEjq2nW3ULd8Oe5zF2zi9l3jaFz29SgyxGRKJnZPOdcXoPzRdNxaWa9gFcjQuNzYIxzbrOZZQPvOedONrM/+ePPRc5XPTjnvuu3/wl4zx9mO+dO8dsnV89X1zoaqrVJQ6M25aWwdVlEkCyA7Z+BqwILQe+zYOBlcMrXEjJACrftZ/yDH3DlyB7812WnBV2OiEQp2tBobJ9GV+fcZgD/j/oJfnt3IPKeEkV+W33tRbW017eOLzGzKcAUgNzc3EZ+pSiF20BOnjdUO3QANi+Gwrdg2Uvwyh3w6g+h99kw8NKECpB+J7Tlm6N68szHa/nWGT055cT2QZckIk2oqc+eqq2n2DWi/Zg45x51zuU55/K6dOlyrIsfv5QM6HkGjP0P+P58+O6HMPoO2LPOC5D7+sNfLoN5T0PJrtjXF2N3jO1Pu7Qwv/7bCp2CK9LKNDY0tvqHjPBft/ntRUCPiPlygE0NtOfU0l7fOuKbGWQPhvPv8QPkAy9Adq2BV26H/+7nBcj8Z1ptgGRmpHD72P58uHIHsz9vGf9sIhKdxobGLKD6DKjrgJkR7d/yz6IaBRT7h5jeBMaZWaZ/FtQ44E1/2j4zG+WfNfWtoz6rtnW0HGaQPcQLkNsXwJT3YfTtsGs1zPq+vwdyOcz/S6sLkGtH9aR35wx+9bcVlFfquheR1iKas6eew+vI7gxsBe4BXgamA7nAeuAbzrld/h/+h4HxQAnwbedcgf85NwA/8T/21865p/z2PODPQBvgdeD7zjlnZlm1raOhL9TsHeFNwTnYvMjr/1j+MuxeC0nJcNJ4OOtfoPvwBj+iJXhr+VZueqaAn39tANeP7h10OSJSjyY9e6olaRGhEck52LwQlr7oHbI6uAf6j4Nz7oacEUFXd1ycc1zz+Kcs37yX9//1XDqkh4MuSUTqEG1o6DYiQTPzrvEY959w5xKvM72oAB4/zzt0tWFO0BU2mpnx7xcNoLi0nN+/szLockSkCSg04klae+/w1J2L4fxfeHsgT3wVnpkI6z4OurpGGdCtPVfl9eCZj9eyevv+oMsRkeOk0IhHqe3gzDu9PY9xv/IuJnxqPPz5YljzYdDVHbMfjjuJ1OQk/uu1z4IuRUSOk0IjnqVkwFe+D3cshgt+Azu+gKcvhqcuhNXvt5h7X53QLo1bz+3H2yu28lHhjqDLEZHjoNBoCVLS4Yxb4Y5FMOFe75TdZy6BJ8fDqndbRHjceGZvundswy9fXU5lVfzXKyK1U2i0JOE2cPp34faFcOF9ULzBu1Dwia/CyrfjOjzSwiHunnAKn23Zx18LNjS8gIjEJYVGSxROg/ybvAsGL7of9m2BZ78OT14AW5YEXV2dLh6czYiemdz39y/YX6bniYu0RAqNliw5FUbe6N2u5OIHYecq+NM58OZPoSz+zlQyM3528QB27C9j6uzCoMsRkUZQaLQGySmQ92343lwYdg18/DBMHQWfvRZ0ZV8ytEdHLh3ajcf/sYYNu0qCLkdEjpFCozVJ7wSX/C/c8KZ32u60yTDtGu8pg3Hk38afQpLB797QKbgiLY1CozXKHeXdXff8n0PhO/BwPnz0MFTGRz9Ct45tmHJWH15dvJl561rXjRpFWjuFRmsVCsOZP4DbPoFeo+HvP4XHxkDRvKArA+C75/Sla/tUfvnKcqp0Cq5Ii6HQaO0ye8HV0+EbT8OBHfD4WPjbv8DB4kDLykhN5q4LTmFRUTEzF20MtBYRiZ5CIxGYeY+dvW2Od51HwZPw8EhYMiPQazsuH9ad07p34N43Pqf0UGVgdYhI9BQaiSSOAx6sAAAOTUlEQVStPUz4Hdz0LrTLhhduhP93uXeFeQCSkrxTcDcXH+TRD4KpQUSOjUIjEXUb5gXH+N/Bhrkw9Qx4/7+hoizmpeT37sSEQSfyx/dXsaX4YMzXLyLHRqGRqJJCMOpm+N4cOOkCmP0r+ONZsO6jmJfy4wmnUlnluGfWUvYeLI/5+kUkegqNRNe+G1z5DFz9VygvhacmwCt3QOmemJWQm5XO98/rx5vLtjL6N+9y7xufsWN/7Pd6RKRhetyrHFa2H977DXwyFTK6eP0fAy71OtJjYElRMY+8X8jrS7eQEkriqpE9uOmsPvTolB6T9YskMj0jXBpv0wJvb2PzIuh/AVx0H3TMjdnqV23fz5/eX8VLCzZS5WDikG7cMqYv/bu2i1kNIolGoSHHp7IC5vwJ3v0VYHDev3un6yaFYlbCpj2lPP7hGp6bs57S8kq+OqArt47py7DczJjVIJIoFBrSNHav8y4GLHwLsofCJQ9B9pCYlrDrwCH+/NFanv5oLcWl5Xylbxa3junH6H5Z2HEeOquqcmwqLmXNjgPsO1hBTmYbcjul0zE9pYmqF2kZFBrSdJyDZS/C6z+Ckl3eUwTH/Nh7HG0M7S+r4LlP1/PYh6vZtq+MwTkduHVMX8YNOJGkpPrDY/eBQ6zesZ/V2w+wZoc3rN5+gLU7D1BWUfWl+dunJZOblU5up3R6dPJeq4duHdsQDukcEmldFBrS9Ep3w1v3wPynvT6Oix6A/ufHvIyyikpemLeRP32winU7S+jbJYObz+nLBYNOZOPu0ppQWLV9f834npLDp/ImJxm5Wen06ZxB784Z9O7clj5dMmibmkzR7lI27CphvT9s2FVC0e5SDlUeDpYk8266WB0i1aHSu3MGJ5/YToEiLZJCQ5rPuo+8jvIdX8CgK2D8b6DtCTEvo6KyiteWbmHq7EI+27LvS9NPbJ9Gny7VwZDhj7elR2Ybko/hD3tllWPr3oNHBEnk+I79h2rmTQsnMbh7R4blVg+ZdG2f1iTfV6Q5KTSkeVWUwT8egA//B8LpMO4/Ydi1MTs9N5Jzjve+2M7yTXvpmeX9j79XVgYZqckxWf+Bsgo27C7h8y37WLhhDwvW72H5pr01eyfdOqQxLDezJkgGdutAWjh2JxSIREOhIbGx/XN45U5Y/xHk5MOZd8JJEyApsQ/RlFVUsnzTXhas38OCDXtYsH43RbtLAQiHjAHZ7Q8HSY9MenRqc9yd+iLHQ6EhsVNVBQv+Ah/cB8XroVNfr7N8yNWQogvzqm3bd5CFESGyuKiYEv/uvlkZKZyS3Y62qclkpCSTnhryXlOSyUgN0Sal+n2IjNQjX9P99tRkL6irHFQ5R2WVw/nj3uDtlVVPr6qKGHcOMyM1OckfQoRDpiBLIAoNib3KClgx03tK4Kb50KYTjLwRRt4E7boGXV3cqais4out+1mwYTcL1++hcPt+SsoqOXCogpJDlRwoq6j1zK5YSTJITQ6RFvZCJDWcRNpRr6k1r0mkhUOkh72Aa5MSok04RHpKyGtPSaZN+Mj2yPl08kDwFBoSHOdg/Sfw8cPw2d+8pwgOvhJG3QZdBwRdXYtSWeUoiQiRmtfyysMBU1bBgUOVlFVUkWSQZEaSgZkRSrKaNjOrc7qZ4ZzjUEUVZRVVHCyvPOK1rLyKgxWVlJVXUVZRycGjXqvnLT1USUl55TE/piUcMtKSQ4RCRnJSEslJXm3hkPeanJREcshq2qvfe+NGciiJUPX38b9TyCLGk8z/3t485k/3xo1QUvX7o9fjffbhepJqbU/2666edrjN+8yQmf/d7Mh5IpZp6LTx5hZtaMSmp1ASixn0PMMbdq7y7mW14FlY8P+g71j4yvegz7mBdJq3NKEko11amHZp4aBLiZpzriZESg55Q/V4aXklpYcqKK1+Xz2Ue0NllaOiylFZ6Sivqqp5X1EZOe6oqKqivLKKkkPuiHmqIg6/VR+eq6xyVDqH89tq2qsP21V545Vx8NjhUJL5YXc41KqHJDvyNXRUIIaSjD9+c0Sz36tNoSHNK6svXPQ/cO5PoeAJmPMY/OUy6DoIzrjNO2U3WVdftyZmRlrYOyzVsYV1aXkB9OWAqqwZd1RWVUVM89/XjB8Oser5qpettf2I6VV+eHmhV1HpasKv0u+DOmLc8aW25FDz/0dMh6cktirKYMlf4eM/wLbl0PZEOH0KjPg2pHcKujqRhKU+DYlvzsGqd71+j1Xvetd6nHIRZPXzrjbv2NN7bd8tpjdJFElU6tOQ+GYG/cZ6w9Zl3p7HqtmwZAYQ8R+ZpGTokBMRJH6YZPqvbU889mtCKiugotR76FT1UFEKVZVep30o5fCQnOq3pfpt+pWRxKbfAAle14Fw6VRvvKIMiotgzzrvDrt71vvDOlj5d9i/9chlQynQoYcXIOlZUHHwyCA4IhgOQnkJVFU0vlZL8sMj9XDAJEcETEpbb68pJT1iPOPwEPk+7M+T4rclpwHmnyBQ1ytffh85j6sCnPfqnD9EtlUdbjt63sjvaEneZ9aMJ/nriZxmtbQneXuGkfNJq6LQkPiSnOp1nmf1rX16eSns2eAHydrDobJ7nRcsyW0g7A8ZXbw/xOF0CPuvyWmHpx89zUJQVe4FV2U5VJZB5SGoOOS9Vg91Ta84CIcOwME9sHcjHCqB8gNeW8XBmG7G+GERIRKKCBU76n1kMPkBSPVLHQEJtQdoXKnj8H+t3QKunul1TTtqvmtfgk59jqG+Yxf3oWFm44HfAyHgcefcbwMuSYIUbgNdTvKGlqSq0guP8hLvtWZ8vxcuNcHi7x3UvHLU+4jX2qbVumdQ117BUW3eB9axJxK5l1LX3kvl4fdV1e1+W1XlkZ9T8z5y+lHfu97vWtf0IDlqDa0697YamteimxbZntym/hKbQFyHhpmFgD8AXwWKgLlmNss5tzzYykSOUVII0tp7g0gLFu/X7ucDhc651c65Q8A0YGLANYmIJKx4D43uwIaI90V+2xHMbIqZFZhZwfbt22NWnIhIoon30KjtYOCXDlw65x51zuU55/K6dOkSg7JERBJTvIdGEdAj4n0OsCmgWkREEl68h8ZcoL+Z9TazFGASMCvgmkREElZcnz3lnKsws+8Bb+Kdcvukc25ZwGWJiCSsuA4NAOfca8BrQdchIiLxf3hKRETiSKu7y62ZbQfWNXLxzsCOJiynqam+46P6jo/qOz7xXl9P51yDp5+2utA4HmZWEM2tgYOi+o6P6js+qu/4xHt90dLhKRERiZpCQ0REoqbQONKjQRfQANV3fFTf8VF9xyfe64uK+jRERCRq2tMQEZGoKTRERCRqCRkaZjbezD43s0Izu7uW6alm9rw//VMz6xXD2nqY2WwzW2Fmy8zsjlrmGWNmxWa20B/+I1b1+etfa2ZL/HUX1DLdzOwhf/stNrPhMazt5IjtstDM9prZnUfNE9PtZ2ZPmtk2M1sa0dbJzN4ys5X+a2Ydy17nz7PSzK6LYX3/bWaf+f9+L5lZxzqWrfdnoRnr+7mZbYz4N7ywjmXr/V1vxvqej6htrZktrGPZZt9+Tc45l1AD3j2sVgF9gBRgETDgqHluBf7oj08Cno9hfdnAcH+8HfBFLfWNAV4NcBuuBTrXM/1C4HW8W9uPAj4N8N96C95FS4FtP+BsYDiwNKLtXuBuf/xu4He1LNcJWO2/ZvrjmTGqbxyQ7I//rrb6ovlZaMb6fg78axT//vX+rjdXfUdN/x/gP4Lafk09JOKeRjRPA5wIPO2PzwDGmtX5oN8m5Zzb7Jyb74/vA1ZQy4On4txE4Bnn+QToaGbZAdQxFljlnGvsHQKahHPuA2DXUc2RP2NPA5fWsugFwFvOuV3Oud3AW8D4WNTnnPu7c67Cf/sJ3mMJAlHH9otGTJ78WV99/t+NK4Hnmnq9QUnE0IjmaYA18/i/OMVAVkyqi+AfFhsGfFrL5DPMbJGZvW5mA2NamPcgrL+b2Twzm1LL9KieuBgDk6j7lzXI7QfQ1Tm3Gbz/KAAn1DJPvGzHG/D2HGvT0M9Cc/qef/jsyToO78XD9jsL2OqcW1nH9CC3X6MkYmhE8zTAqJ4Y2JzMrC3wAnCnc27vUZPn4x1yGQL8L/ByLGsDRjvnhgMTgNvM7OyjpsfD9ksBLgH+WsvkoLdftOJhO/4UqACerWOWhn4WmssjQF9gKLAZ7xDQ0QLffsBk6t/LCGr7NVoihkY0TwOsmcfMkoEONG73uFHMLIwXGM865148erpzbq9zbr8//hoQNrPOsarPObfJf90GvIR3GCBSPDxxcQIw3zm39egJQW8/39bqQ3b+67Za5gl0O/od7xcD1zj/APzRovhZaBbOua3OuUrnXBXwWB3rDXr7JQOXA8/XNU9Q2+94JGJoRPM0wFlA9ZkqVwDv1vVL09T8Y6BPACucc/fXMc+J1X0sZpaP9++4M0b1ZZhZu+pxvA7TpUfNNgv4ln8W1SiguPpQTAzV+T+8ILdfhMifseuAmbXM8yYwzswy/cMv4/y2Zmdm44EfAZc450rqmCean4Xmqi+yj+yyOtYb9JM/zwc+c84V1TYxyO13XILuiQ9iwDu75wu8Myt+6rf9Eu8XBCAN77BGITAH6BPD2s7E24VeDCz0hwuBm4Gb/Xm+ByzDOxvkE+ArMayvj7/eRX4N1dsvsj4D/uBv3yVAXoz/fdPxQqBDRFtg2w8vvDYD5Xj/+70Rr4/sHWCl/9rJnzcPeDxi2Rv8n8NC4NsxrK8Qrz+g+mew+mzCbsBr9f0sxKi+v/g/W4vxgiD76Pr891/6XY9FfX77n6t/5iLmjfn2a+pBtxEREZGoJeLhKRERaSSFhoiIRE2hISIiUVNoiIhI1BQaIiISNYWGiIhETaEhIiJR+/+t9kkeKqMJCQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10205a7b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_model()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A Deep recommender model\n",
    "\n",
    "We can use deep learning models with multiple layers ( fully connected and dropout ) for the recommendation system.\n",
    "\n",
    "<img src=\"images/rec_archi_2.svg\" style=\"width: 600px;\" />\n",
    "\n",
    "To build this model we will need a new kind of layer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"embedding/user_embedding:0\", shape=(?, 50), dtype=float32)\n",
      "Tensor(\"embedding/item_embedding:0\", shape=(?, 50), dtype=float32)\n",
      "Tensor(\"concatenation/concat:0\", shape=(?, 100), dtype=float32)\n",
      "Tensor(\"fc_1/hidden_output:0\", shape=(?, 64), dtype=float32)\n",
      "Tensor(\"dropout/hidden_output_drop/mul:0\", shape=(?, 64), dtype=float32)\n",
      "Tensor(\"fc_2/pred:0\", shape=(?, 1), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "embedding_size = 50\n",
    "reg_param = 0.01\n",
    "learning_rate = 0.01\n",
    "n_users = max_user_id + 1\n",
    "n_items = max_item_id + 1\n",
    "\n",
    "g = tf.Graph()\n",
    "with g.as_default():\n",
    "    \n",
    "    tf.set_random_seed(1234)\n",
    "\n",
    "    users = tf.placeholder(shape=[None,1], dtype=tf.int64, name='input_users')\n",
    "    items = tf.placeholder(shape=[None,1], dtype=tf.int64, name='input_items')\n",
    "    ratings = tf.placeholder(shape=[None,1], dtype=tf.float32, name='input_ratings')\n",
    "    \n",
    "    l2_loss = tf.constant(0.0)\n",
    "    \n",
    "    # embeddding layer\n",
    "    with tf.variable_scope(\"embedding\"):\n",
    "        user_weights = tf.get_variable(\"user_w\"\n",
    "                                      , shape=[n_users, embedding_size]\n",
    "                                      , dtype=tf.float32\n",
    "                                      , initializer=layers.xavier_initializer())\n",
    "        \n",
    "        item_weights = tf.get_variable(\"item_w\"\n",
    "                                       , shape=[n_items, embedding_size]\n",
    "                                       , dtype=tf.float32\n",
    "                                       , initializer=layers.xavier_initializer())\n",
    "        \n",
    "        user_embedding = tf.squeeze(tf.nn.embedding_lookup(user_weights, users),axis=1, name='user_embedding')\n",
    "        item_embedding = tf.squeeze(tf.nn.embedding_lookup(item_weights, items),axis=1, name='item_embedding')\n",
    "        \n",
    "        l2_loss += tf.nn.l2_loss(user_weights)\n",
    "        l2_loss += tf.nn.l2_loss(item_weights)\n",
    "        \n",
    "        \n",
    "        print(user_embedding)\n",
    "        print(item_embedding)\n",
    "        \n",
    "    \n",
    "    # combine inputs\n",
    "    with tf.name_scope('concatenation'):\n",
    "        input_vecs = tf.concat([user_embedding, item_embedding], axis=1)\n",
    "        print(input_vecs)\n",
    "        \n",
    "    # fc-1\n",
    "    num_hidden = 64\n",
    "    with tf.name_scope(\"fc_1\"):\n",
    "        W_fc_1 = tf.get_variable(\n",
    "            \"W_hidden\",\n",
    "            shape=[2*embedding_size, num_hidden],\n",
    "            initializer=tf.contrib.layers.xavier_initializer())\n",
    "        b_fc_1 = tf.Variable(tf.constant(0.1, shape=[num_hidden]), name=\"b\")\n",
    "        hidden_output = tf.nn.relu(tf.nn.xw_plus_b(input_vecs, W_fc_1, b_fc_1), name='hidden_output')\n",
    "        l2_loss += tf.nn.l2_loss(W_fc_1)\n",
    "        print(hidden_output)\n",
    "        \n",
    "    # dropout\n",
    "    with tf.name_scope(\"dropout\"):\n",
    "        h_drop = tf.nn.dropout(hidden_output, 0.99, name=\"hidden_output_drop\")\n",
    "        print(h_drop)\n",
    "    \n",
    "    # fc-2\n",
    "    with tf.name_scope(\"fc_2\"):\n",
    "        W_fc_2 = tf.get_variable(\n",
    "            \"W_output\",\n",
    "            shape=[num_hidden,1],\n",
    "            initializer=tf.contrib.layers.xavier_initializer())\n",
    "        b_fc_2 = tf.Variable(tf.constant(0.1, shape=[1]), name=\"b\")\n",
    "        pred = tf.nn.xw_plus_b(h_drop, W_fc_2, b_fc_2, name='pred')\n",
    "        l2_loss += tf.nn.l2_loss(W_fc_2)\n",
    "        print(pred)\n",
    "\n",
    "    # loss\n",
    "    with tf.name_scope(\"loss\"):\n",
    "        loss = tf.nn.l2_loss(pred - ratings) + reg_param * l2_loss\n",
    "        train_ops = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)\n",
    "        rmse = tf.sqrt(tf.reduce_mean(tf.pow(pred - ratings, 2)))\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model_deep():\n",
    "    losses_train = []\n",
    "    losses_val = []\n",
    "    epochs = 1000\n",
    "\n",
    "    with tf.Session(graph=g) as sess:\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "        train_input_dict = {users: ratings_train['user_id'].values.reshape([-1,1])\n",
    "            , items: ratings_train['item_id'].values.reshape([-1,1])\n",
    "            , ratings: ratings_train['rating'].values.reshape([-1,1])}\n",
    "\n",
    "        val_input_dict = {users: ratings_val['user_id'].values.reshape([-1,1])\n",
    "            , items: ratings_val['item_id'].values.reshape([-1,1])\n",
    "            , ratings: ratings_val['rating'].values.reshape([-1,1])}\n",
    "\n",
    "        test_input_dict = {users: ratings_test['user_id'].values.reshape([-1,1])\n",
    "            , items: ratings_test['item_id'].values.reshape([-1,1])\n",
    "            , ratings: ratings_test['rating'].values.reshape([-1,1])}\n",
    "\n",
    "        def check_overfit(validation_loss):\n",
    "                n = len(validation_loss)\n",
    "                if n < 5:\n",
    "                    return False\n",
    "                count = 0 \n",
    "                for i in range(n-4, n):\n",
    "                    if validation_loss[i] < validation_loss[i-1]:\n",
    "                        count += 1\n",
    "                    if count >=3:\n",
    "                        return False\n",
    "                return True\n",
    "\n",
    "\n",
    "\n",
    "        for i in range(epochs):\n",
    "            sess.run([train_ops], feed_dict=train_input_dict)\n",
    "            if i % 10 == 0:\n",
    "                loss_train = sess.run(loss, feed_dict=train_input_dict)\n",
    "                loss_val = sess.run(loss, feed_dict=val_input_dict)\n",
    "                losses_train.append(loss_train)\n",
    "                losses_val.append(loss_val)\n",
    "\n",
    "                # check early stopping \n",
    "                if(check_overfit(losses_val)):\n",
    "                    print('overfit !')\n",
    "                    break\n",
    "\n",
    "                print(\"iteration : %d train loss: %.3f , valid loss %.3f\" % (i,loss_train, loss_val))\n",
    "\n",
    "         # calculate RMSE on the test dataset\n",
    "        print('RMSE on test dataset : {0:.4f}'.format(sess.run(rmse, feed_dict=test_input_dict)))\n",
    "\n",
    "        # user and item embedding\n",
    "        user_embedding_variable = [v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) if v.name.endswith('embedding/user_w:0')][0]\n",
    "        item_embedding_variable = [v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) if v.name.endswith('embedding/item_w:0')][0]\n",
    "        user_embedding_weights, item_embedding_weights = sess.run([user_embedding_variable,item_embedding_variable])\n",
    "        \n",
    "        \n",
    "        # plot train and validation loss\n",
    "        plt.plot(losses_train, label='train')\n",
    "        plt.plot(losses_val, label='validation')\n",
    "        plt.legend(loc='best')\n",
    "        plt.title('Loss');\n",
    "        \n",
    "        return user_embedding_weights, item_embedding_weights "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration : 0 train loss: 380246.500 , valid loss 94582.500\n",
      "iteration : 10 train loss: 82324.086 , valid loss 21249.504\n",
      "iteration : 20 train loss: 40316.891 , valid loss 10478.391\n",
      "iteration : 30 train loss: 29726.293 , valid loss 8200.178\n",
      "iteration : 40 train loss: 27028.400 , valid loss 7364.033\n",
      "iteration : 50 train loss: 26471.807 , valid loss 7276.563\n",
      "iteration : 60 train loss: 26538.217 , valid loss 7290.890\n",
      "iteration : 70 train loss: 26519.895 , valid loss 7245.939\n",
      "overfit !\n",
      "RMSE on test dataset : 0.9489\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEICAYAAACj2qi6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xt8VeWZ9//PlRMh4ZQdIgUCZqtUBeRk2NCxtXgooq2illpsVbS2dKydVjvt4+E189ja6a/O77Hq40y1YysVrUoZrJU6eKCKrU4VCIoIokOUIBGEQDifcrqeP/YKbEJIdo5rJ/m+X6/12ntf615rXZvafWXd973WMndHREQkGWlhJyAiIl2HioaIiCRNRUNERJKmoiEiIklT0RARkaSpaIiISNJUNEREJGkqGiKtZGZlZnZ+2HmIdCYVDRERSZqKhkg7M7NvmVmpmVWa2UIzGxLEzczuNbOtZrbLzFaZ2ehg3UVm9q6Z7TGzj83sh+F+C5HGqWiItCMzOxf4OXAFMBjYAMwLVk8FzgY+DQwAvgpsD9Y9DHzb3fsCo4GXOzFtkaRlhJ2ASDfzdWCOu78JYGa3ATvMrAioBvoCpwHL3H1twnbVwEgze9vddwA7OjVrkSTpTEOkfQ0hfnYBgLvvJX42MdTdXwb+HfglsMXMHjKzfkHTLwMXARvM7C9m9plOzlskKSoaIu1rE3Bi/QczywXygY8B3P1+dz8TGEW8m+pHQXy5u08HTgD+CMzv5LxFkqKiIdI2mWaWXb8Q/7G/zszGmVkv4P8Dlrp7mZlNNLNJZpYJ7AMOArVmlmVmXzez/u5eDewGakP7RiJNUNEQaZtFwIGE5XPAPwNPAZuBk4GZQdt+wK+Jj1dsIN5tdXew7mqgzMx2A38PXNVJ+Yu0iOkhTCIikiydaYiISNJUNEREJGkqGiIikjQVDRERSVq3uyJ84MCBXlRUFHYaIiJdyooVK7a5e0Fz7bpd0SgqKqKkpCTsNEREuhQz29B8K3VPiYhIC6hoiIhI0lQ0REQkad1uTENEuo/q6mrKy8s5ePBg2Kl0G9nZ2RQWFpKZmdmq7VU0RCRllZeX07dvX4qKijCzsNPp8tyd7du3U15eTjQabdU+1D0lIinr4MGD5Ofnq2C0EzMjPz+/TWduKhoiktJUMNpXW/89VTQCS97fygOvlIadhohISlPRCPytdBv3LV7HwWo9+0ZEjti5cycPPPBAi7e76KKL2LlzZwdkFC4VjUAsmk9VbR1vb+x+/yOLSOsdr2jU1jb9B+aiRYsYMGBAR6UVGhWNwMSiPACWra8MORMRSSW33norH3zwAePGjWPixImcc845fO1rX+OMM84A4NJLL+XMM89k1KhRPPTQQ4e3KyoqYtu2bZSVlXH66afzrW99i1GjRjF16lQOHDgQ1tdpM025DQzIyeLUQX1ZVqaiIZKKfvKnNby7aXe77nPkkH7ccfGoJtvcddddrF69mpUrV/LKK6/wxS9+kdWrVx+esjpnzhwikQgHDhxg4sSJfPnLXyY/P/+ofaxbt44nn3ySX//611xxxRU89dRTXHVV13yir840EsSiEVZs2EFNbV3YqYhIiorFYkdd43D//fczduxYJk+ezMaNG1m3bt0x20SjUcaNGwfAmWeeSVlZWWel2+50ppEgFo3w2BsbWLNpN2OHdb++SJGurLkzgs6Sm5t7+P0rr7zCn//8Z15//XVycnKYMmVKo9dA9OrV6/D79PT0Lt09pTONBLFoBIDl6qISkUDfvn3Zs2dPo+t27dpFXl4eOTk5vPfee7zxxhudnF3n05lGgkH9sjkxP4el6yv55udOCjsdEUkB+fn5nHXWWYwePZrevXszaNCgw+umTZvGr371K8aMGcOpp57K5MmTQ8y0c6hoNBArirB47Rbq6py0NF2JKiLwxBNPNBrv1asXzz33XKPr6sctBg4cyOrVqw/Hf/jDH7Z7fp1J3VMNTIxG2Lm/mtKKvWGnIiKSclQ0GpgUjGss1fUaIiLHUNFoYHgkh0H9eukiPxGRRqhoNGBmxKL5LF9fibuHnY6ISEpptmiYWbaZLTOzt81sjZn9JIg/YmbrzWxlsIwL4mZm95tZqZmtMrMJCfuaZWbrgmVWQvxMM3sn2OZ+C+7da2YRM1sctF9sZnnt/09wrFhRHp/sPsjGyq47l1pEpCMkc6ZxCDjX3ccC44BpZlY/r+xH7j4uWFYGsQuBEcEyG3gQ4gUAuAOYBMSAOxKKwINB2/rtpgXxW4GX3H0E8FLwucPFovFbACxdv70zDici0mU0WzQ8rn4qUWawNNVvMx14NNjuDWCAmQ0GLgAWu3ulu+8AFhMvQIOBfu7+usf7gx4FLk3Y19zg/dyEeIcacUIfBuRkalxDRFqsT58+AGzatIkZM2Y02mbKlCmUlJQ0uZ/77ruP/fv3H/6cKrdaT2pMw8zSzWwlsJX4D//SYNXPgi6oe82s/jr5ocDGhM3Lg1hT8fJG4gCD3H0zQPB6wnHym21mJWZWUlFRkcxXalJamlF8YkRXhotIqw0ZMoQFCxa0evuGRSNVbrWeVNFw91p3HwcUAjEzGw3cBpwGTAQiwC1B88auiPNWxJPm7g+5e7G7FxcUFLRk0+OaFI1Qtn0/W3a3/lm6ItL13XLLLUc9T+PHP/4xP/nJTzjvvPOYMGECZ5xxBs8888wx25WVlTF69GgADhw4wMyZMxkzZgxf/epXj7r31A033EBxcTGjRo3ijjvuAOI3Qdy0aRPnnHMO55xzDnDkVusA99xzD6NHj2b06NHcd999h4/XGbdgb9EV4e6+08xeAaa5+91B+JCZ/Raov8yxHBiWsFkhsCmIT2kQfyWIFzbSHmCLmQ12981BN9bWluTbFvX3oVq2vpKLxw7prMOKyPE8dyt88k777vNTZ8CFdzXZZObMmdx000185zvfAWD+/Pk8//zz3HzzzfTr149t27YxefJkLrnkkuM+f/vBBx8kJyeHVatWsWrVKiZMODw/iJ/97GdEIhFqa2s577zzWLVqFd/73ve45557WLJkCQMHDjxqXytWrOC3v/0tS5cuxd2ZNGkSn//858nLy+uUW7AnM3uqwMwGBO97A+cD7wU/4gQznS4F6q+TXwhcE8yimgzsCrqWXgCmmlleMAA+FXghWLfHzCYH+7oGeCZhX/WzrGYlxDvcqCH9yMlKVxeVSA83fvx4tm7dyqZNm3j77bfJy8tj8ODB3H777YwZM4bzzz+fjz/+mC1bthx3H3/9618P/3iPGTOGMWPGHF43f/58JkyYwPjx41mzZg3vvvtuk/m89tprXHbZZeTm5tKnTx8uv/xyXn31VaBzbsGezJnGYGCumaUTLzLz3f1ZM3vZzAqIdy+tBP4+aL8IuAgoBfYD1wG4e6WZ/RRYHrS7093rf5FvAB4BegPPBQvAXcB8M7se+Aj4Smu/aEtlpKdx5ol5GgwXSRXNnBF0pBkzZrBgwQI++eQTZs6cyeOPP05FRQUrVqwgMzOToqKiRm+Jnqixs5D169dz9913s3z5cvLy8rj22mub3U9T1491xi3Yk5k9tcrdx7v7GHcf7e53BvFz3f2MIHZV/QyrYNbUje5+crC+JGFfc9z9lGD5bUK8JNjPye7+3WAWFe6+3d3Pc/cRwWun/oLHiiK898kedu6v6szDikiKmTlzJvPmzWPBggXMmDGDXbt2ccIJJ5CZmcmSJUvYsGFDk9ufffbZPP744wCsXr2aVatWAbB7925yc3Pp378/W7ZsOermh8e7JfvZZ5/NH//4R/bv38++fft4+umn+dznPteO37ZpusttEyYG4xolZTs4f+SgZlqLSHc1atQo9uzZw9ChQxk8eDBf//rXufjiiykuLmbcuHGcdtppTW5/ww03cN111zFmzBjGjRtHLBYDYOzYsYwfP55Ro0Zx0kkncdZZZx3eZvbs2Vx44YUMHjyYJUuWHI5PmDCBa6+99vA+vvnNbzJ+/PhOexqgdbdbZRQXF3tz85+TdbC6ljE/fpFrzyri9otOb5d9ikjy1q5dy+mn6/977a2xf1czW+Huxc1tq3tPNSE7M52xw/rrjrciIgEVjWbEohHWfLyLfYdqwk5FRCR0KhrNmFgUoabOeeuj8C/fF+mJulsXetja+u+potGMM0/MI81gmW5eKNLpsrOz2b59uwpHO3F3tm/fTnZ2dqv3odlTzeibncmoIf1Zpov8RDpdYWEh5eXltMc95SQuOzubwsLC5hseh4pGEiYWRXh86QYO1dTSKyM97HREeozMzEyi0WjYaUgCdU8lIRaNcKimjnfKd4WdiohIqFQ0kjCxKP6sKE29FZGeTkUjCfl9enHKCX1080IR6fFUNJIUi0YoKdtBbZ1mcYhIz6WikaRJ0Qh7D9WwdvPusFMREQmNikaSJhYdeSiTiEhPpaKRpCEDelOY11tFQ0R6NBWNFohFIywvq9TVqSLSY6lotECsKML2fVV8ULEv7FREREKhotECsajGNUSkZ2u2aJhZtpktM7O3zWyNmf0kiEfNbKmZrTOz35tZVhDvFXwuDdYXJezrtiD+vpldkBCfFsRKzezWhHijxwhLdGAuA/v00s0LRaTHSuZM4xBwrruPBcYB08xsMvCvwL3uPgLYAVwftL8e2OHupwD3Bu0ws5HATGAUMA14wMzSzSwd+CVwITASuDJoSxPHCIWZMSkaYXnZjjDTEBEJTbNFw+P2Bh8zg8WBc4EFQXwucGnwfnrwmWD9eWZmQXyeux9y9/VAKRALllJ3/9Ddq4B5wPRgm+MdIzQTi/L4eOcBynfsDzsVEZFOl9SYRnBGsBLYCiwGPgB2unv94+zKgaHB+6HARoBg/S4gPzHeYJvjxfObOEbD/GabWYmZlXT0LZRj0XxA4xoi0jMlVTTcvdbdxwGFxM8MGnvSe/08VDvOuvaKN5bfQ+5e7O7FBQUFjTVpN6d+qi/9sjN0HyoR6ZFaNHvK3XcCrwCTgQFmVv88jkJgU/C+HBgGEKzvD1Qmxhtsc7z4tiaOEZr0NKO4KKI73opIj5TM7KkCMxsQvO8NnA+sBZYAM4Jms4BngvcLg88E61/2+NVwC4GZweyqKDACWAYsB0YEM6WyiA+WLwy2Od4xQhWLRviwYh8Vew6FnYqISKdK5kxjMLDEzFYR/4Ff7O7PArcAPzCzUuLjDw8H7R8G8oP4D4BbAdx9DTAfeBd4Hrgx6PaqAb4LvEC8GM0P2tLEMUJVf71GibqoRKSHse52S4zi4mIvKSnp0GNU1dQx5icvMHPicH58yagOPZaISGcwsxXuXtxcO10R3gpZGWlMGJ6nGVQi0uOoaLRSLBph7Se72XWgOuxUREQ6jYpGK8WKIrjDmxt0dbiI9BwqGq00fngeGWmmqbci0qOoaLRS76x0xhT2180LRaRHUdFog1g0n3c+3sWBqtqwUxER6RQqGm0Qi+ZRXeu8tVHjGiLSM6hotMGZJ0Yw080LRaTnUNFog/69Mzn9U/1080IR6TFUNNooFo2wYsMOqmrqwk5FRKTDqWi0USwa4WB1Has37Qo7FRGRDqei0UYTi+I3L1yucQ0R6QFUNNqooG8vThqYq8FwEekRVDTaQSwaYXlZJXV13euOwSIiDalotINYNMLugzW8v2VP2KmIiHQoFY12UD+uoS4qEenuVDTaQWFeb4b0z1bREJFuL5lnhA8zsyVmttbM1pjZ94P4j83sYzNbGSwXJWxzm5mVmtn7ZnZBQnxaECs1s1sT4lEzW2pm68zs98GzwgmeJ/77oP1SMytqzy/fXsyMWDTC0vWVdLcnIYqIJErmTKMG+Ed3Px2YDNxoZiODdfe6+7hgWQQQrJsJjAKmAQ+YWbqZpQO/BC4ERgJXJuznX4N9jQB2ANcH8euBHe5+CnBv0C4lxaL5bNt7iLLt+8NORUSkwzRbNNx9s7u/GbzfA6wFhjaxyXRgnrsfcvf1QCkQC5ZSd//Q3auAecB0MzPgXGBBsP1c4NKEfc0N3i8Azgvap5xYNA9At0oXkW6tRWMaQffQeGBpEPquma0yszlmlhfEhgIbEzYrD2LHi+cDO929pkH8qH0F63cF7RvmNdvMSsyspKKioiVfqd2cXNCHSG6WHsokIt1a0kXDzPoATwE3uftu4EHgZGAcsBn4RX3TRjb3VsSb2tfRAfeH3L3Y3YsLCgqa/B4dxcyIFUV080IR6daSKhpmlkm8YDzu7n8AcPct7l7r7nXAr4l3P0H8TGFYwuaFwKYm4tuAAWaW0SB+1L6C9f2BlP1VnhiNsLHyAJt2Hgg7FRGRDpHM7CkDHgbWuvs9CfHBCc0uA1YH7xcCM4OZT1FgBLAMWA6MCGZKZREfLF/o8elGS4AZwfazgGcS9jUreD8DeNlTeHrSpGhwHyqdbYhIN5XRfBPOAq4G3jGzlUHsduKzn8YR7y4qA74N4O5rzGw+8C7xmVc3unstgJl9F3gBSAfmuPuaYH+3APPM7F+At4gXKYLXx8yslPgZxsw2fNcOd/rgfvTplcGy9ZVMH9fUXAERka6p2aLh7q/R+NjCoia2+Rnws0biixrbzt0/5Ej3VmL8IPCV5nJMFelpxpkn5ukiPxHptnRFeDuLRSOs27qXyn1VYaciItLuVDTamcY1RKQ7U9FoZ2cU9icrI01dVCLSLalotLNeGemMHzZARUNEuiUVjQ4wKRphzaZd7D1U03xjEZEuREWjA8Si+dQ5rNiwI+xURETalYpGBxg/fADpaaabF4pIt6Oi0QFye2Uwemh/jWuISLejotFBJkUjvL1xFwera8NORUSk3ahodJCJRRGqaut4e+POsFMREWk3KhodZGJR/UOZ1EUlIt2HikYHGZCTxWmf6ssyXRkuIt2IikYHmlgUYcWGHdTU1oWdiohIu1DR6ECxaIT9VbWs2bQ77FRERNqFikYHiunmhSLSzahodKBB/bI5MT+HpRoMF5FuQkWjg8WKIiwvq6SuLmWfUisikjQVjQ4Wi0bYub+a0oq9YaciItJmzRYNMxtmZkvMbK2ZrTGz7wfxiJktNrN1wWteEDczu9/MSs1slZlNSNjXrKD9OjOblRA/08zeCba538ysqWN0JfXjGuqiEpHuIJkzjRrgH939dGAycKOZjQRuBV5y9xHAS8FngAuBEcEyG3gQ4gUAuAOYRPx54HckFIEHg7b1200L4sc7RpcxPJLDoH69dJGfiHQLzRYNd9/s7m8G7/cAa4GhwHRgbtBsLnBp8H468KjHvQEMMLPBwAXAYnevdPcdwGJgWrCun7u/7u4OPNpgX40do8swM2LRfJavryT+9UREuq4WjWmYWREwHlgKDHL3zRAvLMAJQbOhwMaEzcqDWFPx8kbiNHGMhnnNNrMSMyupqKhoyVfqFLFohE92H2Rj5YGwUxERaZOki4aZ9QGeAm5y96auVrNGYt6KeNLc/SF3L3b34oKCgpZs2iliRfXjGnq+hoh0bUkVDTPLJF4wHnf3PwThLUHXEsHr1iBeDgxL2LwQ2NRMvLCReFPH6FJGnNCHATmZushPRLq8ZGZPGfAwsNbd70lYtRConwE1C3gmIX5NMItqMrAr6Fp6AZhqZnnBAPhU4IVg3R4zmxwc65oG+2rsGF1KWpoxsSiiwXAR6fIykmhzFnA18I6ZrQxitwN3AfPN7HrgI+ArwbpFwEVAKbAfuA7A3SvN7KfA8qDdne5e/yt6A/AI0Bt4Llho4hhdTqwowuJ3t7Bl90EG9csOOx0RkVZptmi4+2s0Pu4AcF4j7R248Tj7mgPMaSReAoxuJL69sWN0RfXXayxbX8nFY4eEnI2ISOvoivBOMmpIP3Ky0jWuISJdmopGJ8lIT+PME/M0riEiXZqKRieKFUV475M97NxfFXYqIiKtoqLRierHNUrKdoSciYhI66hodKKxwwaQlZ6m54aLSJelotGJsjPTGTusv+54KyJdlopGJ4tFI6z5eBf7DtWEnYqISIupaHSyWDSfmjrnrY92hp2KiEiLqWh0sgnDB5BmsEw3LxSRLkhFo5P1zc5k1JD+GgwXkS5JRSMEsWiEtz7ayaGa2rBTERFpERWNEEwsinCopo53yneFnYqISIuoaIRgYlH80ejqohKRrkZFIwT5fXox4oQ+ug+ViHQ5KhohmRiNUFK2g9q6Fj3ZVkQkVCoaIZkUjbD3UA1rNzf1uHURkdSiohGSiUVHHsokItJVJPOM8DlmttXMVifEfmxmH5vZymC5KGHdbWZWambvm9kFCfFpQazUzG5NiEfNbKmZrTOz35tZVhDvFXwuDdYXtdeXTgVDBvSmMK+3ioaIdCnJnGk8AkxrJH6vu48LlkUAZjYSmAmMCrZ5wMzSzSwd+CVwITASuDJoC/Cvwb5GADuA64P49cAOdz8FuDdo163EohGWl1USf0KuiEjqa7ZouPtfgWT/HJ4OzHP3Q+6+HigFYsFS6u4funsVMA+YbmYGnAssCLafC1yasK+5wfsFwHlB+25jUjTC9n1VfFCxL+xURESS0pYxje+a2aqg+yoviA0FNia0KQ9ix4vnAzvdvaZB/Kh9Bet3Be2PYWazzazEzEoqKira8JU6Vywa/zrqohKRrqK1ReNB4GRgHLAZ+EUQb+xMwFsRb2pfxwbdH3L3YncvLigoaCrvlFKUn8PAPr1080IR6TJaVTTcfYu717p7HfBr4t1PED9TGJbQtBDY1ER8GzDAzDIaxI/aV7C+P8l3k3UJZsakaITlevyriHQRrSoaZjY44eNlQP3MqoXAzGDmUxQYASwDlgMjgplSWcQHyxd6fAR4CTAj2H4W8EzCvmYF72cAL3s3HDGORSN8vPMA5Tv2h52KiEizMpprYGZPAlOAgWZWDtwBTDGzccS7i8qAbwO4+xozmw+8C9QAN7p7bbCf7wIvAOnAHHdfExziFmCemf0L8BbwcBB/GHjMzEqJn2HMbPO3TUGJ12sU5uWEnI2ISNOsu/3xXlxc7CUlJWGnkbTaOmf8nS/yxTGD+fnlY8JOR0R6KDNb4e7FzbXTFeEhS08zJhZFWKoZVCLSBahopICJ0QgfVuyjYs+hsFMREWmSikYKiEXj4xoler6GiKQ4FY0UMHpIf3pnpquLSkRSnopGCsjKSGP88AG6MlxEUp6KRoqIRSOs/WQ3uw5Uh52KiMhxqWikiFg0gju8uUFXh4tI6lLRSBHjh+WRmW4a1xCRlKaikSJ6Z6VzxtD+unmhiKQ0FY0UEovm887HuzhQVRt2KiIijVLRSCGTohGqa523NmpcQ0RSk4pGCplwYh5meiiTiKQuFY0U0r93Jqd/qh/LdWW4iKQoFY0UE4tGWLFhB1U1dWGnIiJyDBWNFBOLRjhYXcfqTbvCTkVE5BgqGimm/qFMyzWuISIpSEUjxRT07cVJBbkaDBeRlKSikYJiRRGWl1VSV9e9nqooIl1fs0XDzOaY2VYzW50Qi5jZYjNbF7zmBXEzs/vNrNTMVpnZhIRtZgXt15nZrIT4mWb2TrDN/WZmTR2jJ4hFI+w+WMP7W/aEnYqIyFGSOdN4BJjWIHYr8JK7jwBeCj4DXAiMCJbZwIMQLwDAHcAkIAbckVAEHgza1m83rZljdHv1D2VSF5WIpJpmi4a7/xVo+Os1HZgbvJ8LXJoQf9Tj3gAGmNlg4AJgsbtXuvsOYDEwLVjXz91fd3cHHm2wr8aO0e0V5uUwpH+2ioaIpJzWjmkMcvfNAMHrCUF8KLAxoV15EGsqXt5IvKljHMPMZptZiZmVVFRUtPIrpZZYNMKyskritVREJDW090C4NRLzVsRbxN0fcvdidy8uKCho6eZx1Qdg27rWbdsBYtF8KvYcomz7/rBTERE5rLVFY0vQtUTwujWIlwPDEtoVApuaiRc2Em/qGB3j2ZthzgWw5d0OPUyyjoxr6FbpIpI6Wls0FgL1M6BmAc8kxK8JZlFNBnYFXUsvAFPNLC8YAJ8KvBCs22Nmk4NZU9c02Fdjx+gYZ/8I0rPg0Uug4v0OPVQyTi7IJT83Sw9lEpGUksyU2yeB14FTzazczK4H7gK+YGbrgC8EnwEWAR8CpcCvge8AuHsl8FNgebDcGcQAbgB+E2zzAfBcED/eMTpG/skw609gaTD3YthW2qGHa46ZMTG4XkNEJFVYdxtoLS4u9pKSktbvYOt78MgX42cd1/0XRE5qv+RaaM5r67nz2Xf5263nMmRA79DyEJHuz8xWuHtxc+10RXhDJ5wGsxZCzUF45GLYsSG0VOrHNXS2ISKpQkWjMYNGwTXPQNVemPsl2Lmx+W06wOmD+9GnV4au1xCRlKGicTyDx8DVT8OBXfExjt2bmt+mnaWnGcVFeby6bhs791d1+vFFRBpS0WjK0Alw9R9g37Z44djzSaen8OUJhZTv2M85d7/C797YQK1uYigiIVLRaE5hMVy1AHZvhrmXwN6OvVykoYvHDuG/vvc5Pj2oL//0x9Vc/G+vqbtKREKjopGM4ZPh6/Nh50fw6HTY17kX3J0+uB/zZk/m3782np37q7jiP17ne0++xeZdBzo1DxERFY1kFX0WvjYPKj+Ex6bD/s79a9/M+NKYIfz5Hz/P9849hefXfMK5d/+Ff395HQerazs1FxHpuVQ0WuKkKTDz8fgV449dBgd2dnoKOVkZ/GDqqbz0g8/z+U8XcPeL/8MX7v0LL675RDc3FJEOp6LRUqecD199HLasgd9dDgd3h5LGsEgOv7r6TH53/SSyM9KZ/dgKrpmzjNKtenCTiHQcFY3W+PRUuGIubH4bHp8Bh8L7of7siIEs+v7n+N9fGsnKjTuZdt+r/PTZd9l9sDq0nESk+1LRaK3Tvggz5kB5CTzxVajaF1oqmelpfOOzUV754RS+UlzInP9ez7l3v8L85Rv1nHERaVcqGm0xcjpc/hB89Do8OROqwn32RX6fXvz88jEsvPGzDI/k8L+eWsWlD/w3b360I9S8RKT7UNFoqzNmwKW/gvWvwryvQfXBsDPijML+PHXD33HfV8fxya6DXP7A3/jB/JVs3R1+biLStalotIexX4Xp/w4fLoH5V0PNobAzwsy4dPxQXv7hFG6YcjLPvr2Zc+5+hf/4ywdU1dSFnZ4Ef1hrAAAOtklEQVSIdFEqGu1l/FXwpftg3Yvwn9dCTWrcK6pPrwxumXYaL9x8NpNPyufnz73HtPv+ypL3OvfKdhHpHlQ02lPxdXDR3fD+InjqG1CbOjOYogNzefjaifz2uokAXPfIcr7xyHLWbwtvAF9Euh4VjfYW+xZc8HNY+yf4w2yorQk7o6Occ+oJPH/T2dx+0Wks/XA7U+/9C3c99x57D6VWniKSmtpUNMyszMzeMbOVZlYSxCJmttjM1gWveUHczOx+Mys1s1VmNiFhP7OC9uvMbFZC/Mxg/6XBttaWfDvNZ74DX7gT1vwB/ngD1KXWbT6yMtKYffbJLPnhFC4ZO5Rf/eUDzr37FZ5+q1xXlYtIk9rjTOMcdx+X8JjAW4GX3H0E8FLwGeBCYESwzAYehHiRAe4AJgEx4I76QhO0mZ2w3bR2yLdznPV9OPef4Z35sPAfoC71Bp9P6JfNL64Yy9Pf+TsG98/m5t+/zZcf/Buryjv/9igi0jV0RPfUdGBu8H4ucGlC/FGPewMYYGaDgQuAxe5e6e47gMXAtGBdP3d/3eN//j6asK+u4ewfwpTbYOXj8OxNKVk4AMYPz+Pp75zF/z9jDB9V7mf6L/+bW59axba94c8CE5HUktHG7R140cwc+A93fwgY5O6bAdx9s5mdELQdCiQ+N7U8iDUVL28kfgwzm038jIThw4e38Su1s8/fArVV8OovIC0DvvgLSMFetrQ044riYUwb/Snu//M6HvlbGf/1zmZuOv/TXPOZE8lM1/CXiLT9TOMsd59AvOvpRjM7u4m2jf1SeivixwbdH3L3YncvLigoaC7nzmUW76b6u+9BycPw/G2QwuMG/bIz+acvjeT5mz7HuGED+Omz73Lh/32V19ZtCzs1EUkBbSoa7r4peN0KPE18TGJL0LVE8Fp/QUA5MCxh80JgUzPxwkbiXY9ZfGB80g2w9EFY/M8pXTgATjmhL49+I8avrymmqqaOqx5eyrcfK2FjZbi3ShGRcLW6aJhZrpn1rX8PTAVWAwuB+hlQs4BngvcLgWuCWVSTgV1BN9YLwFQzywsGwKcCLwTr9pjZ5GDW1DUJ++p6zGDaz2HiN+Fv/wYv3ZnyhcPM+MLIQbx489n86IJT+ev/bOO8e/7CL158n/1VmqIr0hO1ZUxjEPB0MAs2A3jC3Z83s+XAfDO7HvgI+ErQfhFwEVAK7AeuA3D3SjP7KbA8aHenu9c/Fu8G4BGgN/BcsHRdZnDh/4lf9PfaPZCeBefcFnZWzcrOTOfGc07h8glD+fmi9/i3l0tZsKKc7503gjGF/SnKzyW3V1uHx0SkK7DuNi+/uLjYS0pKwk6jaXV18Wm4K38H5/4TnP2jsDNqkWXrK/nxwjW8u/nIA6gG9etFdGAu0YF9OGlgLkUDc4kOzGV4JIesDA2ii6Q6M1uRcOnEcenPwzCkpcEl90NdNbz8L5CWCZ+9KeyskhaLRvjTP3yW9z/Zw/pt+yjbvo8PK/axfttenl+9mR37j9w+Jc3iTxksyo8XkZMKcoPiksuQ/r1JS0u9mWQicnwqGmFJS4fpD0BdDfz5DkjPhM/cGHZWSUtPM0YO6cfIIf2OWbdzfxXrt+07vHy4bR9l2/axvKyS/VVHro7PykijKD/n8BlKdGBO8JrLwD5ZdJUbAIj0JCoaYUrPgMv+Iz7G8cLt8TOOSbPDzqrNBuRkMX54FuOH5x0Vd3e27jkUnJUcOUMp3bqXl9/bSnXtka7Svr0yDndx1Z+hFOXnEi3IpV92Zmd/JREJqGiELT0Tvvww/OcseO5H8UJS/I2ws+oQZsagftkM6pfNZ07OP2pdTW0dm3Ye5MNte486S3nzox38adWmoyaaDeyTdbiYFA3M5aTgTOXE/ByyM9M7+VuJ9CwqGqkgIwu+8gj8/ip49ub4GceEq8POqlNlpKcxPD+H4fk5TDn16HUHq2v5qHL/kWISnKm8/F4F2/YeuWmAGQzp35vCvN7kZKXTOyud7Ix0srPS6Z2ZTnZmWvAaX3pnBm0y046OJbzPzkojKz1NXWUiARWNVJHRC654DOZdGZ9ZlZYB464MO6uUkJ2ZzqcH9eXTg/oes27PwWrKtu0/fIZStm0fm3YeZNveKg5U13KgqpZDNfHXA9W11LVisqAZRxeSoMjUF51eGfHX3gnxXocLUBDLSqdXRhppZvElLX7mFf8MaWaYgRF8Tou/NtYmLSFmCesOr09rWfuG9TCZCZUN23iDmzUcu77h9scepL3mcba0vLfmD4LELeo3t4RoY7tMjNW3PTp2bD5HYm3Ltz2paKSSzGyY+QQ8cQU8851419UZM8LOKqX1zc7kjML+nFHYv9m27k51rXOgupZD1fEicqC6loPVdRyoquVgdXw5HKtOiFXVcrCmlgNVdUe123uohm17q46KxQtVat6cUrqn+jry22snMuXUE5pu3EYqGqkmszdcOQ8e/0r8IU5p6TDqsrCz6hbMjKwMi1830rtjB9Pr6pxDNXVHCkl1LYeq66hzxx3q3IMlXszqEmL16xu+1h3+nNi+fnunrq6F7RP2f+y/VYPPjfz9fmyblu0jFXr8WnOZWuJZVf32ibs5Eju23dHHbtt+vJHgifm5zWTfdioaqSgrF772e/jdl+Gpb0L1Qfj0BZATCTszSVJamsW7rLLSyWu+uUiXoaKRqnr1ha8vgMcugz/+fTzWOw/yR0D+KTDwlPhr/giInBTv2hIR6WAqGqksux9c+yx8+BfYXgrb18H2D+CDl+HtJxIaGvQfdnQhyT8ZBo6AfoXxK9BFRNqBikaqy+wNpzbylNtDe+IFZHtpfNm2Lv668Qmo2nukXUZ2/EwkPygoA0ccea/uLhFpIRWNrqpXXxgyLr4kcoe9W44uJNs/gK3vwvuL4rctqdc7klBITj7S9aXuLhE5DhWN7sYM+n4qvhR99uh1tdWwY8ORs5P67q7Sl+LPMT+yExgw7EgRSRxDUXeXSI+motGTpGfGf/wHnnLsuuN2dy1tpLvr5Pg++g+DzJz4hYkZ2Qmv2fEzlaNijb32jr+mwtxLEUmKiobEtbS7a8sa+J8XoeYgbb6WN72JwpLZu5nC06AAZWTHb8ti6fFrXI56TWtBPC2+HNM2vYm4ip90fyoa0rSmursgXlRqq+PFo+ZQwuuBhM8N1x2MX3vSWLyxfRzY0fj66gPxZ5KkDGu+IB1VWBpe/dbadcf9cPztOqrAdcZD3Y6bezP37mjPtklpxf1Y2rIvgIvvh6Kzktxn66hoSNuYxf+yz8gK5/h1tY0XFK+Nr/Pa+JMSj/rckfEG7xu2rXfMj0Vjl/o2tq6p7ZJd1/DGUN7ORaQjz7iO8+PZ6I9vR7ZN8jsm9e/ajvvKPvb5Nu0t5YuGmU0D/i+QDvzG3e8KOSVJJWnpkJUTX0Skw6X0NBgzSwd+CVwIjASuNLOR4WYlItJzpXTRAGJAqbt/6O5VwDxgesg5iYj0WKleNIYCGxM+lwexo5jZbDMrMbOSioqKTktORKSnSfWi0djIz7HDhO4PuXuxuxcXFBR0QloiIj1TqheNcmBYwudCYFNIuYiI9HipXjSWAyPMLGpmWcBMYGHIOYmI9FgpPeXW3WvM7LvAC8Sn3M5x9zUhpyUi0mOldNEAcPdFwKKw8xARETDvjMv+O5GZVQAbWrn5QGBbO6bTXpRXyyivllFeLZOqeUHbcjvR3ZudSdTtikZbmFmJuxeHnUdDyqtllFfLKK+WSdW8oHNyS/WBcBERSSEqGiIikjQVjaM9FHYCx6G8WkZ5tYzyaplUzQs6ITeNaYiISNJ0piEiIklT0RARkaSpaATMbJqZvW9mpWZ2a9j5AJjZHDPbamarw84lkZkNM7MlZrbWzNaY2ffDzgnAzLLNbJmZvR3k9ZOwc0pkZulm9paZPRt2LvXMrMzM3jGzlWZWEnY+9cxsgJktMLP3gv/OPpMCOZ0a/DvVL7vN7Kaw8wIws5uD/+ZXm9mTZpbdYcfSmMbhhz39D/AF4jdJXA5c6e7vhpzX2cBe4FF3Hx1mLonMbDAw2N3fNLO+wArg0hT49zIg1933mlkm8BrwfXd/I8y86pnZD4BioJ+7fynsfCBeNIBid0+pi9XMbC7wqrv/JrjvXI677ww7r3rBb8bHwCR3b+3FxO2Vy1Di/62PdPcDZjYfWOTuj3TE8XSmEZeSD3ty978ClWHn0ZC7b3b3N4P3e4C1NPKck87mcXuDj5nBkhJ/FZlZIfBF4Ddh55LqzKwfcDbwMIC7V6VSwQicB3wQdsFIkAH0NrMMIIcOvBu4ikZcUg97kmOZWREwHlgabiZxQRfQSmArsNjdUyIv4D7gfwF1YSfSgAMvmtkKM5sddjKBk4AK4LdBd95vzCw37KQamAk8GXYSAO7+MXA38BGwGdjl7i921PFUNOKSetiTHM3M+gBPATe5++6w8wFw91p3H0f82SsxMwu9W8/MvgRsdfcVYefSiLPcfQJwIXBj0CUatgxgAvCgu48H9gEpMc4IEHSXXQL8Z9i5AJhZHvGekSgwBMg1s6s66ngqGnF62FMLBWMGTwGPu/sfws6noaA74xVgWsipAJwFXBKMH8wDzjWz34WbUpy7bwpetwJPE++qDVs5UJ5wlriAeBFJFRcCb7r7lrATCZwPrHf3CnevBv4A/F1HHUxFI04Pe2qBYMD5YWCtu98Tdj71zKzAzAYE73sT/z/Te+FmBe5+m7sXunsR8f+2Xnb3DvtLMFlmlhtMZCDo/pkKhD5Tz90/ATaa2alB6Dwg1EkWDVxJinRNBT4CJptZTvD/zfOIjzN2iJR/nkZnSNWHPZnZk8AUYKCZlQN3uPvD4WYFxP9yvhp4Jxg/ALg9ePZJmAYDc4OZLWnAfHdPmemtKWgQ8HT8d4YM4Al3fz7clA77B+Dx4I+4D4HrQs4HADPLIT7L8tth51LP3Zea2QLgTaAGeIsOvJ2IptyKiEjS1D0lIiJJU9EQEZGkqWiIiEjSVDRERCRpKhoiIpI0FQ0REUmaioaIiCTt/wG08wCq072UcQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1c1e107b70>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "user_embedding_weights, item_embedding_weights  = train_model_deep()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First item name from metadata: GoldenEye (1995)\n",
      "Embedding vector for the first item:\n",
      "[-0.03245861 -0.03681468 -0.07690453 -0.06741171 -0.0280375  -0.12563227\n",
      "  0.1472084  -0.1516232  -0.01753164  0.14560509 -0.02281599 -0.08844607\n",
      " -0.11701321 -0.06391956  0.05451636  0.10192668  0.05839488 -0.06322578\n",
      " -0.10411685  0.09739103 -0.11199136 -0.1573483   0.02695784  0.08393615\n",
      " -0.01733899  0.05547168  0.08913539 -0.00134482 -0.15449274  0.00944744\n",
      " -0.05753549  0.11774195 -0.05723855  0.11152874 -0.07524169  0.13244887\n",
      " -0.03645113 -0.07689955  0.06595007 -0.11269008 -0.05746102  0.10201172\n",
      " -0.0324096   0.09458092  0.13133198 -0.12544832  0.15979671  0.11557044\n",
      "  0.06386545 -0.03553589]\n",
      "shape: (50,)\n"
     ]
    }
   ],
   "source": [
    "print(\"First item name from metadata:\", df_items[\"title\"][1])\n",
    "print(\"Embedding vector for the first item:\")\n",
    "print(item_embedding_weights[1])\n",
    "print(\"shape:\", item_embedding_weights[1].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualizing embeddings using TSNE\n",
    "\n",
    "- we use scikit learn to visualize items embeddings\n",
    "- Try different perplexities, and visualize user embeddings as well\n",
    "- check what is the impact of different perplexity value. Here is a very nice tutorial if you want to know in detail (https://distill.pub/2016/misread-tsne/ )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.manifold import TSNE\n",
    "\n",
    "item_tsne = TSNE(perplexity=50).fit_transform(item_embedding_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkMAAAI1CAYAAADVQv5HAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzsvd9vVFe27ztWlRdQpvelyN1Iu1MKSdQPoIvY2AffEySeyEMjnWxQndDdKEr+CHwiS84VtzEtjrBkteg/ItzICbRKZNNX5AGeckT2tlV2I0vwsJUOqNJH4igUV8EFlMvrPphZzJprjvljrVXl+vH9vOxNudZac63lzvx6jO8YI4iiiAAAAAAARpXcdi8AAAAAAGA7gRgCAAAAwEgDMQQAAACAkQZiCAAAAAAjDcQQAAAAAEYaiCEAAAAAjDQQQwAAAAAYaSCGAAAAADDSQAwBAAAAYKSBGAIAAADASDPm8+V//Md/jN55550uLQUAAAAAIDuWl5f/VxRF+2zf8xJD77zzDi0tLSVfFQAAAABAjwiC4AeX7yFNBgAAAICRBmIIAAAAACMNxBAAAAAARhqIIQAAAACMNBBDAAAAABhpIIYAAAAAMNJADAEAAABgpIEYAgAAAMBIAzEEAAAAgJEGYggAAAAAIw3EEAAAAABGGoghAAAAAIw0EEMAAAAAGGkghgAAAAAw0kAMAQAAAGCkgRgCAAAAwEgDMQQAAACAkQZiCAAAAAAjDcQQAAAAAEaase1eAAAAADAqVKo1Wrj1gH6sN+jNYoFmTh6g8mRpu5c18kAMAQAAAD2gUq3RZ3++R41mi4iIavUGffbne0REEETbDMQQAAAA0EVENKhWb8R+1mi2aOHWA4ihbQZiCAAAAEiJmv46cXAf3bn/mGr1BgVEFBmO/VEjkkBvgRgCAAAAUqBLf31+92H75yYhRET0ZrHQxdUBF1BNBgAAAKRg4daDthBKwszJAxmuBiQBYggAAABIQZo0V7EQwi/UB0AMAQAAAClImuYqhHmaO30o49WAJMAzBAAAADii6xM0c/JAh2fIhRJ6DPUViAwBAAAADgijdK3eoIhe9wla+uEn2jnmt51CCPUXiAwBAAAArzB1iNYZpRvNFl29+9BaMaaCZov9BSJDAAAAAPGRn0q1RkS8UdpXCBG9brYI+gNEhgAAAIwUXPSHi/yIDtFvFgvaLtIcARHtKYRUbzS1PxfiCvPKth9EhgAAAIwMpugPF/kRn8+cPECFMN/xs4C5TqlYoO/nP6CVC7+mElNt9maxYI1Ggd4AMQQAAGBkMEV/uBJ58Xl5skSXPzzcFjf5IKCI4oKoEOY7GimeOLiP/Y5pPaB3QAwBAAAYGUzRH13kRxU25clS+3utaMstJAuiUrFAlz883E5zVao1ur5c6/AVBUR05miJypMlazQK9AZ4hgAAAIwMnO/nzWKho2rM5N/RRXMi2hJC386+7/TdO/cfW9cDegciQwAAAEYGLvpz4uA+Oj5/m6YXV4iI6MrZCfp29v2YEKpUa6yJWhfNSeJDUqNRoPsgMgQAAGBk0EV/ThzcR9eXax1T53V9gITZmUMXzeEiP7kgoHdnb9KbxQKdOVqiO/cfJ6omQyVaNgRR5N4hYWpqKlpaWuricgAAAIDecnz+tlawqGkv7ntEW9Ec2SskEALKNKqDO9aG7txJzzWsBEGwHEXRlO17iAwBAAAYOLKMiLiamE2m5jNHt659fP422736x3qDckHQNl4L5F5GPnCVaOcWV2jh1gNEiTyAGAIAADBQqBERLq3liquJ2dR0cfHfH9Hivz2i5makXZNY17uzN7XHJ2nAaBJnaZ/JqAEDNQAAgIEi6948riZm3fcEzVbUFkLqmirVGh2fv03vzt6kXKBv0ygaMM58tdrRgHHmq1W2AaOt4gz9itxBZAgAAMBAkaQ3j4i41OoNyr9KVZWYVJYwVS/cekDTiyu0pxBSEBDV15tUHA+N/h8VEaERx6gpMqLXwmvuxlpMUDU3I5q7saaN7sycPGD1I6FfkRsQQwAAAAYKbt7XnkKo/b6aVhOChEtlqd+Xr/VkvUkBuQ9nzQeBVqzkg4A2o6gjFXbuVVm/CjfbTBZxXPoO/YrcgBgCAAAwUDCZJvZzXVpNoDMvm75P9LrjtE0QhbkgFukRbEYRfT//geUMdoSI4yrL0K/IDYghAAAAA0V9XR8p4T63pYp8qsYENiEUBMRPcSV9xGbveEhPmHs4Pn/baKZ27Z5N5JYyHDUghgAAAAwURUY0FMf1aTJTFZj4uc/3iagtInSE+YB27xhj01tEW8NbVS6cOkQz11ap2Yqf16U6TE71cbimDEcNVJMBAAAYKLhewdznuqnxAt+qMaKtgA8nhIi2KstMQojo9WwymfJkiRZ+c4T2MqIui+owl5ThKAIxBAAAYKB4yggN3ee6qfGCveMh7RzL0fTiCh2fv90uYS9Plujyh4e1oiQgoo+P7adSSmMyl4orT5ZofAeftElbHeabMhwVIIYAAAAMFFyFlO7zuRtr2khIsRDS8+Ym1RvNdk+fz/58r0MQXTh1qEMQFQshXTk7QZfKh63RIxsRUYcAkzEJkrTVYbbjR7X6DGIIAADAQOHaJLFSrbHpqnqjaWzcKLw1sjfpxcYmLf3wU3u6/a4wR8VCaPJJG1EFmIATJAFR6uowk4gb5eoziCEAAAADhUhjlYoFCmhroKpuOGkS/8uP9QZVqjX69MtVrVi6evdhu0P0k/UmvdjYpCtnJ9i0WalYoD8Zft5otujTLzu7THOCJXp1T1xHahfkZ0e0ZQQX6xzlAa+oJgMAADAwqLO7rpydSDS7iytj31MI6bM/32MN0uqnIpqk6wYtIi2iyuvd2Zta71IrirTNH0X5u9zTKMuqr4CI/mnPrpEuqRcgMgQAAGAgEKkreXaXLs0k4NJNe8dDunDqkDbVFgTkNW6DaEt0uUSrTH4ctZKrPFmib2ffp1KxwAowHZVqjSYufkPvzN6kd2Zv0uQfvul4Pr7PcFSAGAIAADAQuA5oFYNRuV5BT9abtHDrAZ05WoqJF65xowkhcoSA+X7+A/p29v1YtEXXW0hGF8niolu1V+k8GTHoVR0fMnPtdRou6yG3wwLSZAAAAAYClwGturEUOmr1Bl1frsWiN6Y5X0REuYBInrBRCPN04uA+Oj5/29r5WddbSEYXOTI1gFTTZQu3HmjHfzRbUXvkSJIht6MAIkMAAAAGApeSettcMRk1IlKp1mj95YbxmM1oK80moklnjpbo+nLNKe1kExy6yJGp+ktdv+n84mc+bQlGCYghAAAAA4FLSb1vhEN8X1dKzzG+Y6ydCrtz/7E27XROaeRIZBccXFfqyx8etq7fdn7xM9e2BKMGxBAAAICBIK1JWYf4vk9ESZTfm3xJRPEoka1Ro6krNVeaL9/vzMkDFObiXY/CfNAWO65tCWTEvb47e5NtFDnowDMEAABgYLANI9WVuHPIERGfiFJxPHS+hkhliTXvHMuxx5mEnKl0XyCuMXdjrW2iFpVz8jNzGegqOF+5R1fvPuxKaX8/ATEEAABgoFF7D505WqI79x/Tj/UGFcdDiqKtuWV7CiEFAVF9vdlhdK5Ua5QzTKGXCXMBRZFf+b2IJJkEVEBbQuP4/G2tAVs2SZuM2j5Cx0alWusQQgJV4A0DEEMAAAD6AlXUuDQDVEUGVyVmO95FCBER/WLXmHf5fS4I6NziCvtzU1PFJM8kKxZuPdA2iSQavuozeIYAAABsO0mbAabtm+PjFSLaiirtKcSn2RNtDXLVeYJsQouLvOieyfTiCr3TI+9ONwfG9hsQQwAAALadpKLG1DfHxfjrG+Eojof0TFN+H+YC+pcjv6SdY6+3VY2X2Zkf6w3tM1EjSN0URN0cGNtvQAwBAADYdpI2A+Q2bGFytkWaTCM7dCXoUbTVxFAllwvo6t2HHd2fNf0PY+fbO66PMpmaLQpsYznSVoDpqt8CIvr42P6h8gsRQQwBAADoA5I2A+T65uhMzjrxwB3/wT93Rnn2jod0+cPD9LSh9wu92Nhk/TU6REk7NyPtxMF95BJYkvskCfEzcfEbmrm2mnr+mK4M/8rZCbpU5vseDSowUAMAANh2XErHdXBVVtOMYVmNNOmOP3FwH11frnWs5Xlzk4jcIjYmCmFea+5W128yL8u8WSzETOR1jWBLWgGWZXVaPwMxBAAAoKeYKqSSVE7pNmxuxpgaadKtxeRfmjl5gGaurWpTZTbyQdDuJq3OMvt29v2O73JiTibMBbT+csNYqSYzbBVgWQIxBAAAoCdUqjW6+PVax8gLtZQ8qyiES6RJV5Zv6gXUFhP+OoiIiDZfVZXprknU2cSQi0Dlg4A2o4j2FLaM3C7jQ+RzAj3wDAEAAOg6ptlfPqXwrnBjJ4io7a359MtVbQQoH+jdOm8WC+xkeBd/jzg+jZfpj787Qt/Pf0C7d455Racwf8wMIkMAAAC6jq2fTzdSOGqkSY0Ecf1/WlFEhTCvjSpx6auItgSX6Hr98/ONDtFkO97FyySnDW3PK8wF7QaRvW7WOIhADAEAAOg6SUvkbfh0aHZtsFh6dR55xtfzja1J9HlmbEepWOjw/Yh11eoNygdBO/qzpxBqDc66+1/64Sf6n0+fU0RE//Ppc1r64af2vZmM3CWIH28ghgAAAHQd0+adNIXjO0TUJfokr+XFxmb7c6F/dEJIt35xfdUfFOYDCnOBNmqk3tvndx+2/92Kova/L5UPs54o1zEkoBN4hgAAAHQdrm9OsRAm2sBtQ0R1cNGnfBB0+IrKkyVrFEl3jIruHM1WRL/YNRbzMqnHf/HdI+11xeecJwpCKBmIDAEAAOgqlWqNri/XOoSL6GSctIFfkiGiPtEUWxRpM4ro+/kPjN/hzlFfb1L19782HmvyMwlGpQdQL4AYAgAA0FW4GVt37j9OfM4kQ0R9ehnZmiu6eJy4c7gcy3mTuEq3LFBbHxQLIc2dPjQSggtiCAAAQFdJOnfMBCc0bENEXaMpovJLF59xHVSqi0SJRonvzt6kPYWQgoC0FV8fvfdWh2dI8NF7b1mvm4RKtRZrJllvNGnmq1Ui0nuwhgmIIQAAAB34VGi5kCZCwqETGiL1RtTZ4fnEwX105/5jr/spT5bYzs4RxcWBS1dttVGiXFWmmr9F+vCL7x5RK4ooHwT00Xtv0aXy4di1ktyfysKtB9q+Rc3NKNEYj0EDYggAAEAbriszUfLoQNK5Yya4lBdRvIJLjrDo7kcnZIj4VJU6ad72zMqTJapUa/Tpl6usF4goPj/sUvlwzFOlu5bt/lwwRelGYYwHxBAAAIA2pg7JScWQ79wx18iULuV1fP62tZeQfD86cTHz1SpRwJuYf36+QZVqreO+TM9MXMMkhAQ24eHSKynJ+zJ5pEZhjAdK6wEAALTphr+HaEu4fDv7Pl05O0FEW4NIj8/fpkq11vE9IRxq9QZF9DrSoX7Pd/3c97Tl75uRcdSFSB3Zrmm6BodNePjenyszJw9QmI+bs8NcMBJjPCCGAAAAtOE24yyiAy5Cx3V2F3f+nGO1lbifpCJPPs72zFyv4ZI6dH0Pvu+rPFmihd8c6UgBFgshLfz2yND7hYiQJgMAACDRDX+PwCUF5xuZksdeBOQ2UF6+H1sJPUcuCOjd2ZttA/P15Rr7zFyusXtHnsJ8jqYXV2jh1gM2Nah7P6b782GU+xYhMgQAAKBNNzsbuwgdn8iUHGki0guhfBDQJ8f2s/ejmw7vQiuK2tGt68s1OnO05HUNEb8qFQv0ybH9tBltVZfZUoO692O6P/Gcjs/fpndnb2pTk0np1nm3A0SGAAAAdNCtCIFLib1PZMrFi7MZRcYu1+I+uTJ6oq3qsSgietpoUk5TYdZotujO/ccdg1qJOo3gxfGQdo7l6Gkj3lNIZ/o2maBd30+lWusYNkuUTXWgOHfWVYfbCcQQAACAnuAidHwqz1y8OC7emfJkqaPzssze8bBjdMa7sze155DXohMhT9abVAjzdOXshPPojzSmdVWsyKStDiTqTtXhdgIxBAAAoCvIkRHRbbnRbLX795QYoeMa+bB5cXy8M1zVu/q5LbqVRITsKYQdwkn+XIdL6wFb1MzmwUoqRAe1JxHEEAAAgMxRRYG82beiqC1UfKIIus7LqnFZmKg5ocXxVCNGdJ/boltJRAhXACd/zhnFufSUTZSYPFguqa9udBXfTmCgBgAAkDk2UeBaLi/QleVfX67Rf9q/pz28NB8E9PGx/fS3+Q/o29n3nX01x+dvs1Vo6uZuM5jbRIgu2lPXpOfkz21Gcd2zNIkSHw8W9550pvCsqg63A0SGAAAAZI5LusQnpcJt1P/jP35qi4NWFNH15RpNvf2GU3drMSvM1GDx2YvObtNE5jSeLXVXbzTp+PztjqiVLcriYhRXnyVXgr93PKQLp/ST6H1SX75dxfsdiCEAAACZ49Jbxyelwm3UXJREtymbUncc9UaTZq6t0tyNNW0lmIpLHyA1/WRLvSUxistipVZvtH1a4zv0275oWKkbGcK9p2HqS4Q0GQAAgMyx9e/xTalkIZx8xmLINFuRUw8gotdptLylE7acfrKl3mz3zj1LIbQKYb4tctT1V6o1mvzDN3RucUUrhAY59eUDIkMAAAAyR02jiGqy+jofXTFVMumiJ1zHaU48ZFXp5FJC/g+7xqyRJ3k9uiiLqbu2q1Hc5gMyRbHyQZBZw81+B2IIAABAV/BJo9gqmXQeFdMYDJ2wcknduY70MJWm29JkAlPERz1PRMkq5Uw+IFukbDOKYp2sh8UjpAIxBAAAoKfoNlWXJn46cTX19huxcxGRVlidOVqKiacwF9Avdo21I1Y6gaWDEzKcwFBFli39pDuPEEJqp2vbOjlztk/5/bB1nFaBGAIAANAzuE2VEx+2DVsnkLjxFnfuP6bLHx62RjdkgVUcD+nn5xvU3HwtZUxCxmT0Lr0SIC5RFe48vkNlTeZskYLTod7jsHWcVoEYAgAA0DO4TTXvWclkwpQackndqd/xSQ9xkZisIjrBq/W4ChBbCbxOiBYLIc2d7iy/H7aO0yoQQwAAAHoGt3mKrtQuA1ptpOmOzAkfk/hQB7KGuSAWSTpxcB8dn7/tHBmaOXmAphdXYv6liMg7GqMTd2Itewoh7QpzRmM70fB1nFZBaT0AAICewW2eopycKy8XG/i7szfp+PxtbWm7+I6ovpJxEVa6LtemMnrdMU/Wm0QBUSF8vb0GFNHivz/yOm95ssQaubMY4CrWUm806Xlzk66cnTB27R62jtMqiAwBAADoCZVqjZ692Ih9Ls8pc2mWqDPvZlF9xaXwzi2u0LnFFW36SHdMsxXRhtTVer25GbuWi9+mxERjsh7g6rIW8bOLX69tCT4i2jk2PPGU4bkTAAAAfYsQK2rvnb3jobWXjcvMLFv1VZoydEG90aSZr1Y7IjqunbGTXG/m5AEKc/Hmjc9ebsSiSrqo1vTiCp2v3HO6pmu06bkk7OqNpjXCNShADAEAAOg6XMn5+I4xq1AxVVaJtBlXFeWTUnLxvzQ3ow4RlsYzExGxKT+irWjML3bFEzjNVkTnFlc6juXE4NW7DzvOz63XthbuGr4Dd/sViCEAAABdJ01EgksLEVE7CsINv1A3f5P3yDZCRLdm3THmQRyd2PxD3ER79VhThEoWK6Z7tK1lmCvKIIYAAAAYcTEv2+AiEi6RFcuYLyJ67RGSUQ2+NoO0PCPMhLxm3Vyxj4/tjwmOMBfQ3nG9qDNFV2zPRxxr+p4cNbPdY5K1DENFGcQQAAAAliQVVjrSVCOZoiMywiOkq0Yj2jL/2tI85ckSfTv7Pv3p7ASF+bgKC3NBTGCppuVL5cMxgbTw2yNU/f2v2agRF12ZOXnAGmn6sd4wfk/0JlLvMclahrWiDNVkAAAAWLLqPGxr/mfCZaYYkbmxYaVaa1dBqeg2f131lFpN5lLlJu5XCC7ffj3lyRKdW1zR37B0bHmyREs//ESf330Y+znXmyjJWuR7cn2HgzDTDGIIAAAAS5Y+EZ/BrTIzJw/QzLVVarb4Gi2XWV8cus1fbOD19SZbmu8zEd40H822dq7EXj32UvmwVgwRdb4vcW/Ca8XNTEvSgFJlUGaaIU0GAACAxccnksRb5HJMebJEC7850uG5KYQ52jsesikxFZN4U4WIa2rQdyJ8o9miz+8+pF1hjooF97VzJfZERGeOdooTzgsk3pd8b0SdXit5LVmlRwelAg2RIQAAACymQZ8ySSIAPsckjSoJuJRQsRA6R3w+/XK1Y21JJ8I/WW9SIczTlbMTbeFhGtVRnix1pOtk7tx/3PFv2/uy9WOSu3irJEmPDkoFGiJDAAAAWHTVUnL0QER1Pv1y1TsCkEXUwDUaxZl/504fin3XND9t5trrposmQ7FrFZhrBIYzkatrNb0v0739WG/EokYu17MxKBVoiAwBAAAwoovKqFEd3cR5IvPmmTZqoIsszXy1She/XosNHvUx/5oM281WRBe/XrOekzMzy9TqDfr0y9XYs/ONQum8PZyR3HQerjGm+j0fdJGqgLbu/fj87b4xU0MMAQAA8KJSrWk3cR2mzTPtJHTtXLDNqJ1OUtNurqk23QYuI6eruHOq6SsdAfEishVFHWvn1vTk2Qua+WqVmptb57GlJ01ptGlL1VqSMnpZMKqG7X4yUyNNBgAAI0jSRooiGuMihGybZ9q+NS4RpCRmXZFqSoNtbWoVlw557WJNauPG9eZmWwjpjlMR5ylKXb13hVtSwCRCXYzeHKKvUalYiN1zv5ipERkCAIARI025sy2Vkg8C2oyiWBqqUq1pe/Zc/vBw4h40rv2HdMLE1vumPFmiuRtrscGyYu1p1mYqlTetvTxZooVbD9h+SdxxOl5svB64+mS9SdOLK+3KMrXUPqkIcl1TP5ipERkCAIARI41x2bRxFcI8/fF3R+j7+Q86JsVXqjWaubbasYmLCfBERN/Ovh87xgWX7sxE+vlkLhPe504fipW0h7mgw3TNRdhOHNynHQ/yp7MT7SiJC+raXYWDKcrDVZSJ/6srtc+CfjZTQwwBAMCIYZoCb0uXcRtXPgjYjXPh1gNtw0R1Arwv5cmSNdUkm3V9JryLyFFzM6L8q+FoYqyG2oFaFlUzX63S//F//7/0+d2HHWsLqLMnkMtQ2EKYpxMH93WIrSIz30w9zpRqtAkqudQ+Sy9PP4/zQJoMAABGDFMKx5Yu4wy4pghCkooyG0KsmODMui4T3tVKObFpy/fIGbhVD484t2yqVivR9hRCarY26dnLrfMVCyH9y5FfdnSrrtUbFOYCCvNBh7gM8wHt3jFGTxtNp1SjS3qxG6mrNCNZug3EEAAAjBimaik5XWYaxeCzoe0phFrvDZE9RaLz9hCRsdqrEOZp51gudk15wjsnBkzdo9WGg76CQdcTSO72LF/zxcYm/evq37Viq1gIaffOMW9BYRrDocJ1GDfNanMhbfPMbgExBAAAI4bYjLgBoCKKwhmsfTe0wGDsMaVIKtVarGx85qtV+sWuMVYIiTliXJn4j/UGXTk70TYMq5i6R6ufuxq45e/r4MQXd49PG02aO30oNgTW9E5UwSWbpU3zyeTj1flwsu+rHwWOD/AMAQDACFKeLLEm3nwQOBmsXcvzue7JYh0cczfWYiknuY+QSkDU9rmYzLrlyRJ9fGy/1uBs6h4dEXXcp4vvRz23Dt8I055C6D03zDSG48rZCbZjtXx8N3xf/QIiQwAAMKJw/h8uIqFOPnctz+ciKLaKKi61xiGLGNuMrkvlwzT19hvt6EpxPKQoIppeXKHieEhhLtB6f3T3Kft+nr3ciImGveMhXTjFp5O457N3PKTnzc3YPQQBOaXxZEzRLpdIn0mw+UTH+hVEhgAAoM85X7lHv/rsL/TO7E361Wd/iZWAJ4WbY2WbfE7kV57fjSoi2/lsM7rEd76dfZ+unJ2g581NqjeaFNGrDtMB30+o0WzRucUVOj5/u31/bxYL9LTRpF/sHOuYSP+nsxNU/f2vtWJDHoqqi1JdOHVIew+uc8pk0pa1m76XN+VBBwREhgAAoI85X7nXMeOqFUXtf18qp+uSTMQbWm2T6n0a6CWtIto7HmpTYiLSYjufq7dJWxXWimj3zq0KLc5oLDxMFFA7GqROpOcweXhKyv3oUla+Y0xskTIbMycPsB4zl27k/Q7EEAAA9DFffPeI/TwLMaTDRbz4zhVLUkV04dShmGk3zAftlJPtfLYu0wKTsLOZpHWpNFvKisjs4VGHrKr3ceLgvo6SeyK7sElb1l6eLHVUksm4NpDsZyCGAACgjzEN8uwmNrGRNtJA5DYSg8i+gbuU3yfxNIlzmcr4OWymaNfIms6bdX25RmeOlujO/cfsc+GebZqqrwunDqV+5/0KxBAAAPQx+SDQCp80Pg3XiImJtJEG3SY/vbhCSz/81BHxsm3gnJF751hO62n69Mt4KTgneJ692CAiossfHqZPv1z1EqByhEz3vF0ja5w36/O7D9uVYDpxmHT2nIl+bpqYliDyeLlTU1PR0tJSF5cDAABARvUMCT45tj9RmkzX4C/LYZyuCOOwSkBEHx/bb4x6uJzHhO5+1YaC6ne5vkREFOsILZ+fe95njpa0qS51Xe/O3jQ2R9T5jLhnokvBDTtBECxHUTRl+x6qyQAAoI+5VD5Mnxzb344E5YMgsRAiSjekNUtMIzGu3n0Y66FzvnJP29MoydgI3f2WJ0s0viOeLJG7VusYD3O08JsjbNUa97zv3H9srXYjsld7qeNGKtWacfacS1+oUQSRIQAAGCG4SENARN/Pf6A9Jou0mnqOZy82vPoI6bokX/7wMFtZpevRo1J61W1a3BMX/QmI6MrZiY5u2IIwH9DCb46wzyPJ85bRRZZMCDMzF3XTPcNhSHNxIDIEAAAghm+/mfOVezS9uOLV7VhFN9392csNr3WrgkJEbLgeRqJHD+etEtPs5Xvaw/QVEl2rf7ErHjlqtiKau7HGRlxMz9ulg7fcL8mFH+sN7TPRzSLbjohgvwIxBAAAI4RPA8RKtUZX7z5MvYmyfXx25GPNBn2/VuTrAAAgAElEQVRs4aJ7MpduKk+W6KP33tIeq7unIDA3c+SaHdYbTVYscs/7xMF9ziM1RHPIP52dsI7/EMJNfSZcDsglzejS9NN1NEu/gmoyAAAYIXwqghZuPUi1idq+u/6yRVfOTsR66Cz+2yNt/x4VEXUxVZzduf/YeZ319WZsPfKzcR3MKvcZ4p73xa/XvEdqyOfSTZ6XhZv6TDhTtc2T5NL0s1vVa70EYggAAEYM134zScY7cN/lNmJ1LZVqjRb/vbPRZC4gyufiFVsu/W18RJtYD9Fr8SJPhPfpOSRfV3eP3LBZ23rlc5m8XFk0aiRya/rJmcR1bQz6FYghAAAAWjgRExB5NdrzadCom46+GRH9bzvGaPfOMarVG5QPgo5UnWmzNd2DLqriEuUQIsMUuzKJRVOK0UdkcqLW1qjR5xm6NP3kBFwrigYmQgTPEAAADDjd8mtwRtyPj+332txchqYKuI31aaPZXo/YiF3M3Jxn5+Nj+7XrsbUeEP6d7+c/YE3NNrFoiv5wx/m8Y1M5v+8z5Azo8ucmATcoJm1EhgAAYIDppl8jTcdhXQpn5uQBbfpJxpRSMwkVNQ01d2OtXbq/e0eeioWQnjaaxnuoVGusJ0gnYHQRL5tYrFRrlGO6ihcLoXOkx/SOTaM+XJ+h4KP33tI2/ZSN6bb0YZJeUL0GYggAAAYY383NlyTzrHSb98y1VaLo9WBTbkM3pdSmmanp8mZbqdZi/YCevWxRmN80TpIXa+bQRT98xaK4hk4IFcI8zZ0+pD3O9x2bBKXrTDSB8AV98d0jakUR5YOAPnrvrdjIFCJiR5b4pP62C4ghAAAYYHw3t17AldKrNJotmrux5jyclWuwKG+2C7ceaCvRmq3IKBB1axaEuYDWX27Qu7M3Y4LHVSxWqjVWLOSDwNj80Pcdc4LyxMF9bVGjYhIsl8qHrR3PxdoHdZArxBAAAAwwrgM/fUjbcdpHiNUbTapUazFBpKa9Jv/wjbYCS91sTddO+jMKqH3tJGlIU0SIiGgzihKZwLl3rBOUopqMi0plIVgGeZArxBAAAAwwPpVaLrj6U0yCybUfj8AUsalUazRzbVUbWdo7HtKFU4c6jjVd2yQQuePyQRC7ti0NKT+bPYWQ/r/nTTK1TbIJV86bdOLgPvYYXZ8hXeTLFpXyIYuxLdsFqskAAGCA8anUcsFlkKtuvIat63KY53tLm6IyulJ7wfiOsdh9zpw8QGEufq0wHxgFIld1xkVzuDWrz6beMAshF+FanizRmaOlju7cERFdX645Vw5y67VFpVyx/U70O4gMAQDAgJPE5Mzh4k+xGXpNXZd1qS5TZMQ3tVWeLNHSDz91jBHZvSNP//2/6gWiHM0ojoe0cyxH9Uaz3Ycnz1R+cWs2eY9U8kFAZ45ueaGmF1eM0ZQ79x+zY1Fc3n030qky3TbydxuIIQAAAG2K46FWsBTHXw8xdRFMnEDzTemZ0l4RbaV/1M7L15drHcKBi8yoKcEn600K8wGFuaBtwvb12LimBwthns4cLXV0hTb5kdIa5bNOp7quYxDK6omQJgMAgKElSTNGJivU8blpErsJke6RG/aJ6AG3tpmTB4wpNjUd45LmE3BVb6a5aAERnTnKR+K4JoXqdy5/eJju3H/svNakz1yQdTo16/VtN4gMAQDAEJK0GePThn5mlvy5b5RBpKJ0w0VtaxP/5lJsRK/nYE0vrrAjMnQRmyRRi4jMw185j5GgEObbIsSlb5Igi8iOKZ2a1vzc7chTt0FkCAAAhhCfCImMy1/4PlEG2VhLFBdCLmsrT5ao+vtf09/mPyAu7tKKIuOssODVWrh78sEkorgRHeJn8nPyiabIz5yIOmaL6aJqPlHBLMzP3Y48dRuIIQAAGEKSeji4qir1L3x5Rte3s+8nambouzai5AImoviAVK7qTVeN5roG7vl9cmw/ERFNL660xYnrsxaUJ0tOs8V8xU1S4axbn8vvRD8CMQQAAENIGl9Pln/h+6SiXISOTkAkXYvuXhd+c4TO/ue3WO+PLfWjO6cwSqvihIjYZ81FdlyEi6+4GXTzcxbAMwQAAENIGg+H8JYIH8n04got3HqQqImeawNGeW0m/4pats8NPeXWIqO7DhHFOjULn1NJ46Xh1mpreCjEiS6CYvJ7uQgXX3HT7bL7QQBiCAAAhpC0oxGSGrBVbBPNiTq7ILtcVxYb6vc5VCHIXWfnWC52LiGEvp19v+Nz12fkK05MkR0X4eIrbgbd/JwFEEMAANAnZD3OwGeIqHrdrJroyaJMV00mV1eJ7/lcVz0/x64w1xHh4q7DiSqdcHFZa6VaY6NXnDgxiacrZyeswsVX3AzyTLGsgBgCAIAeos6tCgKi+nqTiuMh/fx8o93jJmkkJsl6dNENH1FgQ43kmDbdJP4Vcf7j87e1giig+KBVV1O3QCdcbGs1DWg1iRNTZMdFuCQRN2m7mJ+v3KMvvntErSiifBDQR++9ZZ10309ADAEAQI9QhUdd6t2j66HTi3EGXHSDG0Mhd6JOgm3TTepfqVRr9OzFRuxzXV8j0/3tHQ/peXOz45mE+YCevdigd2dvdggL21q5SjrbcFRbZMdFuGQ5osXG+co9+vzuw/a/W1HU/vegCCJUkwEAQI/wKTMXdLuihzt/K4q0nZ+fNprO/WeSdMD2LTcn2tqMpxdXOsQl0Zaw4azVrSjSXufCqUMdFV57x0OiaEu4qmXqXGm+EE5c2s42HFVUpO2VhOfOsf7drr/47pHX5/1I/z5dAAAYMpIIm25X9HDnLxUL2n47m9FWN2gbaRr57Qpfb03FQmiMolSqtY6hrDLjO8bYJoiijF1X1i73yxnfMRYbzyFH7EzCiWNPwS269ry52f7/642mdyPEJGI0CVw1n2uVXz8AMQQAAD3CV9j0oqLHFIlZlzZjGW4shszFr9e06TeTkBICSj7/s5cbNHdjjd3QF249YIXHj/WG8f5cmgRy0R3xuU046XAYX5aqEWKlWqPJP3xD5xZXUnWVdoXryeQyp61fgBgCAIAeYWsYGOYDKhbCno4z6MYYhUq1xgqmJ+t8mo0bnKpLUQlM0TZhOE5zfz4bvWvkr655NmoUhxNhtmvoBKUgSVdpFz567y2vz/sRGKgBAKBHqFU+cjXZdpYzc2bbYiGM+XDE5yZsGy5nCncRE6qpnDMxB0RehmMVUfXmmgIyldCr6Jo/qhV9OuO37lgVmy+tGx40YZJGNRkAAAAnsqrySdKTyPeYudOHaOar1Y7UT5gLaO70IeN1bBuubydk0/FcU0d5FlkSIWQrv5e9SKYSehVd6lMnYCKKV8K5pE1tz75bHrRL5cMDJX5UkCYDAIABI4k5Ockx5ckSLfz2SOfsrt8esYoL24Zr6oTsMndMPl6d5i4nr5L6ZGzRFVWUmEroPzm235qi4wSg6Hztk94zPXudmOqVybrfQWQIAAAGjCTdoV2P0UWP1DEUNkwjOHw6IY/vyNOzl/FzrL/coEq1FhvPofPacM/FFCUzRVd088m4729GkVO0hOt5lA+CRM9+5toqNVvx8505Woq96yxGrgwDEEMAADBgJOnS7HJMpVrrSIvV6g2a+WqViPjN0SQqxIgMsdnrhISKPCRWbMwqT9abiWaAibWqnhxVBHDpOt18MqL0g06zLE0vT5Zo7saa1ut15/7jjn9nNXJlGECaDAAABgxukzVtvi7HzN1Yi5WGNzcjmruhL4fnUm/nK/faAqlULNAff3eE/mYoX9dhS1XpKqNM9yivlUjflVqcz7fxY5JGkTJcL6S9Cbt9P9UIIaK4WEwiqocViCEAAOgjXDwcSTZfl2N00QTT51xk4erdh6n727hsyOp3TPfo0v1bnM+3HN/1+9y7nTl5QNvt++fnG4k8PK5iOYmoHlaQJgMAgD6gUq3Rxa/XOvrDcB6OpIM4fY+xwQkWLuricy2XyjJ50xYpMHnumJyWm15csV4zIqLj87fbx/isVzeMdnpxpf2cicjoz9GltpqbUaKUlevUet/p9sMMxBAAAGwzplJuTkgkKdG3HbN3PNQ26+PSNa6l8ET+qReTCZuoc9NWn18riiggohMH91n7EalwAlQVq8VCSHOnD2mjPzrRs3MsZ/TnuKa2XHAVvur3iuMhRRHR9OIKLdx6sG19r7YDpMkAAGCb0Y2ukOmVh+PCqUOkjiPLBVuf69ClpbgBDL6pF1PJ/N7xznllXJ+eq3cfdqSiXNeq+pEq1RrNXFvtEIr1RpNmvlrVjgfRiR4u1SjebdYpK5dRI/L3rpydoOfNTWO372EGYggAALYR0+gKQS89HHlFDan/ltF5ZT4+tt/oTfLpa1OeLLVFjJx6e67MTDOl6+TGi+par5ydYAWRfM6FWw+0peoijeWyFg7xbtOasNOSZhbaMIA0GQAAbCO2zabXG6K66TdbZt+KLvU29fYbsRQNEdHkH75x8kSpa7KVf5tSYLI40a1VlNqryALUp2WBT+pQfrfd8HT5wN2j670MOogMAQDANmLaaIuFsCfDWm1r8Y12qCkaIko8PNRlTTMnDyROz7lEZHxaFrh20c4HQce7TTJeJUu4ewxerW3YgRgCAIBthNuEioWQVi782mtDTDtaoVul1mmGh7qsqTxZoo+P7Y8JIlNUTTyr6cUV2hXmqFgI2bJ4rvQ9zAWx86vpOI7NKOoQQr6jUrKGE5RyqnGYgRgCAIBthItM2IahqmSxoWblW1FFmS3VsqfANxfkjM8nDu7r+OxS+TBdOTvh1Otn4uI3dG5xpf2snqw36cXGJl05OxEzG4uITbMVdZjLi4WQndMmR8a4hoqymOsHv055shRriSAYhSaM8AwBAMA2kpVXxHe0gm2MRtK16ErL1enrKoEhhFKeLNHSDz/R1bsP2+eIiOj6co2m3n4j1n/JtFbfFgbq9zejLXFoS13Kz7Y4HlKYCzo6e6sC0+TXeXf2pvU9ZJViK6UcKzLIQAwBAMA2k6RnkIqr36dSrcUa/KlG5jRr4crcTYKobqmmu3P/sXcjR51A8E3XJZndpQqoJ+tNCvMBFQshPW00tWLFZLqWo3xEcaN5lsNWR7kJI9JkAAAwBLh4a8TGqet5k1VaxlTmnmdCQLbIg6+xm0sZ+nS0TnJdIr2AarYi2r1zTNvzp1Kt0frLDeO6iPj3k2WKzXcMyTABMQQAAEOAi98njZHZlSLTrXrveEh//N2RRJ4kX2M3JxA4Mcatw/e6lWrNqcRf/j5XZed6DlOKLYkB27VZ47ABMQQAAEOAy1/1NrGj2+R9K9QiJhcWRckjD5yJulZvaNfE3WcrirRl72pHa9N1OfEmhA2H7tly4tQngmaKqo1SB+m0wDMEAAADhMksa/P7mLwpuk0+iR+Fm7ElPk86U43odYNE2X+kWxN3nyXJO+RiNvYxlJuibpyAsok2F++OaYZbkgG5owrEEAAADAhpzbIzJw/QzFerHZVNRFuRETF/7Pj87fbG/+zFhreBmBMiptSSi9gQIkpXqq+uyWQETjON3oQp6sZFv7IQbeKzc4sr3usCr4EYAgCAASFJdVMMJQMT5oO2EFKFFodpg/WpSEoi7lxMzT4RnfOVe/TFd4+oFUWUDwL66L236FL5MHt/HCZhw91LVqKtPFlyGisCeCCGAABgQEg7LsM0e4yIjOZqGVuUR5iVW1HUjnLoNva5G2tdizy5iInzlXv0+d2H7X+3oqj9b19BlKQsPct5ZKNcFp8FEEMAALANJGmU55uCUsli9phrlEf4Xrj7qlRr2hJ/23pmTh6gmWurMVGndqTmkJ871/foi+8eeYuhpMImix5Taa4PtoAYAgCAHpPU+5P2r3+bmOJSY3vHQ6qv6xsGCrgU3tyNNfb7pnWaaG3GZczivz2KdaRWMXWg7jg/VxJnwaUDdjfFSlbCaruHxm4HKK0HAIAek7RRXtqmeKZScdPk9/Ed+oaBMlw0p95oasu7bdEfjoVbD0ijhai5GVmfn63PkoArbU8zCLcfhrG6UKnWaOar1Y51zny12nfrzBpEhgAAoMekSVel+evflkpJU5FkKtvXeYC47+dMo94tazF1pOYMxjo+eu8t7TmSVvJVqjX69MvVWMSpH0vf526sxaoNm5sRG+EbFiCGAACgx6T1/nBw6Q3XtEeaQZ0zJw+wYkp3Tq4/zmZERpFhEl1c00iX1JjMnfuPqVKtdVw/aSWfuD6XepMFXFaVbWngfFzc58MC0mQAANBjfDobu8KlYc5X7sU+P7e4QhMXv4mlPrqxLiJ92kmk/HQ/E14jHTMnD1CYjx8T5gLtOm2psUKYp0+O7e+4b10KK2k0z3Z9IeBEZZsQTaKy7XyF72oNsgNiCAAAekw3BmJykYsvvnuk3YzrjWZsw0+zLpNfpxVFWo9NebJEm0zEhPMalSdLtPCbI7RXmoFWLIS08NsjsQGougaNMuL+7tx/bPVw+c4pE5jEkiw0v/jukfY73OfdYi8zW46Ihto3hDQZAAB0iTSjM3wxjXbg0KV5kq7LFiHhPDa+XiOXNbqkxkrFAn07+z4REU07eKWSVvJx95cPgg6hyb2npJVtSblw6hCb7uw3f1OWIDIEAABdoNfVQ1yEwjSpnSj5dHPX68voKuZMYiLpKAmX1Jh8XZeoT5ZDZgthnv74u85IFveebO8va0z3M8yjPSCGAACgCyQtn08Kt+l+9N5b2kntMjqR5ltGrru+DnVDLU+W2NRMUkO5KTWWDwI6c7QzsuTilTpfuUeffrlVcp4LAjpxcJ9TlMRVROkq2Eyfd5NSwpTgIIM0GQAAdIEsuj27YhuDMfX2G3Tx6zV6sq6vCFLTZecr9+jq3YfGyfAqatk+BUS6DI9uQ71w6hCbgkrSAFA8Ax2tKKLry7WOBo22lgNpx3a4pB7FeZJUk2XdJHEUR3sEkUc+cmpqKlpaWuricgAAYDjgzLuyVyULdP6YQpjXRh8q1RrrBwmI6Pv5D6hSrdH04op2VIXr2kXjPrVfTZgPaOE3r9ND8iZeHA8pioieNl53uiai2L0FRPTxsf1GkfDO7E3rGn3ew68++4tWXOWDgP7j8n9xOoeOLESMz/vv9dr6gSAIlqMomrJ9D5EhAADoAmn+uvbZiHz637hMN1+49YCd2WWKaslrzjGRmd07xjqEkPx8nqw3qRDm6crZifZ3js/fjt1bRERX7z40jt7g+iXJCK+UywbfDXOzronj9OIKnVtcMQ63VUna/8hG1gb/fgeeIQAA6AJJDbe+xmvfdJzNH2MSPKZp9fKaOZHwVGrc5+Kp4tYSkbmU39W/5Gpo74a5WXf/alrSZW29TMcOMxBDAADQJcqTJfp29n3rXC8ZX+O1b/8bm0jjjguIr/xynfkln9tlEzcZdk2bvXyPRMTOXHM1tHfD3GwTK65rS9r/CHQCMQQAAH1E1pEeHSaRpjuf8On4Dmk1rcllEzcNj7Vt9uIe/zb/AV05O8F+z2Xtl8qH6ZNj+9uRoHwQ0CcW35INF7HisrZudQ0fNSCGAAAgBWkmmevIOtLji+58V85OGDd+U48jbk1cKuvZi432MyxPlujjY/tjgsh3sy9PllKXi0+9/Qb9055dFBDRP+3ZRVNvv9H+WZLfAZdUnsvautHNfBRBNRkAACSkG5U8WZ2zl0M/k675fOUe/T/fPSSl6Cx2rIuh3PYd3RoD2vLp2AzLpvsjile8ub4v+R2pmCoCh6HKq1e4VpNBDAEAQEK6VT6fdsNT++Koa/M9XxZiRHdO08gMn2foKsbEGmv1RlsImb4vML1nIn2TR9v6k4izbpXRDzMorQcAgC7TrUqetGXNpuGeLg0UZXQl4PLxssDIBwFblq9iM12LZ+gislzLy8Vz1YmbRrNF5xZXaOHWg9g1krznJNPshRDiRFS3yugBPEMAAJCYfq3ksfW/8RkLYtqA5ZJ6+boupeE2sZALAjpfuefUZoA7V63e0Hp4TNfWXYN7n9Grdeow/Q5UqjW2D1ISgYUy+vRADAEAQEL6tZLHpf+NHHkxmX9NG7ApumMTXDbB2Ioiunr3oVObAdO5fMQNdw2T2Znz+8i/A/Iznrj4Dc1cW2WvbVpbv4rvYQBiCAAAErKdlTwmEePS/+bNYsGpwaNpA7ZFJEw/d6mmcu2EbTuXj7jRXUPtW6SDq5xTn3G90aRmS39nNiHdr+J7GIBnCAAAUpDU35PGJG3z8ahDP1XEBuriQTGNFeFGewhMEQt1OKrPYAv1vPK5XNJP4vuffrnKphR11yhPlujd2ZvatW5GEX0//0Hs87kba04NKYnIKqRtA2VBciCGAACgx9jEjA0XEXOpfLgtijjhNc0MbdUJh7kba1R/NU5jV7iVVNAJJYEuVaRbgzyHTCdkdFVfukiIyRxNpBc33P2Le9PxJjP3LBcEsVlnlWqt/cxslIoFp3c/ajPDegXSZAAA0GNsIzfkFNjkH76hiYvfdKTDfI20XMdpLnIjNnaZFxub7f//yXqzLd7k9JHwKtlSRbp0HJcC+vjY/nYashDm6MXGVtXXrz77C52v3Iut3SeVxN3/3vGQFRwnDu7TdsVuRVHsnlxN6sGr84LtA5EhAADoMSYxo5vmLhAiojgednwukDd3lzQcF9kRGzvR60n3nHhzmbnmEsmypYDU3kmtKGr/W24maTuP/FyK4yGFuYCaUtfHQpinC6cOae+jUq3R9eUam9JT78m1yisiouvLNZp6+w1tXySkxLoPxBAAAHiQxQbFpVreLBas/XcazRbtHMtRIcxrfTxijWoabnpxhc4trnQ09DN5Z+SNPW1Jt+vxphQQ1zvpi+8exTprc+fRCc0wH1CxENLTRtP6Pl0G0qqDZvUpNYp13VaFVNpUqgCCyg2kyQAAwBGXdI8LplSOi8B42mgaq9i4hn70as0z11Y75n9tMiZisRZTnx2XWVxZlIRzRmdbTyUZ3XNptiLavXNMO7RWxeXdqINmde9ZFUK689tSqS5k9fs6CkAMAQCAI1lsUETmknwXgfDmK7MtN3netmk3WxFd/Hqt43zcdYjMpeguG+zMyQMU5judNmE+MJaEq60DckzrJJeeSgJTc0YXbO9G9SZx79llaGwWDRaz+n0dBZAmAwAAC/LICR1JOgBzqRxThRaRW18ZLj0jI3uOTOXzYq1EfOm600gINRpiCOjoUkTcX+4uPZUE3HMJXl3TNkvt2YsN7bGmeWLcezY9b9NafaJpWf6+DjsQQwAAYMA2UJQo2w7AqgG4OB5SFJGTp0VgE1S2a+quIzb1d2Zvas9hEl8Ltx50mJSJiJqbUcwjI66fC4JY+muTtqrJXm5E1IoiygcBffTeWzG/kHou+V5mTh6g6cUVrS4ziTnud2DveEgXTh3y9uC4PG+bQLVRqdZibQkE6FgdB2IIAAAM2Eyz3egAnLaXjEsTwmIhTHTNvEaoiM85bCkfVWxwPqBGc5NKrzpf/9OeXTT19hux79iMx+cceiupcL8D4zvGEr8n2/NO22Bx4dYDrRAKiO+hNMpADAEAgAHTJsmlRmSyqubxPY/YbNWSdCKiMBfQ3OlDidaWxMhsS/m4VGkRbW3k4jxcdZWtjL+UIP20XQNS04hibm0R+VWjjQoQQwAAYIDbyEvFAn07+77x2CzLo5OcR/TFkQmI6Ox/3vLZJDknJyZMc7tsKR8XUaFL+ei8SrYJ9icO7qPryzWv9BP3O1AcDzXf1qMTnkSvo3ci4uYisF0w/d6COKgmAwAAA2mGY2ZVzZP0PFyJ/Z37j73PKaq7avVGrAOz6XkIEdBottgO1VxURh5+6jq01SRQavUGXV+u0ZmjJa/hurpqOCKin59vOJWp60rcZ75apZlrq23BIiJrWZW/Y6irH4gMAQCAgTTejazSK0nPk+Q43c/UyFRE9ioqcdzMtdX2lPZWFLVL6l3MwrJIcZk5VqnW6Ofn8YovmUazRXfuP2ajelzqUJ7NJlBN4Bza/kZcsyFyrM6zgKGufkAMAQCAhaTejSzKo9Ocx3ac6zm5CJMtVXjx67W2EBKIHkdqpZq4TprqKl3Vmg5ODJrSkdzAVZceRUm8RVn4kZL83o5qx2qkyQAAoEtklapIeh7TcbqfhbmA1l9udAyFJbL7cLiUjm5+Gve5qYmk+Lmp63alWkvdPNGUOuSq5VyaPiYpZd+O8vdR7liNyBAAAHSJrFIVSc/jcpz42Z5CSM9ebrSFihjbMXdjzdQfMbEpPAm2mWMumESkKa3IPQOXcSC6qFaYC4gCikXObGvsJi4DdYcViCEAAOgiaXsG+ZyHS3Fwx8k/Oz5/O+6JaUVsekiG2zCLhVB7vNrjKC2m0vwcEdGrwaj5IKAzR/nnkSSt6FKdxYlS8VmW1WTnK/foi+8eWRtT6tiuFgL9AMQQAAD0CFWsnDi4j+7cf5yJP4ObVL/0w09Om2HaDU93/NzpQ/TfvlzpGEyaC7Y+1yGPPfERB8a1SxPiW1FE15drNPX2G86jUOQozcxXqx2epDBnnq8mowqihVsPaObkAWt7Bh/UnlKtKGr/2+V3wCQGu/m72w/AMwQAGFrUYZ/b6X3Q+TE+v/vQ25/B3RNncr5696HxnOJ87rPf9ZjK42U2I6KlH37SrkM8HyK/UnOTv0b1U5vaB9h8SZvK95ubkfZedPTCj/PFd4+8PlfhPGYnDu7L5He3n4EYAgAMJf1mBnXpsmzrHWS6J1PHYVPvIFmAuFAshM5mbq66SyfQTM/H9lx0m7gJUySJM3Jf/HqNWpp7+dwiNgW9mCCfpDu4DCcG79x/nPp3t99BmgwAMJT0mxnUNQ3lOyNL3JNpUj13TpMA2Tse0s/PNzrETCHM078c+SX96+rf28eZhpXaBJpL52iXn4vzcHPHVJJUanGVcURbQslmbu+FHyfJ3DgVncds2vG5DrK3CJEhAMBQ0m9mUNcNOOmMrJmTB2KdoW3n5EwcT10AACAASURBVM4XEFH197+mhd8e6YgSnDlaouvLtQ5T9POmmjyyX5doK6olR1T2WEzVtudXniw5GbO7Uan1ZL0Z6y49+YdvOlKZ3PqzLKH/6L23vD53JYvf3X4HYggAMJT0YvPxwSWV4zIji/u8PFmij4/t9xqVYXtGaspIjggJTOkRk0Ajoo60pSl44SJgKtUaPXsZ7z6dC7ZSe66jNzh8KuCamxE9WW92pDJPHNzX9fEYl8qH6ZNj+9uRoHwQ0CfH9jtXk3Fk8bvb7wSRYy6RiGhqaipaWlrq4nIAACAb1OoqoviIB9/z+fT5MQ3mTFqRo7snos5Ulc86decLcwH9YtcY1debHcdXqjU2DRUQ0ffzH2ivvfTDT3T17kPWoC26WL87e5P9zp/OTljfGTeuY+94SNXf/9p4rAuVai1WGeeDqIpL0nOqHyq5+mENSQiCYDmKoinr9yCGAADDSlajBVxEiO37aYSYem7dnKyk55efkWi8KDcCFOcVJe86xEbP3TMR7+cRQooTM7aRHwJOTInzu2Iq7yfa8gcJ/5CIFrn0YvJdh7we3e+eTFa/W8OIqxiCgRoAMLRk1fCQMxo/WW9quy9307xdnizRwq0HsQ1YTlf5CEBb40VxXpPXSkQ8uHv+dvZ9VkyJlJzL7DETWcyBU4WHWt5/+cPDsSiTi1jxXYeMTxUixFBy4BkCAAALJiGg88x027xtmhWmlt5PL67QO459lkzr5jbzveMhlSdL1nu2zVez9fghMveNmjl5gMJ8p/EozLs3RSRKVt6vrrtYCGPrSOOnyaIKEdhBZAgAACyYytaJ4htRVtPqfdeTDwJt40UitxlipnVzkZsLpw5Zj5WvaYpamSJ5pony7WPUPJmnvydpeb+8bjWNaWo94ILtd0/+HkgOIkMAgIGlVx2mbdU06kaUdMq86/1w57c110vSvFBe967w9ZZRLIQdkRvdsQF1TrZXq9OIyPn92ZoW6ho8Njcjr0aANkFh+7kQbK6tB3THq89jFCq5+gFEhgAAA4lTpCAjxPk447K6ESWZMu9zP9z5TSZngUvzQl0VnDqX69mLDfbYWr1BAemjUi7f0T0n7r7E51mkJnXRL4GL4EjjFePe/+UPD7cN7FlXcmVVYDAMoJoMADCQpK0+SkoWG4juHJyQ8bkfFzOvfD7XlM7ExW/Y6fMrF+Jl66Yy9+fNTef1yfzqs7+w3ZX/4/J/cfp9cHl3SYfFEqWraOv173M3Kx77CVSTAQCGmu3qMJ22Qo2LAHACwed+TNEZos7oRqVai0V7nqw3aebaase5iPjSce5zbs2mkRa2Y21zt2zVaK6RN9X/I0+ZV7+rksYr1uvf534bV7PdwDMEABhI+q3DtCvcJsTNj/K9H+HL+dv8B3Tl7ARbncUNUW22/Hw2WazZ5diSoZqNyF6N5jsoNcmg36ReMaLe/z7327ia7QaRIQDAQJK2L41KFh2m1TJw3c9NkY9CmHe6H9e16qIc04sr3tVxe8dDbVRHCBEV3bsJiKgQ5mjdYCg2vb8TB/fR53cfxj5/ut5sm7NNUTvfzT9J5CSJV0yQ9e+zjW5XPA4aEEMAgIEkzcajGy1wfbnmbMa2pVxMP+c2IddxDUmM47pj1BSajLohXjh1iGaurXZ0pg7zQbusXqU8WYqN4YhoK+oU5oOO84h12Hw5d+4/1n6+SeSU2vHd/JNGTpKmUdP8Pus4X7lHX3z3iFpRRPkgoI/ee6tjRlmvxVe/AzEEABhYkmw8OmGgm51ligLYogamn5s2IZf7SRKx0B3DCaF8Tt+o8Bc7xzrGUMydNvfOuXP/cewaIi0njMl7x0OKIqKnDuMsTCLEJbXDPfcTB/e1zcuyYXpPIdR6oroZOcmqY/r5yr2OKForitr/FoIoa/E16EAMAQBGCh9h4BsdEJ+bfp52E0oSsfDxgfzDzrFYuk8VES827L1zTNdsRRGFuYB+fvF6DpotwmVK7bkIFN1zVyOC8viNMB9QmAs6fFW6yInvYNxeiI8vvnvEfi5Hh7ISX8MAxBAAYKTwEQYmU6sp5eLSjdlnE5I30dyr6IXrWk3r0aFGaZJWHdmuqTNvm847c/JALFVHRBQykSwd6nM/Pn+breJrtrYiV+M7xoy+MFu6VLy34nhIPz/faN93N/ti2SrvQByIIQDASMFt0qYydCKKbWymqIFLmbdPNEE3PFTG5vXgDM26rVEVVUm9M6YGhiZMIy+I4lPjbem6JNcS1NebscGsMrYKNfn+dQb0bpWy5xnBzFUsApTWAwBGDK78+eNj+9mybLXM+sl6kyjY2ox13zeVefuWbHPDQ/NBwA40VdGt5+Nj+53KwLmIU3E8NI7SKE+W6MzREvluv6YIV3myRNXf/5r+9KplwNNGkxZuPUg8hiXt+A2TUHSZNm86Rxo+eu8tr88BIkMAgBEjiWdHt7E1WxHt3jmm7cAsrpPEfK3CbZabUdTR1dgUbeJ+NvX2G9b2AOrYDcGT9WY72sGlfHQmakGYC4gC6kh7uc5vc62ms0Xg0o7fMKVDXUVONwzZwhdkqiYzkaYL96ACMQQAGDl8PTtZNqjzPZdLSTgnEJZ++Ilu/vXvHSkaVTyYxlGYyu9VdILO9HwWfntk6/969mpyFZOVaq3DY1SrN2LdtdWO3b4bPyem1l9uUJHpzSQT0Fb/pKxQn9cff3ckdbWlbCrvlsepH4AYAgAACz49amzRCN9+Nyb/kSxcVBrNlrZlgPiZLhKlboS+dltV/Jh6KomUIUfasSUXv16Lma2brYgufr3Gjt/whRvg+2S9SWEuiPVUyucCakk+s4iIri/XaOrtN1ILjKwGF5vSe8M8rgOeIQAAsOA6ZsHFD+Q7soHzHxFR+1ocJjGji9q4+lw4ikpHatO92p5V2rElXFRG/bxSrRm9TzbKkyXavTMeV2huRrR7x1jHe/sHzfdMI0F88B03wmGLdg7ruA5EhgAAwIKrz8glhcP1u5FHZajn1kUvTGXhLugiUWk3OrWAyfTcdOuXn1UWY0tsJPEf6dJp3FqfNpodnrJ3Z29qv1erN+j4/O1UnpysUrm2lgjDOq4DYggAMPTYRhO44JJOcd2Q1JlhSdIbtk3O5PfhxINtI9SZnmV0naS552Z7VmnHlhSZDtLFwuvolY//yOSj4fxBqnAwPd+0nhzu3LkgoHdnbzo3eUxrKh9UkCYDAGw7aVMVJsRoArGBidEE5yv3MruGIMnk8bkba97T1I/P3zamwLjSeaItMcCV4uvSWiIpVSoWaOG3R2jhN0ecU1UmbM/KlGIrT5bo29n36fv5D+jb2fe19zJ3+tCWeJMIcwHNnX49T81VvNp8NFFETqlP3T2p50qaMuPO3YoipxYOAjktS/S6N5Gu3US3/je7HSAyBADYVrIyfnK4jibIAt/hl5VqTRu9INJv1LrRGDKFMN+xYdlK51V82g6kbSppe1Zpx5a4HO9qZrdF4Z42mnTl7IR1rWr1mo6kqUr1fnWdyhvNFp1bXGnPyOOepXoutbqu2/+b3Q6CyKM999TUVLS0tNTF5QAARg0xJFOlVCzQt7Pvpz7/O4xPg4job1Kfnqzw6S7N3TuR/v5t3+9lHxjuPnWCTRVptnOoP+9WvxvXtZqeO1Gy31Wf3/skM83enb1pjB5y74RoK5qqViKKtGupWKD1lxvatGBW/5vNkiAIlqMomrJ9D5EhAMC2kmUPHx1ZjiZw2ZR8SrVN96iLJnHfD4i0m5BOTPiICtP9cj2KPv1yVRuR0JVkm55Vkn43vuLJFj1y6bmU1EfjGkVMGoWx+b9M7RV0LRnEv03nHORKM3iGAADbShKfjQ9ZjSbQlYJPL66k8h5x97h7R54Wbj2I+TF8npW8XiLq8EwR2T0kvmNDxPe5YaC+G6VLvxtuvUTu98n5j9TzyXclrEguo1A4TCNbZJKWzNv8SUR6YbNw64F3fynBIFeaQQwBALYV3747vlwqH6ZPju1vR4LyQUCfHNvv7RfSbUoREV29+zCxeZTbsJ69bGlFiM+zcukZZNpUfTdh2/V8N0rffje+4smG6Xw7x/L0p7MTrHnbFRcjeNLIqWqE1qGLjiaN7gx6pRnSZACAbSWtUdaFS+XDqc3S3CYR0VZFWNLoABFZUzFiMxepMJdn5bqpcUZtX4Ov6Xq2jVKXjvPtd5Nls0DT/ROl68Ts6/8xlcxXqjWrIb48WWJ9c7oonu25qwSvjhn0uWUQQwCAbSfNSIReYdok6o2mdWPiEPduM+mKzdz1WbluamL6vNwA8voyH+kypep018sHgTGVxHlizhwt0fXlmnO/G9v9uogHeT02XMWVLH72FEJ69nKjY16azf/D9f1pRZFzBVfJ0LPJ9XrcefvNMJ0UpMkAAMCBmZMHyGS5TjtSwba5+qaZXDwjYT6gn59vdKTkrt59aCzd5yI8XArPNiyUS8f96+rfjf1uiKijz82Jg/uM9yvEgy2l6TqSxOV9qL6reqMZa1hpS+GJdJcupeWa/vNJr+q8TMd/9Ubsd3/Q02IqiAwBAIAD5ckSLf3wE31+96H252kraUyRjSQbDzeRXa6yevZiI9bnyGSeNUV41HTnnkJIQUA0belrwz03sS6uSk6NJl1frtGZoyW6c/8x+xxd0lsu79H1fbgKKxf/z/TiSqJjxfFiPS7pOa5SsJup7O0GYggAABy5VD5MN//6d6fRC75w6Ym94yFdOHUoVQqOg5uVpUNMmne5nk85uEkEcsKFiybduf+4LZ64PjsuETgu3bcZRWz5vU4kuApkNVWpExquDSI50qaiByGVnQaIIQDASOP7F++FU4e8ukz7XPfyh4e7+te3es09zPwu1cjte3+uM7+ItkTgOc+oh8vnScUD1/+HaxppEn0uvi2RqhQCmxOOvt3NgR/wDAEARhbfXjpE7v1hklyXiKyl1vI5fGZD6a757OVGbH5XIczTx8f2s/fncl0fEVOeLNHe8VDzbf8eVGIoKechchEPPu/X1n5A59UJcwHtHQ/b5969Y4yam3YfURa/d4AHkSEAwMjiE8GQSZsy4K776Zer7fObSNKVWHfNZiuiveMhje8Yc4pGuV7XNyrjG20zVViJdckeIt9Im+v7tYk+F68Ol6rkhCPET3eAGAIA9BW9NGpyKQyfPitJ4DZR13LpJCKONSqvN6n6+1+7LNv5ur4pnSQGX/n73FBS2UOUJeJ3lDOby6LPJmDSeoF8GHYTdBoghgAAfUOlWqOZa6sdfVhmrrlFS5LAzS0LArIaWsV6k2wuJi9Jmoonk2E3i03X9bq6SjY59cPNCVOHtJregfx9n+iKC6b3qhvuKuPr4+mVF2gYJ81nCcQQAKBvuPj1WqwPS7MV0cWvk3V4tsHN0Yqi19EhbtNIs7nYGtslrXgyCRvTpusq6nyuK45P8ox8nm2lWos7vl9RZLxIJmzXNpXLuwy/VTFFxZKIbd0xROQ1QHcUgRgCAPQNupJ10+dp4Trzqug2jaR+IyJ7zyJbtObEwX3aY08c3Ge8pli3HK25+PUa/fx8o23iNQkP3+smfUauxwnhwmha9nP5+Itfr7V/v4qFsH0t7tqcUA1I3xPJBa6vj6+Q1B0z89UqUcAL/0GeNJ8lqCYDAIwsLl2aBeqmkXSAJtHWpsWNvHBJkdy5/9jrc0F5stS+Z7E5PllvOlUzJbmu6zNSK9Rc56LZmho+1bQNkK85c221Q2jXG01tqwH52r4VbklJMq1ea5LfjGLRVplBnjSfJYgMAQD6hiLT90b8xe6DS4pBl6LQdWUmim8aaTw43CZum+MlSCPE0nRF9r2u6RmJ96MOqDUNrPUdzmp6Fwu3HhhFgsqeV7+DvfL4JHnHvlEe9Cl6DSJDAIC+Ye70oVjfmzAX0NzpQ17n8ekfVJ4sdfT2mTt9yKk/jS6qFJA5VSUwVZO5kCY64bph6s7le11uJtaJg/va74coLnwiIqdZWKb7tW30vsJBjAbLqs+UrVdTknfsE+VxFd6jAsQQAKBvKE+WaOG3Rzo2moXfmgd96kiSYpDX4L7ZRbF/XV+uWRsgmjYtl2GiPoM3fa4tCPOB9ly+1+We5Z37j63RqejV903vgEtzBkR05qi9pN2HupROUwV0Fg031Xee5B1zTR7DfLyxpm2A7qgRRI5/iRARTU1NRUtLS11cDgAApIebSxUQ0ffzH6Q+v628ulQsGM20aY8X50hS1m+7NtHWBsqJ0Cx61XDvR8blGRARna/c05q6TfdAFG/jYMN1PTY4T5Tu/FlWk41qf6EgCJajKJqyfQ+eIQDA0NHtRnY2343LFHIi8prJpat8mjsdH+Bq20BdGhY2NyP69MtVml5ciZ0jiy7ItpldPl4Wzrzd3IyMVWvic/WZ/suRX9L15VrXPEG+o0p8nzV3zKiIn6RADAEAho5um1xdjbsmYSJ61riINl0Uo95obpVN0+uNzrUcW2yYlWqNFWTyaIukzfm4+9e9H2Ga9u3Vk8ZQzAmHqbff6FokpZcdp4E7EEMAgKEj60Z2KqbIhtzI0CZMXEUbV/mkRj98+vqI9bmQpDmfy/1z6ZzpxRVauPXA6d2Y3kVSgWGKyKT9/cH0+f4EYggA0BN6PRcpq0Z2OrgO0nvHQ7pwait1dXz+tlaYzN1Y64gOEdn9HK7RD58UjGuJvXoOl/dYqdbov325Qkr7og5RpRu/keTdcI0gw5zeBJ6GLH5/fOewgd4AMQQA6Dr9MhcpTddoGZcNjR2M2mhSpVpjPTi6mVyu0Q+fFIxvabnoDWR7jyKlpwoh23WTvBuueWUhzNHlD/+5IxIoum63oijR2Iyka9SRhe8qSzDAFWIIANADstpE0pKmWaGKbUMzCRjuvjmxceZoiRb//VEsVaZGP7gUzImD+5wF1t7xkJ43N7VpHJf3aGtmyKWukrwbLrr1xu6dbSEkPw+dD0qcx0UIZPn70y/0yx8q2w36DAEAug4nClzmgmWJqZGdSyM8H5I0/OPExp37j2nhN0dorzR4tFgIY6Xjur4+/2n/Hrp692Gsr82Jg/u0fWwunDrE9llyeY82YcA9lyRNBm3ixJQKFClL1+actjVm/fvTK9L05BomEBkCAHSdvKZ8W3zeS0yRE5+/jl1Hfchl2zJJoiOuqRX5e1y1mBBYlz88HBvcKozLup463Hsk2uqfY0vpFQshew9JjMW2tKBNmOnGrpgiltwa3/nfCzS9uNIxUmR6cYXOLa4kTsn1imGMdiUBkSEAQNfhNlDX8RO+cH+l+3REbjRb9OmXq7Fz+Iz6uHDKbbSHOG+OEYdJq6LmbqyxPxMCSx3carof0/uSI05qx2Mi+1iVJGMubF2akz43Tgjo1njmaIn+x3/8pB0pQmSPNmXJ+co9+tVnf6F3Zm/Srz77C52v2KsFezV4tt9BZAgA0HVKzF/wpS78B9fmgdBFWKYdeu1ML67Q0g8/scJJF01wrRwSa9aJjTRl19wEdqLXm52Pn4t7j/JxIqVnahDJRdZ8jcW258tV/RFtPdddYc4rcieuKa/x+PxtazftXvjj1E7crShq//tS+TB7HEr9t4AYAgB0nV7+BzeJWdvWEZlo6y/9q3cfshufKZpg2wRN3hbbjK2kiGfvkyaZOXmgIx3EHWfr05OlYdd0LVks6arJiCj176VrOqnbaacvvnvEfm4SQyj13wJiCADQdXr5H9wkHghTBEEmIt43kzStUKnWjEKMGzfhwt7xUBv52L0j3372Jt+NLoJji4LYnkOvKgvVtf/p7AR7/jS/ly5CWnyvm6RJRfdbqf92AM8QAKAnlCeTT/r2IYkHQvWCmIzdrShKPDFexaULdJqIwoVTh2L+nTAf0H//r68jBZzvRpjKVW9UsRASh0ujw14YdnW+rnOLKzRx8ZuYd0f+vRTtA3wqwnTPT6UXaSfT7+wgVbdtF4gMAQAGFl3kImlKTq3C4tJBIsWSRZTLpQt0cTz0aoqnfvfs//kW3bn/2Hlw65vS/ekiOLvCHBXCvH7dDsWBLo0h0zQBrFRr9OmXq9qISL3RZFNySdN3uud34uA+4zPvBh+995a2EzfR6PYO8iGIPKo5pqamoqWlpS4uBwAA7FSqNZq7sRYzCBfCPF3+cCvqkUasbG2Mf6VGc1N7fpdzuWzo787etKadCmGOiIKYuNOtQ93QfdfssraAiK6cnWAFRxAQXfkdn5KyrTHNPeiO1VEqFmKtA47P32ZN/ro2A/3I+co9+uK7R2xqbJDuJSuCIFiOomjK9j1EhgAAA4VpwxPekzRpuNfn7xRC8twx3zVyf5m7+E3UdWx9pvfYZOnHMUVwypMltgIviohmvlolIn0UwuYf8x02K59n/eWG07w1XUpuGPrtXCofpkvlw6yQHaR76TXwDAEABgpbaintf/C584/vGHMWFK5dfV38JhxpN3Rbx+Q0PXyam5Gxg7HJP+Z6DzpfkM4srkO39mHqtzNM99IrIIYAAH2HaaO2iZ20/8HPIkLARXvUc8jGbR2FMN8xgkNGNwKiaPiujEvjSFsTRJsHq1ZvJBpN4bqRu/itdHD+MZv4GySG6V56BdJkAIC+wpZiMqWWsvgPvs/kdx2Vao0CIm2agjvH+suN2GcBbfUYmnr7DecRImEuoDAfdAxK1T0T11SUrYePzrclIwstcYzNS+VqgHcVp2E+oN07xuhpo2n0j/Vzvx1fQ3k/30u/AjEEAOgrbBs11xPIx9PDUanWtMLER2Qt3HrAGo/Vc5j8TxFt9RgSDfNcqr2amxEVCyHt3jlm3ASz8sfMnT5EM9dWjVPqiTpThDYvletGzolWl/vPqgN2t5DXt6cQ0rOXG+1nrHtmpvuRfz69uAJhxAAxBADoK2wbdbf+6uWEiTpKwga3/ojihmJX/5PPCJF6o0m7d47RFUOTwSLTjNE3xSjOzw2klfmx3sgkIiWYOXmAZr5apebmayEm5p+Zjs26A3bWqOuzDZM13Q8RxaJ3/Xa//QLEEACgr3BJU9k2yyR9ajhhsnunu3HatP694yEdn7/dsaY0/idO0BCZN7xKtUY/P49Hv8K8vWGiDt2sLu79ZV6xpfY1cuhz1KsO2Elx9UKJZ8bdz8Wv1+h5c9NYddkP99svwEANAOgr0po/fabKy9g2alv1lWn9YT6gn59vxLs5M4Znoq0ox/rLDfZ6thZxuuo1oq3NU46mCHZ7VMuZML2/LKucFm49iKXnmi1zFRtRf5bQy79bLqM9iF4/M27dT9abXa26HDYQGQIAdI0kEZq0abCkf/nbZnTZUivyvRbHQ9o5lmubdp+92IilOxrNFu0c03dzLoQ52tiM2pEfMU5i7sZaOw301GBcFviU37ucT2B6r7b359Md3HSdpKImrUE+a1wbRcrIz8wUITSBMvtOIIYAAF0hjTcjjZE16SZpqmKyCSz1Xp+sN6kQ5tu+nXdnb2qv+bTRpCtnJ7TmaN2GLY+TcGnYyPXTSVst52KCTluxpbvO9OIKLf3wE10qH058H0nHtaQZEWLCJS0W5gL6xa4xqq93VsSZUp67d4yxlX4os48DMQQA6Arb5c1IukmaNmrOrPxjvcHOwpLv1dbN2dUcLZ+Xq6oT6Da8SrVGz174+YVcujz7vFdXoav7/YmI6OrdhzT19hupZtCJ87sKm26ark0iPSAyrs+U8pw7fahrVZfDCMQQAKAruERouvHXdtJNkojfqDkxs6cQ0md/vsfOgqq9Eku+a7JFfX6sN2KbenE8pCgitp+OMR3D+I90IsC0piwxVeWJkStEydKpvpHHbgp77l27zBEzpTzRa8gPiCEAQFewRWi69de2vAnU6g3KB0GHmTjJuTkxEwRkTXF89ud7dPnDw3T5w8PsxqSKwhMH99H15Rp7bvEMfTZ1UzpGjM/wLf1X15SluDUJQlPLgW7QTdN1GvFu+99Yv/RNGgRQTQYA6Aq2qjDX+V1JEM0ZC2G+HbVxrSrjzqcbTVF3MK7KEQTdPC5d9dv15RqdOVrSjuJI6vewbdy6TdV1s5c7YvtW8XFVejMnD7CV8r02/3Zz1pdt7IkJjN3IDkSGAABdwRam73aJc9apDd1f2ZzRWcX0HW6dd+4/purvf+0dbeG+b0u9Ba+Olc/t0+U5yfO2RQeXfviJrt592JHFs232/ZZ6dSFpBAepsOyAGAIAdA3Tf+S7XeKcRGz5bqQ2E7MgH3TGOOTrcO2CkqSCTOLCtlbhxZGvdeLgPq0Y0XV5NpnMOWwC6lL5ME29/YbzO3FNvQ7TrC+kwrIBYggAsC10+69tX7GVxMOkbpKcsJEN1q59ZZI2IuTEhWw4tnlxxDqvL9c67kkMj/UxmZvuw2UdaX1RanQqqVcNomO4gWcIALAtpPFKuODrp0jqYZK9QCVm45c/d+ork3A0hstct29n32fX+aZlnRER3fzr37XH+j7vSrWWuSfIJRrYTa8aGFwQGQIAbBvd/GvbN7XhMo7Ddi6XaJeLJyrpaAzX6EyadT5Zb8a8RUT+z3vh1gNtJC14tb4kuNx/P47jANsPxBAAYGjxEVtpx3HI/79JELh0jvYZjSHgGirqojNp18mZon2et6mPkO0cnDB1EXlpvGrd6kINth+IIQDAyFOp1mj9JS8kTJPBdRGStKZrnzRRpVqji1+vaedTmboNu6zzHGOKrtUbdHz+dlf6CHEpPIGLMDUJljTjOLrVhRpsPxBDAIC+wvbXd9Z/nXOG5mIhbFdNcZVSXMrIhPju//Xnv9J6czP28zDn7heymbGfrDcTN5ssT5Zo7sYaO9/KJAaySinqmLuxZjRJ20Re0sqw7RovA3oDxBAAoG+w/fWd5V/nYsPmUkG7d7727SRJGdmINPbhQpijyx/+c/tebRu2ixnb9ozk55APAmpFEZVeXe9fjvwyVlovoxMD3Dta+uEnunP/ccf9mLpy66hUa6w48/H8JPGq9bPXCOm79AQRrrNn6gAAIABJREFUM1NHx9TUVLS0tNTF5QAA+pFe/cf2+Pxt45wm289d105E1lRVQETfz3/QPgeXMhLX93k2tvtwiVYREb07e5MVKty5ZUyRpTAXEAVEzZb5CvJzMt1bQBTrV+RbPcidm8htllcakvzu9QLdO0zybIeVIAiWoyiasn0PpfUAACO6cRHqmAVupIIvtr++ff8659auS7WoyL6d8mSJioX4aAyirU3edwSF7T64iE+90ew4v4+3SHdN27wymxDSrcFkjJZJUs5uisJ0ewRFv46+QKuAbIAYAgAYsf3H1kUsuWKbAeU7I4pbO5dqEeg2ubnTh2KboRrtEOeXNyKdULTdh2nTl8/vsxHrrpk2xaN7TmkFmgnu3HvHw65HQbrdFysp/Zy+GyQghgAARpJEMZL+ZWr769v3r/MkGwK3yek2Q9soDU4onji4bysNJSEbp22CQm6iyEWsZLhnlKS5YT4IjGJA946yaq7Ivf8Lpw55nScp3LDd7aJSrVEu0D/dXg+zHXRgoAYAGLH1ZcnyL1NbpY9vJRC39r3jIT1vbnr7LFTjLecjEc+GE4r/uvr3mEJobkZ0bnGFFm49oBMH99H15RqbwpI3urnTh2KekTAf0O4dY/S00TQ+I1OZv84z5PqMxL2Ld6S7nyQppn6eEdZrhNBuaXy//ZC+GzRgoAYAGLEZNPvVWEpkXjuR26ZqMo/bno2PwVmmEObpzNES3fzr32P9g3SCJI3B3VRNRhR/RrrPXK6Fiqds4f53lw8C+uPvjuDZvsLVQA0xBACwkkYQbDdphYLt3kznN1U/2ZAry3ymtndTcKR9193oETWqAosT2mp136gDMQQA6BnDuilNXPxGa7Z2FSqceNgV5rQdo2V8NzXdtQIi+vjYfrpUPux8HnEu3X2liQJmLZr7XYR3m8k/fKP9HeqHiGw/4SqG4BkCAKSmmwNXs0IdW6H27NF939Tgz6UBJOdxIbL3ORK+IFehyU2Zv3r3IU29/YZzClBdm3xfXJTLJfrF+afmbqwlEtJZdIQeVBFfqdbo5+fx8TFh3r17OegEYggAMPRUqjWaubbaYQauN5o089UqEek7M5uq4d4sFpw3Y5NQFF4dXUPCmZMHvDpum/r76AQCd+5dYY69L+EpUslLFU2cwODWV28026LTp6N4WuP+IM8aW7j1gJqb8fewe8dY36+9X0FpPQCgr8iqgaPMwq0H2gaCzc2IFT22Bn8um7HpXkSZ9t/mP6ArZye0/Wt82haYSqldGy42mi02ffdjvaEVQkTU/tzUc8q11Nu1LYNvzymVQW5WyEXinlr6ZwEeRIYAAH1Dt/5aNwkb7memsnwhVLiy+kq1FhtyaroXLnrkE/2YOXmAphdXtKbaLBouinOYJs2bBIapjD/J2pIOerVdw3Rtl7Rat1Nv5yv32J+ht1ByEBkCAKQiy0hOt/5aN20S3M9sDf64n584uI8++/M9rd/I9158oh/lyRJ9fGx/rMFhFg0XxTlsTS9NAkPXtHLvuL5hpMva0naENj1b3e+061iarLqx66hUa3T17kPtzwLq/kiSYQZiCACQmKz/49+t0QIzJw9QmI936pW7PqvYNlv550RbvplGs0VffPfIGP2o1RvOwtG34/al8mE25eZybh3yOWzPxCYw1IjJhVPxESc+0Z00HaFtYlb9nb74dXyenSpuu516W7j1gO1bFVH/e536GaTJAACJyaKiR8bW7TopYi0+1WTiOJduy3K6hvPVyMibrHwe7vw+aRfXyr7yZImWfviJvvjuEbvmgChWpm06P5e6EgJDTX9e/vAwXf7w8LZUdHHPlvud5gSuLNS7PSfMdJ4SUmSpgBgCACQmi//4yxGD4nhIYS7oqJTJarQAt4mrEYsTB/fRnfuPnTdn0+R3Gy7CUV63WOv04komHaGvL9eM4s1XhPoKjIVbD7Z1xpfud2J6ccXrHHLUi3uSWXl5uD8WkCJLD8QQACAxaSM5qmH6yXqTwnxAxULIztXK0qCqltzX6g36XPJkuERv0v7VL3oWuRhz1ejKzLVVooja4lG3Xm7cxrMXG0YRx4lQF/GoRpM4geF6772E+50uFkJ6sRGfZ6dGvVSynBOmi7yJxppIkaUDniEAQGJ8PS0quohBsxXR7p1jWh+IzqM0vbhirLAxcfHrNW3JvYzN88EJP3m6+59e+Xh07CmETr4r7lmp/Wbk9crPi+h1Cq9Wb7ANJYl4r5Hu+X9+96F17dwzKo6b770bbRZscL/Tc6cPaf1Sd+4/ZoWQr6nbhs6zdeXshHeHcRAHkSEAQGLSThH3TbMl6bJswjYSw7YeIt4no9sEdd8LAnLyXfnMOBPrTZLCM41zcDmfbu3cM4oi/t6J+E7Y3YyC2H6nXdNqOr9VVutDFCh7IIYAAKlI8x9n3zSbb5flrDCl/VwFIfc9UwpJhuv+bFqvbwrPFtVzPZ8u/XXmaCmWTjPdO+cz+vTL1Q7PVDfeuc/vdLdM/6C3QAwBALYN38Z53MZDlMy7UyyExnSRbT0Cnwou9Xum5o0yrkJIXu8ey/3t3pGn4vgO56ie6fnLiNSfHNW5vlyLRctM9869TznV1w/jM1x+h/vNFwXiwDMEANg2VA9EsfD/t3c2MVJdWZ4/LyIfSSTVQ0KLkcopYzNegIRog8iZYooVXpgFbStlqhqVqO3soa2UskZWASVrnBIqUfve2iqlbUohKFrCC1i5m+qBysxCKcGihjZW2KNBA4nGZACRmW8WyY28ceOee899H/H5/0mtLicR791734t3zzsf/xPT1rhEZ+cWrDki08f3tokKKtK8iZ9/fz/FpdYjlqKNcaQR8lOE5LrYclTickTPXqy2fJ/LOdoxFlt1f6rzNXr2sr2Zp87L1XWaPr5XrNMj0Sbyhf58x1OGhOR69kL7DJ/2UtFCjCAfokT4tkFENDk5mdy+fbvA4QAAhhWzWorInnvzUfUufXbrYVtj07SJqnm/tUvnwY1hfCymH56vtskLnDw8QZfv1LzH1avHJJSjiNaTRDR3s83IjrGYTvzdj63hL9vOEhHRg9kT7Nz1MdjW0YbtmL3E0dkbbAuTInKKQCtRFN1JkmTS+zkYQwCAXiBk0yjCgLGVn4cc96PqXaeAoXTzO3jhK2toS43HNW+pAcFhGlcSI81m5OVlAOjnp4jItrS9blTsmbkmNgxB/kiNIeQMAQCadDO3IaSyLM+KGtOASJOT8lH1bos+kQ1JTlN1vsbm+Kj+XkUJQBK1VoLZNKBcn1fjV0ZlRNTmvQvV21HzNfWgFK52Kja6cX8jwbo/QM4QAICIup/bENKUNE9cBoQ0J+X3f/rW+xnfPKrzNfrw88XU3yfKp+1DaFm+2uhNTaOEqJnflVVv5+L1+1Y9qB9tHREfs1v3d1YtLtAZYAwBAIio+CaTPrq1afgMCImB4av08s1DbdSu40jWwWUwTYxX6JdHdjcTfcuRPRU9tCw/ok2Pi00DSoWxQowWM/mcG8uyUCeKqHv3ty/BGvQGCJMBAIio+CaTPrIKOKbFVy4u8ci4NIAkuUc+L8xYXBKtQ4gAJJforYwuaRm90njKq0+dTWhxfCy2hulCvIbdvL8hlNj7wBgCABBRb+Q2dGPTsBkQCqln6hc/ed2aM/TLI7tFrRJ8G/Kop5xdYRqU2ysxRdGGSvLF6/etSsqc8elaF9v4Q+4fLneH896MjpSoEpfFelQ2euH+Br0LjCEAABGFCyBmpVeE6HSjIKSazBz/0bd20q3/9YTWkoTKUUS/+MnrNPnGTjo6e8M7R58XJiQcpCcd+9pZuIxPm2HFJXeruUnuH9e4OKPwab1Bl04dzHS/dPr+Bv0FSusBAE06ZaCk0eLpJSTjD5mjryQ+Tfl43vo23PEiIrp06mDTAPPdPy7pACJ7D7a8yueLur9t+kvn3tvfF/fyoIPSegBAMJ0KU7mSWdOev5OeJsn4Q+ao/lvfUBWVuEzH9u1iPUzcvPPOkXH1hZN4mtRYXdIBl04dLNR7U8T9XZ2v0fQXiy36S09WGjT95WLznKD3gTEEAOg4eW/UkpBQVnSjg/On6+MPnaMe3tKNmzf/ttKiuK3PjYjv7C7JkQkxILnjcW1CbLgqt14br+SeRK8LYarQpSSHK4SL1++3GEKKxlpSaPNgkC8whgDoAtX5Gl24utSskBmvxHT+/eFxq+edzFqEp0lHquysjz/tHHXvRXW+Zm1toZeEc/P25ciEGpBZcm4kLULUcfLy3phCmGtJ0vzvPA0ilwHfqUpMkB3oDAHQYZSarl4qvFxv0D9+vjA0zRvz1hTiNp3acl3cMNWFRIDQHH8ec7x4/b7TC+XyPvn0bXy6O6beDxGl0ssxxRht7BiLc38R4IQwJQKZIbiMW1Sq9Q/wDAHQYTg13fWE6IylBHoQyTscwnlhItpMyM0SOnO94UdE1vHnMUfXeV9zJByrfzO9TBev36ezcwvO6rXvluus1+iTDw4EJzL7DMlKXKZz7+0POqYETvfJJ5AZyvTxvW05Q0REcTmsVQjoLjCGAOgwPtd5EfkuvUieyay2EI7ZG4sofeiMMx5Ux3eONHPU83hKjJhjRJthpbTl7Lb1IdqYa55hR5dHKLQZbgicECanvJ0WW/I7qsn6DxhDAHQYibKvHq7oBS2eXsfmhXF5Pji4hGJOgFDa1FWaqMw1jdWJiOj0kd0t3/cdm2uVwTVTPTu3YF0fFXYMuR9dRkmR3eY5Icxf/OT13M8Fhen+B8YQAB1m+vheOsNsNjpqgy2yQmqQMDckTheHy+OQJBS7PDac5yQkUZkLKSkPlMQAuf3N4zbjyFUWPzFesSpB5xV27FS4ykQlSRddTQYGA4guAtAF3py55v0M90adlwDdoBMq7BgiUrhn5hqb2GzmEOVx3IiIHsyeaPu7pMqtEpdpdKTECh3a7iXbcbmwmu9+zFv8EYAQILoIQA8z4QmVmX2YdFCuu4kr/BSawByiC+QKwyUkazFRW67TnplrLeMKLceXVLnVG2u0NQ7r7ZVX2JGoN9pg9ErrF9C7wBgCoAu4En5VUikXqkC57gZZe2+ZhBgikiamKmwWYjiF9Pfy6fboLK/4e3vZDAbdcxMadlRwRqk6piSPKosh0wlBTtD/wBgCoAtIvRbdfqPuZfIWWgzxYJjXz6UFZGsxYaLGrYwPn9Ey/eWiVZ6BQ6k7m8dRxsj4WEw/PF9tlofbDIYsHh7buSUGSqghYzOcihbkBIMBjCEAuoTPa+EymHTPgLTL+qCRd0uP0LCafv1cXpMQw8k8ro0LV5eCDCFJqb0uAKowDQb1/3Xl9NGRdLq9UgMlxJDhDCeEm4GEgTOGEBsG/YjLuDGTTLnya/Xwv/3NY7p571FbSGLQfhd5t/QgSl8i7fOaSA0nhes5ZjNcFBPjFTq2b1fb9ZdWrZnYDIbnjfXm/16uN1KFnFx5VG/OXGOLB1zf5wwn7lgINwOdgTKG8ooNw6ACncRn3BCR921ZUW+stTX1nP5ikSiipjdhUHImeiExVxHiVeLyjZ69WG22C0n7HJNWZ0m9IqbBkFfIyae15Su7txky3Jxsx0K4GZj0nTHkMlTy+KEi2Q50Gp9xY96/vo3MfPTbOmoPQs5E3i09OLhnji/h2DduPdxEtOll2RqXnM+x8UpsLZMfr8TiOUmEP20GQ5rQpG2dJAnoIeMiks2JaPiaIgMZPW8M6T+k7ZWYnr1cZd9wfc0abQ9K84e68nIVyXago/iMG/PfpQ/9rOftB4pW/rW9HJ2dW6Avbj+kPz98GvTSZD5rbM6PemPNm+Ny/v391l5YUbRxDsl6+IyRiIhOHm5f29DQpKvH2ScfHPDmUZljSuNxM9k2OoJnOWijp40h84dkexuqN9bow88X6ezcAtvHh+hVuODLRTp/ZYme1hv02qvY+uU7tZYfKscgbBygN/EZN+ZG43roc8J4kuMOE9JQONfG4uu/Pm77rOulyWYUhKI3XyVq7YVFtJFLJPVi614121gSIrp571Hb30NDky5v/dcz73jzqBQSgUbfnBR4lgMb6UoBCkSVe+6ZuUYffr4ocqOuJQkl5I8zN9YSWq43mtoen956KHbTDvPGAYpl+vheqsRl67/ZNpqpQxP0yQcHaOLVPakaT06MV+j0kd0Ul1obUZZoo4O277jDgjJMaq88EspbofJ1dEI3Tu7z0oRloo0wjnk/RER0bN+u5n9PHZqgbaPt77J6Tzui1ufp0dkbLXOcOjRBX8+8Q1zbUttc9Hsvoo17jlP0JuKNPvPvob8BDjWnCcfzGs9yYKOnPEOSJoXdQiU3wr0K8sZ8o5WUynPhoY+qd9vCJ+VyRKf+8+veCqNeJs+ihpDcwtCQJLfRSo2qSlym8+/vp9vfPG5JhE+I6PKdGk2+sbM5Rl/+jjT/kZtjKYraFLLVd6VrL+0cn+Y34MK13sP6EgDc9JQxFPL21GnSlpACICGP3JfqfI0+s3TpbqwldPPeo6YA3XfL9ab3oB/u5byLGly5hSaqgzvXL8zW8d0GZ3DsGItpbMuItVGqeU7TYPPl70iNPi7s6qtqlBDSpDXP/C/XevfDPQ86T08ZQ70ey0UiNehlbBuowhSgy6tKshMyFGmrRDntpu1MNVZE7QnIU4cm2rw0RBuGz8nDE2JvG5drc+49e1WTxOuz8nK17d91g0xa+WVW5dlyL+uNNTozt0AXr98PusZcDz49jFXEPeRabwBs9JQxlKZKplyKaM1SOlwUeVTxAFAErpeJchTlWiVZna+1lYYXJUORtpyb024y86cUCZF1PT6eOkCTb+zMtGHbkp63xnzKpsvrw3WqN0vGQyq/dK/Mnplr7LhCr7Ev4drl9SNKL5vQKdkFMDj0lDE0fXxvcM+dThpCRBvlqwD0ItzmFxEfrnDJTnBwmzFRMd7TNErTrpC76/nCGVh5hXBerG6qN3PVX9X5Gj17wXt9uLkt1xst4c+0opS+l9KQa+wzSjiv34WrS/S8sZ7Jk1m07AIYLHqqmmzq0ARt22K3z3rFBkkSslad9AOuyhLQ/9gqciIiOn1kt7O6xlVNZcOX25d3uNs2L9+mnnYMZkuMPH8vrnCffs5f/eFuWxhvx1jcrNpyzU2/lqrySxdjdHmjFK7KLkXI+qoKrwezJ1rK6V3HebLS8K4VAHnSU8YQkV1LiEiundIJ+vEHGVJODPoTW9nzpVMH6eOpA94NLmSj8W2EeZcuh5ZzS8awY6y9fN0Wvgn9vbgMKEm4jzM0x7aMtIS/XJjX0uaNcs3DlG6wkZfRGHqv9HpeKehfeipMRsSXYvYS/Zg3lFdPIdDbcKEBiSBdSL8q7hhF6ReFhjxcwpR6Im2erX18VW+ScJ/EYJIoLavP++bBJS+r/7OFRKU5P1lyikZHStYXY2gEgaLoGWNI/Sh73RAiatfI6AfSJKGCwUJtcJKu6S64zbiXej5JdWu4sUp/L7oxwVVhKcNDksMjMZhUhdvv//Qt+7xUn3fNQ5q8vL0S09a4RMsrjdz7QXI5RUSUKt8JgLT0hDHkSsjsRTplsOVZcpomCRUMJlm7vfdLpU6WBFrJ70UqEquOI1k3ybWpztfo8p0ae764HDU/75oHZ8icv7JEL1Y3k5eX6w2qxGW6dOpg23rm8ZLluk69fo+BwaEnjKFeFlu04Yql50XeJadZN0AwOORhzPRjpU7Iy4Xk9yJ9bumeZN+6Sa6N97yajeSax9m5BevXuR6QIQrdrpcs/TqMj8WUJNTsF5lW6RqArPSEMeTTR+m10JneI6go8i457Ze3+WGGEwks4jp1a6PphEgjd97Q3JbRkVLz8zvG4jaBRKn3Qz2/pHP3XRvfeRvrSdNwMX/3yvjwNbaWnjf0Jcu8Dp3QqfLRrXsS9BY9UU3GvUVMjFdovQBDSC81TcO1v3yf00h4iig5dZW4gu6iVy8RtbdCGISqv25WNErK2s1x6h6S5431ts9JQ8wTmlCiPvezcwv0ZkEVWPrzQ/3uL506SM8b681m1TZDqBKXaceY/fnIiTWGVPr5vFqdLp9HlS1Q9IQxZCv7jUsRrbxcLaSk/u/f/nGm7+tvM0WBktPhwrVJDIq+SohBkjchuS3ScUr0eFxCierZFroBTx/f69Vdsz0/uHusHEVNQ+bkYbsRY8tbUuX0qkWH5CVL8pzq5LOsm/ck6C16whgy3y7GKzFRVIzREUVEN+89yv24ecMJzXFeLSRC9ze+DWAQjN1uVjRyvw/b3yXjVKGVemOtmRM0MV6hX74SuDS9JL45hmzAU4cmnC+JXJiKG8N6ktCD2RM0fXwvXb5Ta3vujlfiFm9PFm+K5DnVyWcZqmyBoidyhoha4+RHZ2+w4ovjlZiiiGh5pUHjY3GwwZQk2XWCsobZJKDkdLjwtUAYBGO3mxWNIbktvnHaqsjUsTiviKTvoq/Xmv4s2ME8+6JoQ2X6rKWpapou90RE20ZHWuaVpZzep5EU8izLI9cHVbZA0ROeIRPuoRAR0cK5d2n+1+/Sg9kTNMa07nAxXolZF7N6o3MRlyI6/35nOh/bcnzSqPGC3scVchkUYzdNW428CPnd+MaZJrQiCamVosiq4mzzxPzwfLWt4WxcjmgkiujJSsPqsfHNS+ol4Yw6yUumeR12jMXNZ3LIsyyvXJ9u3pOgt+gZz5CO1FoPdWVW4jJFkb21R0TUjO1zP+qiKntCQcnp4CEVCexnul3RyP1ubB6GTz44wI4zTWjFvL4RtT+HzKR59T2b8dVYT2i8EtO20ZHmGJ+9WG3zqOseG9/6S5+7XIWvVIw2j+dXXor63b4nQe/Qk8aQ1KXN/XgrcYl2bhu1biqctkZCmz8M27lD3ljwwwJpGAYjt4g5ZvnNcSX3n3xwgL6eecf6HS487wut6HMPUa7mjKyn9QYtnHu3+d97Zq5ZP2dWlmXRViLixSVDJQSykGeuzzD87oCfnjSGpNY69+P1lXbaDCglpJjlTSFrnx4AQBhZf3OhHobqfI1+eL7a9ndd9dn2Ha7/F5HfiJF6bLIKIL72qprs5r1HzmffBHMeXUKg6Gcgcn1A3vSkMUQks9bTGC6St5+0bwpohgpAZ8n6mwv1MFy8fp8a6+2ekW1bRljjyWcc+DZ2qccmqwBibblOl+/UvF5w7jzH9u2iDz9fdHq58gKK+iBvetYYkhJquBQZI0aZJgDtFBU2qc7X2Pw+6W8u1MPgClnZkBhrLuPi6OyNpnL06EiJntYbtP1VRa1ZMRb6bEtrSNpUrZ831ujTWw/Z7+T9DESuD8ibvjeG0lBUjBiuWwBaKSpsoo7LUYoiqs7XUpV6uzwMeRlPZh4PUevGfmzfLrp8p9bStiIuR1SJSy1J0uZ6hiSJZ3l5U+fZvL7tCt06RTwDkesD8qQnS+v7FZRpAtBKUQq/vrYOa0kiKrUOlaoI/Y1LxR5NGY2b9x61V5CtJbRiMTp868mVoY8HtN3gkDSrxTMQ9AND6RkqCrhuAWilqNCx5PvSXJUQD0PobzxNHo9L3oPDtR6cQTo6UqJKXM6Ud+O7DuUoKkQHDVW7IG9gDOUMXLcAbFJU6Fii6ExUTL5eUcaTGVIMwbWerjynS6cO0oWrS02pgNGRsGCB6zqESJKEgKpdUAQIkwEACqOo0LFE0ZmoN/L19BCYEna1KU1LQk42fOvpC9U910Jvy/VGcNPYuGQXWzx5uJgXQzRXBUUAYwgAUBhFtY+xtXUwN+Vey1XxtZBwebFs8yNqb6Jqw2WQZjUspg5N0I+22gMMRTXERtUuKAKEyQAAhVJU6Ng8bhF5JHke01fKzoWcJsYr9PXMO6nH4grVcYr8IYbFskWNO/QYIaBqFxQBjCEAwECQt9GVNjeFM1p8Hg1fsnWW+dm+W52vWVuBEIUZFp02TiC4CIoAYTIAALCQJoTkCoVJyuz1BOZStHm+0G7sPtQ4bYZQqGHRaUmRokKvIVTna3R09oY19wv0J/AMAQCAhdDclOp8jW1Hof5udqtXRoOtkkx1/SiiWopL1k5TCt8NSZFuVu2imm0wgTEEABg6JPk3IeEfl6eFaLOje0LUNIgmtPMenb3hrCTzaSaF5hNxBt16kgRv6FnyqvpRLwg9KAcThMkAAD1PnmEJX1WXwhb+iV59PktZvG4IqTL7LJpJ0vnoSJWxfaQ5dx7fDSXP+wfVbIMJjCEAQE+T96YpzQXSc1OIqCXEFVIWb0N9X81JAmeopMltyivPJ0tpfqf0giT3T4ixlJchCXoLGEMAgJ4m700z5M1eCSZOjFfajBZ9DKEbYTmKggQWXYZKGk+FLwlZahyEnNs8JucNy9vD4rt/bMbS2bkFepOZO3pQDibIGQIA9DR5hyXSlIJz56ot12nPzDUafyWK2FjfNJkqcZlOHp5o6T6v4HKLiDY8UNsrMUXRhoaPL5cmbWm7q8O9NEFYem7bMc1kcum4Q/HdPzZjyfQAEm3OHT0oBxMYQwCAniZvHZs0OjWuHlwJET1ZaVBcjmi8EtPTeqsBM/nGzpb+Xy6UwKKU6nyNnr1Ybft7Fk9FSIKwdC05g4OrrssT3/3jM6ptc3dVs/VjUjhAmAwA0OPkHZZIo1Mj6YXWWEto2+gIPZg9QV/PvNPiSRjb4n/vDJ2T8rYs11uNrKz6RBJPnAp5nZ1boK1xicYrsXMtuWOqZPIQvaDQZGjf/SMxqlXSvO+cnUwKB/kCzxAAoKcpIiwRqlNjjoELcqUJ6UVEqebEVbBl1SfyeVLMkNeTlQZV4jJdOnUwOJSXxhMWqvHju39s3i0TVUXoOyfnVTt/ZYkuXr9PteU6lV+pfk/Aa9RTwBgCAPQ8eYrsZenxpT7HJQC7Ko0kxkDI2CQ5U6H6N5KwWxqdnbxaaKTV+HHdP7qxZMtlsuU2cefkrslyvdH04Kl8MYg19hYwhgAAPUvW/Avz+8f27WpJaE67IYVu7pJAWLE1AAAgAElEQVTPh3o9XHlMOtJEc5sKNhHRjrGYzr23vzmGtNVrRHLvXtr+bmnRjSXz3CFVb9JrooBYY+8AYwgA0JNkbXtg+/6ntx62fU66IZmb5MnDE3Tz3iPR5i4xBkK9Hsf27aLPbj306hRJE825sNvYlpGW8+ddvWbiuu6daAprjpPzAm6vxG1/k4TcTCDW2BvAGAIA9CRZ2x6EqEL7NiTbBn35Ts2qzcMZOz5jIFSz5/KdWpshVIo2c4aIwkJRLvkAfV6mdy30PD5c170bHeunj++l6S8WW2QTiIievVyl6nyt7RqrOaj1Wnm56qwkhFhjbwBjCADQk2QNiYS8cfs2JG6DPjO3QBev388UflMeJ87DU4oi2jNzrcXA4gy97ZWYxraMpAorcl4XM3n48p1ai1dsfCymJCE6+2otsiYFu657SLitOl9rkTQYr8R0/v39wWObOjRhlUZorCVWw9w0ernwIxHEGnsJGEMAgJ4ka0gkJH/j2Yv2t3wdl2FVW65bw1WmF8uWB0NE3rCKLeGWTdRdadC59/Y3z3P+yhJduLokEm88tm+XNYxom9fNe4/o65l3Cung7rvuknBbdb5G018uUmNtc/TL9QZNf7GYamzLjGdHYnCbCdqoJutNYAwBAHqSrCGRkPyN5XojU7Kyr9SeMxpGR0rs+NSmqaMMLG4842Nxy3l0DSKfoXLz3iN2fty8iujgbrtueoNciQFx8fr9FkNI0Vi3e3N8hBrmNsM3REIAdB6ILgIAepI04ojc94k2NlQXrn5nEtFFG2qz5IwGUzBRh2vZ8d1ynRUSTBJyGn+uOaYJKxZR3RXSIJfDdf40YwsR/oTwYn8CzxAAoOeoztfo/JWlprGwYyxOFVKwlUy7PDzcRmmGOmy4Wkuk2YBtniGiDUPEFnqpN9ZEXjBuLK6cIW5eRVV3qfl9+Pki6x3j8oRc+VdpxxaSq1SEtwwUDzxDAICeojpfo+kvFlu8Jk9WGjT95WKmt2vVgd7lIXJtlOr7vzt10OolOH1kN+vF4o67Y6y9PFuxliROb8TUoYmmx8LV+NVEJWTrbSVcYouueRXVwV15V1zeMe47LmM3LkWFJywXpYUEigWeIQBAT3Hx+v22MmYivnonFJcHRLJRml4CVU312a2H9Np4xdqWgstfShKibVvK9Oxlu0dHJdiGahP5MBOyb3/zuK1UXrE1LtHkGzvp46kDorXIqzGpb14249L3nbTVZERhmled0EIC+QNjCADQU+Sd72HCJeiePrKbpg5NiFSvVfhNukmq/22WaHM5Q8q7klabSM1peyWmKNqohioxCdm//9O3rAfmyYo7sVz9Pe/wj2tenOeJ+05ERA9mT2QaT0joqxtaSCA7MIYAAD1BUfkeNrbGm1VcuscgtFQ8ZJNU+kCcAJ/KzVEeISJyijgShTVA3TNzzXpeX4hNku+StW2KCTevchSxSfRFemRCQl9FectAsSBnCADQdUT5HuXs+R7qPLpB8mJ1vfm/XcaNjdD8EJfHQxlCyoiRVCSF5OxwRkE58tXZucddRPXUsX272nK7KnGZfvsPbzubwRaRv0TkbsBrQ+WXPZg9QV/PvNNzhpBSSzdzx4YZGEMAgK7jy/fYMRbTxZ/xG2GW8+jGjtSIUZsJ51NRm6S56dj6WdnOIzXKQuQHOGPhFz953Ssb4PKuhBqQNvR1OvSbr2ju375t6xx/8rA/HLc13tzSxiuxV4pBahTY1k7XPgoxJrptiKD03w7CZACAzNhK4fVO5z6KzPeQnEf9XRJqcbVXINr0RthCbnE5orgUWRPE9fO4+oSZrTmkOTuu8M3kGzubZfpmKX1cjujZi9W28yqyVk+Z62QLIybkFoW0XRPd4yf5jiskapNW0LWPpr+UKVsXodgdCme8fvh5OnXuQQGeIQBAJvIohQ8NQ6TFdx5JqMXlxdI9M7bPNdYS+tHWERq3eIhM/R6OLG/zXPhGlw3QvVfbtpSJko1Eb+68rjWVeEGkFXEu4yqNd8plFNjGqdbIJofQWEvowtUl3xRy8aJlhVvHtSQZag8RjCEAhoSi3PO+UngJReZ7hJxHEnZyebGUgVGdr7H5T8srDVo49y797tTBIP0ek7w3UeW10I3alZdrbdfWPC+3psf27RKFY6QeJC70WJ2vpfJOpTUKuAR4V2d63zk7qUHkMrRdxuCggzAZAENAke75PErhO1WBIzmPL+zkC6WptXZ931d9ZY5T0vsszdrp37OV3vvOaxurOr+00k7SUNcVevzVH+7S+FhsNUZca+06b1rFaFezXzWebmsQ+Xr2KWOQaLhCZjCGAOggeZcgSymyRYBrUwl5yHNGSFFrlhDR/376nM7MLdDF6/fFx/XpyLjCPspjMv3FYtPjUluuW7up6+txdPYGu4mmNXTN74WoWJvX1Xbtzs4tWL9rGsi29SyXIlpfTyihjWo3lTx9dPaG9T4eHSlRJS63XRPlnbKtjc8o4Az58UrM6kP51r2TGkTc72bq0ATd/uYxfXrrIfvdYWwfgjAZAB2im1UcRbrnp4/vpbjUXp6dZyl8XmtmlvCbasyS4/pCaa41/eSDA/THxe/bQk+N9YTOX+FzTlzhvbR5KNJcHVuJu+S6SvPAzPXcMRZTiTa9UmtJQpfv1JzhsKf1hvWa3Lz3yPkS8MkHB1hpAW7859/fb73f9WNzZG0+LMX2uzkzt0CHfvMVfVS9S5fv+O/zYWsfAs8QAB2imw0ci3TPq7FnqSbjOH9lKdc1cxkAIcd1hdJcQohThyboDOMxcXWwV2/zulq0Gi/nlfNtZpLNrhKX6eThCbp571GwZy7EC2J6wcyQl5qr6z5O6536D5WRtvPpYTnOK8ldR/3YLu9M3vhCnkQbeU2f3XroFDZVDFv7EBhDAHSIbiZPFu2eL+IBX52vsQZC2jXzfc+Xu6LDbXRFrHV1vkaX79TaNjhbKbzCt5n5cnWyGrRcD7eznrCk63dy6dTBoLV1GU+cRIJSJCciZ/iRM0TVup/+p3+lr//6uPn3kDy90NBwSMhTYggNY/sQhMkA6BCdKh+30Sn3fJ64wg1p18z3PYkaM5E9DHF2boHenLlGF6/fp5OHJ9i15jrVuzrYuzxaCaULZfkq1p433Do9ElQ5+ukju2l5peEs0Ve4fieh93FoeJGIaNvoCCuNoIfBXMf+qHq3xRCyfZ9DSVXo99b0F4v0UfUuWw2apmGvjX55NhQBPEMAdIhuN3Asyj1fFC4vTto1k1TSSLBtProI3+U7NXZDOffefpr+cpEaa5vnissRnXtvP3s+n0dLJRqvJUlLwrELm5Cgji9sKPVeVOdr1tBM2kanIfexq3rQF0LzeXJdx1YChq7vc5y/smTNKdMTnk0vU17e5WHuowZjCIAOgQaOYbjCOOrtOnTt1Oc//HzRavhE5C+PJvJvaLoHgLvetr+nKQNX41bzUQnHk2/sFFfnnZ1bsIZPuHmGVLC5mu+mbXQaEkbijCdfHh3376Uoat4j3LFdRrXPO+nKHdPRjUlurLYQKhdWJdq8jre/eZwqT6yfiZKAcsrJycnk9u3bBQ4HAAA2kLS9SJvcW52vsQaAreO7CVfq7qMSl1mPkW2+6vNExK4Ft7nZ5sGdY2tcsur0cGvBzd/2+T0z19jNV30+xLj5qHq3zdPkWlcO13orw5Rbc9+999av/pk1iH536qBznG/OXBPPgWhjDY/t20WX79TYe0dfW9tnTcx7Ks369gpRFN1JkmTS9zl4hgAAqSlSN0kSxtE3xZAEVVdVlyTk4Au3cehv8+barbxcZXNUlIGh1kKFxCYcHiPbPLg8GE6nh6uoCikGcHktXEKKRO3XMTTk5sLngXJ5EX333pH/tMOaM3T0Lbu3TmcHIyDJocKypnF2bN+ulrldMowwV1VZHuvbb8AYAgCI0TfG7ZWYnr1cbea+FNF0UoUhOO+C7aEtbTjJGRKS5GzTUHOFHky+W65bDQDX59U5bXNyCTJyxzJ5Wm/QpVMH2wwDIntFlUvx2cRmOEZEdPrIbqeQom3zDQ25+fDlH7lyiziDgYjozw+ftn3+6Fs76bP/9l+9Y7LllJVLEZWI2Aa/9cYa3bz3qGk0+wzMm/ceie9XRe3VfTuoBhGMIQCACPMBa8ttKOoNUtKyQaG3EyDi3/zTJLRznjD9775N5rXxSlD1j88447xUteU67fnVNUqSDcNv+vjeYJ2eUMVnTkOIiL8OIV4ml8ETUmEY4tEMufe+W66z1/bf/6+sdQq3XupvEk+gT9MsbcL1ILfpgDEEQJ/QrVYeijy6i6eF8y5whke9sUYXri7R88Y6+3YcmtDue9vWRQO5DUsZDJy3gfu8C3XeC1eX2rw1Krqjxnry8IQ1t4Q7R6gnKTSJmShMENQXcpMQ2r4k5N57bbziNO6k5+bWS3nSfOvlMzBDDDydQQ6XQWcIgD6gm608FKHdxdNg60hOZNdJ+ulbO9v0dXSerDSsb8d62wulg/Ng9kSz47w+jjdnrtFbv/pnenPmGn34+aKo7QWn3zMWl2hrXKKzcwtUEugZRUR08vCmgWXTl9HnMbbF/W6rQikhOj0+zR/b2oXi0uuRfFYPuUkIbV8ydWiCTh6eaGpQlaOIfvrWTnbMrjVL2zpFR7JePk2z6eN7nb8dF4PapgOeIQD6gE628khb3k2UTTdJ4nnRjZVf/eFucN4D0UZ4z5X7wKn5ctVB5uZg8zipCh7luZHoGSVEdO0v37d4clxeDMkm9d1yXaTTo+4BWz5UHtpY5j0mrQrMQ54iVAneVP9eSxL688OnzjHbwq/H9u1im6OGGBiSNZBoNXEFBD4GtU0HjCEA+oBOtfJwGSS2B2xciuhHW0doeaWROXQXYvD5QnaVuEyjIyVWs8VlRIaq+do2B9PgsOXeEG14GdaThDXqbEnK3JpIjFXXRsYZQErhOqHN3KPQa+xLvHeJVJqEiC7aCO3Tx92Xn956SBOWKi2XMewak05oXpGpuyUxmFyViByD3KYDxhAAfUARjVZtD1yXQaKXdxeRt5RXIq3asIlkzTRD/s0koo3N/OjsDedacMdcTxJ6MHsiWLfIdrzp43tp+otFtuLItZGZRrB5BKVwbdt4fXQz8d5GaOK8636Q5vxwxrB+bs4YtZ3D1FqyfcZnNHKJ9+OVmP7+7R/TzXuP2mQcBll8EcYQAH1A3q08OA8Q98D2lXe7zsNVxUjDcCGJtKbony2pmDum79gmvg1Lckw9h8N2fTnvFueNItpo56C+E0XUUk1GtLExKw9NFBEtrzTYLuc66t9DJRS6mXhvIzTU5rsfJIaca242UU2Xzk9eWkt5hBwHCRhDAPQBeT+4OA9QmdkU03igbAbX9BeLRBFZtYlCDD7pZ8+9tz/YiJQIKtpCDK7NSJLDQWQvpza9PXEpYsfvMlZdHhppTzZFyMabV+J9ntWUIUa95H7wzZEzqMpR1Eyo910DdY48tZayhhwHCRhDAPQJZgLxxev36ezcQq5JpGtJItaP8WEzuGwhnDRhOJdxmDY51zw2179swlM+7TqmZLx6Dkp1vtbWkr6xntDtbx4Hb2J5dTZXSMKDRDJPm0TfKaQcPk/0a8fNI60WlC8533aOvLSWQCswhgDoM/LYGFxhJpU7lPUNPOQtNU0YzvZZ29rYknN1A2R8LKYk2dDOkVQFqfUJzeGSjle/lhev329RIlZ8dush24iVI9RroMKAnLfQNl4brkawRLKk7E5WU9pQ147rZ+bKxdLvs9GREj195ZEL8cXp58hDawm0A50hAPqMEK0STrfHpVWSl35MyFtqXm+0krX5qHqXzs4tNDWbnqw0aLneaNNvsmkbKaPKp/XCrXvoeDkDJnn13RAka1yOouZcL506SP8+e4J++w9vW3WTbOO1MXVogt34IyLRPdapakofrnvCxNQGe7LSoKev7jOJIaQcguY58tBaAu3AMwRAnyHdGCQepBD1Zb00WiXeur7HleLrOUNE8jdriZfKtzZc8qmO7nHgPFW+sJfUc5dFKdhnCJhr5+tWznUml4SJfGPJ0gdOfc72/fGxWPT9PJF6L22GrtQIKlprCbQTJQGJc5OTk8nt27cLHA4AwAdXhm1WUrnKtc3QhMvosIUGdLhNlDsuEYlCVFxIwqVH41sbaQl7REQPZk94P5dmDCGfrc7XUoWYuLXT86dsRi2Re5OVzI275qHX0pyP2byUaMO4vvjzt53H6FYbG665sIsdYzHN//rdQsYzrERRdCdJkknf5+AZAqCHseW2LNcbIlVgiT7K7W8e0x8Xv2+pLKot12n6y0U6f2WJntb9Zdeu3A3zLdqczw/PV5tJ1bZ8mZA8kep8jZ69WLWO8dmLVarO1zrSUiQkpCOpihvbUqZnL+2GKOd14tZO72xuIvFo+cbLHeOTDw7QJx8cSG2UTB2aaJEMUDTWE2feUJb8ujRGlP4dSYWYTlyO6Nx7+8WfB/kCYwiAHsV8kOt6ORJVYIk+ChcyaqwlzY1H8kCXGBmu+ehj8nXWtv3d571arjfoV3+4S+NjsfW8OllVdn2aQtJqN9+cFDYDMU2OjcT49IVofKKdWTSqODXxrHPizm0aUWfnFujM3ILYGyf53SgFcoS6ug+MIQB6FF8ptDKEuDd9Vy8k/Rh5IPGkhIrvhYgwSo5db6zR6EipTTqAiKgSl+h5Yz2XTYlb92P7domr3aRzUpgGQRrFcs6oqC3XW3q5uXJm8kp0tq2Tq1M8R9rxuPJ9QrxxRJvVeDZvrjRMCIoHxhAAPYq08SbHzXuP8hwOi9STEhqisrWX4AQHpcd+Wm/QpVMHReGP6nytRcF6vBLT+ff3ezeva3/5nv37zXuPxJ6KEAPCNAgk4SwzId5lGOubvyt85DLCQsJOnDES2jQ2bRsb39qHeOPWk4T+ffZE13KXgAwYQwD0KFkbb/oe6NybNody6UuryWxjDRbfMwQH2/474Njqc5JqIFvC7nK9saGgTa0eAXOT48JwT1YatMz8m+1aufRkfAZBSLUbF37S0cvnXTk4nBF2bN+uoNwdl6SAEr2U3Huc2KHKIeO+K7mfQr1xUHvubaAzBECPYtMT0ZG8FXNMjFfo9JHdzuOb5/rtP7xND2ZP0MK5d2n+1+8G6xDZ5hOXIxqvxFbNFpvgYGMtsWra+NZKzUGaC8SJHaqEXYWpJZPWeLX9ndMyOn1kt1PnRmkcnX3VpPbSqYMt1ymtErVq0urSReJ0eFweMRvcOqmwsPTeU+PZYZTgqxwyTv9Jcj/ZvHEu7SnQ28AzBECPYr7du8rQbXBv6frmOfnGTqd+DJFMIbiI+YTke9g8Icf27Qpqw+E7h+3fQgyL8Uoc1H9NqiejjB9JhZ5vbi6kmkc2D4gyzFzf08mzMbGqTDQ9dr4qSKJNbaWs3jjQ+8AYAiBH8s4LcJWm62/j3HeJ3A9ndfw3Z66xY+AStCXY1kPp5/jCJqH5HlwYQu/jdvH6/bY1sI3RtfHr55caFnEpask3MvN1uLH5QiuhFXpq/D4Plm3zP7ZvF1t96MvBSXMtifIzLNIkUutrL/1dIxTWv8AYAiAnim4mmeb40ocz13+qHDFJOhnHKyl5zsM74FszroT6p2/tpP/z/55bRf7083Ob/I6xmMa2jFg3T3VNqvO1lgTx2nLdmpPkmhvXTNZE3/R9XdhNcUY1fq5buqQnVpprmadhkTaRuoixgN4ExhAAOVF0M8kijq/eeLkN1fx71oogNV7Xm7pZ6bQ1LgUna0vGwAk7JkT0L399TKeP7KZrf/neWk2mxsiFUM695686O39lqaVSjmgjJ+kM4yXSUUacVNRP3/Q575RvjblQV0J+482X0F10aCnPsBsYTGAMAUDZwlv6xmgjr2aSoSKEklwTn6jfhLaJhnqmXON19ZoyK50qcZkunTrY3DhVfkwevcpcVUs37z2ytkYw10Ev+R73hL10XFVcvrUNyVUyN/209zp3zSYyeFeK9qbq5yZCPg/ggTEEhp6skv0+gyKvjuxSV79UPde3oZqbaKhnyjVe7k09SchZdcRdJzU+c6PzrVmaRqicN2m8EtOL1fXcNnbX2roM7Lgc0bYtI6J+b7XlOp2ZW6DzV5a8GkpFeFeK9qbqINQFXKC0Hgw9vnLh0O/qSDcL5fHYM3ONjs7esJb8Skt3Jeq5vj5d45W4rWQ7NAnVNV6uBPupo+UCd50uXF1qK29Xc/St2fTxvZx0EWvEcvNdrjeC7iOz3DvkXNzYylFEF3/2Ni2cs0sfcPerr9SciC+bz2Jg5KVYDUBW4BkCfYHPtZ8lzJXlgez6jLQkXeqZkrr6peq5Lq/IttGRtuPmXRFke1Pnwo2vvRLas+GqoFKVcK4x3P7mcVuVlMuIlQo8Krhxn3tvv7UTu3kuGxLZhJCxEG0alq7v5+1dyZrYDEBewBgCPU+aiqCQ8ESWB7Irj0JSks5VBHGhAslmJFXPvXTqIJ0J0H/pREWQrXxbncOnh2Si5uAbw8dTB5p6SxJjmhvj1rhkNcwkxqJtXq61TZsD47s3nqw0nMrMeYPEZtArwBgCPU+aiiCbMcF5j7I8kLN811cRlDZU4CudJtpsS6H33jL/3YQTNlQaPmmTUl2VWRERnTy8aczY1np0pGRNRg7xLkiNtup8jS7fqbWVmEeU0JOVRnDvLG5e5ryzjFlHcm9IqtnyAonNoFeAMQR6nrQVQfrfP6rebXmbt3mP0jyQsxgIvnyjtKEC0+Pg2qDPvbc/yJgzheh8HjlbQ1C9hJuo1RAwjQxV1WWbVzmKqN5Yo61xieJS1FKmbptDHiXc3DVbaaw3x6vWWxom5XK8imi0q8bCGcGKoqq6uDHB+AHdBsYQ6HlcZdhHZ2+wzUaVMVGdr9Gntx62/bvuPcryQA41EBQuz08acUGb0jP3b3rODFE6Q9DnkXM1BFXrMjpS8paIm60eiFoNqCcrjWaPM661R14l3BJvnTKEpMrdnU4iVvfrwQtfOcv701Z1oTs76EdgDIGex+baj8sR/fB8lX27LUWbqrgXri6xx857wwkpFeaMvHIUBVXp2JSMzxil9NzGzG1ckg3Nt4n7PF/1xppIK8f0kNmO21hLaNvoCC2ce7el/YaunpxHCbc0eTrkvupWEvH599u9giahv49O6QYBkDcorQc9j62kd9uWkTb1Xh39n1zhgLw2HFUaH6JZw5V9//Yf3g7aOGxKxgq9zNw2ZltJ+ul/+lc6O7dgLVXX8XVgz8PQtHnIfOrVtjnlJYgp6WZOFHZfdavbuf674gj9fWSRqQCgm8AzBPoCM4y1x9FYVCF5689jw5EKL9q8LZ98cCBzSMEV6iDiPSDcxvX1Xx+LjuFLHpd4UXaMxfS8sd5yDF/OjcuTws2J672mb/a260PUHkLUr9n2SkzPXq62lMeH5ivZ8nhGR/J7T/Wd2xbSNOchDX1BNwj0KzCGQF8iLR8n2hAQtBkMY3FJpAHk2wQkwovH9u2yhg8++eCANYSVd96FbTMK3aDMz/vyjSQNQc+9t995DBsuI4zrn7WWJFSJy87N3rw+018sEkXUNHS4a+YLNZpJ7Fzo6PmrJGyiDQP3zNwCXbi6JOpzxpGHhlVI6Au6QaBfgTEE+hJp+TjRRm6EnlNDtNF9/H988HfOc0g3AYnwYkjOiuu8RO0b1o6x2BkKJLJvRqHigVy5PbdRmxusqyFoyGbv2rg5zR79OtiMLmsekiX0aLtmtjWw9S9zHYczqJ+sNDLl3ITcd9y1DDkGdINAvwJjCPQlIYJ1aSumpJuARHiR81jYDCnuvOevLFl7X508PEFz//NbVsmY24yUJ0XW95zo2YvVYEG+osqmueMe27fLWjl4bN8u51hCvGSSz0oaqerH8SlDSxO9TS9VHrlSIaEv6AaBfgXGEOhbzJJ21wM4zaYs3QS4t+Fj+3Y1O6yXBDkrvvPaQn31xhrdvPeILv7s7ZacE4nWzdShCVaBmju/z0vR7bJqTpvHp9kT4iXjQj763CUGpn4c3/klxovNo2hqTJnnllyvNG1YYPyAfgPGEBgIingASzcBTnjx8p1ac2OyGUKcxyZt7ys95ySh1qaoHBOO8n7bmOuNNfrw80UiajeIeqGsOm0Cr1W+oRTROhGt6eHVcmS9ZpIkeh3TWB4fi9uEI3UkOTeceCMnusldr9vfPKab9x6x97J+DAAGBRhDoCfptoehOl+jlZerbX+PaCPkYmIaY0dnb1g3xnIU0XqSOOfEeZpcva/S6ui4Gn5yIbS1JLEaOWnHkOe1TpvAyxm0c//2LbXMKLGPeeXlqtcQ0r11poGhhCMrcYnqmlFL1G54cOvFGXzqnObnbfdovbHWptR++U6NTh6eaDGQEPoCgwaMIdBzdNvD4HrLT4jo8p0aTb6x0zkWbmNaTxJ6MHvCeX4u74LI3pvLVUXl84ikSUYmshs5abwyeV/r0ARelyF2dPZGm6emsZ7QhatLLXIAPi9eRGQ9tk048j/+zVZnordrvUKbBruMJx0VipUqagPQj8AYAj1HXmrBOiHeB4lysm8sWUuMTSNFNc7kdIk4w0VyPi7E6KvYMzfTNHPO41qb11bqxfAZYpyx4Kvc0wk1RL5brjtDvq71CjUEQ8Kx0AkCgw4UqEHPkbdwG6dKbFNllp5HkoOSRVXYNubpLxbpwtUlVtMnbxVjpVBcjiLrv5tGTpoxZL3WtnW6fKdG08f30oPZE/T1zDupDAui7No4PkMk5O8KnxFlKrW72rrYrpf9SrvHpdTX98xco6OzN9jfFQC9DDxDoIUQD4r+2fGxmJKE2EaZIeQt3BbqfZC8Masmsa7qNXXuNHkWnO6N8kqYXoyiSprV9yUeB1d4j1urrNc6i2fJZ4hxnpbRkZK1sm+8EtO20RHR+nMJ2ysvV2nPzLXUlV0hhQSSxH81Z1eYsdtJ8wDkAYwh0CTkwTCDrlcAABMzSURBVGZ+Vg8ddDrvwxyXuRmHeh+mj++l6S8XWd0es0msS9U3jTaMa8w65qZfpKYPkczQMsfgu6eyivRl8SxJDAsiu3FnE/E8/75cKdomSPnspf+eSmtEucZhfnbyjZ2ZQsouY7TbhREAcESJpXyWY3JyMrl9+3aBwwHdhGs0ast7cDUldX1PivnQPLZvlzcPhOuvxFVhucZ38MJX1rf/chTR32wdsf5bmvlyY+a8DyYRkTchWzKGojYoyT2V5fwh96yCa5NBtFlJ56t+M43luBzRxZ/ZG+xK5hcyD/14XG803xzyYs/MNWvFoe2+5O71To0VDCdRFN1JkmTS9zl4hkCTkLfsPPJqXJiCihKPFfeWOjpScvalsvGUMUTWk4T9tzTz5ca8NW4fs42seS1Fhzkk91QWj1aa6jGzTYZEoFLn4vX7bV7DxlpCZ+YWmonMofduiFK0vl5HZ2+0Gc1Ziw10qvO1FjHP8Urc4gELCXMWURgBQF4ggRo0CUnqlGzCts+kSbb0JboquI33ab0RlFjKjV39PW3yqw1WbXqldcyVuP2nmofwnXRt0yJdq7RJuKFJw5wwYQguo9dMzpesb3W+lipx2TWWPKq/lAdM96ou1xs0/cUiVedrrBZXXLILU3Jj8nmYkaANOgE8Q6BJyFu2r+xaPRDNJOsfnq82cy2kXgjpA9/1lsp5H7gQhm8tsjajVOflNmJ9zMq7oBMR0cnD2XOEitxMiWT3VFbvlL5OF6/fp7MWD41CYsj4zutLsNe9HZL15e6DiMh7TxXZJd7mASPaSOQ3++S1wFh23FgjIrbnHRK0QaeAZ2iI8L1hqbfsHWNx82+jI/ZbZOrQBJ08zD+MVpONsMHZuYVm2fOTlUabiJ3ECyH1LoSWdrtK7l0eh1BvhOu8Nswxc94MX78tCXl6uWxI1ioP75RUPsE3L8l5bfeZiTJ2JOvrEj/03VNFSCr4xkW04SHiXoQaa0mb58uVY5gQsWtetOcSAAU8Q0OC7Q1L6dYsr2yWwxO19rjimnNW52v0+z99y55P5eVLQhDmQ9eWPC0p9w0tL/flMLhyWbLkubhEHVWrBuXhyKvzOEfWai4JvrVyeU+kydXSfBSfR9M1Hn0+6pzctRl/9UIhWV+XcrSPoiQVXOOSoNZQ2rMt1EOp7g1XPhMAIcAYGhLkujVJW28kc1NRDzhbI8806G/JNqMtpDdSiJFSdIgo9LwqLKJXKrk2ozy8N0VuplK4TXd7JRaHSKTXUn1P30Rt4/Ghh+ZsMgw/PF9tCf24tJe2V2KKy1FbRZikH5k+lrzhJCbiUkQ/2jriVOJWa+hTczc/b/s7d2+YY1P5TEQIoYFwYAwNCL43aKluDUdtud58eJeYjuZpMLuAc2/4aXoj2dZEncM1j7xCRByuPI8LV5dYfSOTZy9W2VyLEIraTKVw3pMoar8nueqjkNyZqUMb7Utsm7kkT8c81vkrS20VXaqHmX7/XTp1sCUHTM1tud6guBTRjrG4xUsbWpGWNzbDUXlfiNrz5hS6ISd9sXjy7AUdvPBVm2ir697g8plQnQbSAGNoAJA8LLO4vIk2Ngn1/bwMISJqi6Pl5a3hwoKkPURt88g7RGTDFTo5wzRcJSLaMRa3VfYMQjIp5z0JaT4bGu7LkqdjwulBPVlpWEUUOS/t2JYRmv/1u23H6WZJus9Qtuk1bdUqH7nnTiki0tMHVxrrtPLKI217fknvDSL0UQPpQAL1AMA9LM/MLTQTpSVJny5yNH9aUG9yirwSerkNx/Y2WY6iVInQadGTitX5JUmhY1va313qjTX68PPFvi83njo0QV/PvNPST4y75tsrcdvfQpPauWNL8nR0XGXxJuoap8mPCfl7p1DX7HenDtJW7dnyZKXRTF7nErz/w9b2a6jj+z24ngdFe3bBYAJjaADwlQufmVug//6Hv9DJwxPNzWL8VZ6CjvShnjf6+Lnmkcf27Up9TB/rSUKXTh0kIqKzmgFZJCoEUInLTQ+Vy3M3XonZOa0libPxbL8yfXwvxaX2u/LZy1XrXG0GlevYXBWWXnV56Ddf0cELX7EVmC55BBvKu2Ejr7/bCNXqCfm8z3N18vBEs9lvOYro5OEJVrhURyVJ26oEj+3b1fb8IuI1jgDwAWNoAJA8FFca6/TprYe08nKVLp06SAvn3qWLP3u75U369JHdXTGI9PGrh6c+joSILt+pBW32IRuFStSVdrXPC2lyqep75ZrTIJYbTx2aoB9tbfeGmaXbaY9t8yQREU1/udgiB7Fcb7D3Rah3ZnslZg1+lZdn3ndZy+elsgNpP++r+Lp8p9Y0+NeShC7fqTWr7Vy8Nl5x5hBe/NnbLTIg45WYLv7c3hIFAB/IGRoAJOXCCuXCJmoXq/vs1sPCwmEctof6zXuP2sYRmiPBNbQkI/EyNFE3L6rzNacnaGK8Yk2Gd13nbodNimCZqVjKY662fJhDv/nKmcBu3hehuXhR1F6ar+fb2PJlslb8cQbFh5/bK69Cc5S2xqW2ClSiDcMvpEWOjnouuPLGup34DwYLGEMDgET3REd/sEl1QMYrMW0bHaHvlus0tqVMz176DS8fXC+oPHIkXCXNWRJ188CmKK3DNRlVc/rHzxdo3bJfS962+428FJalekWucnGFGdY9O7cgfolQxp3ayG1ihDbDI8vG7wuvquPrDWylx6nO16yGENGG4edqkXPp1MEWdfokobZqMm48yAsCeQNjaEDQvTySh7N6SElCNZW43BQzU8f3EUWbwos2ylHEbkh5bYDcBmJ7E5acL6/u7q4194U/uFJuIvd69yt5iELmXZpuhnVdFYAmpShqkUPoRHK0y3ulh1d9L0Vc81UOJRMQ2iJHpxOioAAQIWdo4Jg6NEE/fWun93Oq8afroWurypEmjPo2ZlfSb94tBnzJoJLzheZRuHCtuaSajUs+lSSl9htZW58QhbV0GLdUqunY7sOQCrS1JKGzcwv0UXXDGCu6HQqRv33Id8t170sR9/tz3cvqhSHLbzmP6w+ABHiGBozqfI3+/PCp93P11Q3XtqsVgC1Uk+cbK5eH4MqRCPXOmArBteU6TX/ZmishycnIU+vFtea+Y1Xna10Ti+wWWXNDQtp9/P3bP6a5f/u2rYceER/W9elDmSRE9NmthzT5xs6OtUMhIvrw80X2vnH9rrl5q+9yzVf170h/s9zvG8YPKJooCfCtT05OJrdv3y5wOEAK99BwNUQ0iWgjyfHZy9W2pGLu7evgha9Ykbk0RET0YPaE6LO2/CbXWIk2EmJteSA7xmKrwB23rntmrrGdxaXjzzIP7nsh3x9WuN/EeCVu67xeicvi1i86b85cCx6XeuHQc3XKrwxdlwGSFtd9x4WKuZci1zEjIjp9ZDd9PHUgt/HhvgZpiaLoTpIkk77PwTPUh7hyIEI8NwltqOeWiNhWACZRzrX3Id6MNN4ZLiHW9nfXuuaVx0SUvjrIFco4eRhvzxyh7T7M1i8qzOq6VhMpFN7V522Vgnm03LAZ9srwMedy+5vHbdWkEg9V1ko3HVfVm2pa3Om+eWB4gDHUh7iMgjRtN9aJL2E2kX5OQmg4oOhkU9e6+sIZoeG7NK5/1zz/uPh98Jv4sGBWWyrFb4lEgbWty5eLdP7KUkvlU4i8hSJ6dXxVNSUx9KX3mW3cqrBiQuuTpj57+U6txRCKSG5g5xXGclW9EW3O4fY3j3Gvg9xBAnUf4jIKbAmLJdroBeQiefV/vsTgUE+I3uril0d2Z0qETJNsyiXE2v7uWldXIieXXP1R9W6Q6q8P1zyX642BU6DOE5viN4e+zta2LmtJmxAjEbXcH2WBCzWhzWosiaEfksRvG7epZaS+d/7KkvWzN+898s4hTyTPFpVvhXsd5A08Q32Ir1yViHdbS3IbXKEnl66KLh5HJM+DkXpU0iSbnn9/P01/sdiSEKsUnc0xcFukekhzb8DcW70edlCNYi9cXaLllQarq+LCp2mDbt1upDIS+v0k8TrWG2t0/soSLZx7t7n+e4Q5ROr4kjBsiPfI5x1WYybiG80WKeJp+91LvWvKiMS9DvIEnqE+xFeu6urRtEMozMe1Bpg6NGFt21GJy3Q60PMTWq6ue2eIWhucur5z8eetbUd0yX59DDYkoTxXB3SdxnpCT1YaojYP3FxOH9kdPA6wQaiMBJHcE2p65qTfU5+TlKBLvUeqWlIyZmUQucaWN9zvnkjuXcO9DvIGnqE+JEvS4rn39reUmrvgkjg/njpAk2/szJw06XvTdXmNQpJNXTkNLm/BxHiFju3bRRev33cmcKbJ0zKRlul/PHWArv3le2sC+KCW1udFqIwEUVirG/36Sb5nvsCoY3C/KYn36MLVJdFvW+GqDH3y7EXTk7xjLKZz7+33jlGC63evv7y5BGRxr4O8gTHUp6RNWjQfuuNjMf3wfNWqq0Lk1gLK6qb26b9wBg/3MD1/ZSm3pGSlkyIxumwbnxkyzDIWk3Pv7YcqbwrShFltvxeuQlG/fjbj5ti+Xc6Sfd9vSjJ+STsRKStam40nKw36x88XqFyKWjS70lS8SQshslS5ARAKdIaAtydRGi0dCZz+iwqD2f5th2MzIiL6nVYlU9QYbJ4E04t1bN8uunynFlRd5NN0cZ0PJccy8lg3Trsq5PqlxXaf6QZWqIdyx1hMzxvrmQz5chTRepKI19P1m7OtH+51kAWpzhCMIdAk9CGlSPuw4kTWTh6eoE9vPUw1h9ANySX05koUlxiH+rrYxC11IC7XP2QVB8yyuevf9Xl1dTba70TWMRO1erCyhHzTiofi/gdFAWMIBJPmIWX7TlyOaNuWEVGllLkxvPm3FfqXvz4ODjEp0ipCh6h5p/UAmBtZaDUZ6B3yfgGQGAIuBXIXcSmiiz9/m4hk+T4hKvY2JL8PeHtAp4AxBFIR+pCSPDhDHvau0nEJeYYq8AYL8iaLgR1ipChFbEl7D1vozdafrRRRS84QR1FhdQDSgHYcIBWhidFSHRZJpZRL60dCUQ0u8QYL8iKLinpIOfmzF6sUl/3JzrZChct3anTqv7xOf1z8vlltZqsmG7aGwWCwgTEEMuGqrtHJ+2GvUMmeRTS2JMqv1QAARNl63IXk89hK5m0vJVxl5s17j2jhXHsjY6JNY4rznKLSC/QjMIZAJqRRVtvD3nTPb6/ETt0TRdEGEAB58lH1Lv3+T9/SWpJsiAmWIlrTQlBSA8JWWq/y8yS/G6JNMVV1Ps64kryYwHMKBgkYQyATTwUPYdvDXinl6m78sqOBml6+q5cTq95OeACDbsLl2n1UvdtSGZkQ0dp6Qtu2lGnl5VqQAeEyPvbMXBOHmFWzWdcXpKEueE7BoABjCGRC4rofHSnR2bkFunj9ftOQsX1nbT2h0ZESvVxdZ3ucucQY8VAGRWMzeoh4RfTf/+lb63GeN9ZTJRlzxkdoSbwrCRqhLjCMoDcZyIStp5JORNTSg+vTWw+dD+0Xq+t06dRBtseZS8ofgCKx9dQ6M7dAZ+YW2HvSlmBMROzf02L7HcalSNyLUAfVkmAYgWcIZEI9NM9fWWrLW0jTkkIdk3sYZ6nGASALkq73Ot9ppe02qvO1tvtczy8qRxH94iev08dTB7zncoXQQkvyYQiBYQTGEMiMMl7MEEIa4bbxivtNNks1DgBZCDW4VX4bp6ZuhnfN/KK1JGn+t9QgshkyXOI1JdSiJYTwGBhmECYDuTF1aIK+nnmHHsyeoK9n3mn295ISlyI6//5+52ds4QA8xEEnCDG41T358dQB+uWR3dbPmOFdLr+I+7tOdb5GR2dv0J6Za3R09gZV52vNf5s6NEGffHCgJfR88Wdv08Wfv82GowEYNuAZAoVheyM1CS2TRzkv6BbTx/eKFNLNe/njqQNtndcVurcpbX7RR9W7Lce3FRVwXiP8bgDYAMYQKAyb4WJ22U5jyKCcF3SDqUMTdPubx6xhQ7Sh1GxrqyEJ73L5ReWIl5yoztes45GqvgMANoAxBAoFhgsYJD6eOkCTb+y0FgzE5ajZssLE5iU1w7u/+Mnr1vyiX/zkdXY8rhY2KCoAQA6MIQAACIArGHB5OSXhXZUkHVJN5jJ4UFQAgBx0rQcAgD6FK5uPiOjSqYPwyoKhR9q1HtVkAADQp9iqKyMiOn1kNwwhAAJAmAwAAPoUVFcCkA8whgAAoI9BkQIA2UGYDAAAAABDDYwhAAAAAAw1MIYAAAAAMNTAGAIAAADAUANjCAAAAABDDYwhAAAAAAw1MIYAAAAAMNTAGAIAAADAUANjCAAAAABDDYwhAAAAAAw1MIYAAAAAMNTAGAIAAADAUANjCAAAAABDDYwhAAAAAAw1MIYAAAAAMNTAGAIAAADAUANjCAAAAABDDYwhAAAAAAw1MIYAAAAAMNRESZLIPxxFj4jom+KGAwAAAACQG28kSbLL96EgYwgAAAAAYNBAmAwAAAAAQw2MIQAAAAAMNTCGAAAAADDUwBgCAAAAwFADYwgAAAAAQw2MIQAAAAAMNTCGAAAAADDUwBgCAAAAwFADYwgAAAAAQ83/Bwk1CNMrllMWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1c1e38ab00>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.figure(figsize=(10, 10))\n",
    "plt.scatter(item_tsne[:, 0], item_tsne[:, 1]);\n",
    "plt.xticks(()); plt.yticks(());\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using item metadata in the model\n",
    "\n",
    "Using a similar framework as previously, we will build another deep model that can also leverage additional metadata. The resulting system is therefore an **Hybrid Recommender System** that does both **Collaborative Filtering** and **Content-based recommendations**.\n",
    "\n",
    "<img src=\"images/rec_archi_3.svg\" style=\"width: 600px;\" />\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"embedding/user_embedding:0\", shape=(?, 50), dtype=float32)\n",
      "Tensor(\"embedding/item_embedding:0\", shape=(?, 50), dtype=float32)\n",
      "Tensor(\"concatenation/concat:0\", shape=(?, 102), dtype=float32)\n",
      "Tensor(\"fc_1/hidden_output:0\", shape=(?, 64), dtype=float32)\n",
      "Tensor(\"dropout/hidden_output_drop/mul:0\", shape=(?, 64), dtype=float32)\n",
      "Tensor(\"fc_2/pred:0\", shape=(?, 1), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "embedding_size = 50\n",
    "reg_param = 0.01\n",
    "learning_rate = 0.01\n",
    "n_users = max_user_id + 1\n",
    "n_items = max_item_id + 1\n",
    "meta_size = 2\n",
    "\n",
    "g = tf.Graph()\n",
    "with g.as_default():\n",
    "\n",
    "    tf.set_random_seed(1234)\n",
    "    \n",
    "    users = tf.placeholder(shape=[None,1], dtype=tf.int64, name='input_users')\n",
    "    items = tf.placeholder(shape=[None,1], dtype=tf.int64, name='input_items')\n",
    "    meta = tf.placeholder(shape=[None,2], dtype=tf.float32, name='input_metadata')\n",
    "    ratings = tf.placeholder(shape=[None,1], dtype=tf.float32, name='input_ratings')\n",
    "    \n",
    "    l2_loss = tf.constant(0.0)\n",
    "    \n",
    "    # embeddding layer\n",
    "    with tf.variable_scope(\"embedding\"):\n",
    "        user_weights = tf.get_variable(\"user_w\"\n",
    "                                      , shape=[n_users, embedding_size]\n",
    "                                      , dtype=tf.float32\n",
    "                                      , initializer=layers.xavier_initializer())\n",
    "        \n",
    "        item_weights = tf.get_variable(\"item_w\"\n",
    "                                       , shape=[n_items, embedding_size]\n",
    "                                       , dtype=tf.float32\n",
    "                                       , initializer=layers.xavier_initializer())\n",
    "        \n",
    "        \n",
    "        \n",
    "        user_embedding = tf.squeeze(tf.nn.embedding_lookup(user_weights, users),axis=1, name='user_embedding')\n",
    "        item_embedding = tf.squeeze(tf.nn.embedding_lookup(item_weights, items),axis=1, name='item_embedding')\n",
    "        \n",
    "        l2_loss += tf.nn.l2_loss(user_weights)\n",
    "        l2_loss += tf.nn.l2_loss(item_weights)\n",
    "        \n",
    "        \n",
    "        print(user_embedding)\n",
    "        print(item_embedding)\n",
    "        \n",
    "    \n",
    "    # combine inputs\n",
    "    with tf.name_scope('concatenation'):\n",
    "        input_vecs = tf.concat([user_embedding, item_embedding, meta], axis=1)\n",
    "        print(input_vecs)\n",
    "        \n",
    "    # fc-1\n",
    "    num_hidden = 64\n",
    "    with tf.name_scope(\"fc_1\"):\n",
    "        W_fc_1 = tf.get_variable(\n",
    "            \"W_hidden\",\n",
    "            shape=[2*embedding_size + meta_size, num_hidden],\n",
    "            initializer=tf.contrib.layers.xavier_initializer())\n",
    "        b_fc_1 = tf.Variable(tf.constant(0.1, shape=[num_hidden]), name=\"b\")\n",
    "        hidden_output = tf.nn.relu(tf.nn.xw_plus_b(input_vecs, W_fc_1, b_fc_1), name='hidden_output')\n",
    "        l2_loss += tf.nn.l2_loss(W_fc_1)\n",
    "        print(hidden_output)\n",
    "    \n",
    "    # dropout\n",
    "    with tf.name_scope(\"dropout\"):\n",
    "        h_drop = tf.nn.dropout(hidden_output, 0.99, name=\"hidden_output_drop\")\n",
    "        print(h_drop)\n",
    "    \n",
    "    # fc-2\n",
    "    with tf.name_scope(\"fc_2\"):\n",
    "        W_fc_2 = tf.get_variable(\n",
    "            \"W_output\",\n",
    "            shape=[num_hidden,1],\n",
    "            initializer=tf.contrib.layers.xavier_initializer())\n",
    "        b_fc_2 = tf.Variable(tf.constant(0.1, shape=[1]), name=\"b\")\n",
    "        pred = tf.nn.xw_plus_b(h_drop, W_fc_2, b_fc_2, name='pred')\n",
    "        l2_loss += tf.nn.l2_loss(W_fc_2)\n",
    "        print(pred)\n",
    "\n",
    "    # loss\n",
    "    with tf.name_scope(\"loss\"):\n",
    "        loss = tf.nn.l2_loss(pred - ratings) + reg_param * l2_loss\n",
    "        train_ops = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)\n",
    "        rmse = tf.sqrt(tf.reduce_mean(tf.pow(pred - ratings, 2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import QuantileTransformer\n",
    "\n",
    "meta_columns = ['popularity', 'release_year']\n",
    "\n",
    "scaler = QuantileTransformer()\n",
    "item_meta_train = scaler.fit_transform(ratings_train[meta_columns])\n",
    "item_meta_val = scaler.transform(ratings_val[meta_columns])\n",
    "item_meta_test = scaler.transform(ratings_test[meta_columns])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model_deep_meta():\n",
    "\n",
    "    losses_train = []\n",
    "    losses_val = []\n",
    "    epochs = 1000\n",
    "\n",
    "    with tf.Session(graph=g) as sess:\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "        train_input_dict = {users: ratings_train['user_id'].values.reshape([-1,1])\n",
    "            , items: ratings_train['item_id'].values.reshape([-1,1])\n",
    "            , ratings: ratings_train['rating'].values.reshape([-1,1])\n",
    "                           ,meta: item_meta_train}\n",
    "\n",
    "        val_input_dict = {users: ratings_val['user_id'].values.reshape([-1,1])\n",
    "            , items: ratings_val['item_id'].values.reshape([-1,1])\n",
    "            , ratings: ratings_val['rating'].values.reshape([-1,1])\n",
    "                         ,meta : item_meta_val}\n",
    "\n",
    "        test_input_dict = {users: ratings_test['user_id'].values.reshape([-1,1])\n",
    "            , items: ratings_test['item_id'].values.reshape([-1,1])\n",
    "            , ratings: ratings_test['rating'].values.reshape([-1,1])\n",
    "                          ,meta : item_meta_test}\n",
    "        def check_overfit(validation_loss):\n",
    "            n = len(validation_loss)\n",
    "            if n < 5:\n",
    "                return False\n",
    "            count = 0 \n",
    "            for i in range(n-4, n):\n",
    "                if validation_loss[i] < validation_loss[i-1]:\n",
    "                    count += 1\n",
    "                if count >=3:\n",
    "                    return False\n",
    "            return True\n",
    "\n",
    "\n",
    "        for i in range(epochs):\n",
    "            sess.run([train_ops], feed_dict=train_input_dict)\n",
    "            if i % 10 == 0:\n",
    "                loss_train = sess.run(loss, feed_dict=train_input_dict)\n",
    "                loss_val = sess.run(loss, feed_dict=val_input_dict)\n",
    "                losses_train.append(loss_train)\n",
    "                losses_val.append(loss_val)\n",
    "\n",
    "                 # check early stopping \n",
    "                if(check_overfit(losses_val)):\n",
    "                    print('overfit !')\n",
    "                    break\n",
    "                print(\"iteration : %d train loss: %.3f , valid loss %.3f\" % (i,loss_train, loss_val))\n",
    "        \n",
    "        # plot train and validation loss\n",
    "        plt.plot(losses_train, label='train')\n",
    "        plt.plot(losses_val, label='validation')\n",
    "        plt.legend(loc='best')\n",
    "        plt.title('Loss');\n",
    "        \n",
    "         # calculate RMSE on the test dataset\n",
    "        print('RMSE on test dataset : {0:.4f}'.format(sess.run(rmse, feed_dict=test_input_dict)))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration : 0 train loss: 292926.438 , valid loss 72812.727\n",
      "iteration : 10 train loss: 31816.936 , valid loss 8456.081\n",
      "iteration : 20 train loss: 27847.322 , valid loss 7550.862\n",
      "iteration : 30 train loss: 27442.869 , valid loss 7475.166\n",
      "iteration : 40 train loss: 27260.652 , valid loss 7570.971\n",
      "iteration : 50 train loss: 26871.463 , valid loss 7337.713\n",
      "iteration : 60 train loss: 26462.799 , valid loss 7285.252\n",
      "iteration : 70 train loss: 26302.744 , valid loss 7242.453\n",
      "iteration : 80 train loss: 26205.311 , valid loss 7229.100\n",
      "iteration : 90 train loss: 26126.928 , valid loss 7228.513\n",
      "iteration : 100 train loss: 26030.127 , valid loss 7219.623\n",
      "iteration : 110 train loss: 25869.744 , valid loss 7210.592\n",
      "iteration : 120 train loss: 25694.607 , valid loss 7194.119\n",
      "iteration : 130 train loss: 25403.344 , valid loss 7176.284\n",
      "iteration : 140 train loss: 25105.943 , valid loss 7166.001\n",
      "iteration : 150 train loss: 24717.309 , valid loss 7178.116\n",
      "overfit !\n",
      "RMSE on test dataset : 0.9477\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEICAYAAACj2qi6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XuUVOWZ7/Hv0xdobt00t24EkkaDRiEI2AIzJsZLokAmYhJMyJhIHBMcYyYmZzJHk1lzTDLjjHOOox7XRDMaMThHJRyMkZnBOEQxxhNFGqPIxQxEUVoQGpCLXBS6n/PHfqt7V1FdXV1d3dV2/T5r1ara73733m+B8qt9fczdERERyUZJoQcgIiLvHwoNERHJmkJDRESyptAQEZGsKTRERCRrCg0REcmaQkNERLKm0BDJkZltNbNPFHocIj1JoSEiIllTaIjkmZl9zcy2mNleM1tuZieFdjOz28xsl5ntN7N1ZjYpzJtjZhvN7KCZvWlm3ynstxBJT6EhkkdmdgHwD8DngdHA68CSMPsi4FzgVGAo8AVgT5h3L3C1uw8BJgFP9uCwRbJWVugBiPQxlwOL3P0FADP7LvC2mdUBx4AhwIeB5919U2y5Y8AZZvaSu78NvN2joxbJkvY0RPLrJKK9CwDc/R2ivYkx7v4k8M/Aj4CdZna3mVWGrp8D5gCvm9mvzeyPenjcIllRaIjk13bgg4kJMxsEDAfeBHD3O9z9LGAi0WGqvwrta9x9LjAK+AWwtIfHLZIVhYZI15SbWUXiRfSP/ZVmNsXM+gN/D6x2961mdraZzTCzcuAQcBRoNrN+Zna5mVW5+zHgANBcsG8kkoFCQ6RrVgBHYq+PAX8DPAzsAE4B5oe+lcA9ROcrXic6bHVLmPdlYKuZHQD+HPhSD41fpFNMRZhERCRb2tMQEZGsKTRERCRrCg0REcmaQkNERLLW5+4IHzFihNfV1RV6GCIi7ytr167d7e4jO+rX50Kjrq6OhoaGQg9DROR9xcxe77hXFoenwk1Lz5vZS2a2wcx+ENrHm9lqM9tsZj8zs36hvX+Y3hLm18XW9d3Q/nszuzjWPiu0bTGzG2LtabchIiKFkc05jXeBC9z9TGAKMMvMZgL/CNzm7hOIbla6KvS/Cnjb3T8E3Bb6YWZnEN3kNBGYBdxpZqVmVkr0LJ7ZwBnAF0NfMmxDREQKoMPQ8Mg7YbI8vBy4AFgW2hcDl4bPc8M0Yf6FZmahfYm7v+vurwFbgOnhtcXdX3X394geIz03LNPeNkREpACyOqcR9gbWAh8i2iv4A7DP3Y+HLo3AmPB5DLANwN2Pm9l+oge2jQGei602vsy2lPYZYZn2tpE6voXAQoAPfOAD2XwlEXkfOHbsGI2NjRw9erTQQ+kzKioqGDt2LOXl5Tktn1VouHszMMXMhgKPAKen6xberZ157bWn29vJ1D/d+O4G7gaor6/Xc1FE+ojGxkaGDBlCXV0d0cEH6Qp3Z8+ePTQ2NjJ+/Pic1tGp+zTcfR/wFDATGGpmidAZS/RIaIj2CMYBhPlVwN54e8oy7bXvzrANESkCR48eZfjw4QqMPDEzhg8f3qU9t2yunhoZ9jAwswHAJ4BNwCpgXui2AHg0fF4epgnzn/ToqYjLgfnh6qrxwATgeWANMCFcKdWP6GT58rBMe9sQkSKhwMivrv55ZnN4ajSwOJzXKAGWuvu/m9lGYImZ/R3wO6Iax4T3fzWzLUR7GPMB3H2DmS0FNgLHgWvDYS/M7BvA40ApUanMDWFd17ezjbx78pWdvPLWQb5+3oe6axMiIu97HYaGu68DpqZpf5XoyqfU9qPAZe2s6ybgpjTtK4jqEmS1je7wm827Wbpmm0JDRJLs27ePBx98kK9//eudWm7OnDk8+OCDDB06tJtGVhh69lRQW1nBofeaOXj0WKGHIiK9yL59+7jzzjtPaG9uzlxcccWKFX0uMKAPPkYkV7VVFQDsPHCUIRW5XYomIn3PDTfcwB/+8AemTJlCeXk5gwcPZvTo0bz44ots3LiRSy+9lG3btnH06FGuu+46Fi5cCLQ90uidd95h9uzZfPSjH+W3v/0tY8aM4dFHH2XAgAEF/ma5UWgENZVRaLy1/10+NGpIgUcjIql+8G8b2Lj9QF7XecZJldz46YkZ+9x8882sX7+eF198kaeeeopPfepTrF+/vvWS1UWLFjFs2DCOHDnC2Wefzec+9zmGDx+etI7Nmzfz0EMPcc899/D5z3+ehx9+mC996f1Z0VehEdQmQuOAbiISkfZNnz496R6HO+64g0ceeQSAbdu2sXnz5hNCY/z48UyZMgWAs846i61bt/bYePNNoRHED0+JSO/T0R5BTxk0aFDr56eeeopf/epXPPvsswwcOJDzzjsv7T0Q/fv3b/1cWlrKkSNHemSs3UEnwoOK8lKqBpTz1n6Fhoi0GTJkCAcPHkw7b//+/VRXVzNw4EBeeeUVnnvuubT9+hLtacTUVlbo8JSIJBk+fDjnnHMOkyZNYsCAAdTU1LTOmzVrFj/+8Y+ZPHkyp512GjNnzizgSHuGQiOmpqpCh6dE5AQPPvhg2vb+/fvz2GOPpZ2XOG8xYsQI1q9f39r+ne98J+/j60k6PBVTW9lfh6dERDJQaMTUVFaw+513Od7cUuihiIj0SgqNmJrKClocmt55t9BDERHplRQaMa33augQlYhIWgqNGN2rISKSmUIjpkZ7GiIiGSk0YoYP6kd5qfHWAZ3TEJHcDB48GIDt27czb968tH3OO+88GhoaMq7n9ttv5/Dhw63Tc+bMYd++ffkbaI4UGjElJcaoIbpXQ0S67qSTTmLZsmU5L58aGr3lUesKjRQ1lf0VGiLS6vrrr0+qp/H973+fH/zgB1x44YVMmzaNj3zkIzz66ImVqLdu3cqkSZMAOHLkCPPnz2fy5Ml84QtfSHr21DXXXEN9fT0TJ07kxhtvBKKHIG7fvp3zzz+f888/H4getb57924Abr31ViZNmsSkSZO4/fbbW7d3+umn87WvfY2JEydy0UUXdcszrnRHeIraqgpeeSv9c2ZEpIAeuwHeejm/66z9CMy+OWOX+fPn861vfau1ct/SpUv55S9/ybe//W0qKyvZvXs3M2fO5JJLLmm3/vZdd93FwIEDWbduHevWrWPatGmt82666SaGDRtGc3MzF154IevWreOb3/wmt956K6tWrWLEiBFJ61q7di333Xcfq1evxt2ZMWMGH//4x6muru6RR7BrTyNFTWUFO3UiXESCqVOnsmvXLrZv385LL71EdXU1o0eP5nvf+x6TJ0/mE5/4BG+++SY7d+5sdx1PP/106z/ekydPZvLkya3zli5dyrRp05g6dSobNmxg48aNGcfzzDPP8JnPfIZBgwYxePBgPvvZz/Kb3/wG6JlHsGtPI0W87Ksq+In0Ih3sEXSnefPmsWzZMt566y3mz5/PAw88QFNTE2vXrqW8vJy6urq0j0SPS7cX8tprr3HLLbewZs0aqqur+cpXvtLhety93Xk98Qh27Wmk0L0aIpJq/vz5LFmyhGXLljFv3jz279/PqFGjKC8vZ9WqVbz++usZlz/33HN54IEHAFi/fj3r1q0D4MCBAwwaNIiqqip27tyZ9PDD9h7Jfu655/KLX/yCw4cPc+jQIR555BE+9rGP5fHbZqY9jRQq+yoiqSZOnMjBgwcZM2YMo0eP5vLLL+fTn/409fX1TJkyhQ9/+MMZl7/mmmu48sormTx5MlOmTGH69OkAnHnmmUydOpWJEydy8sknc84557Qus3DhQmbPns3o0aNZtWpVa/u0adP4yle+0rqOr371q0ydOrXHqgFapl2d96P6+nrv6PrnTLbuPsR5tzzFLZedybyzxuZxZCLSWZs2beL0008v9DD6nHR/rma21t3rO1pWh6dS6PCUiEj7FBopVPZVRKR9Co00VPZVpPfoa4fQC62rf54dhoaZjTOzVWa2ycw2mNl1of37Zvammb0YXnNiy3zXzLaY2e/N7OJY+6zQtsXMboi1jzez1Wa22cx+Zmb9Qnv/ML0lzK/r0rfNksq+ivQOFRUV7NmzR8GRJ+7Onj17qKioyHkd2Vw9dRz4S3d/wcyGAGvNbGWYd5u73xLvbGZnAPOBicBJwK/M7NQw+0fAJ4FGYI2ZLXf3jcA/hnUtMbMfA1cBd4X3t939Q2Y2P/T7Qs7fNku1lf15ZceB7t6MiHRg7NixNDY20tTUVOih9BkVFRWMHZv7RT4dhoa77wB2hM8HzWwTMCbDInOBJe7+LvCamW0Bpod5W9z9VQAzWwLMDeu7APjT0Gcx8H2i0JgbPgMsA/7ZzMy7+WdHbazsa1mpjuCJFEp5eTnjx48v9DAkplP/IobDQ1OB1aHpG2a2zswWmVl1aBsDbIst1hja2msfDuxz9+Mp7UnrCvP3h/6p41poZg1m1pCPXyQ1VSr7KiKSTtahYWaDgYeBb7n7AaI9gVOAKUR7Iv+U6Jpmcc+hPdO6khvc73b3enevHzlyZMbvkQ2VfRURSS+r0DCzcqLAeMDdfw7g7jvdvdndW4B7aDsE1QiMiy0+FtieoX03MNTMylLak9YV5lcBezvzBXORuCtcJ8NFRJJlc/WUAfcCm9z91lj76Fi3zwDrw+flwPxw5dN4YALwPLAGmBCulOpHdLJ8eTg/sQpIlLhaADwaW9eC8Hke8GR3n8+Athv8tKchIpIsm6unzgG+DLxsZi+Gtu8BXzSzKUSHi7YCVwO4+wYzWwpsJLry6lp3bwYws28AjwOlwCJ33xDWdz2wxMz+DvgdUUgR3v81nEzfSxQ03W7YQJV9FRFJJ5urp54h/bmFFRmWuQm4KU37inTLhSuqpqdpPwpc1tEY801lX0VE0tP1pO2orarQ4SkRkRQKjXaoVriIyIkUGu2oCc+f0uMLRETaKDTaUVtZweH3mjn47vGOO4uIFAmFRjta62rovIaISCuFRjvabvDTZbciIgkKjXa0PkpEJ8NFRFopNNqhsq8iIidSaLRDZV9FRE6k0MhAZV9FRJIpNDJQ2VcRkWQKjQxqK/vr8JSISIxCI4N42VcREVFoZKSyryIiyRQaGajsq4hIMoVGBir7KiKSTKGRgcq+iogkU2hkoLKvIiLJFBoZqOyriEgyhUYHVPZVRKSNQqMDtZXa0xARSVBodEBlX0VE2ig0OlBb1V9lX0VEAoVGB1rv1dB5DRERhUZHVMFPRKRNh6FhZuPMbJWZbTKzDWZ2XWgfZmYrzWxzeK8O7WZmd5jZFjNbZ2bTYutaEPpvNrMFsfazzOzlsMwdZmaZttGTdIOfiEibbPY0jgN/6e6nAzOBa83sDOAG4Al3nwA8EaYBZgMTwmshcBdEAQDcCMwApgM3xkLgrtA3sdys0N7eNnqMHiUiItKmw9Bw9x3u/kL4fBDYBIwB5gKLQ7fFwKXh81zgfo88Bww1s9HAxcBKd9/r7m8DK4FZYV6luz/r0SVK96esK902ekxFeSlDB5br8JSICJ08p2FmdcBUYDVQ4+47IAoWYFToNgbYFlusMbRlam9M006GbfSomiEVvLVfjxIREck6NMxsMPAw8C13P5Cpa5o2z6E9a2a20MwazKyhqampM4tmRWVfRUQiWYWGmZUTBcYD7v7z0LwzHFoivO8K7Y3AuNjiY4HtHbSPTdOeaRtJ3P1ud6939/qRI0dm85U6pbayv0JDRITsrp4y4F5gk7vfGpu1HEhcAbUAeDTWfkW4imomsD8cWnocuMjMqsMJ8IuAx8O8g2Y2M2zripR1pdtGj1LZVxGRSFkWfc4Bvgy8bGYvhrbvATcDS83sKuAN4LIwbwUwB9gCHAauBHD3vWb2t8Ca0O+H7r43fL4G+CkwAHgsvMiwjR4VL/s6umpAIYYgItIrdBga7v4M6c87AFyYpr8D17azrkXAojTtDcCkNO170m2jp8XLvio0RKSY6Y7wLOheDRGRiEIjC7orXEQkotDIgsq+iohEFBpZUNlXEZGIQiNLKvsqIqLQyJrKvoqIKDSyprKvIiIKjayp7KuIiEIjayr7KiKi0Miayr6KiCg0sqYb/EREFBpZ06NEREQUGllT2VcREYVGp9RWquyriBQ3hUYn1OgGPxEpcgqNTqgNN/iJiBQrhUYn1FRFZV+PqeyriBQphUYn1FZW4A5NB3VeQ0SKk0KjE2qr+gO6wU9EipdCoxP0KBERKXYKjU6o0aNERKTIKTQ6IVH2dafKvopIkVJodILKvopIsVNodJLKvopIMVNodJLKvopIMVNodJLKvopIMeswNMxskZntMrP1sbbvm9mbZvZieM2JzfuumW0xs9+b2cWx9lmhbYuZ3RBrH29mq81ss5n9zMz6hfb+YXpLmF+Xry/dFSr7KiLFLJs9jZ8Cs9K03+buU8JrBYCZnQHMByaGZe40s1IzKwV+BMwGzgC+GPoC/GNY1wTgbeCq0H4V8La7fwi4LfQrON2rISLFrMPQcPengb1Zrm8usMTd33X314AtwPTw2uLur7r7e8ASYK6ZGXABsCwsvxi4NLauxeHzMuDC0L+gVPZVRIpZV85pfMPM1oXDV9WhbQywLdanMbS11z4c2Ofux1Pak9YV5u8P/U9gZgvNrMHMGpqamrrwlTqmsq8iUsxyDY27gFOAKcAO4J9Ce7o9Ac+hPdO6Tmx0v9vd6929fuTIkZnG3WUq+yoixSyn0HD3ne7e7O4twD1Eh58g2lMYF+s6FtieoX03MNTMylLak9YV5leR/WGybqOyryJSzHIKDTMbHZv8DJC4smo5MD9c+TQemAA8D6wBJoQrpfoRnSxf7tF1q6uAeWH5BcCjsXUtCJ/nAU96L7nOVWVfRaRYlXXUwcweAs4DRphZI3AjcJ6ZTSE6XLQVuBrA3TeY2VJgI3AcuNbdm8N6vgE8DpQCi9x9Q9jE9cASM/s74HfAvaH9XuBfzWwL0R7G/C5/2zxR2VcRKVYdhoa7fzFN871p2hL9bwJuStO+AliRpv1V2g5vxduPApd1NL5CqK2sYOOOA4UehohIj9Md4TlQ2VcRKVYKjRyo7KuIFCuFRg5U9lVEipVCIwd6lIiIFCuFRg70KBERKVYKjRwMG9SPfqUlCg0RKToKjRyYGaMq++vwlIgUHYVGjmpDMSYRkWKi0MhRTVUFOw/oklsRKS4KjRxFz59S2VcRKS4KjRzVVlZw5FgzB46q7KuIFA+FRo5qQjGmXTqvISJFRKGRo5ohuitcRIqPQiNHKvsqIsVIoZEjlX0VkWKk0MiRyr6KSDFSaHSByr6KSLFRaHSByr6KSLFRaHSBHiUiIsVGodEFKvsqIsVGodEFKvsqIsVGodEFKvsqIsVGodEFKvsqIsVGodEFKvsqIsVGodEFKvsqIsVGodEFKvsqIsWmw9Aws0VmtsvM1sfahpnZSjPbHN6rQ7uZ2R1mtsXM1pnZtNgyC0L/zWa2INZ+lpm9HJa5w8ws0zZ6G92rISLFJJs9jZ8Cs1LabgCecPcJwBNhGmA2MCG8FgJ3QRQAwI3ADGA6cGMsBO4KfRPLzepgG72Kyr6KSDHpMDTc/Wlgb0rzXGBx+LwYuDTWfr9HngOGmtlo4GJgpbvvdfe3gZXArDCv0t2f9ahu6v0p60q3jV5FZV9FpJjkek6jxt13AIT3UaF9DLAt1q8xtGVqb0zTnmkbJzCzhWbWYGYNTU1NOX6l3Kjsq4gUk3yfCLc0bZ5De6e4+93uXu/u9SNHjuzs4l2SKPuqBxeKSDHINTR2hkNLhPddob0RGBfrNxbY3kH72DTtmbbRq7Teq6ErqESkCOQaGsuBxBVQC4BHY+1XhKuoZgL7w6Glx4GLzKw6nAC/CHg8zDtoZjPDVVNXpKwr3TZ6Fd3gJyLFpKyjDmb2EHAeMMLMGomugroZWGpmVwFvAJeF7iuAOcAW4DBwJYC77zWzvwXWhH4/dPfEyfVriK7QGgA8Fl5k2EavMqoyev6U7tUQkWLQYWi4+xfbmXVhmr4OXNvOehYBi9K0NwCT0rTvSbeN3qaivJRqlX0VkSKhO8LzIKrgp3s1RKTvU2jkQW2Vyr6KSHFQaOSBHiUiIsVCoZEHNZUq+yoixUGhkQc1KvsqIkVCoZEHKvsqIsVCoZEHKvsqIsVCoZEHuitcRIqFQiMPVPZVRIqFQiMPVPZVRIqFQiNPdK+GiBQDhUbC4b3w2tM5L66yryJSDBQaCb+8AZZ8Cd59J6fFVfZVRIqBQiOh/ip4dz+s+1lOi6vsq4gUA4VGwrjpMHoKPH835LC3oLKvIlIMFBoJZjDjamh6BV77dacXV9lXESkGCo24iZ+FgcNh9b90elHd4CcixUChEVdeAWddCb9/DN7e2qlFVfZVRIqBQiNV/Z+BlcCan3RqMZV9FZFioNBIVTUGzrgEXrgf3jvUqUWjsq8KDRHpuxQa6Uy/Go7uh3VLO7VYbZXuCheRvk2hkc4HZkLtRzp9+W10g5/uCheRvkuhkY4ZzPhz2LURtv4m68VqKivYc0hlX0Wk71JotGfS52DAsE5dfltbFZV93aWyryLSRyk02lM+AM5aAL9fAfveyGoR3eAnIn1dl0LDzLaa2ctm9qKZNYS2YWa20sw2h/fq0G5mdoeZbTGzdWY2LbaeBaH/ZjNbEGs/K6x/S1jWujLeTqu/CrCsL79NlH3dpZPhItJH5WNP43x3n+Lu9WH6BuAJd58APBGmAWYDE8JrIXAXRCED3AjMAKYDNyaCJvRZGFtuVh7Gm72h4+DDnwqX3x7usHttle4KF5G+rTsOT80FFofPi4FLY+33e+Q5YKiZjQYuBla6+153fxtYCcwK8yrd/VmPnjd+f2xdPWfGn8ORt+Hl/9th1+qB5fQrU9lXEem7uhoaDvynma01s4WhrcbddwCE91GhfQywLbZsY2jL1N6Ypv0EZrbQzBrMrKGpqamLXynFB/8YaiZldfmtmVGjsq8i0od1NTTOcfdpRIeerjWzczP0TXc+wnNoP7HR/W53r3f3+pEjR3Y05s4xg+kLYed6eP3/ddhdZV9FpC/rUmi4+/bwvgt4hOicxM5waInwvit0bwTGxRYfC2zvoH1smvae95HLYEB1VpffjqpU2VcR6btyDg0zG2RmQxKfgYuA9cByIHEF1ALg0fB5OXBFuIpqJrA/HL56HLjIzKrDCfCLgMfDvINmNjNcNXVFbF09q99AmHYFvPIfsG9bxq4q+yoifVlX9jRqgGfM7CXgeeA/3P2XwM3AJ81sM/DJMA2wAngV2ALcA3wdwN33An8LrAmvH4Y2gGuAn4Rl/gA81oXxds3ZXwUcGu7N2E1lX0WkLyvLdUF3fxU4M037HuDCNO0OXNvOuhYBi9K0NwCTch1jXg39AJw2B9Yuho9fH938l0a87GvVgPKeHKGISLfTHeGdMeNqOLIX1j/cbhfdFS4ifZlCozPqPgajzoDVP2738luVfRWRvkyh0RmJy2/fehneeC5tF5V9FZG+TKHRWZM/DxVV0d5GGir7KiJ9mUKjs/oNii6/3fRvsP/NtF1U9lVE+iqFRi7O/ip4CzSccMEXoLKvItJ35XzJbVGrroPTZsPa++Dcv4LyiqTZtZUV/HbLHq6873nqRgxi/IhB1A2P3k8aOoDSkp59wruISL4oNHI14+qoQNOGn8OUP02adfmMD3Lw6HFe232I1a/t5fB7za3z+pWWMG7YAOqGD6JuRPQaP3wQdSMGclLVAEoUKCLSi1lfe9xFfX29NzQ0dP+G3OHOmVDWHxb+OrqyKm03p+ngu7y2+xBb9xzitd2H2Ro+b91ziKPH2uqJ9ysr4YPDBibtndSNGMjoqgGUlRhlpUZpiVFqRllJCaWlRllJW5sCR0RyZWZrY3WR2qU9jVwlLr/9j/8G256HD8xop5sxqrKCUZUVzDh5eNK8lhZn58GjUaDsPhxC5RBbdx/i1//VxHvHW9KuM9OQEiFSVlJCiUFZaUmYDuFSYhhQYoZZNL4SS0yHeSVhmrb58ffE8iUlUFpSQqlF7/FtlJVEIRZ/T4Rba9iZRcuXJC9fVmqUmKWZjn2X0kR4tvWJxhK+o7V9xyhLU74Hie8f79f2nZOno/5Y4nPUJ1pr2/zE74b4tLX+3cTW1cMFKEXySaHRFZO/AL/6ATz/L+2GRiYlJcboqgGMrhrAH5+SPK+lxdm+/whbdx9m18GjHG9xWlqc4y1Oc3hvm25pbW+OzY/eW2huIamPe/SM+RZ33J2WFnCcFo/2jNyjeS2hn7uHvm3tzS3Oe81Oc0tzynZbaHGi7TY7zZ48ntQxFrv2ggViQZXU15KWSyxDapilzLfQydKsLxGQEP1gsFjAJvrHgxXiPzrSh21Jyrrj723BHfuxEsI9uU/UrzT0iX5cJNqt9XNpSeJHTNsed2K5kpQ+rT+cLPGjheT1lbS1t/WJrdtI/hGU8kOp9YdL+GGU+PGTukxZGMP7kUKjK/oPhqlfikLjwA6oHJ23VZeUGGOrBzK2emDe1tkbxYOw2Z3m5kTQRdPHm9OHTmufE0IyCtHkYIzeSZpuC8i2wAzzSQ5Pj4UntD0MwInPa5smtnzUL838sKJEbKabn2hLNLT1bX+7rfNT5iWmE2txP3F+S/iQ+DM44c8o6cdFfDvJPzDaflxEn5tbPLZOb122Jbbe+N9PS2x+i7f90En8N+He9vce79P2nvt/jz2pxGgLHYsHULT3XVZSQkni3RLTKSEU9vgT875z0alMHju0W8et0Oiq6V+F5+6MLr+94K8LPZr3nZISo5/OxUietbS07eW60/q5JREuYQ87eveU4EkXRG172Il1H2/xpL3pdHvS0Q+hFpo92tuP7/W3JPU5cT3Z9Em8jjQ3c7zFOdbc/Ymp0OiqYSfDqReHy2+/E50YF5GCKikxSjDKSws9kr5HN/flw/SFcKgJNjxS6JGIiHQrhUY+nHIBjDg149NvRUT6AoVGPiQuv93+O2jsgXtEREQKRKGRL2fOh/6V0ZVUIiJ9lEIjX/oPgSmXw4ZfwMG3Cj0aEZFuodDIp+lfg5Zj0HBfoUciItItFBr5NPwUmHBRdPnt8fcKPRoRkbxTaOTb9KvhnZ2w8dFCj0REJO90c180h4iGAAAGV0lEQVS+nXIBDDsFVv5NdN9GeQWUVUQ3/ZUNiN7Lw3tZRdurvCJ5Ot6vtB+tT8tLPDko3ef4s2wyzXcnei5F/J0M82Lv8T4AVhJ7lYYHGpWc+CopjU3rDnCR9yuFRr6VlMAnfwjP3Ab7XofjR+HY0eg98Wo5XuhRFl5S0JSkCTkyTEPKh46Xydgn2+UyfqEs+mS7rmzlcV1Zraqr3zHD8p1dJuOfYy7LdLBsVjLco9Xu/VupP9iymZdhmUvvhLqPZhhj1yk0usPpfxK92tN8PATIu3D8SHiPh0tKW/N7JO8RtLN3EP98Qt/Y/KQ9j9R3kj9n6tO6/hZoaQ5PsWtJeTXHPqfMb2lOno6Ljz/jdBf7ZLtcJlnfz5nHGz/zehNpNt8x2+210y/j8p1dpov/OHd62XTr6q4ws+zmtdde0b0PKwSFRmGUlkHp4OgpuSIi7yO9/kS4mc0ys9+b2RYzu6HQ4xERKWa9OjTMrBT4ETAbOAP4opmdUdhRiYgUr14dGsB0YIu7v+ru7wFLgLkFHpOISNHq7aExBtgWm24MbUnMbKGZNZhZQ1NTU48NTkSk2PT20Eh3ucEJlzi4+93uXu/u9SNHjuyBYYmIFKfeHhqNwLjY9Fhge4HGIiJS9Hp7aKwBJpjZeDPrB8wHlhd4TCIiRatX36fh7sfN7BvA40ApsMjdNxR4WCIiRcu8j5UnNbMm4PUcFx8B7M7jcPJF4+ocjatzNK7O6a3jgq6N7YPu3uFJ4T4XGl1hZg3uXl/ocaTSuDpH4+ocjatzeuu4oGfG1tvPaYiISC+i0BARkawpNJLdXegBtEPj6hyNq3M0rs7preOCHhibzmmIiEjWtKchIiJZU2iIiEjWFBpBb6zbYWbjzGyVmW0ysw1mdl2hxxRnZqVm9jsz+/dCjyXBzIaa2TIzeyX8uf1RoccEYGbfDn+H683sITOrKNA4FpnZLjNbH2sbZmYrzWxzeK/uJeP6X+HvcZ2ZPWJm3V+WLotxxeZ9x8zczEb0lnGZ2V+Ef8c2mNn/7I5tKzTo1XU7jgN/6e6nAzOBa3vJuBKuAzYVehAp/jfwS3f/MHAmvWB8ZjYG+CZQ7+6TiJ5uML9Aw/kpMCul7QbgCXefADwRpnvaTzlxXCuBSe4+Gfgv4Ls9PSjSjwszGwd8EnijpwcU/JSUcZnZ+USlIya7+0Tglu7YsEIj0ivrdrj7Dnd/IXw+SPQP4AmPhi8EMxsLfAr4SaHHkmBmlcC5wL0A7v6eu+8r7KhalQEDzKwMGEiBHrzp7k8De1Oa5wKLw+fFwKU9OijSj8vd/9Pdj4fJ54geWFrwcQW3Af+dvBZ+z14747oGuNnd3w19dnXHthUakazqdhSSmdUBU4HVhR1Jq9uJ/qdpKfRAYk4GmoD7wmGzn5jZoEIPyt3fJPrV9wawA9jv7v9Z2FElqXH3HRD9UAFGFXg86fwZ8FihBwFgZpcAb7r7S4UeS4pTgY+Z2Woz+7WZnd0dG1FoRLKq21EoZjYYeBj4lrsf6AXj+RNgl7uvLfRYUpQB04C73H0qcIjCHGpJEs4RzAXGAycBg8zsS4Ud1fuHmf010aHaB3rBWAYCfw38j0KPJY0yoJroUPZfAUvNLN2/bV2i0Ij02rodZlZOFBgPuPvPCz2e4BzgEjPbSnQo7wIz+z+FHRIQ/T02untib2wZUYgU2ieA19y9yd2PAT8H/rjAY4rbaWajAcJ7txzWyIWZLQD+BLjce8dNZacQhf9L4b//scALZlZb0FFFGoGfe+R5oqMAeT9Jr9CI9Mq6HeFXwr3AJne/tdDjSXD377r7WHevI/qzetLdC/7L2d3fAraZ2Wmh6UJgYwGHlPAGMNPMBoa/0wvpBSfoY5YDC8LnBcCjBRxLKzObBVwPXOLuhws9HgB3f9ndR7l7XfjvvxGYFv7bK7RfABcAmNmpQD+64Wm8Cg2iuh1Aom7HJmBpL6nbcQ7wZaJf8i+G15xCD6qX+wvgATNbB0wB/r7A4yHs+SwDXgBeJvr/riCPojCzh4BngdPMrNHMrgJuBj5pZpuJrgi6uZeM65+BIcDK8N/+j3vJuAqunXEtAk4Ol+EuARZ0x96ZHiMiIiJZ056GiIhkTaEhIiJZU2iIiEjWFBoiIpI1hYaIiGRNoSEiIllTaIiISNb+P30YpuRmWFa7AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1c1caba3c8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_model_deep_meta()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}