vrnn_tf_v4.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import seaborn\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "from functools import reduce\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "GLOBAL_SEED=1\n",
    "VERBOSITY=1\n",
    "TESTING=0\n",
    "\n",
    "np.random.seed(GLOBAL_SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# useful function for recovering functions in deleted cells\n",
    "def rescue_code(function):\n",
    "    import inspect\n",
    "    get_ipython().set_next_input(\"\".join(inspect.getsourcelines(function)[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# wrapper around tf.get_variable that checks if the variable has already been defined.\n",
    "# useful for playing around in jupyter. allows you to call cells multiple times\n",
    "def get_variable_wrap(*args, **kwargs):\n",
    "    try:\n",
    "        return tf.get_variable(*args, **kwargs)\n",
    "    except ValueError:\n",
    "        tf.get_variable_scope().reuse_variables()\n",
    "        return tf.get_variable(*args, **kwargs)\n",
    "\n",
    "## do matrix multiplication with weights and add bias (as in fully connected layer)\n",
    "def fc_act(x, next_layer_size, act=None, name=\"fc\"):\n",
    "    nbatches = x.get_shape()[0]\n",
    "    prev_layer_size = x.get_shape()[1]\n",
    "    with tf.name_scope(\"fc\"):\n",
    "        w = get_variable_wrap(\"weights\", [prev_layer_size, next_layer_size], dtype=tf.float32, initializer=tf.random_normal_initializer())\n",
    "        b = get_variable_wrap(\"bias\", [next_layer_size], dtype=tf.float32, initializer=tf.constant_initializer(0.1))\n",
    "        o = tf.add(tf.matmul(x, w), b)\n",
    "        if act: return act(o)\n",
    "        else: return o"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sample Data\n",
    "I want to create 100 bi-variate time-series sampled from the same distribution and split them into train/validation/test 70/20/10, i.e. split into groups of time-series size 14/4/2."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def gen_time_series(n_points):\n",
    "    time_series = np.array([[np.random.normal(mu*i, sigma*i), \n",
    "                        np.random.normal(mu*i, sigma*i/4)] \n",
    "                       for i in range(n_points)])\n",
    "    return time_series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(100, 100, 2)\n",
      "20000 data values\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAEFCAYAAAAc33cJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzsvXmYJFd5p/tGZORWlVl7dlXvrVZ3\nH6klgXYhQEI2i8xmbGNfGBvs8YVrzwx+DGNfjMcGI2/XhjHYY2OwDTa2MRgwO8KAxCohJLSitfv0\n3tVL7WvumbHcP2LJyKysPbuqq+u8z6NHVZGxnBPZ9Ysvfuc739Ecx0GhUCgUlwb6ejdAoVAoFK1D\nibpCoVBcQihRVygUiksIJeoKhUJxCaFEXaFQKC4hlKgrFArFJYQS9U2CEOIFQojvCiGeEkI8I4T4\nuhDiqjW6tiOE6FvG/ncIIZ6Z57M/EkL8cutaV3funxZC/PUS9vuoEOKGZZ77JiHE33k/3yiE+NxK\n23mhEUJ8TAjxsvVuh2JlGOvdAMWFRwgRB+4GXiGlfNzb9ibg60KIy6SU1ro2cBlIKf/gAp77K8BX\nlrDry4G/X+bprwJ2eNd5FPj5ZR6/Zkgp37rebVCsHCXqm4M2oAtIhbZ9EpgFIkIIB/hL4AVAGtCA\nt0opHxBC/DNQAK4B+nFFbwJ4LTDg7fcdb78icC2wBbgH+E0pZTXcECHEW4D/gfuWOAH8hpTycJM2\np7xodh8wDfyalPKId51npJR/0XDeXwNeK6V8rff7FcC3gV3ArwC/DsSAHuDPpZQfEUL8V+AtQDsw\nA/wL8PNSytcIIV4AvB+IA1uBe6WUbxFC/CmwDfik98ZwGPg/3v2Jetd8p5TSDLVtJ/BHQKcQ4uPe\ndT4kpbx6Gfc3BrwPeAkQAZ7w7u9sw324AvhHIOF9jx+TUn7Y++z3gdd79/4U8D+klOeFEN8DJoEr\ngI94+3xISvk5IcQLveu2Axbwh1LKu4UQA8C/Av4b2NeklO9p8j0q1hhlv2wCpJRTwO8A3xBCnBBC\nfAL4VeBbUsoKcAuuUN0qpTyIKzq/GzrF9cBPArcDvw3kpJQvxBWz8H634EaxB73/fj3cDiHES3AF\n9jYp5XW4ovnFeZq9E/iglPJa4FPAJxbp5r8DL/bEBq9/HweSwP8DvMq75hu86/pcBdwhpfyJhvO9\nHfgDKeUtXl9+Wghxg5Ty94HzwC9JKX+E+zB8TEp5A3Adrsj9VvhEUsozwB8A90spf7VJ25dyf38X\nMIEbpJTP99rw503O9U7gq157XgXcLoTQvQfQNcDN3j39T+BjoeOmpJQHpZR/428QQnR79/DNUsrr\ngdcBHxFC7PLu6Qlv+23AfiFEZ5P2KNYYFalvEqSUHxRCfBQ30rsdeBfwLiHEzVLKB4UQ7wZ+XQhx\nOXAHkA0d/lUv4h4WQuSBb3jbj+NGvj7/LKXMAQgh/hX4GeBDoc9fjRt5/1AI4W/rFkL0SCknG5r8\nlJTyh/55ccVkXtGQUmaFEF8A3iSE+Evgl3AfHjkhxGuAVwsh9uO+SYTfWJ5qjHY9fgV4lRDi93Aj\n2GTDcT6vAW723kDw9lsuS7m/r8F923q5d+9iwGiTc30R+FchxM3At3Cjedu7BzcDj3rHR3Df4Hzu\nb3KuW3HfUr4U+r4c4HleG//TE/hvAb8rpZxZbscVrUeJ+iZACPEi4IVSyv+N663f7YnVM7giUcSN\nCj8AfBnXUnhT6BTlhlNWaY4Z+lnHfV0PEwE+IaV8l9cuHfcNYarJuRqPdcLXFUL8NK6lAXBeSvkq\n4KPAPwCHgENSypNCiB3Ag972HwCfwxVIn9w8fbkPeApXvD6L+xaiNdkvAvyClPKQ164ur63LYSn3\nNwK8XUr5de86KVyLpQ7PGtmP+8b0UuC93qBuBHiflPIj3vFxoDt0aLP7EMG9j7f4G4QQ24AxKWVV\nCHEZ8DLct4yHhRCvlFI+tqQeKy4Yyn7ZHIwB7xZCvDi0bSvQCTyNKwBf9f7gH8WNsCMruM4bhBBx\nIUQCN9L9asPn3wT+ixBiq/f7f8P1oJvxfCHEtd7Pvw78QEpZ8D+UUn5FSnmt99+rvG0P4QrvH+AK\nPMCNuP3/E1yf/zUAQoh5++cJ803Au6SUX8Ad4NxH7Z6YuP6536f/KYTQPKH8CvAbTU4bPmYlfBP4\nDSFEzHsYfhT4syZt/xTwBinlp3HHLmaBy73j3yqE6PB2/SMWt7QewrVVbvfOfS1wFNguhPhz4D1S\nyi/hWlXPAlevon+KFqFEfRMgpTyCK9T/n+epP4cbff6qlFICfwfcIYR4Gngc97X/Mk88lkMB9zX+\nae//H29oxz24g273CiGeAn4R+DkpZbPI9hBulPkk8NO4D4ml8FFgL/Al7/d7gLOA9M65C1fk9813\nAinlNK5gPu6lVv4u8EDomC8A/yaEeAXwm7iDiE/jRvZPU+/Z+zwE7PUsopXwx7iDm08Az+E+vH57\nnv1+ybtvP8K1Y+7D9c/vBh4SQjyLa6H814UuKKUcwx00/d/e+T6B66+fAv4KuNa7P48CJ4FPr7Bv\nihaiqdK7ilYwX1aKQqFYW1SkrlAoFJcQKlJXKBSKSwgVqSsUCsUlhBJ1hUKhuIRY1zz1sbHsir2f\n7u42pqYKi+94ibEZ+70Z+wybs9+bsc+w/H5nMulmcyaADRypG8ZK0qg3Ppux35uxz7A5+70Z+wyt\n7feGFXWFQqFQzEWJukKhUFxCKFFXKBSKSwgl6gqFQnEJoURdoVAoLiGUqCsUCsUlhBJ1hUKhuIRQ\noq5QKC4pbNvhC/ed4NzYfOufXNosOqPUW0zgo4DAXY3mV3FrOf8z7govzwBv85bMei/ukmUm8A4p\n5cMXqN0KhULRlLNjOe7+4SlKZZNffPmB9W7OmrOUSP21AFLKF+GuKPNB7793SylvwxX41wkhrsdd\n//IW4I3A316QFisUCsUCVC277v+bjUUjdSnll4QQd3u/7gZGcKPx73vbvg68AndlmXu8VWwGhRCG\nECLjrZ7SlO7utlVNj81k0is+diOzGfu9GfsMm7Pfq+3zaLYCQMSIbKj716q2Lqmgl5TSFEL8C/Cz\nwM8DrwktQZbFXeuyA5gIHeZvn1fUV1O4J5NJMzaWXXzHS4zN2O/N2GfYnP1uRZ8nJvMA5PLlDXP/\nltvvhR4ASx4olVL+CnAA119Phj5KA9O4C9ymm2xXKBSKNcOyXdvFsjbnAkCLiroQ4s1CiP/l/VoA\nbOBRIcQd3rZX4i4y/ABwpxBCF0LsAnQp5fgFaLNCoVDMiy/mylOfny8AHxdC3AdEgXfgrsr+USFE\nzPv5c1JKSwhxP/Ag7sPibReozQqFQjEvpifqphL15kgp88D/1eSjlzTZ9y7grlW3SqFQKFaIb7+Y\nyn5RKBSKjY9lb+5IXYm6QqG4pPA9ddNUoq5QKBQbnsB+sZX9olAoFBuewH5RkbpCoVBsfMxNntKo\nRF2hUFxS1CYfrb2oHzkzzTs//ACj08U1v7aPEnWFQnFJUZt8tPae+uHTU0zMljk1NLvm1/ZRoq5Q\nKC4p1jOlMVesAlCqWGt+bR8l6gqF4pKiNvlo/US9rERdoVAoWoM/UOo4NYFfK4JIvapEXaFQKFpC\nuDqjaa6tr64idYVCoVghZ0dz/M+/+QFHztRX+A5H5+Y6RepK1BUKhWKZnB7JMpOvMDhSv7iEZYcj\n9fWyX8w1vW4YJeoKhWJDUiy7wtlYjTFsv6zlBCTTsoOsFxWpKxQKxTLxBbRxMDRsuaxl+d28F6WD\nGihVKBSKZVOsLB6pr2VaYzYk6ipSVygUimVSKrvC2SjcdZ76Gop6Xom6QqFQrBw/Um9cYDpc82Ut\nUxpzyn5RKBSKlXOxReoXi/2y6BqlQogo8E/AHiAO/AlwFvgqcNTb7SNSys8IId4LvBowgXdIKR++\nEI1WKBSKku+pNyyGERby9bJf1jNSX1TUgTcBE1LKNwsheoEngD8CPiil/IC/kxDietzFqG8BdgKf\nB25qfZMVCoUCikuI1NcypdG3X2KGTqViYTsOuqat2fV9liLq/wF8LvS7CdwACCHE63Cj9XcALwbu\nkVI6wKAQwhBCZKSUY61utEKhUNQ89flFvdFvv5D4ot7TkWB4skC1ahOPRdbs+j6LirqUMgcghEjj\nivu7cW2Yj0kpHxNC/D7wXmAamAgdmgU6gXlFvbu7DcNYeaczmfSKj93IbMZ+b8Y+w+bs91L7XKm6\nYh4xInXHaHotOk62xdbsHla8B8i2TIrhyQLtHQm604klH9+qdi4lUkcIsRP4IvBhKeWnhBBdUkq/\n4MIXgb8BvgyEW5XGFfp5mZoqLL/FHplMmrGx7OI7XmJsxn5vxj7D5uz3cvpcKLmRcb5QqTumXK5N\n0Z+cKqzZPZycKRLRNZIxN//k/NAMZqm6yFEuy/2uF3oALJr9IoToB+4B3iWl/Cdv8zeFEDd7P78U\neAx4ALhTCKELIXYBupRyfMmtVCgUiiViWjYVr66LNWegNJT9Yq+l/WLSnoySiLmx8notlLGUSP33\ngG7gPUKI93jbfgv4KyFEBRgGfk1KOSuEuB94EPdh8bYL0WCFQqEIC+bcgdJwnvoaDpQWKnSm4iQ8\nH728ThkwS/HU3w68vclHL2yy713AXatulUKhUCxAKWSxzCkTsA556rbtUCiZbM+kiEc9UV+nSF1N\nPlIoFBuOcKQ+J/tlHWq/FMomDpBKRoOMl/WyX5SoKxSKDYefzggLR+rVNUppzBYqAKSSBomoEnWF\nQqFYFv7EI5i7utF6zCjNF92HTCoZCyL19fLUlagrFIoNR2mRSD3i5aqvlaj7E49Sddkv67P6kRJ1\nhUKx4SiGBkqbeep+BspaLZLhi3p70lj37Bcl6gqFYsMxX0qj4zjYjhNYIGuV0hiO1OPKU1coFIrl\nUZwnpdEfJPWFtdFvv1D4op5OxmqRuhJ1hUKhWBp1KY323IHRQNTXOFJvTxrrPlC6pNovCoVCcTHh\nR+rtCSMoFwC1SH29PPVUMooRcWNlZb8oFArFEvEFM5WM1nnq/sSjIFJfw+wXDWhPRNWMUoVCoVgu\n/uSjVDKK47jT9CHkqcfWVtTzxSptCQNd19B1jVhUX7fVj5SoKxSKDUepbKFrGsm46yD74u2nNxoR\nnYiurdnKR7lilVQyGvyeiEZUpK5QKBRLpVgxScYjgX/te+d+qd2IrmFE9DXx1B3HcUW9rSbq8VhE\n5akrFArFUimVTRIxAyPizRy150bqRkRbE/ulVLGwbIdUIiTqUUMNlCoUCsVSKVUsEqFI3R8gtRoj\n9TVIaQxnvvgkYq794jhrt0iHjxJ1hUKxoXAch2LZIhkziHiRuh+hB6IeWTv7pZajXm+/2I6zZgO1\nYZSoKxSKDUXFtLEdpy5S9710P2KP6GtnvwSzSdvqB0phfXLVlagrFIoNhb/qUTJmYOj+QKld9/+I\nrmEY+pqKemOkDuuTq65EXaFQbCj86DcRi4Tsl3pP3ZjHfqlUrZYLfeCpJ+aK+nrkqitRVygUGwp/\n4lEyboRSGn1P3YvUm2S/mJbN//qHh/j3bx1taXvyzQZK13FW6aK1X4QQUeCfgD1AHPgT4DngnwEH\neAZ4m5TSFkK8F3g1YALvkFI+fGGarVAoNiv+qkd+fRcITz6qZb9EIzqW7Zbi1TWNYtlkKlvmxNBs\nS9vjr3rUzH65WCP1NwETUsrbgFcCHwI+CLzb26YBrxNCXA+8BLgFeCPwtxemyQqFYjMTeOpxo7bC\nkd08pRFqmTF+1DydK7e0PYWy56knajHyekbqSxH1/wDeE/rdBG4Avu/9/nXgZcCLgXuklI6UchAw\nhBCZVjZWoVAowp56o3AHA6URPfisajp1x83mK0GtmFaQL9UeMj7rOVC6qP0ipcwBCCHSwOeAdwN/\nIaX070oW6AQ6gInQof72sfnO3d3dhmFE5vt4UTKZ9IqP3chsxn5vxj7D5uz3Yn2OxMYB6O9LM5kt\nAdDWniCTSdPWPgVAd2eS9rYYAJ1dbXSl40zk3YjacSCajNHTkWhJe6uWg65r7Nzehaa5bw6Z3hQA\n0bix5O+wVd/1kuqpCyF2Al8EPiyl/JQQ4v2hj9PANDDr/dy4fV6mpgrLa22ITCbN2Fh2xcdvVDZj\nvzdjn2Fz9nspfR6byAFQKVcpFSsATE7lGRvLMj1TBKBQKGNZbpQ8MjpLtZRgaLTmpZ84PYk10BoR\nnc6WaIsbjI/ngm2VkvsAGZvML+k7XO53vdADYFH7RQjRD9wDvEtK+U/e5ieEEHd4P78SuB94ALhT\nCKELIXYBupRyfMmtVCgUiiXg2yjJZmUCgjx1fU4Oe9gKmWqhr14om3V+Olzk9gvwe0A38B4hhO+t\nvx34ayFEDDgEfE5KaQkh7gcexH1YvO1CNFihUGxu/FWPErHQQKnvqYcHSg3PU7fqPXWAmVaKesmk\nJx2v2+Zn5qzHjNKleOpvxxXxRl7SZN+7gLtW3SqFQqGYhyBSjy1QJiCiBRUc/eg9LLDTuUpL2lI1\nLaqmTVto4hHURH09yu+qyUcKhWJDEUTq8VDp3YbJR0Y4+8W3X6qtj9T9zJc59stFntKoUCgUFw31\nZQIaPfW5eep++d2SNxMVWhepFzxRny9SVwW9FAqFYhGKZZOooQcLYUBzTz0aqZ+YVG+/tCZSD0Q9\nXh+px6LKflEoFIolUapYJL1IuDHDpb72S2Ok7gqspsFMvjWRer40dzYp1OwfFakrFArFIhQrJgkv\nMg5SGufUU9fmFPvy/e2+zgQzudbMKq3ZL3NzThLrtE6pEnWFQrGhKHmrHgFB6d3Ggl5GRA+lNNYP\nlPb3tGE7DlmvuuJqKJSbe+rgDpaWQz5+cEzJ5K/+40mOnl1wbuaKUaKuUCg2DLbtUK5awUBkLRr3\nC3qFFsloqLVeKptoQH9XGwDT2dX76r79Ml+k3sx+efL4OE8dn+DU0IWZLaxEXaFQbBhKlfriWY25\n6GbDGqVQi9RLVYtYLEJX2q0JM5NfvagX5klpBHdWaTP7RQ669WkO7Oxa9fWboURdoVBsGIJ0xrgb\nqUcaI/WGeupQP1CaiEXoSrmzP1uR1jhfSiO49otpzV18+vDgNMm4wc4tqVVfvxlK1BUKxYahGFqf\nFMAI6qk3ZL/o+hxrplyxSETDot5C+yXe3H6B+rTGqWyZ0akiB3Z0onttbzVK1BUKxYahGJp4BE08\n9SZlAsyQ/ZKIGXSmXPulpZF6E1FvVtTLt17Eru5VX3s+lKgrFIoNQylUIgDmeurBwtN1kbqN7TiU\nKxbxkP3SilIB+ZJJMm40jbr91Y/Cg6WHB92Mlyt2Xxg/HZSoKxQblmLZrJv6vhkIF/OCuZ66GUw+\nqlVpNC2HSrUW4bcn3AWrW2G/FMvVpoOkEIrUq/WRejIeYdeWC7f4iRJ1hWKD8r5PPc5ffvbJ9W7G\nmlIsN89+abbwdPizcL0YTdPoSsVaYr/kS2ZT6wVqRb38a09ly4xMFTmwo+uC+emgRF2h2LBMzpaZ\nnC2tdzPWlGlven+6zc020TUNjbn2S3hGadWyA1/bF9rOVMxdq9RZ+axSy3YfFs1y1MGt9w41T30t\n/HRQoq5QbFhMy6Zq2ovveAkxPu0uV9fXmQRA0zQiET1UT90momtoWi2l0aqL1F2h7UrFsWyHXGHl\ns0prOepz0xmhZr/MFtwHkTzj+uli14Xz02GJa5QqFIqLD9Ny0LTNJepjgajXFo02IlpdlUa/dID/\n/6rpBGMPvtB2tdfSGjvaYytqS61EQHMZ3Zlx89D/7Z4jlCoWh097fnr/hclP91GRukKxAXEcZ3NG\n6jMlulKxoLQtuGmN4XrqEa9yYzSU/eIPVvoDrP6s0tX46gsV8wLYt6OT33z980jGI3z620cZmSqy\nf0dX0L4LhRJ1hWID4nvHpuWsyhe+kIxPF5lqQX0VH9OymZgt0deVrNseCUXqlm0H65bWsl9q9osf\nqXe2r34CUq3uS3P7BeDa/X380Vtu4XmX9wLw/H19K77eUlH2i0KxAQlPPTdNuy5yvVj4i0//mI5U\njN970w0tOd/kbAnHgUxnvagbuh4q6FWzX2q11p2aqEfrI/XV5KovVPclTGd7jLf//PMYniww0NO2\n4ustlSWLuhDiFuB9Uso7hBDXA18Fjnoff0RK+RkhxHuBVwMm8A4p5cMtb7FCoQhEDNzsjotN1B3H\nYWK2xGyhguM4aNrqU/jGZtxMn0xXom67EdEoefaKZdlB6QDDaJbS6A2Utq++/stCs0kb0TSNrb3t\nK77WcliSqAshfgd4M5D3Nl0PfFBK+YHQPtcDLwFuAXYCnwduamlrFQoFUB+pX4y+etW0sWwHq2KR\nL5mkkvNbFEvFHyTNNNgvRkTH8gTWtJ0gGo/oOprmeereQGki8NTXxn5ZD5YaqR8Hfg74hPf7DYAQ\nQrwON1p/B/Bi4B4ppQMMCiEMIURGSjk230m7u9swjJVHGJnMhZuVdTGzGfu9GfsM8/fbidT+btId\nSTJrFAUulalsLX/e0vRlfX/z7VuouA+v/Xt66/aJxw3sXJlMJo3juL65/3k0ooOmoXs6M9CfJpNJ\n0+c4GBGNQsVa+b8tz97ZvrWjJf8+W/VvfEmiLqX8vBBiT2jTw8DHpJSPCSF+H3gvMA1MhPbJAp3A\nvKI+NVVYdoN9Mpk0Y2MXpsj8xcxm7Pdm7DMs3O+Rydrfzsholoh9cUXr4fYdPTVBZ2JpwdtCfT59\nfgYAw7Hr93Ecqqa7rWraODbB55GITrFkMjnjRvnFfDn4LGZEyBeqK/63Neb1sVKsrPrf53L/jS/0\nAFhp9ssXpZSP+T8D1wGzQPhKaVyhVygULeZit1/CRazGZ1oz63VsuogR0QLrxMfQG7JfIjX/PhrR\nsOzajFLffgGIGnqwgMZSuOfhQe76+MNBHZmFlrJbT1Yq6t8UQtzs/fxS4DHgAeBOIYQuhNgF6FLK\n8VY0UqFQ1FMn6ssQprXCr9ECMNEiUR+fKdHbmURvGHSNRHQcx13qzrKcYKDU/6xqzh0oBVfUTXPp\nC0M/fXKSwZEcJ4dmASh4nvpi2S9rzUpF/b8DfyWE+B7wIuBPvMj9fuBB3EHSt7WkhQqFYg512S8X\nYaReDFWPHPOsj1Wdr2ySK1bnZL5AaOaoZXspjTVZi0b0uoHSeLQhUl/GvZv16s4cO+faQPmSSSxa\nK/F7sbDkR4yU8hTwAu/nx4EXNtnnLuCu1jRNoVAAnB3N0ZWO12WQmOZFbr+UaxFwKyL1IPOlIUcd\navnovi0SCUXqhqFTKJuUKpa7xJ1RL/jLecvxa7gcP+dG6sWSOW/dl/Xk4nrEKBSKOipViz/+10f5\n5L1H6rab9sUt6uFIfXymhLPKWa9j036OehNR9yJ1vxRA2FP3/XZ31aP6wdqooVOpzr13+VI1iMp9\nbKdW/Ov4+RkcxyFfqi4pR32tUaKuUFzE5IpVqqZNtlAvMqYZnny0dF94rfA97JihU65aZIsrr4YI\nMD7j56jPtV98+6PsCbQRqq1iGL790lzULdvBtusfOH//5Wf5s397rG5boWQGpRmyhSqj00UKZXPe\nui/riRJ1heIiJueJodkQjV/s2S/+QOn2jJs/v1oLZqyh5G4YPzL3M1zq7JeIHpQJiMfqBTjq5a43\nWjBjMyVGpoqBnQM1P90/97MnJ3Gc+cvuridK1BWKi5i8N1OyUXgaa79cCMami7zzww9w6NTkso/1\nPfUdXvnZ1aY1Lmy/+JF6E/vF+zlfqtYNkgKBv974UKx6GTHTIQvGF/UrdrsLXDx13J2SoyJ1hUKx\nLPJepF416y2Ctch+OX5uhonZMg88M7zsY31PfccWT9SnV5cBMz5TpD1hNBVR324JRD1sv3iC7zg0\ntV9g7v3zffZwsS9/kPTqy3owIjqHT7urGClPXaFQLAt/gstCkfqFylP33xLk4PLnEPqe+s4WROq2\n4zA2Pbfkro8fmVeaROrRULrhHFEPLXcXxv89XOwr6w2Sdqfj7BlIU/EeBCpSVygUyyK/jp667+dP\nzJaCgcql4nvqQaS+ClGfyVUwLbup9QIh+8V7kNRPPqr9PG+kXq0faK56kfp0qBb8jGe/dLTF2Lut\nI9iuPHWFQtGUqmnxkS89w5Ez9VFxzpu1aM6J1Ftnv1i2jRycmpN2mAtlrDS2azFKFZOooZNKRmlP\nGMt+KISp5ajPzXyBmm9eamK/hCP1uQOlcyN107KDRUfCFRz97KOO9hj7tncG21cSqVdtky8d+0+G\n8yPLPnYpKFFXKC4CTg5leeTwKA8+W+9f+zW7G4W7lZH6vY+c5X2femKOzZIPifpyLZhi2QqWjuvr\nTDKxilz1+Uru+vgzSJvZL4axgP3SxFMP/xwWdX+gtKM9xuULiHrVqvLE6NPYzvzfyROjT3Hv4Pd4\nbvLIvPusBiXqCsVFgO/ZZhtWtw/sl4WyX1bpqT913C3RNNYwmOm/JcSiOnKZkXqxYpLwBhH7uhJU\nTJvZwspy1Ycm3GqIme757JeFJh+FRH0J2S8V0wbdAt2s89RnCxUiukZbwqA7Haenwy0q1mi/fGvw\n+3zsmU9waPIo8/H46FMAHOwR8+6zGpSoKxQXAbmiKyCNk4zyoUg9HOmuxH75/o/P8bt/92CQyQGu\nEPq1TBonCOWLVaKGzhW7uhmdWt56o6WKRdKzO/o822SlFszhwSkiusberR1NPw+yX7x663XZL0ZN\n4ONLitQtYvsfJ37woXr7JV8l1RYNion5Fkzj4h9PjT8HwFRpqmlbi2aRQxOSbe0DDLRvma/Lq0KJ\nukJxEeBH6I3RrB+pO9QWm4aVZb8cPTvD6HSRxw6PhrZNBw+IxgdKrlilPWEgdnYBIM80F6pGbNuh\nXLGw00McnjwaTBgan17+YGmhZHJyaJbLtnaQnCd9sDFSNxomH/nMm/0SEvVcuYjeMYHelmO6UKsJ\nP1uo0NEWC37/2dv38ksvP8DW3tqaozPlLIPZswBkK3ma8dTYc5iOxfVbnr9Ar1eHEnWF4iLAH5TM\nzROpw/yWy1Ijdd9zfiQk6s+dqgl1o/WTK7rL0B3Y5Yr6kTMzS7qOn8442fUwn5FfXFWkfuTsNI5T\nm/TTjMgCk4/qB0rnidRD9/LY/ELoAAAgAElEQVT07Bn8yr4lspQrFpWqRali0dFeE/X+7jZeesOO\nurVXn5s4HPycreaattW3Xq7fcs28/VktStQViosAX1DzJbNOsP11MKFevFci6n5tFHlmOhj4O3Rq\nKpj6HhZ1y7Ypll1R392fJh6NIAeXFqmXKiZgY+sVJkpT9HS4YriSUgH+JJ8rFxB1Y06ZgJqs1ac0\n1kf6Mb9MQOj+nc6dDn7W4gWm8+XArupoWzh98ZmJQ8HPucpcUS9UixyePML21Fb6L5D1AkrUFYqL\ngmyxFqHnQoOj4RWEwj76Sjx1P1J3HHjsyBjZQoXBkSz7d3QSNfQ6r91/Q2hPRjEiOvt2dDI0UZhT\nvbAZxYoFEfd4y7EwEu4xYysQ9UOnp4gaOvu2N/fTYeEyAXWTj5YwUHq2MBj8rMWLTGfLzObd7yMd\nsl8aMW2Tw5NH6Um4D59sE1F/evzCWy+gRF2huCjIhaLkIGpvGLismmGBX76nXq5agbXw6OFRDg9O\n4wAH9/SQbovWWT/+tf3sDt9XX0q+eqlsohk12yhnzZBKRpddKiBbqHBmNMe+7Z1B8a1mRBrqqRv6\nPCmN8fpzGA2ibtomw+XzOJa7XY8XmM5VgoddZ/v8on5s+iQlq8zz+g6SNJLkqnM99cdHnwQurPUC\nStQVF4hnT04yuoqFxTcb2TpRb54JU11tpG7atCeiXL6tg8ODUzzk5cQf3NNDOhmra4P/tuBndxzw\nRN3PlFmIYsWESO1c48VJMl0JJmbLwcSepeDnxi9kvUCzlMa5tV+ABQp6uccNZs9hOSbWVD/gReq5\nMlnv7WShSP1Zz0+/uvdK0rH2OZF6oVrk0ORRdqa2saUts2B/VosSdUXLyRWr/OVnn+Rz3zu+3k3Z\nMIRnb/qRYa5h4NJs4qnHo5Gle+oVi3hU58YrtuA48MTRcZJxgz0DadJtUSpmbYHmfNGNtH1R3+lN\n9z871nwAMEypbNVF6hOlSXo7k5iWzUxucfvG57kl+OlQE+5S09K783vqjbVfjk+fBMCe3kLEiaHF\ni8yEIvWOBSL1ZyYOEYvE2Ne9l3Q0Ra6ar5uAdGz6BJZj8bzMVQv2pRUoUVe0nOGJgrtSzCoXRtgs\nVKpWEGVCLWr375+fG904nR1cS2HpkbpFLBrhRlEbpLtiVxe6rpH2BgH9twP/2u1JVwiTcYO+zgRn\nx5qn6oUplptE6ivIgDl0eopELMKerekF9wsidasIOHPqqfssNqP0+Iwr6la2m3a9Ay1eZCpXCjz1\njvbmA6WjhTFGC+Nc2b2fqG6QjqVwcMhXa2+qk2X3raP/AkfpsAxRF0Lc4i00jRBinxDiB0KI+4UQ\nHxFC6N729wohHhZC/FAIcfMFarPiImfEs13KDYWSNisTMyU++tXn5tgpPr6AdqfdWYq12aV+hOiK\nSV2kbtpouIN/y/HUY9EIvZ2JoCjVwT09QM1a8CcgBfZLaMbkjkyK2Xxl0cHSUsVCi9RH6rW0xqUN\nlk7OlhiZLHBgZ1ddNkszIhEdLZGjeuU3iWTO1gn5QvZLzBP1imljOzbHp0/RpnVANUFXrBstYjFZ\nmK19D/PYL/6Eo6v6rgAgFXPfasIWzHTJta264l2Ld36VLEnUhRC/A3wM8CvqfBB4t5TyNkADXieE\nuB54CXAL8Ebgb1vfXMVGYHjSFfVw5sZm5vEjYzz47DAPPde8gJMv4tu8iSy5Bk+9M+WKfV2kbjtE\nIjpRY2mRuuM4VKp2IGyvuGknfZ0JrtvfBzAnUvdTKdtDMyb9VYwWs2CKFRPNqI/U/bK5Sx0sPTy4\nNOsF3Ehdb8uiaRDpHm0aqceiOnpoO9RH6sP5UQpmkW59K1AT38nKVFChsZmnbtom3zvzAFE9yvP6\nXGslHXVFPRfKVZ8u+6LeOeccrWapkfpx4OdCv98AfN/7+evAy4AXA/dIKR0p5SBgCCEu/LuG4qJj\nZMr9w61cBJH60ER+1bVRVotfPfDk0GzTz/10xq19rmj6s0p9T73L83IbI/WooRE19CWJul//OxZ1\n/+RvvrKf9//3F9LT4cZpQaTekHkTngbvr2K0mAVTKtdSGqN6lNlKls60+zBZaqR+cigL1AZoF8LQ\ndYi6U/r11BThwD7qWTON6YxQW87ONO3AeunAHSTtirkPk5w5Q7ZQIRk3MCIapm3WnePh4SeYKk/z\nom03k/Yi9FTM/R7Ds0qnyzNoaHTGF7aSWsGS6kZKKT8vhNgT2qRJKf1h7CzQCXQAE6F9/O1j8523\nu7sNY4FUpcXIZC78DboYudj7PTHr/uFWTLtlbV3JeU4PzfLuj/2It/z01bzu9stb0o6VYHiCMjiS\na9oPzZupuW9XD9957CylqkUmkw4skIFMiiePT5BsiwfHO5pG1IjQloxiWjZ9fam62Y2N+Kv4dKTi\nTduwY8C1Y2xNJ5NJ481TYveObro8W+ga4QDPMpEtL/h9OLoWROp7urZzdPIUPf1u22YK1UW/y0wm\nTcl7CO3f00t3R/OSuz5lB7So+2DUDBMrmSeT2QvAaNbd3p6MzbluvM3tlxbROVt0p/dnYjuBCXb1\nDsA4WJE84xMlejoSfOPcPXzv1EP8wR1vZ0/3Tizb4tsPf4+IHuEN172a3jb3/NuLXiwbN4NrzppZ\nOhNptvbP/+bRqr+VlS7bEQ4N0sA0MOv93Lh9XqZWkfKWyaQZG8uu+PiNysXeb8dxOO9FcsWy1ZK2\nrrTPTx0ZwXHg2ODUut6zSc9yOD+e59SZyTmV/c4PuxG8btu0J6NMzpQYG8sGVkjcizYnpvJBP8pl\nE13DnUkEDA3PLJjL7Q9QOpbT9F7YXlrf8FiWsbEsE9Pu32YxX6JactsRw8GIaBw7s/D9nJ4tBZH6\nQHIrRznFqdHzdKZiDI3nFjzW/67HJgtoQLlYZqy88ID77EwRLVorvnV84jhjY3sAyOfcAMOIaHOu\n62f65AolTg4fJhVtp5pzHyAJy422tUSBUsWiLRHhB6cfJVfJ86ff+xDvvPE3OD59kuHcGC/adgt2\n3mAs757fKbnfw9DkOGNjWRzHYbIwxdb2gXn7vtx/4ws9AFaa/fKEEOIO7+dXAvcDDwB3CiF0IcQu\nQJdSjq/w/IoNynSuEgyQmpaNZa+f9eFXFWysp7LWhAeMTw3N/cP1ByfTbVHSbbFaBopnhfieel3t\nF9vGiOhNi1I1w193s7H+iU/NU/cHSk2S8cicQcetve2cG89j2/PnmxfLJpqX/bIztQ2AcW+wdHK2\nvOCxPrP5Cu3J6KKDpH67/EgdYKh8tu4zaN5vv4JjVh9mppLl+ZmrAourL+kOIGtx92GYSJWYLs+Q\njqWYqczyt0/+I18/9W10TecVu++oO2+6YaA0bxao2ibda+Cnw8pF/beBPxRCPAjEgM9JKR/DFfcH\ngc8Db2tNExUbiZHJ+rcvvxzqejA564n6BUytPD+e5/4nzy+4Tzk0YHyiia/uP3RSbTHSyWhQ/yXn\n1fBOe752tcFTNyL6vIsnz2mD92CJGc3/5NNJ31OvDZQ2W6ptR6adStWeU3s9jD+jVENjW2oAgIni\nJJnOJJbtLKmEb7ZQWXAGZ5hIREOLlnGsCE41yvnimeAzP92xMZ0R3JmoEV1jNn4CgJsHbqitPRqL\nk9DaA1G32l0X+bV77+QlO17EUH6E4cIoN/ZfS1+yt+68/kBp1ptVGmS+JNZG1Jdsv0gpTwEv8H4+\ngpvp0rjPXcBdrWmaYiPipzNqmusMlKvWui3OO+l5+411wlvJ1x48xYPPjnDl7u55F0YOR+onz88V\n9SBST0ZJe0KWL1bJFqq0JSJ8Z+rLGAM6prUvOMa0nGWJuj9o3ZjW55OMR4joWtCWfLHKNm/gNow7\nWDrC2bEc/T1tFMsmf/Zvj3PrVf288gW7Abf2i26YtBnJQPAmipP0dh4EXCuod56l6dy+2eRLZjDh\naTEMXXdFvRrDKaTJRkeZLE3Rk+gOIvVmA6UA0ZhNMXGW3kQPl3fu4ZvmM+72qE5ntIuifQ40m7wx\nBA5c0X2AW7feRLaS5bkJyZ27f3LOOduiSTS0oKhXkPkSu7gjdYWiKSOTbmSztdcVhPXMVa/ZLxdO\n1HPezEt/cLgZfqTemYpxYmh2zrJu2UIVDXeij2+DzBaq5IoVkqkqp4tHMXYcYbpSy0MwLRsjojUt\nH9u0DdX67JdGNM2dgDSbr1CpWm5JgeTcSH17QwbM/U8NcXYsxxPHak5rsWyCUSVpJEhF24lFYoyX\nJoPl6MYWqaseXjpuKei6A0YFpxrHzrkDkce82aFdqRjb+9oRu5oPUEa6R0G3uHngejRNCx6OMUOn\nN9GNprkWzJRznkyyl95kN7qm85ar38SfvujdTRe60DWdVLQ9KL87VV7bSF2JuqKl+JH6rn73j7+8\njrnqk9ma/bLS9TEXo1h2RX0qN7+l4E760bl8Wyez+coc+yFXrNKWMIjoemC1ZAsVsoUqsXZ3X013\nOGw+GBxjWg6GsRxPfeFIHdy0xmyxOqfuS5gdoVx123a497ETRHqGGMnVHjiligWRKsloEk3T6Ev0\nMFGcDJaAW2xW6ULT8o9Nn+TPH/4rRgu1pLqKXUbTHajGsbKueB+fOQW4aYt//NZbeOkNO5pfrMf1\n328euM49l3efooZOf8rN4Y/0DGNSQfTsrzs0YcTn7UM6lgpSGtcyRx2UqCtazMhUMZhSDusXqZuW\nHUR8lu0E4ttqihX3vNPZ+Qdjy1WLeDTCZd5098Z89VyhEuSJ+0I2Nl3Eth0iSTeqdWyNMecUR6aO\nYdsOtuNg6NryPfUFRT1KuWIFa3OmEm6OefiB2J2O0xY3OD09wocf+Qz5vd8ktu9JSr3PBPe4VKmA\nbpE03Mi8N9lDySqTSrltXCxX3Z+W3+ipV6wqnzj0Wc7kztetAZoz3YjYtV86iOpRTkyfWvAa4Iqt\n3TaGVugOimxVTZuIrhHRdbalPVHPuB79ld375z1XI6lYiqJZxLTN0GxSJeqKDYZtO4xOFejvTgYR\n4XrNKm2Mhi+Urx5E6gsM/tVE3c0FDw+WujVyTFKe7eKLuz8rV4u5/zfPuoLy+aN3UzHdaxrGMjx1\ns1YAbD5q13YjzHJ8hP/1gz/mO2fuD/bRNI0tW6vkdn6LQ/nHwY6Ao6HHi4xNF6maNqbm3us2w32w\n+5kkluGmKS4u6s1ncN5z+ruMF903goniZG3/ipdKWI2Do7OzfSfn88N1tVea8cjwE+58+MlaFF8x\n7eCeZtrc8QA97rb3QPfS5zqko+4bTa6aX/NIfX1GsBSXJJOzJUzLYaCnLRCP9ZpVOtngcecKVRaY\n97FilmS/VCy60nH2eBN8woOlhZKJ7TiB7eL/f3jCFSTLcAXWmhxgYLvF2dxxfjT8OOAOEC7ZU6/4\nkfr8cZx/7SHv2qP6EbDgP0/ey00D19ERS+M4DqW+p9F0m8qpgxxMP59zfV8lHy0zOlWkKx0P0hmD\nSD3hivp0ZZrujviK7JeRwhj3nv4u7dE28tUCE6GFnWui7u5/WcduTmRP8LWT95Kv5jk+fQoHh/Zo\nG21em6q2yUhhFByd6kR/cK6qaQcZQuGslp2p7bRFa+uRLka4/stUeYb2aBuxyMIrJ7UKFakrWsaw\n56dv6U4GecHrHan7NlDj+putwHYcd0o8ML2ESL0tYbC1t41Tw9mgrrifQugPkPrZL36kXo3k0NFx\nKgkGKtcT1Q3uPfNtYJmeurkET92/9kQBNIth8yQaGiWrzNdO3AO4JWZn9PNY031Yo7t4xU27SUXT\naNEyw5N59/v2yu76AupH6hPFSfo6Ekxly0HO/ee/f5w//pdH6+Yz+JG6b784jsNn5BcxHYv/Il5P\nVDeYKM0TqQN7Oy8D4PtnH+DRkR9j2iYRLcJkaYqj0yc4On2Cs7nzgEY6f4Bq2QgspqppBZO4OuMd\naJ5EXtlzYMH7O+deRsOiPr1mUTqoSF3RQvzMl4GetqB40np56v4g6e7+NOMzpbrl4lpFuWLhu83z\n2S+mZWNaTiCml23tYGhimOGJAtv62kODkq6A+eLuZ4iUyNIZ7ySPjm4muWXgBn5w/kfoXaMYkYEg\nUj+VP44+Osy186yqs1RPHWBosoDeNUbVqfCTO2/j2QnJA+cf5sXbX8AXj30NDY3q4BXsyLRzcHc3\n3eMdjFWGGJqeoVTumzdSHy9N0tfVz5GzM0zOlkjEDb758BlMy2ZitswWLztmNlQVsWJV+drJe5BT\nx7iq9wquzVzNV72BV59GUT/QvZfX7r2ThJFAdO9joG1LUELBsi00TUPX3Pv2gc/8mFEmMS2HqKFR\nMe1ggFjXdPqS3YwVJ7iyd+l+OkDaq/8yVpygYlXWbOIRqEhd0UL8iUf9PW3BZI/1tl/8LJwLMQEp\nPPg6nWu+qk9j1onvqx8/7/qs/huEL6ipRBQN9y0A3aLsFAJRrJo2L9nxIgCMgdMYER3D0NFiRb43\n9WU+/ty/U7Wa9zOYUbqQqHsPlpHJApEed1Wkmwdu4Of2vRoHh7954qOMFNxp8S+/5kp++aeucLNb\n2t2iW8PZKa+WuntfklH3LanXi9SH8iMY6Wn07mGeGz3JfT8+H0Ts4clMfqR+rnySP/3RB/jW4Pfp\njnfxhgM/g6Zp9Ca7KZhFiqZ7TFDi1rNfohGDn9rzUu7Y8SK2tvfX1cSJ6JFA0N196990KqZdt67p\nZZ276Yyluaxj97z3rem99OyXM9lzwNr56aAidUUL8asz9ncnAzFbb/tlZ7+bcXIhctXDom7ZDrlC\ndU4anp8f7j/k9u9w/7gPn57mtudtm5M+qOsa7ckouWIVLebeT9++qFo221IDXJbay0lOUKlOEzW6\nMXZKLCyw4dTsIPtDA3pfPv51Hh35MbuqrwYW8dS9B4tFlWj3KH3xPnaktrIjtZUruvdzeOooiUiC\n1+x9BekrahODuhLug2qiME2xYgXFvHz7JR6JkY6lODFzihOcIr4fvnDuGYzDdwbnqBf1KsltZ/n7\nZ76Brum8dNftvGrPy4MUwj4/8i9OsTOdZLZcH6mHF55ejMYxiWrVJhq6R794xc9j2ibRZfrhc0X9\nwtdR91GRuqJljEwVSLdFaUtEA0993eyX2TJRQ2erV6N8tZ667ThzIvFiub5vzSyYkpfy6NseO7ak\nSCWjHDo9ieM4IU+99jDwHwxa3H3z8UXdr0tyfc9NAIwYzzJunsPoHcbAPebo9IngPI7j8KOhR5ks\nTTHpuGl5C2e/uMIV6R5F022u2/I8NE1D0zRev/+1tEfb+Jl9rwwEy6cz5j4482aO2XwlWCAjadRm\njb5u7yu5sf9aru+6BWs6g4XJrD4UvEmFRX2mUEbfcpKoHuVdN/4mP7fvNXU54X7kP+n56rOVLJoV\nBSeCrmnBSlFLIbxOqWXb2I5DLFQYLaobdf1YKinPUz+fd9941mriEShRV7QI07IZny7R3+2KqC8e\n6yXqU9kSPel4YCms1n752Fef4w8//kjdNj9HPRl3X3ibZcBUGiJ1XdO4cnc307kKw5OFOfYL1LJQ\ntIQrdJlkDxFdC6LJ3cl92KU2xrTjPDh9L44D18fdqPeoN5MSXEGZ8fzmad2dZBNrUsnRcRwK1WLw\nthDpGQLglq3XBftsSw3w/tvu4rbtt845vjPuRupatMyZkRwY9Z46wK3bbuJXr/pFXnf5qzDPu2Vx\nI12jvOEn3NIH/hiCZTvk9VHsWJ7nZ65iR3rbnOv5dpTvq89Wsuj28qN0qF8ow/+uovPUx1kOvqfu\nr1OqPHXFhsNfl3SgZ/1FvWrazBaqdKfjoZomqxsolWemOTOaq6uU6Nsv/opFzTJgGiN1gCv3uLmV\nh05PBQ+bdGj2pi/wuhep9yZ7MQwd03TfFCzbwRzZhYPFtDmJNb6dLmcH29oHODlzOljI4bkJGZwz\nZ5wFnKb2y6MjP+ad97+Xv/jxB4ntfg69cxyKHWxN9c/ZtxkdXqSuxcqcHskGkXqbMbcWTnc6jlbo\nxqlGifdOcGBXF1FDDyL1bL5CpNe1LF4wcGPT6/Um3Ps3XprCsi3y1QIR271WRF+5qIdLBKyWRCSB\nodW+87X01JWoK1rCkbNu6fx9nmcc2C/r4KlPZd2or6cjgaZppJLROZ76I4dHg7TBxaiadiDY+VLN\nR/dF3V+xqJn90uipAxz0lmg7dGoqiNRT4Ujdt19Cnno0ogcPFNOysca3EyFKTI9TPXOAqmmzr2sv\nVbvK6Vk3Kj80eQSAq3uvxI6UMdIzdaV0fZ4YexpwJ8pE+gfRdIdEfueS7g3URJ1omcHRbLDodDPb\nIqLrdKcTWDN9WJEi5/PD9HUmgmXuxmdyRHqGiTltiJ59c46Hmv0yUZwkV83j4BCxE975VxGpm1bd\nttWgaVqQqw7Q5b3NrAVK1BUtQQ66oi685cfWM1L3xdWvNZJqi9Z56pOzJT7ypWf4j+8eW/A8jww/\nwYNDjzI5WwpSF8M2ju+pb/OKlzWzX8pNaq5kupL0diQ4PDjFbL5C1NCJGToPDz/ObCUbRO2RZJGY\nHiUVba9bts60bLCi3Bp/HW/c/WYw41RNm/3drq1xbPoEZavC8emT7Ext44XbXA8+2jN3eQPbsTky\ndZzeRDfvv+0u0kO3Uzl1JT3lKxa8N2GCSD1aplK10QzfU29etfKmK7awNermkj8zfohMV5J8ySRf\nqvLwuSfRDJPtEVGXpRKmzUiSiCSYLE0F6YyG416r2UNrIcLZL/79XWixkeXgjz0kjQSJFfjyK0WJ\numLVOI7DkTPTdLbH2NJd++OK6Nq6ROp+HfXutLf+ZjJKoWwGke6QF6GfH59/rc3Dk0f55+f+nX87\n9Fl+cLbmpedDol7wI/UF7Be//2FR1zSNK/d0ky+ZnBnNkUpGOZ09w78892m+cPRr3qCpA7Eifcle\nNE3DiNQ8ddNyHzG9sa3sSG8H3OyNfV2uUB6dPsHRqeOYjsWVvQLRvR9sHa1zdE77zmTPUTSLiO59\nRPQIPdo2rNHdpJPNBbkZET1CKtpeW30oUkVDIx5pXmXxF35iH7/zmlegazrPThwKqjeOT5d4fPQx\nAPa3XzXv9fy0xvHSZCDqUU/UV+ypW621X6A2AWktrRdQoq5oASNTRWbyFcSurrqc4Hg0si6R+qRv\nv6T9SN2rUe5ZJ34+/dh0qelMzFwlz78+92l0TSdpJPje+DfQ2t03kbColzxR97376YUi9YZFGq70\nLBi/RICf+vb0+LO0JTV3sDFi0pt09zMiDZE67qLK4eyNjlia/rYtnJg5xTMThwE42HOAhBFHK/Th\nJGaZKE7VtUNOuW8rotu1OvwsnPbk8rKdO+Md6DFvndCISdJILrhmalu0jcs6dnNq9gwdnjNxemKM\nM4WT2LlOdnQMLHi9vkQPFavC+ZybXRJjpZ66+7249ot3XxdI+1wO/gLUStQVG44jZ+qtF594LLIu\neeqTgf1Si9ShtsKQP/PVdhxGG1bwcRyHTx7+HDOVLK+97E7ectWbsLGJ738coiVypbD9Ust+6UrF\n6zz1ilVlMHu2qf0CNVEHd2D0bM7NOClZZaY4F6y442d6RI16Tx0g0mQ5u31dl1G2Kjw09CjxSIzL\nOt1JM860W4Xw2YlDde2Qk66o7/dEvcMX9SarHi1ERyztTjrSTTSjGhTzWohr+q7EwaEYdxeieGji\nARwczPHti9ZS9331k7ODAMQ0921pKcvfhakbKA1Wh2qR/eJF6muZ+QJK1BUtQA660d+BRlGPRtZl\nRulUYL94kbov6l6UPRJa8HyowYL5wfmHeGr8WQ50Xc7Ldr+EK3sPsK18I1qsQlw8yli+FukWvQdW\nMm7QnY6TL5lUqhYz5SwffPzDvO+Rv+Zc6RQA8YborysVD1YWSrXFOO+JOsCpkgwyX/yiUvUDpU6w\nrbFK4/4u11ev2lUOdO/D0N2IuzLpivrT4zVRr9omx2dOsbW9n86464sHM1ub1FJfiJqvXoGIuaTi\nV1f1ur79keKTxK9+gNPm00SdJNbEwKKi3uNlwJyaOQ1A3Bf1ZdovvtVSMa1apN4q+yWm7BfFBuXI\nmWlSyeic5c/isQilNRb1mXKWU7H7iKUKtHvL6KUaFlX2Z75CzV8HmCxN8YWjd9NmJPnlg28IBuqc\n8T2YI7vQ23LcV/wMh71a3n6knohF6PYWhz42fpYPPPahwE45a7ki2mzhYz9aTyUNzuWH2dY+QG+i\nm2PZI2x3rfIgfc+N1N0JULVIfe7KR/5gKcDBHgF42TKlBLFqF0emjlHwStKemjlN1a4G1gusXNSD\nXPVYES1iNU1nbGRrez+9iW6GiufREnk6CvvYm/9psGLBG8N8+BOy/Dz8BK6oGyuM1M26gdLWyKL/\n4Mm09bXkfEtFibpiVYxPF5mYLXNgZ9ccD9WN1O2mNVEuFJ8/+hXKqUFie5/G8XJWfKHKFqtYts34\ndDGIBIcnapH6fxz5ChW7yuv3v5buRO2tY2K6jDl4JZWTBzGdCh/68cf4zuB9FMomMUPHiOhuydm2\nGT4m/5GJ0hSvvuzl9LdlmOA0RKpNZ3Jes9cVpkS6TMWqsD21jWu3XEPZKmN2uRGobzP4WR2WZQcz\nS6Mh+8Xf1hXvDKL7g71uZUH/bamzehmmY/G5o18F5vrpADeILbzw6gGuP5BZ1n0PIvWkez+XMgtT\n0zRes/dObtjyfKLHX4Jz9hpyWfdB1Wyh6DC+LeWTiKxwoDRkX/kpja0aKL1uyzX8+jW/wo3917bk\nfEtlVbVfhBBPADPeryeBvwf+D2AC90gp/3B1zVNc7Mh5/HSgrqhXInbhywwdmTrGY6NP4jhgJab4\n0dBj3Lrtptqs0kKFiZkSlu1wcHc3j8rRoHb40+PP8dT4s+zruoxbBm4IzlmpWszkK2zPpDg3tovd\nmV1M9fyQzx+7m3b7ZSTjboTYnY4T3XmEsl3iTVf8ArduuwkNnbtPfpNI98g8ot7Lb73h+RTiZ+Ew\nbE8NcKD7cr49eB9TZXKdTtsAACAASURBVPe++uJlhMTHtN2HVSSio+uaO9s0NOC707wRpzAUHOvn\nyg/YV5NMj/Gj4cd4Xt9B5NQxNDT2ddWi+472GG99zcFl33s/UteTbnGt+dIZG7l54HpuHrieP33i\nUU4NZTFth4622IKDrFCLggHao21ELS8NdKV56nXZL63x1HVN53mZ+bN4LhQrfiQJIRIAUso7vP9+\nFfg74BeBFwO3CCGub00zNy8/PjYeVK274Nc6Os67/u6HjE4tbVIO1ES90U+H2ixKX1RayeRsCTk4\nFdTBtmyLzx75MgCVo9ehOwZfOv6fddPfs8Uqw3554N42+nvaGJosUDLLfPbIl9E1nTeKn6sTFH9B\n6d1eYTAn18Xr978WgGLqRFAiIJIoEumcoEffyq1eXvhN3rqXkb7zTSNPTdO4+rJeRktuBseO1DZ2\npXcElktHPBXUPKmJjxOK1LXgs7ConzvWwdlntgWD1EGlSMPgVw6+kahu8Cn5eU7NnmFXxw7aoktP\nX5yPIFJPeJF6dHl52ZmuJJbtMDlboqN9cesnYcRJeasLpWNpDO9erG7yUWvtl/ViNa1/PtAmhLhH\nCPEdIcTtQFxKeVxK6QDfBF7aklZuUoYm8vz1557iyw+cXHznFvCoHGVsusR3Hj+35GOOnJkmGTfY\nuSU157OEL+qV5uuDDo5k56zXuRDlisWXf3CSP/z4I/y/H/4h7/vUExw+7Q5c3nfuQYbyIxxMX4s9\n3c9lkRvIVfN87eQ9gf2SK1SDQdItXUkyvRoVY5JPH/oKk6UpXrbrJWxtr58a7y+91t+dJBmPkCua\nPD9zFaloO1bXGRJenanT1ecA6DVriyn0JXuIVzLo6UnyVnbefvmZL9tSW9E0LaiJvqW95sXWInUL\n065lv4An6qHyBX5JBH8Jv3AGzkD7Fn7m8leTrxawHbvOelkNvqinut37tRRPPUyms7b/Yn66j/8m\n0hFLB/cnsszJR0aTMgEbXdRX805cAP4C+BiwH/g6MB36PAvsbXJcQHd3G8YqXnUymfSKj90InBpz\no57hyWJdXy9Uv/1Bw4eeG+G//fzzm86ssyybbz0yyKmhWYYnCoxOFbnxyn76++dOg+7yUgrbUok5\nbT49NMuff/JxIhGdT/7hTwV/jKZt8fFvPMYjz4zx12+/k4QXCWcyaf7+C09x9wMnMSIaO/vTnBnJ\nMpGvQluFr526h/Zokms7b+cxjnD7ztsoTJzk++d+yMvF7QCUTZts0QSjwtdnP8F4xxiJq+GRMci0\n9fDmG3+GuNFQOveoOwvzsp3dpJ8doVQx2drfzR2XvZC7j9yL3j1KT28bh3NP45gG7dbuur62l/ZQ\njo1xpHiYg3vupBnDxRE642n27XCLV71Uv5VvD97H1vSW4FwdXiZPR2cb0Zj7kMr0pchk0sRjBpbj\n3iO38qMr5kbMIJNJM55zf+/qTJLJpHl93ys4PCt5euQwt+69tiX/ntKme9+KtvtvNtPVtazz7t3Z\nDZwCYEtv+5KO3da1hdPZM2xJd9PheP/WktFlXTfnvUVGohGinkXo39e1plXXXI2oHwGOeVH5ESHE\nDBAevUhTL/JzmFrGa34jmUyasbH5o59LgVPn3Ns3ODzL6OgsmqZdsH6bls3gsHve2XyFex88xU1X\nbJmz3zMnJvjQfzwZ/N6eMLjligzDI9PcffIerum7kr2dewCwvehxeCRLKpTSly9V+eN/ftSzBywe\ne3aIY5XH+fbgfWSr3srwOyJ86YdbeOnVV5DJpBkZneW+Z4/RtuM873z1nRhmivf844/40dlH+dLE\nwxTNEm848DOcPeLaK13xOD97+Wv58JP/yD888m/Eo1czOVOiWrWI7jjCeHGM/uhOzp3RuXbPDt5w\n7e3MTpWB+glEJ73vIK5DMhZhaLzE2FgW0XY1d3MvM4mjfPfwI8yUZ7EndjFmluu+H3tqAFI63z3+\nIC/svXWOV1w0i4zlJ7iie39wXCe9vPXqN3PtHhFsMz0rZXQ0S9abXJWdLTI2ZhDR3IlQY2NZimUz\niDgHz8/Q0xZlxDuHVTWD8/2K+EWO958kw0DL/j0lInFKlnv/rJK2rPMmQvFDLLK0Y1OaK4IxJ0HZ\nmz9gmfayrpvz7uVstozlvdEU8uU115bl/l0v9ABYzXvG/w18AEAIsQ1oA/JCiMuFEBpwJ3D/Ascr\nFiFcRGr2AizyEGZoooBlOxzwCnLd9+T5efcDeONL9/M377iNv3nH7dwgtvD0xCHuOf1d/vPkt4J9\n/dzs8KzSr5/8Nu+/5wuMThfZPeD+wzw8OMV95x6kaJXYndqNNZVBi1h84/zdQelSeXaC8o4HcbY9\ny/uf+CCfOPlPxA48zono97FskzeKn+W27bcG7RvobeOqXsFN/ddxevYM8e2D5IoVzhfPYmw5y7b2\nAd6875epDl5JOndF3cBbmHGvJGxfZ5JUMkrFtKlULVJaJ9ZMLwVjhK+ddNfvTOb3zinqVS1F0PP9\nDOVHODY910Y7582I3J7aWrf9ui3XsKW9tvCxYbgPg6plBymNRth+8YTcr88OtYVBKk2WsksaCa7u\nu3LRAcnlEBT2YgX2S9cK7JdkE/tluZ56k9ovrcp+WS9W0/p/BLqEED8APoMr8m8FPgk8/P+3d+bh\ncZx1nv9Ud1d3q0/d92FLtsu3HVvxFecg5GAJAYcJCUeYASbM5IFhB3Y4FpZZYGfY2eWBCU8GeDiG\ncCTATgITQsjhhNy+YhvH8ZnyIR+6JevultRn7R/V1aqWumVJbsXuzvv5S+qj+n1LrW/96ncCr6uq\n+tqlL/Hti7nsfHKRTLZp7dGthKuXVbCo1s+xM/3JznlmehI53kvrC1OqDnd17AXgzNC5pBA7Erez\nRsCue7SXP57ZTq/zDVY2FvOZ9+u+42OtXfSPD7CkqImN9m2ET64n1l/BqK2HF87tRNM0Hj35GBZX\ngIaCxSwvUWgPdmAt7EUKFvHlDZ/j2hrdCu7sC+Jzycng6J1L3otX9hApPc5QdIBgyUEA7lbuoCrR\niMu4EKTjwtA4NquE32NP5r0Hx6OMhWLEemsBaA900uCto8RePmWsXSgSo2B4ERISPz78C9pGUi+W\n7Ql/+mRRn4w5oGcUHxn+YLNP3dy4zPCtZ6pqzTY+x4SozzT7xaDQ60gGOy9WeGSwunQ5K0qWsqZs\nRTKV8VJ6v+RLoHTO7hdVVcPomS6T2TT35QjMDIxMWF2dfUGWNqS3JrPB+W7d7VFX7sEhWznVNsSO\nw51su7aRuBanM9hNtbuS7kFdAM2W1cD4YLJ393gsRHugkzpvTdJSNyzFV9t2AyDZotxxUxnFPicV\nRQW0DLZCMdR7a3nzSCJF0rKVk9E/8MSZZ5AcUbo5iRYs5DNb/5ICu4ORcIDvPbmbk6oFz8363UUk\nGuPC4DiLTZk4HtnNXco2fnrkYWzKbiQ5QnG0Kdn8qtjnmLYFb9/QGCU+J5ZEC1/QK1NHQ1FiAxXI\nOIkwzpbqq3mj1UFLfDhlrF0oEqMkVsn7l32Ah48/yr8d/Al/f9XfUu3Re5u0B3SRv5ioJwOlZks9\nYZXKib4wmqYlhzbDhMAbYjXdKLts4LdPxFVmOy3IIkmU+gvo6h+dsaj7HT4+teYTANisenB/rtkv\n0WiciDV7rXcvJ7m9+jzHfCvfcWHu8YeZ0NozIepXLy3Habfy6qFO+kYHeOD1H/O/997P862v0DMw\nhs8lJ1P5AHZ37kNDSwql4WYwt98NxcLs7tyffE/nuC5mSn0hEVnPYKn31HD83AB+j507tiwjcm4Z\nUS3Cfx5/Gi0qsyT2DgrsesDQa/ew0LcAkJIB3u7+MTQmhlYYrCtfjT/agCRH0GJW1nq2Jp+rKnYx\nMBJKmTdqEArHGB6NUOrXBcq4MwmORfRmXpqFpfYNLPQ10FyxlsJEMNP4u8U1jXAkjkO2sqmqmQ8t\nfT+BSJAHXv8xZxLl7e2BLiyShUr31PiFGbP4JEXdZKmD3j7AbKkb7pd0nSLnA7OlPpc0ScNQmKmo\nmzEucLNuvWtqiBbOcp765UKI+hXMYCCU7F/S0Td/7hdN02jtCVBW6KTAYcNht7JxeTlDtnN8c+/9\nnBxsQUJi+9kX6QuMUF40IZpxLc6ujn04rHbuWrINgNNDZ4GJ0vjxcIz9Xa8zHhsnNqBXKp5NNGJS\n6ouwuPW0Rnu0hOFgmGX1RTRUeCmMLkQbqgAkwqdXs2nxwpR1G1OWuhLuE+McVZaktisA3fKPDZUQ\nObOShuKJaknDBZPOWr+QyFEvSaTbTbbUAZZ71/H55k/jtDnxJ8TIsJbDkzo0XlO9kbuX3MFIJMB3\n/vwDfn/qKToCnVS6ypM9WjJhiFU0ZnK/WAxRn+g0OJJiqSfWEZ3qU58PLsWnDrBlZaWeSVU0+/fO\n1adutUhI6HczIqVRMK9EojECYxGWNRQhSbr7ZSY8cuJxQtEQH1l2Z8YhA5MZDIQJjEWoWxDjV8d/\nS2ewm3ZHF47FISJxGx9U3k8wEuSJlu1IZWcpL6pNvvfN/pMMhAbZUrWBanclfruP04Nn0DRtwlIP\nRznQvgsJici55chFOyZEva4Q6fwQ1riT9g5dKJc2FCFJElcvreCZ19ZgL4gghZysbkrtoZEU9YQg\nG+JeVTK1mVSxy094v14UVFE88bzx2q6+URZWpaZl9g0lZoQWJiz1AsOnHkkKq/mOxbAwhwIJX3Ya\nC/m62s1Uuct56PijPHf+JQBqPFPncE7GHNBLtt5NBE9tJr9wqk99ap76fGK4X6ySFdkyu94xABuX\nV/Ce6xfNKfMk6VOfZe8XSZKQZd19ZbO+NW6q+Sa3V5/HDCSEocjroLrEzWAgzOh4+iIeg8HQEK+0\n7WJP136ePfdiynPtF4IcOt2X9n0tXQPY6t7krPdpdnXu5dxIKz6Hl2hfJY2Bd3NtzSZuqN2K0+LC\nVnWGwsIJa8gIkF5TswFJkmgqXMBweITesb6khdoT6dCDic7FaOECiiwVdAS6GI+GkJ1RLI5xYgEf\nx87qbhij0VWzUg5YCI85WNZQhMuZaoNUlqRa6oYbJp2om2eAlpssQcOq7+yfetE0hiGXJNwv6Sz1\nAseEUE621DOJ6eKiJr6y4XNclxjibPRomQ5zQM/cehfMgh9LfrZFkiYs9chbI1aG+6XA5sxqVs1M\nmCg+mv3nylZLIlAaw2qRZn1huNLI7dXnMUY6Y5HXYcrSmN5aP9hzBA0NCYk/tjzLiYHTAJxuH+Kb\nv9zPA789NMV3fH64jf/X9u/IVWfxWn18as1fc//1/8zXN38Bua2Z3i5dSJ02B4vt65GsMXrkw8Ti\nMXZ37ufQhWNUuytp8OozLZsSfvXTQ2eTYtYaPwpAnVXvg1FZUI2GxvmRNs6P6PM0IyNe3jh9gRKf\nM+lbXVjlTY6kS9dgyu+2U+CwJsW880IQu82S7KNuxhDkIq8jRWSNC0B779Rz2zc0kc4IJp/6eHSi\nl7o9s6VuZP2k69DotDm4W7mD+6//JldXXDXl+cnYrGafuobVImGRJtoEgOF+0a3zsqKCZKvht9pS\nn4vr5VKZa5sAmEgJjUTiybueXCb3d5CnGME2ve+2LjyGzzgSj/Ive7+b7HVicKDnDSQk7l15D5Ik\n8bOjv+bw+Q7+9ZGDjIdjxDUtxXc8HB7hh4d+xmh8hEjnQj6z6u9YUaJgs9iQJIm6cg/d/aNJN4Jn\ndBFa2IE6epBv7PkWDx9/BIB3L7w5aZk1+ROiPngGp2xFcg3TJ7VQ6a5AHteDgcYF4Ozwec4P61kL\n8aAfTUsdHiFJEu+4qoZCjyOtqEuSRGWxi56BUaKxOF39o1QWu5JiZ8ZoFTDZX+t32ykvLODImf4U\nfzTABcP9ksZSHzcNyDAfC2AoqP/tDAt5OjG1W+UZWbXm3i+RWDzFIk0R9WAYh91KsdfBWCim9zTJ\n8vCHTExY6pdD1FPvWmaDIerhaDznc9RBiPoVy0BaS32io2BboINX2nbRPdoL6K6X00NnKbVVc/h1\nO4ssG3XRPvgQ49EQqxr1QhbDVRHX4vzs6G8YCo/g7F+B3LOc6qJUn3JduQcNaLugZ8b0DYSJtDcR\n1aIMhUe4rmYL39j8Ja5K9CoBvdOg0+rk9OAZrDawLzwCksadi25PNiZbVLwAgLND55OWejyof/ay\nSWmbt21ewEPfeFfGjIjKYhfRmMbJ1kHC0ThVpVODpAAlPicSUDupR40kSdy4vpZIND6l4KpncAzZ\nZkl+trtgIvtlNI2oG68z9jke0V+TzlKfLeZAaSwWTxEvs2tmZCyCzyVP9LsZiyQbqmVjHdPhtrlY\n6KtP6en+VtFU4+em9bVsXF5x8RdPQrZZE8VHsbwQdREovUIxCo+KvI6kO8IYlGz4sTU0nj37Ih9d\nfhevdehFNe2n/JzvaQd82BdVYC3upmbjYbZWL+NwS1/SVfFky7OcGDjFyuJl7N9Xw6IazxSLsa5C\nF8DW7gBN1X56BsdwBBfyt6s3UO+tTbZbNWORLDQWNnCsT2V723Ys7mG8Y40sK1nC9qDeXqC+qIxC\nh58zw+exSBZ8di8edyG9g+OzzsU3gqUHEj1aqorTT9wpLSzgKx9dn7xAmtm6qorHXm3hhQPt3Lqh\nHpvVwrGz/ZzvDuiB3MR5cTlsSOiibvSkMfvUbVYLbqeNoaARKL24pT5TjI6MkWicSExLaVyV9KlH\n4gwHwzRUepOzRkdGwyZLfX4FS5IkPt/8d/P6GZlwyFY+fPPFYxPpMHzqFglcsxzjdyUiRD1LxDUN\nTdOyFmQxu188BTI+l0xnX5ALwX7e7D/JAl8947EQe7sPsNqziT8e3Y1WAI3uJXzwr1YyGooyGFjK\nodBLHBl6g8e7H8biXcyJ4TEeP63y7LkXKXEWc0PpbezTjqTtslhfrt9Ot/YEiMc1egfHqCv3sqp0\n+n7bTf6FHOtTebVjF1rYgXdwDaD7mu2yPgBhga+eg72HAVhZsoyNNy6mZ2AsmcI5U4xA58GTvYnf\nM49Ra6pJP1bM5bSxdWUVzx9o4/WTF1i7qJSHnj2BJOntEAwsFgmX00ZgPEockKSpgu33OBgKGO4X\nXUwvNvBhJhhpixOW+lT3y/BomFiiH7l5MEgoEkMi91P15gtZthCJxJGAwjw4R0LUL4G4ptHSPsze\n493sU3tAg3+6d+OsR4Gloz8wjq24i+8f/R5bazdRVeLmROsgfzq9Cw2Na6o3IFtkfn7sN/zkwKNo\nvn6KpGq+9IHNWEzBoi3ah3nqTBlPnf0TjmV7aQVaz4HdaufeVfdwtkV3EaQT9epSF1aLxPmeEQZG\nQkRjWkrmSCaMIiQA2lYRseiCNBgMUeh2IEkSC3x1SVGv99awrnF2k3YMDMu8LzGXtDqNJT4Tblxf\nw/MH2vjT/lY6+4J094/yzvW1yf40Bu4CmeBYBEnSg6ST7278bjsdF4JEovHkKL9sZJ0ke79E40Ri\n8ZSLiSHW/Ylz4HHJyWwf3VKPY5etb3lGSq4gWy3ENY1QOJYXFz4h6pfAr547wYuJ3uM2q0Q0pvGn\n/a1su/bSfIp9Y/10+15Gru6mcxR+e+IPLCm7Ba0VXmzZhd1qZ135amSLzGMnnmHI1w3ArYs3pgg6\n6LfEtzXeQpWnkl/t3klwyMGn372ZBb46vHYPr3Tr5f115VO7vsk2K5UlLtp6gnSZ+pBfjAZvLWUF\nJSwpamLf0UpC9hjxuMZwMMyihLVsTLkHqPfVZjrURSkvKkACNEACKornFqSrKnGzqrGEwy19tHQM\n61Wtaf6OngKZvqFxrFYpxZ9uYPjVR0bDyQCzU770fzPZ1CYgFtOQnVN96v2JYindUjfWoVvqkwdf\nCyYwzp9GftzN5P4OLhPRWJxdR7oo8jr47AdW893PbMVTIPOn/W0XzSdPR1yLo/af4udHf8M/vfZt\nYp5u7OPlfGLFR7BarLTYXsJa1sZAaID15Wtw2pxYLVZKQ3qQUkJibfnKjMdfV76axVxLuGMBVbYF\nyUnnpzuG9f7kaSx1gPpyD6FIjKMt/QAzstRlq8zXNn2RDyl/kRw+PTIaRtMmMkTqvTXJ4qh679xF\n3S5bk3nkpYXOtD3gZ8pNzfo6YnGND71z8ZS8eNDTGmNxjaFAOK2oT2TAhE2phNmw1E09SiZnvyQE\nvz/hsvOaAqUjoxHC0di8V5PmMmYhz4fzJER9jpzrGiEUjrGmqYTVTaW4nDK3bqhjNBTlhQNtydfF\n4jG6gt3JsWvpeLP/JF/f/S0eOPhj9nW/js/uJ3x6NY3jt7C+Yg0fXHIHEULYF+r53lsS49I0TaPt\nhB/GfKwtW5VSpp2OySXxoUiMtp4ADRXejBaKYcHvV3sAqCjK7LM2I0kSkiQlhk/HksFDv0f3mdut\ndpYUNlHjqUobcJ0NRrA0XRB0NqxYWMyyhiI2LCtP20sewJOoKo3FtZQgqYEvnahnYT5rqqWePvul\nz2SpexKWemAsQigcm/cc9VzG/N3PB0tduF8SxOMa//HCKRbX+mnO8A9t5s3zevWjOVvjxnW1PL3n\nPM/ua+Xm5jqsNo0fHfoFx/pVajxVXF+zhebKq3BY9X+4uBbnuXMv8UTLdiyShc1VV7Opqhl5vIRv\nvLKfogZdADdXX83J/nO81rMXW9THAm89AOe6RxgcibApvo17V2W20g0M8evsH2VlYwlnO4eJxbWM\nAUSYyIAxxrrNxFI347BbGQ/HGEwU5BR6JlIT71v9MTQyX+xmSmWxiyNn+tNWks4GiyTxhQ9NXwjk\nNsVLprPUh4NhU5uA7FnqRutdaxpRH0iIeqqlHtbzr4X7JSPmC6RIacxRegbHKPU5U/zPe9/s5rn9\nrbx8sJ26Ck9aizQcC7O36wC9Y30c6B3GWhLFUzLhdy1w2LipuZY/7DzLCwdaaffs4Fi/SmlBCZ3B\nbn6t/o7fnXqCancVle5yhkLDHOtXKXT4uXflPUk/8xun9PQ8cybIh5dv4+T5AJ1n3RxpGmB1UwkH\nTuivW7f44hchmNorpaVDb6Q1raib3DJOuzUpFjPFIVvRtIlCHr97Yk+yNTvpY0bueW1ZehdSNvGY\nUt5cM3W/ZCP7JSE8RpVq+uwXvYLU67LjccpI6BeXSDQuLPVpMBdlCUs9B9l5uJOfPnmczSsq+OTt\netl6LB7n8R1nk93afvbkcb74kXXJysRAOMjL7bt4pW0XgUiinNwDdg98/8hhFvjqubFuK2vLVnFT\ncx3P7jvPk21PoBW3Ummv5b7lH0d2xtnRvocDPYc5N9LKmWG99eqSwiY+sfIjSR83wEBgIp3RwGax\n8TfNd/K1Q3v5/astrGos5vWTvdisFlY2mqcIZmZyV8NT7UMANFVndn/4XHb8HjtDgbAelJxlBoUh\nJsZwDbOlni22rKzELlsyukyyidlSd04TKB0OhLPa8tYQm/HEEO90eeoGXpeMxSLhLpCTLpl88BXP\nF6nul9w/T3kt6pqmpYjQua4Rfrldz/bYfbSbdUvKWK+Us+doN939o1y/tprAaIQ/n+jl+T+3cXNz\nHXvbD/Fr9VEihHDZCnjXgnfii1Xz0PNHWNrkwlPRz5ELb/Lg0V9jk6xIkgXr2jhxYsSDPs7sX8pX\ndu3nb967gvcsvZX3NN5KNB6ld6yPYGSUhb56rJbUL5K574uZunIPW9dUs+ONDp7b10p7b5DVTSU4\nZ+izddityaEQmqZxumOYIq8jba8UM/XlXg4H+lJa7s4UQ9C6E3cHc+mVfTFsVgublldm/bjpMDo1\nAml96klLfTSc1Z4rRk+T8ZBhqWcWIiPzxeuSk3dl+eBWmC/M/V7y4TzlpahrmsZz+9v4w44zNC8t\n431bG5FtFr73n4eJROPcfeMifvdyC7/crtJU4+cPiQn179m8AJvNgto6yO9ePsHeoRfokI6ixS3E\nOxQ+e9tfUFNSyBO7zhIfrOCGupU0Ly2nZ7SXl9p2cXZIbyeLBH7ZT3PDjXSWRHlqzzl+9PhRLBKs\nV/Te2VXuzOXM5mrSyXz41qXsPNTBf7x4Ckjf6Go6KotdHDs7QHtvkOFgmGbl4u+vK/dwuKVvTn2u\nk50aBw1LfXbFRVca5hqEgjQXU48r4fYIhJIFStm4pZckCZvVwljSUp/qfgH9QmP87i2Q6UyELIT7\nJTMiUHqFE4nG+OUzKjuPdCEBr7zRye6j3ZT6nfQNj/PeaxZw64Z6NA0eeekE3/j9YwTkcVasLacz\ncoYLI/00bDjNqcEWOqRxGPewNP5ODnaEeHp3B/e+p5A3z+lB0iX1+ti0clcZdy15X/oFNcHS+iK+\n88hBfvj4Ue57HyyuLSQ4HiEwFmEwEGZgJMToeIR3rKvF77Ynx9ilE8C6Ci+bllew+2g3ErBmUemU\n10yHIeq7jugDjxurM/vTDZY1FPHUnnM0Vs0+S8WZEJPewTGsFgnPLH3yVxrmuazpAqVWiwWvS2Yo\nGMYhW3FksehHtlkYS2Opmy1Nr2los8f0s3C/ZCYlUJoH5ymvRL1ncIwfPX6UM53DLKzy8qltqzjY\n0sETh16nl34WLqvghqtLiWtxiup68Vy1g7BtFDtwkmOcPPRS8lh2h506+yru23InLoeTr3fvY/eR\nLm5qruVU+xA1Ze4ZTz1fVOvncx9Yw/2PvMH3HzuS8XUnWgf5/AevYmAkhNNuTSsaAO+9ZiF7j/fQ\nVONP3u7PFMOvvvuoLuqLpgmSGqxYWMy3P7Vl1iX8MFFNGY1pFHkdaTso5hJmSz1doBTA53bQNzyG\n12XPahMt2SolW+umy34BUgLZ5p+FpZ6ZFEt9Dl0erzSyKuqKoliAHwBrgBBwr6qqp7L5GekIhWM8\nueccz7x2nmg8yqrVEvWLuvmRuoOOQBdanYYMdHGS/7FrB26bi2B0FKtsReppZEXFQlYs8jEWHaPQ\n4afBV0e5qzRlctCdNzTy3UcP8YPHjhCJxllaP7vGU0vqCvmHu9fy1J5z2Kx6EMvtlPF77BR5HOw8\n3Mkbp/t4as+5/r+Q6QAACrZJREFUlDF26agodvGPf9WcYpXNFCOXeygYxmqRaKicWcbIxfzumTD7\n+2d7AboSMVvqzjQ+dQC/W6atN4BEJKt3JrLNkkwAtaUpPgJSDA2zqIuUxsykiHoenKdsW+rbAKeq\nqpsVRdkEfAfI4JeYO22D/fzwtcfpCfQSjA8TjI4QI4a8GpxylFNEOdWmZ4w0FS5gUWEjdd4aeoK9\nnBo6Q9tIBxsr13PbwlsodhbO6PZ4VWMJS+oKOdGqT7ufraiDbrH/1ztXp31uaUMRX3twL79/9Qxx\nTctY4WlQXzF9oVEmKk1dDOsrvPMe7TfnaOeDqBc4rFgtErG4Nq2lDjAaiiaHVmcD81Bl20ws9YKJ\n8y0s9czYRaB0WrYCzwCoqrpHUZTm6V5cVOTCNgdR+cHuxzk6tlNv9mEFLW7HbXfgczkpkB0sK1vM\n2qrlLC9bgsOWPSH55LZVfOHfXkWS4Jp1tXOylDNRBnz+nvV89Ye7AKgq81BWll64Mz0+E0pKPNgT\nVZ6rFpVe0rFmQmnJxMWpcpo9XYz5Xuds8LrsDAZCVFX40q6r0pQv73bZL2nt5veaUyh9XmfyOa9p\nmlVF6cQ5rjZd+IuLXFfUOZyOt3qdJcUTM1FLi+f+Hb1UsvW52RZ1HzBk+j2mKIpNVdW0zVAGBqZO\ncJ8J96y9hcM9i3FqBVT7Syj1utNanMMDIXQvUHYoccu8a2O93oEvGGI8mL1jA1T5nbx7UwNP7j6H\nx2FNO4C3rMw7p8G8ZiqKCmjtCVBdXHDJx7oY4fGJaUIOqzSnz8vGnrNJgcPKYADGx8Jp1yWbbvys\nEnNe++R9m+8nw6FI8rlYPJ583MbE52mxWNrXX8lcjr/12Ggo5efLcZ5mu+/pLgDZFvVhwPxplkyC\nfin4CgrY1rzxspz8u96xaF6Pv+3ahVSXuGdcUDQXGiq8dFwIzihIeqmYA4X+HE9nNDAKkDK5X/ye\n+XF7mLNczO4Xq8WSdAkJ98vsSddHJ5fJtqjvBG4HHkn41A9n+fh5j9ViYfPK+S2kuevGRbxzfe2c\ng5+zwSwm+eBTB1hWX0QkGp/Gp24S06xmv6QXddAFPxaO4XVnCpQKUc+E+S5/vue4vhVkW9QfA25W\nFGUX+t3ix7N8fEEW8BTIWRnkMRNSRH0eWgRcDu64rpE7rsvcM9988cqmhSynWOqpwX3ZaiFELDkc\nAyalNOaBBTpfiOKjaVBVNQ7cl81jCnIbs6Va6M4P98vF8M+TpW7LUHAEE2JkvkuQbVYcdiuhcAz7\nPA+dzmVS+6nnvqjn/g4EVzTOPLTUL4a7QE4WWc2bpW5JL+qT78AMy92RB26F+SLfLPXc34Hgisbw\n5XoK5Cl+4HzFIkn43AkxzWag1ORyMWaWGvjddkp8jinn2Ei7FZZ6ZkSXRoFgFhiili9B0pnic9sZ\nDISzGyg1Cc5kS/2Tty8nEo1PfkvSry6yXzIji+IjgWDmWCwSm1dUUF16aaPmcg19GEggqwOfUy31\n1OOW+tN30Lz16joWVHrx5XgjtflEpDQKBLPEGEbydsK4M3HI2fsXS01pnFljtGULilm2YP5qHvIB\n4wJpSbQ3znWEqAsE84ARFM7U9Gsu5Fs3wSsFQ8ytM7xQXukIURcI5oHr1lQTi2kodYVZO2ZKFakQ\n9awi2yzJ6VK5jhB1gWAeKCss4K4bs9tSQljq84dss8zYpXWlI0RdIMgRUi31/BCgK4X1StmUjKJc\nRYi6QJAjCEt9/vjoLcrlXkLWEN8MgSBHkIWlLpgBQtQFghzBJix1wQwQ3wyBIEeQRfaLYAaIb4ZA\nkCOY+70IS12QCfHNEAhyBOFTF8wEIeoCQY5g+NQlyJtCGUH2EaIuEOQIhqVutVqQJCHqgvQIURcI\ncgQjT122CUEXZEaIukCQIxgVpdY8qXwUzA/i2yEQ5AiGqOdLjxLB/DCnNgGKokhAG3Ay8dBuVVW/\nrCjK7cD/BKLAg6qq/iQ7yxQIBIb7JR96fgvmj7n2fmkCDqiqervxgKIoMnA/cDUQBHYqivKEqqpd\nl75MgUAgW4WoCy7OXEV9PVCjKMqLwBjwOcABnFJVdQBAUZQdwLXAo5kOUlTkwnYJg17Lyrxzfm8u\n83bc99txz5C671hMn0HqdNjy+nzk896mI1v7vqioK4ry1+iibebTwL+oqvqooihbgYcTrxkyvWYE\n8E937IGB0dmt1kRZmZfe3pE5vz9XeTvu++24Z0i/b4skgabl7fkQf+uZvz4TFxV1VVV/CvzU/Jii\nKC50vzmqqu5QFKUGXcTNn+QFBme8SoFAcFFuvrqWimLX5V6G4Apmru6XrwF9wLcURVkDnAeOAYsV\nRSkGAsB1wLezskqBQADA3TcuvtxLEFzhzFXU/w/wsKIot6Fb7B9TVTWiKMp/A7ajp0o+qKpqe5bW\nKRAIBIIZMCdRTwRDb0vz+BPAE5e6KIFAIBDMDZEbJRAIBHmEEHWBQCDII4SoCwQCQR4hRF0gEAjy\nCCHqAoFAkEcIURcIBII8QtI07XKvQSAQCARZQljqAoFAkEcIURcIBII8Qoi6QCAQ5BFC1AUCgSCP\nEKIuEAgEeYQQdYFAIMgjhKgLBAJBHjHXfuqXDUVRLMAPgDVACLhXVdVTl3dV80NimPeDwAL0GbD/\njD6M5OeABhwBPq2qavwyLXHeUBSlHPgzcDN6z/6fk/97/jLwXsCO/h1/mTzed+L7/Qv073cM+CR5\n/rdWFGUj8H9VVb1BUZRFpNmroihfQ29tHgU+q6rq3tl8Ri5a6tsAp6qqm4H/DnznMq9nPrkH6FNV\n9VrgvwDfA/4V+GriMQl432Vc37yQ+Gf/EfpQc3h77PkGYAtwDXA9UEf+7/vdgE1V1S3A/wK+SR7v\nWVGULwL/DjgTD03Zq6Io69D//huBDwLfn+3n5KKobwWeAVBVdQ/QfHmXM688Cvyj6fcosB7dggN4\nGrjprV7UW8C3gR8CHYnf3w57vhU4DDyGPmjmj+T/vk8AtsTdtw+IkN97Pg283/R7ur1uBZ5VVVVT\nVfU8+vkpm82H5KKo+4Ah0+8xRVFyzo00E1RVDaiqOqIoihf4LfBVQFJV1ejtMAL4L9sC5wFFUT4G\n9Kqqut30cF7vOUEpuoHyAeA+4FeAJc/3HUB3vbwJ/AR4gDz+W6uq+jv0C5dBur1O1rdZn4NcFPVh\nwGv63aKqavRyLWa+URSlDngReEhV1V8DZv+iFxi8LAubPz4B3KwoykvAWuCXQLnp+XzcM+iD3Ler\nqhpWVVUFxkn9Z87HfX8Ofc9L0GNkv0CPJxjk457NpPtfnqxvsz4HuSjqO9F9cSiKsgn9ljUvURSl\nAngW+JKqqg8mHn494X8F3c/+6uVY23yhqup1qqper6rqDcBB4C+Bp/N5zwl2AO9SFEVSFKUacAPP\n5/m+B5iwSvsBmTz/fk8i3V53ArcqimJRFKUe3Wi9MJuD5qLb4jF0S24XenDh45d5PfPJV4Ai4B8V\nRTF8638PPKAoih04ju6WyXf+AfhJPu9ZVdU/KopyHbAX3dj6NHCG/N73/cCDiqK8im6hfwXYT37v\n2cyU77WqqrHE+djNxPdgVojWuwKBQJBH5KL7RSAQCAQZEKIuEAgEeYQQdYFAIMgjhKgLBAJBHiFE\nXSAQCPIIIeoCgUCQRwhRFwgEgjzi/wO/yIiif+zOAgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x129e2a6a0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "mu=1\n",
    "sigma=1\n",
    "n_series=100\n",
    "n_points = 100\n",
    "n_features = 2\n",
    "# multi-variate time-series: \n",
    "# x_t ~ [Gaussian(mu*t,sigma*t), Gaussian(mu*t,sigma*t/4)]\n",
    "# Points X Time X Features\n",
    "x_data = np.zeros((n_series,n_points,n_features), dtype=np.float32)\n",
    "for i in range(n_series):\n",
    "    x_data[i] = gen_time_series(n_points)\n",
    "\n",
    "if TESTING: x_data_test=x_data[:10]\n",
    "plt.title(\"Sample bi-variate time series\")\n",
    "for i in range(n_features):\n",
    "    plt.plot(x_data[0,:,i]);\n",
    "print(x_data.shape)\n",
    "print(reduce(lambda x,y: x*y, list(x_data.shape)), \"data values\")\n",
    "\n",
    "x_train = x_data[:70]\n",
    "x_validate = x_data[70:90]\n",
    "x_test = x_data[-10:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Building Variational RNN\n",
    "\n",
    "note that all gaussian distribtutions: p(z), q(z|x), p(x|z) should have the same dimensionality, i.e. share z_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# First I'm going to define the loss functions necessary because they don't depend on anything\n",
    "def negative_log_likelihood_gaussian(y, mu, sigma):\n",
    "    \"\"\"\n",
    "                a                  b              c\n",
    "    (1/2)*[ ((y-mu)/sigma)^2 + 2*log(sigma) + log(2*pi) ]\n",
    "    learn more: https://www.statlect.com/fundamentals-of-statistics/normal-distribution-maximum-likelihood\n",
    "    \"\"\"\n",
    "#     sigma might be really small. in that case, a & b blow up\n",
    "    sigma=tf.clip_by_value(sigma, 1e-3,1e10)\n",
    "    a = tf.square(tf.div(tf.subtract(y,mu), sigma))\n",
    "    b = 2*tf.log(sigma)\n",
    "    c = tf.log(2*np.pi)\n",
    "    return tf.add(tf.add(a,b),c)\n",
    "\n",
    "def kl_gaussian_gaussian(mu1, sigma1, mu2, sigma2):\n",
    "    \"\"\"\n",
    "            a                  b         c           d\n",
    "    log(sigma2/sigma1) + (sigma1^2 + (mu1-mu2)^2)/2sigma2^2 - 1/2\n",
    "    \"\"\"\n",
    "    # Note: if sigma1 or sigma2 go to 0, this explodes\n",
    "    sigma1=tf.clip_by_value(sigma1, 1e-3,1e10)\n",
    "    sigma2=tf.clip_by_value(sigma2,1e-3,1e10)\n",
    "    a = tf.log(tf.divide(sigma2,sigma1))\n",
    "    b = tf.square(sigma1)\n",
    "    c = tf.square(tf.subtract(mu1,mu2))\n",
    "    d = 2*tf.square(sigma2)\n",
    "    e = tf.divide(tf.add(b,c), d)\n",
    "    return tf.add(a,e) - 0.5\n",
    "\n",
    "# I will also define a simple wrapper around reshape to flatten the 1st two dimensions of a tensor\n",
    "def flatten(x):\n",
    "    return tf.reshape(x, [int(x.shape[0])*int(x.shape[1]), -1])\n",
    "\n",
    "def tf_sample(batch_size, layer_size, mu, sigma):\n",
    "    epsilon = tf.random_normal(shape=[batch_size, layer_size], seed=GLOBAL_SEED)\n",
    "    return tf.add(mu, tf.multiply(epsilon, sigma))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class VRNN(object):\n",
    "    \"\"\"class for VRNN.\"\"\"\n",
    "    def __init__(self, input_shape, rnn_size, x_size, x_1_size, \n",
    "               z_size, z_1_size, phi_size, prior_size, theta_size, lr=.01):\n",
    "        self.batch_size = input_shape[0]\n",
    "        self.ntime_steps = input_shape[1]\n",
    "        self.nfeatures = input_shape[2]\n",
    "        \n",
    "        self.rnn_size = rnn_size\n",
    "        self.x_size = x_size\n",
    "        self.x_1_size = x_1_size\n",
    "        self.z_size = z_size\n",
    "        self.z_1_size = z_1_size\n",
    "        self.phi_size = phi_size\n",
    "        self.prior_size = prior_size\n",
    "        self.theta_size = theta_size\n",
    "        self.learning_rate=lr\n",
    "        \n",
    "        self.inputs = self._define_input_placeholder(input_shape)\n",
    "        self._build()\n",
    "\n",
    "    def _define_input_placeholder(self, input_shape):\n",
    "        return tf.placeholder(tf.float32, shape=input_shape, name=\"x_t\")\n",
    "\n",
    "    def _tensorboard_additions(self):\n",
    "        # get latest values from VRNN so that you can visualize how the last time-step changes over time\n",
    "\n",
    "        with tf.name_scope(\"prior_sample\"):\n",
    "            prior_mu, prior_sigma = self.priors;\n",
    "            prior_t = tf_sample(self.batch_size, self.z_size, prior_mu[:,-1], prior_sigma[:,-1])\n",
    "            tf.summary.histogram(\"prior_t\", prior_t) # should be a unit gaussian\n",
    "        \n",
    "        with tf.name_scope(\"z_sample\"):\n",
    "            z_t = self.calculated_posteriors;\n",
    "            z_t = z_t[:,-1]\n",
    "            tf.summary.histogram(\"z_t\", z_t)\n",
    "\n",
    "        with tf.name_scope(\"x_sample\"):\n",
    "            x_mu, x_sigma = self.likelihoods;\n",
    "            x_t = tf_sample(self.batch_size, self.x_size, x_mu[:,-1], x_sigma[:,-1])\n",
    "            tf.summary.histogram(\"x_t\", x_t)\n",
    "        \n",
    "        \n",
    "    def _build(self):\n",
    "        self._init_cell()\n",
    "        # transpose input so T X N X F instead of N X T X F\n",
    "        # this allows you to do one time-step at a time for all batches in tf.scan\n",
    "        self.inputs_t = tf.transpose(self.inputs, [1, 0, 2])\n",
    "        self.outputs = self._scan_sequence(self.inputs_t)   \n",
    "\n",
    "        self._tensorboard_additions()\n",
    "\n",
    "        with tf.name_scope(\"negative-log-likelihood\"):\n",
    "            self.nll = negative_log_likelihood_gaussian(self.inputs_t, \n",
    "                                                        self.outputs[-2], self.outputs[-1])\n",
    "            self.nll_r = flatten(self.nll)\n",
    "        with tf.name_scope(\"kl-Divergence\"):\n",
    "            self.klgg = kl_gaussian_gaussian(self.outputs[5], self.outputs[6], \n",
    "                                             self.outputs[8], self.outputs[9])\n",
    "            self.klgg_r = flatten(self.klgg)\n",
    "\n",
    "        with tf.name_scope(\"objective-function\"):\n",
    "            nll_mean=tf.reduce_mean(self.nll_r)\n",
    "            klgg_mean=tf.reduce_mean(self.klgg_r)\n",
    "            self.objective = tf.add(tf.reduce_mean(self.nll_r),tf.reduce_mean(self.klgg_r))\n",
    "            \n",
    "            tf.summary.scalar(\"negative_log-likelihood\", nll_mean)\n",
    "            tf.summary.scalar(\"KL_divergence\", klgg_mean)\n",
    "            tf.summary.scalar(\"ELBO\", self.objective)\n",
    "        \n",
    "        with tf.name_scope(\"train\"):\n",
    "            self.train_step = tf.train.AdamOptimizer(self.learning_rate).minimize(self.objective)\n",
    "\n",
    "    def _init_cell(self):\n",
    "        with tf.variable_scope(\"lstm\"):\n",
    "            try:\n",
    "                self.cell = tf.contrib.rnn.LSTMCell(rnn_size, state_is_tuple=True)\n",
    "            except:\n",
    "                self.cell = tf.contrib.rnn.LSTMCell(rnn_size, state_is_tuple=True, reuse=True)\n",
    "        self.state = self.cell.zero_state(batch_size, tf.float32)\n",
    "    \n",
    "    def _calculate_prior(self, hidden_state):\n",
    "        \"\"\"\n",
    "        calculate your prior based on your previous state information, i.e. z ~ p(z)\n",
    "        \"\"\"\n",
    "        with tf.variable_scope(\"prior\"):\n",
    "            prior = fc_act(hidden_state, prior_size, act=tf.nn.relu, name=\"fc_prior\")\n",
    "        with tf.variable_scope(\"prior_mu\"):\n",
    "            prior_mu = fc_act(prior, z_size, name=\"fc_prior_mu\")\n",
    "        with tf.variable_scope(\"prior_sigma\"):\n",
    "            prior_sigma = fc_act(prior, z_size, act=tf.nn.softplus, name=\"fc_prior_sigma\")\n",
    "        return [prior, \n",
    "                prior_mu, \n",
    "                prior_sigma]\n",
    "        \n",
    "    def _encode_x(self, x_t, hidden_state):\n",
    "        \"\"\"\n",
    "        infer the latent values given z, i.e. z ~ p(z|x)\n",
    "        \"\"\"\n",
    "        with tf.variable_scope(\"x_1\"):\n",
    "            x_1 = fc_act(x_t, self.x_1_size, act=tf.nn.relu, name=\"fc_x_1\")\n",
    "        with tf.variable_scope(\"phi\"):\n",
    "            phi = fc_act(tf.concat([x_1, hidden_state], axis=1), self.phi_size, act=tf.nn.relu, name=\"fc_phi\")\n",
    "        with tf.variable_scope(\"phi_mu\"):\n",
    "            phi_mu = fc_act(phi, self.z_size, name=\"fc_phi_mu\")\n",
    "        with tf.variable_scope(\"phi_sigma\"):\n",
    "            phi_sigma = fc_act(phi, self.z_size, act=tf.nn.softplus, name=\"fc_phi_sigma\")\n",
    "        epsilon = tf.random_normal(shape=[x_t.get_shape().as_list()[0], self.z_size], seed=GLOBAL_SEED)\n",
    "        # z = mu + epsilon*sigma\n",
    "        z_t = tf.add(phi_mu, tf.multiply(epsilon, phi_sigma))\n",
    "        return [x_1, \n",
    "                phi,\n",
    "                phi_mu,\n",
    "                phi_sigma,\n",
    "                z_t]\n",
    "    \n",
    "    def _decode_z(self, z_t, hidden_state):\n",
    "        \"\"\"\n",
    "        generate the mean and variance for your data from latent values z, i.e. x ~ q(x|z)\n",
    "        \"\"\"\n",
    "        with tf.variable_scope(\"z_1\"):\n",
    "            z_1 = fc_act(z_t, self.z_1_size, act=tf.nn.relu, name=\"fc_z_1\")\n",
    "        with tf.variable_scope(\"theta\"):\n",
    "            theta = fc_act(tf.concat([z_1, hidden_state],axis=1), self.theta_size, act=tf.nn.relu, name=\"fc_theta\")\n",
    "        with tf.variable_scope(\"theta_mu\"):\n",
    "            theta_mu = fc_act(theta, self.x_size, name=\"fc_theta_mu\")\n",
    "        with tf.variable_scope(\"theta_sigma\"):\n",
    "            theta_sigma = fc_act(theta, self.x_size, act=tf.nn.softplus, name=\"fc_theta_sigma\")\n",
    "        \n",
    "        return [z_1,\n",
    "                theta,\n",
    "                theta_mu,\n",
    "                theta_sigma]\n",
    "    def _rnn_recurrence(self, x_1, z_1, cell_state, hidden_state):\n",
    "        \"\"\"\n",
    "        use x_1 and z_1 and the previous hidden state to update lstm cell\n",
    "        \"\"\"\n",
    "        try:\n",
    "            return self.cell(tf.concat([x_1, z_1], axis=1), [cell_state, hidden_state], \"rnn\")\n",
    "        except:\n",
    "            tf.get_variable_scope().reuse_variables()\n",
    "            return self.cell(tf.concat([x_1, z_1], axis=1), [cell_state, hidden_state], \"rnn\")\n",
    "\n",
    "        \n",
    "    def _vrnn_step(self, vals, x_t):\n",
    "        \"\"\"\n",
    "        A single step on the time-series (or batch) using the VRNN. \n",
    "        \"\"\"\n",
    "        # I chose to store them all but you don't HAVE to do this. \n",
    "        # This is a convenient thing to do however so you can check values later on\n",
    "\n",
    "        prev_hidden = vals[1]\n",
    "        # mu_z, sigma_z = f(x, ht-1)\n",
    "        encoding_vals = [x_1, \n",
    "                phi,\n",
    "                phi_mu,\n",
    "                phi_sigma,\n",
    "                z_t] = \\\n",
    "                    self._encode_x(x_t, prev_hidden)\n",
    "        \n",
    "        # mu_phi, sigma_phi = f(ht-1)\n",
    "        prior_vals = [prior, \n",
    "                prior_mu, \n",
    "                prior_sigma] = \\\n",
    "                    self._calculate_prior(prev_hidden)\n",
    "\n",
    "        # mu_x , sigma_x = f(z, ht-1)\n",
    "        decoding_vals = [z_1,\n",
    "                theta,\n",
    "                theta_mu,\n",
    "                theta_sigma] = \\\n",
    "                    self._decode_z(z_t, prev_hidden)\n",
    "            \n",
    "        output, state = self._rnn_recurrence(x_1, z_1, vals[0], vals[1])\n",
    "\n",
    "        return [\n",
    "            state[0], # vals[0],\n",
    "            state[1], # vals[1],\n",
    "            output, # vals[2],\n",
    "            x_1, # vals[3],\n",
    "            phi, # vals[4],\n",
    "            phi_mu, # vals[5],\n",
    "            phi_sigma, # vals[6],\n",
    "            prior, # vals[7],\n",
    "            prior_mu, # vals[8],\n",
    "            prior_sigma, # vals[9],\n",
    "            z_t, # vals[10],\n",
    "            z_1, # vals[11],\n",
    "            theta, # vals[12],\n",
    "            theta_mu, # vals[13],\n",
    "            theta_sigma # vals[14]\n",
    "        ]\n",
    "   \n",
    "    def _scan_sequence(self, sequence):\n",
    "        with tf.name_scope(\"scan_input\"):\n",
    "            return tf.scan(self._vrnn_step, sequence, initializer=[\n",
    "                    self.state[0], # cell_state - 0,\n",
    "                    self.state[1], # hidden_state - 1,\n",
    "                    tf.Variable(np.zeros((self.batch_size, self.rnn_size), dtype=np.float32)), # cell_output - 2\n",
    "                    tf.Variable(np.zeros((self.batch_size, self.x_size), dtype=np.float32)), # x_1 - 3\n",
    "                    tf.Variable(np.zeros((self.batch_size, self.phi_size), dtype=np.float32)), # phi - 4\n",
    "                    tf.Variable(np.zeros((self.batch_size, self.z_size), dtype=np.float32)), # phi_mu - 5\n",
    "                    tf.Variable(np.zeros((self.batch_size, self.z_size), dtype=np.float32)), # phi_sigma - 6\n",
    "                    tf.Variable(np.zeros((self.batch_size, self.prior_size), dtype=np.float32)), # prior - 7\n",
    "                    tf.Variable(np.zeros((self.batch_size, self.z_size), dtype=np.float32)), # prior_mu - 8\n",
    "                    tf.Variable(np.zeros((self.batch_size, self.z_size), dtype=np.float32)), # prior_sigma - 9\n",
    "                    tf.Variable(np.zeros((self.batch_size, self.z_size), dtype=np.float32)), # z_t - 10\n",
    "                    tf.Variable(np.zeros((self.batch_size, self.z_size), dtype=np.float32)), # z_1 - 11\n",
    "                    tf.Variable(np.zeros((self.batch_size, self.theta_size), dtype=np.float32)), # theta - 12\n",
    "                    tf.Variable(np.zeros((self.batch_size, self.x_size), dtype=np.float32)), # theta_mu - 13\n",
    "                    tf.Variable(np.zeros((self.batch_size, self.x_size), dtype=np.float32)) # theta_sigma - 14\n",
    "            ])\n",
    "    \n",
    "    @property\n",
    "    def loss(self): return self.objective\n",
    "    \n",
    "    @property\n",
    "    def x(self): return self.inputs\n",
    "    \n",
    "    @property\n",
    "    def optimization_step(self): return self.train_step\n",
    "\n",
    "    @property\n",
    "    def cell_states(self): \n",
    "        return tf.transpose(self.outputs[0], [1, 0, 2])\n",
    "        \n",
    "    @property\n",
    "    def hidden_states(self): \n",
    "        return tf.transpose(self.outputs[1], [1, 0, 2])\n",
    "\n",
    "    @property\n",
    "    def priors(self): \n",
    "        \"\"\"\n",
    "        mean and variance for prior on z p(z)\n",
    "        transposed from T X N X F to be N X T X F\n",
    "        \"\"\"\n",
    "        return tf.transpose(self.outputs[8], [1, 0, 2]), tf.transpose(self.outputs[9], [1, 0, 2])\n",
    "\n",
    "    @property\n",
    "    def calculated_posteriors(self): \n",
    "        \"\"\"\n",
    "        mean and variance to infer z from q(z|x)\n",
    "        transposed from T X N X F to be N X T X F\n",
    "        \"\"\"\n",
    "        return tf.transpose(self.outputs[10], [1, 0, 2])\n",
    "    \n",
    "    @property\n",
    "    def posteriors(self): \n",
    "        \"\"\"\n",
    "        mean and variance to infer z from q(z|x)\n",
    "        transposed from T X N X F to be N X T X F\n",
    "        \"\"\"\n",
    "        return tf.transpose(self.outputs[5], [1, 0, 2]), tf.transpose(self.outputs[6], [1, 0, 2])\n",
    "\n",
    "    @property\n",
    "    def likelihoods(self): \n",
    "        \"\"\"\n",
    "        mean and variance to reconstruct x from p(x|z)\n",
    "        transposed from T X N X F to be N X T X F\n",
    "        \"\"\"\n",
    "        return tf.transpose(self.outputs[-2], [1, 0, 2]), tf.transpose(self.outputs[-1], [1, 0, 2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# test computing latent variable\n",
    "batch_size=10\n",
    "\n",
    "rnn_size=20\n",
    "x_size=x_data.shape[2]\n",
    "x_1_size=x_data.shape[2]\n",
    "\n",
    "z_size=15\n",
    "z_1_size=15\n",
    "\n",
    "phi_size = 25   # N ~ q(z|x)\n",
    "prior_size = 25 # N ~ p(z)\n",
    "theta_size = 25 # N ~ p(x|z)\n",
    "\n",
    "tf.reset_default_graph()\n",
    "vrnn = VRNN([batch_size, n_points, n_features], rnn_size, x_size, x_1_size, \n",
    "       z_size, z_1_size, phi_size, prior_size, theta_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# define session and model saver\n",
    "def get_new_session():\n",
    "    sess = tf.Session()\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    return sess\n",
    "sess = get_new_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import os.path\n",
    "import shutil\n",
    "LOGDIR = \"/tmp/test1/\"\n",
    "if os.path.exists(LOGDIR):\n",
    "    shutil.rmtree(LOGDIR)\n",
    "merged_summary = tf.summary.merge_all()\n",
    "\n",
    "writer = tf.summary.FileWriter(LOGDIR)\n",
    "writer.add_graph(sess.graph)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train(sess, data, model, summaries, writer, batch_size, run, tensor_freq=10):\n",
    "    i=0\n",
    "    nbatches=max(data.shape[0]//batch_size,1)\n",
    "    print(run, )\n",
    "    for j in range(nbatches):\n",
    "        batch = x_data[j*batch_size:(j+1)*batch_size]\n",
    "        if i % tensor_freq == 0:\n",
    "            s = sess.run(summaries, feed_dict={model.x: batch})\n",
    "            writer.add_summary(s, run*nbatches+i)\n",
    "        sess.run(model.optimization_step, feed_dict={model.x: batch})\n",
    "        i += 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n"
     ]
    },
    {
     "ename": "InvalidArgumentError",
     "evalue": "Nan in summary histogram for: x_sample/x_t\n\t [[Node: x_sample/x_t = HistogramSummary[T=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](x_sample/x_t/tag, x_sample/Add)]]\n\nCaused by op 'x_sample/x_t', defined at:\n  File \"/Users/roger/anaconda/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"/Users/roger/anaconda/lib/python3.6/runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/ipykernel/__main__.py\", line 3, in <module>\n    app.launch_new_instance()\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n    app.start()\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/ipykernel/kernelapp.py\", line 474, in start\n    ioloop.IOLoop.instance().start()\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/zmq/eventloop/ioloop.py\", line 177, in start\n    super(ZMQIOLoop, self).start()\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/tornado/ioloop.py\", line 887, in start\n    handler_func(fd_obj, events)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/tornado/stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/tornado/stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 276, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 228, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 390, in execute_request\n    user_expressions, allow_stdin)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/ipykernel/ipkernel.py\", line 196, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/ipykernel/zmqshell.py\", line 501, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2717, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2821, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-8-740527d5784c>\", line 17, in <module>\n    z_size, z_1_size, phi_size, prior_size, theta_size)\n  File \"<ipython-input-7-5d411a2d9d8c>\", line 20, in __init__\n    self._build()\n  File \"<ipython-input-7-5d411a2d9d8c>\", line 51, in _build\n    self._tensorboard_additions()\n  File \"<ipython-input-7-5d411a2d9d8c>\", line 41, in _tensorboard_additions\n    tf.summary.histogram(\"x_t\", x_t)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/tensorflow/python/summary/summary.py\", line 192, in histogram\n    tag=tag, values=values, name=scope)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/gen_logging_ops.py\", line 188, in _histogram_summary\n    \"HistogramSummary\", tag=tag, values=values, name=name)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py\", line 787, in _apply_op_helper\n    op_def=op_def)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 2956, in create_op\n    op_def=op_def)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 1470, in __init__\n    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access\n\nInvalidArgumentError (see above for traceback): Nan in summary histogram for: x_sample/x_t\n\t [[Node: x_sample/x_t = HistogramSummary[T=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](x_sample/x_t/tag, x_sample/Add)]]\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[0;32m/Users/roger/anaconda/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1322\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1323\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1324\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/roger/anaconda/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1301\u001b[0m                                    \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1302\u001b[0;31m                                    status, run_metadata)\n\u001b[0m\u001b[1;32m   1303\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/roger/anaconda/lib/python3.6/site-packages/tensorflow/python/framework/errors_impl.py\u001b[0m in \u001b[0;36m__exit__\u001b[0;34m(self, type_arg, value_arg, traceback_arg)\u001b[0m\n\u001b[1;32m    472\u001b[0m             \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mc_api\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_Message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 473\u001b[0;31m             c_api.TF_GetCode(self.status.status))\n\u001b[0m\u001b[1;32m    474\u001b[0m     \u001b[0;31m# Delete the underlying status object from memory otherwise it stays alive\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mInvalidArgumentError\u001b[0m: Nan in summary histogram for: x_sample/x_t\n\t [[Node: x_sample/x_t = HistogramSummary[T=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](x_sample/x_t/tag, x_sample/Add)]]",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-31a4bf6cefa0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msess\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvrnn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmerged_summary\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwriter\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensor_freq\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-11-e76966f78706>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(sess, data, model, summaries, writer, batch_size, run, tensor_freq)\u001b[0m\n\u001b[1;32m      6\u001b[0m         \u001b[0mbatch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx_data\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mtensor_freq\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m             \u001b[0ms\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msummaries\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m             \u001b[0mwriter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_summary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mnbatches\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m         \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimization_step\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/roger/anaconda/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    887\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    888\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 889\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    890\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    891\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/roger/anaconda/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1118\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1119\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1120\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1121\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1122\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/roger/anaconda/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1315\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1316\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1317\u001b[0;31m                            options, run_metadata)\n\u001b[0m\u001b[1;32m   1318\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1319\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/roger/anaconda/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1334\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1335\u001b[0m           \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1336\u001b[0;31m       \u001b[0;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1337\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1338\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mInvalidArgumentError\u001b[0m: Nan in summary histogram for: x_sample/x_t\n\t [[Node: x_sample/x_t = HistogramSummary[T=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](x_sample/x_t/tag, x_sample/Add)]]\n\nCaused by op 'x_sample/x_t', defined at:\n  File \"/Users/roger/anaconda/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"/Users/roger/anaconda/lib/python3.6/runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/ipykernel/__main__.py\", line 3, in <module>\n    app.launch_new_instance()\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n    app.start()\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/ipykernel/kernelapp.py\", line 474, in start\n    ioloop.IOLoop.instance().start()\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/zmq/eventloop/ioloop.py\", line 177, in start\n    super(ZMQIOLoop, self).start()\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/tornado/ioloop.py\", line 887, in start\n    handler_func(fd_obj, events)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/tornado/stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/tornado/stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 276, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 228, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 390, in execute_request\n    user_expressions, allow_stdin)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/ipykernel/ipkernel.py\", line 196, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/ipykernel/zmqshell.py\", line 501, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2717, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2821, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-8-740527d5784c>\", line 17, in <module>\n    z_size, z_1_size, phi_size, prior_size, theta_size)\n  File \"<ipython-input-7-5d411a2d9d8c>\", line 20, in __init__\n    self._build()\n  File \"<ipython-input-7-5d411a2d9d8c>\", line 51, in _build\n    self._tensorboard_additions()\n  File \"<ipython-input-7-5d411a2d9d8c>\", line 41, in _tensorboard_additions\n    tf.summary.histogram(\"x_t\", x_t)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/tensorflow/python/summary/summary.py\", line 192, in histogram\n    tag=tag, values=values, name=scope)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/gen_logging_ops.py\", line 188, in _histogram_summary\n    \"HistogramSummary\", tag=tag, values=values, name=name)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py\", line 787, in _apply_op_helper\n    op_def=op_def)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 2956, in create_op\n    op_def=op_def)\n  File \"/Users/roger/anaconda/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 1470, in __init__\n    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access\n\nInvalidArgumentError (see above for traceback): Nan in summary histogram for: x_sample/x_t\n\t [[Node: x_sample/x_t = HistogramSummary[T=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](x_sample/x_t/tag, x_sample/Add)]]\n"
     ]
    }
   ],
   "source": [
    "for i in range(2000):\n",
    "    train(sess, x_train, vrnn, merged_summary, writer, batch_size, run=i, tensor_freq=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for i in range(2000,4000):\n",
    "    train(sess, x_train, vrnn, merged_summary, writer, batch_size, run=i, tensor_freq=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def reconstruct(sess, data, model, batch_size):\n",
    "    mus = []\n",
    "    sigmas = []\n",
    "    nbatches=max(data.shape[0]//batch_size,1)\n",
    "    for j in range(nbatches):\n",
    "        batch = x_data[j*batch_size:(j+1)*batch_size]\n",
    "        reconstructions = sess.run([model.likelihoods], feed_dict={model.x: batch})\n",
    "        mu, sigma = reconstructions[0]\n",
    "        mus.append(mu)\n",
    "        sigmas.append(sigma)\n",
    "    mus=mus[0]\n",
    "    sigmas=sigmas[0]\n",
    "    return mus, sigmas\n",
    "def plot_reconstruction(mus, sigmas, data, index):\n",
    "    mu_series=mus[index]\n",
    "    sigma_series=sigmas[index]\n",
    "    actual = x_test[index]\n",
    "\n",
    "    reconstruction = np.random.normal(mu_series, sigma_series)\n",
    "    print(\"reconstruction\", reconstruction.shape)\n",
    "    print(\"actual\", actual.shape)\n",
    "    plt.plot(actual[:, 0], label=\"real\");\n",
    "    plt.plot(reconstruction[:,0], label=\"reconstruction\");\n",
    "    plt.legend()\n",
    "    plt.show();\n",
    "    plt.plot(actual[:, 1], label=\"real\");\n",
    "    plt.plot(reconstruction[:,1], label=\"reconstruction\");\n",
    "    plt.legend()\n",
    "    plt.show();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "mus, sigmas = reconstruct(sess, x_train, vrnn, batch_size)\n",
    "plot_reconstruction(mus, sigmas, x_train, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess2 = get_new_session()\n",
    "j=0\n",
    "batch = x_data[j*batch_size:(j+1)*batch_size]\n",
    "cell, hidden, priors = sess.run([vrnn.cell_states, vrnn.hidden_states, vrnn.priors], feed_dict={vrnn.x: batch})\n",
    "cell = cell[0]\n",
    "hidden = hidden[0]\n",
    "priors = priors[0]\n",
    "mu = priors[0]\n",
    "sigma= priors[1]\n",
    "print(mu.shape, mu)\n",
    "print(sigma.shape, sigma)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# TODO\n",
    "# loop over learning rates\n",
    "# relative is computational time\n",
    "# wall time is actual time\n",
    "\n",
    "# play with embedding. look at the inferred z for different sequences. write them somehow?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}