Part1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: CUDA_VISIBLE_DEVICES=[]\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "%env CUDA_VISIBLE_DEVICES=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tue Jul 25 09:10:34 2017       \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| NVIDIA-SMI 375.26                 Driver Version: 375.26                    |\r\n",
      "|-------------------------------+----------------------+----------------------+\r\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\r\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\r\n",
      "|===============================+======================+======================|\r\n",
      "|   0  GeForce GTX 1080    Off  | 0000:02:00.0     Off |                  N/A |\r\n",
      "| 17%   48C    P8    15W / 200W |   8110MiB /  8113MiB |      0%      Default |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "|   1  GeForce GTX 1080    Off  | 0000:03:00.0     Off |                  N/A |\r\n",
      "| 18%   49C    P8    15W / 200W |   7924MiB /  8112MiB |      0%      Default |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "                                                                               \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| Processes:                                                       GPU Memory |\r\n",
      "|  GPU       PID  Type  Process name                               Usage      |\r\n",
      "|=============================================================================|\r\n",
      "+-----------------------------------------------------------------------------+\r\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'numpy.ndarray'>\n",
      "(55000, 784)\n",
      "(55000, 10)\n"
     ]
    }
   ],
   "source": [
    "print(type(mnist.train.images))\n",
    "print(mnist.train.images.shape)\n",
    "print(mnist.train.labels.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build the graph"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 1 Going forward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# data placeholder\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "\n",
    "# weights and bias\n",
    "W = tf.Variable(tf.zeros([784, 10]))\n",
    "b = tf.Variable(tf.zeros([10]))\n",
    "\n",
    "# logits\n",
    "y = tf.matmul(x, W) + b\n",
    "\n",
    "# placeholder for correct labels\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 2 Labels' predictions, accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "labels_hat = tf.cast(tf.argmax(y, axis=1), tf.float32)\n",
    "labels = tf.cast(tf.argmax(y_, axis=1), tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "accuracy = tf.reduce_mean(tf.cast(tf.equal(labels_hat, labels), tf.float32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 3 Nn-training "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "loss = tf.nn.softmax_cross_entropy_with_logits(logits=y, labels=y_)\n",
    "train_step = tf.train.AdamOptimizer().minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 4 Executing computations on the graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_accs = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i in range(300):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    test_xs, test_ys = mnist.test.next_batch(200)\n",
    "    test_accs.append(sess.run(accuracy, feed_dict={x: test_xs, y_: test_ys}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztnXmYXFWZ/z+n9qqu6r07naU7GyELYQmETUEiCAZE0XEZ\n9DejuOGoqDM6juCCiDOj46jjzOg4g4rbjCACalQQkF0IkISEJSF7SNKddNL7VnvV+f1xl75VXd2p\nTlKpdNf7eZ48qbp1u+q9dbvP97zLeY/SWiMIgiAIAK5yGyAIgiCcPIgoCIIgCDYiCoIgCIKNiIIg\nCIJgI6IgCIIg2IgoCIIgCDYiCoIgCIKNiIIgCIJgI6IgCIIg2HjKbcBkaWxs1PPmzSu3GYIgCFOK\nDRs2dGutm4503pQThXnz5rF+/fpymyEIgjClUErtLeY8CR8JgiAINiIKgiAIgo2IgiAIgmAjoiAI\ngiDYiCgIgiAINiIKgiAIgo2IgiAIgmAjoiAIglCAlzsGeH5fX7nNOOGIKAiCIBTga/e/wld+t6Xc\nZpxwRBQEQRAK0DOcZCCaLPjaj/68h2u+++cTbNGJYcq1uRAEQTgR9EdTpDLZgq999feGB9E7kqS+\nyncizSo54ikIgiAUoC+aZCieLvhayOcG4IX9/SfSpBOCiIIgVAiZrOaGXzw/ZZKnnQNx3nv7c/SN\nFA7hlJJ4KkMinSWZyRJPZca83lYfAmBjge9y3au9fOrOjWSzuuR2lgIRBUGoEHpGEvz+xYOs3dVT\nVjsS6Qx7e0aOeN7Tu7p5YnsXL3YMnACrculz5BIKeQtJM6y0sYCn8Ni2w/x20wEG40b4aU/3ka/1\nZEJEQRAqhMGYMbhFk4VDIieKr/5+C5f862NH9AD298YA6BpKnAizcuiPpuzHg/HUmNet7/LF9rGC\nZf3sYCzNrzd2cMW/PU7/OAnrkxERBUGoEAZixmAVTY4Nh5xINpmz611dwxOet683CpRHFJyewq/W\nt/Pzta/az7XWDMZSeFyKgVjK/l4tLFEYiKXY2zNCKqM5NFj4Gg70x/j7X71QUHgs0pksX/7ty+zr\niR79BU0CEQVBmIYUnt0ax2JlFoXZtUHgyKKwvy9XFJyz7YFoCq2Li9n3jSTZfmiIHYeGSI9TTZSP\n01P4wZO7+eaD2+0cQTxl5BpOm11j2NmbO1j3xww7B2Ip2/aekbGioLXmqv94krs3tLNhb5/9XvmD\n/+7uEX66di+PbD1UlO3HSklFQSm1Wim1TSm1Uyl1Y4HX5yqlHlZKvaiUekwpNaeU9ghCJXB4MM45\nX32Ip3Z25xy3hGKkzKLQEPYDsPPwEUTB8hSGE9z53D7OuvUhNu3vZ8PePs689UEe2nLkQVJrzRu+\n/ThX/NsTXP5vT/Bvf9pelI1OUchkNQOxFHvMPIj1PS6fVZ1jp0XfyKinYImCdczJs3t67c/pHU6y\naX8/F3/jUV73r4/m5H0ODsQNm2LjexPHk5KJglLKDXwPuBJYBrxbKbUs77RvAj/TWp8B3Ap8rVT2\nCMKJZu2uHh7c3HnCP/fAQJxURo8ZdAdsT+HE5BQ2Hxjgjuf2jTmeSBmz9R0TiEIinaFz0BgMd3cN\nc+O9LwHGAHyn+Z7bDw0d0YZEOkvPSJJrzprFgsaqgjmAQvQVyAFs2meEvazvcbnlKfTlioL1+mA8\nRfew8T69BTyFA/0x+3HXcII1mw7Yz7d1DtqPOweM85xCVUpK6SmcB+zUWu/WWieBO4Fr8s5ZBjxi\nPn60wOuCMKVwhjS+++gOvn7/1hNuw7BZLdOTl8gdiE4+p5DNaobixYVq8kswr/vxOm669yUOmYO7\nhVXiuePQ+KLQ0RdDa3Ap2HxgdIBMprOs3W3MopVSR7TJmtWvnFvHma21R/ROLPqjSfLffuN+I8Rj\nDfpz6oLUBL127sPCEpTc8NFYkXEO8ocG49z/8kHesLSZoNfN/r5RwbA8hUJCVQpKKQqzgf2O5+3m\nMScvAH9hPn4bEFFKNZTQJmGK8JOn9nD1fz5Z8s/pHUky78Y/8OuN7cf8Xu+9/Tne+d9r7eddQwkO\nDsSLjn0fL4YTxmCTX92Tn2jOZjWXfeuxgrN5MGbrf/H9pzn9lgf58prNE37mP9/3Chd/49Gcmn5L\nJB7I85as6qeO/ti4lVC7u4xQzZKW6pzjrxwcpN0cMPMTvIWwykkjAS+nNIc5OBBnaIKkrkV/NEVz\nxG8Lg9uleGJ7N4l0xhbXmqCXtvqQXSUFxndmfb/90RTdw4Yo9BYUBUN45tQFeXx7FwcH4qxePpPW\n+mCO0HRa4aNp4CkUw98DlyilNgKXAB3AmGmMUup6pdR6pdT6rq6uE22jUAZePjDIyx2DJNLGr0PX\nUGLM4GIRTab5zcaOoxp8X2g3QgJ3rTt2UXhiexfr9/bZg1XXUIJYKmOXLx4NvSNJvvvIDn769KtF\nL4YaTmTsn3VizZqtRPOBgRi7ukbY4piJP769y66r/94jO9m0v5/GsI+N+0br8V9qH+DF9tz6/Nue\n2E1Hf4zvPrLTPub3GMPLfS8dzDk35hAO54Dq5OGth6jyubl0STMAi5rDALzqWN8wUUnrPRva+Y+H\nd7DDDDFFAh77PYrxFvqiKepCPsJ+oxPQx1YtZF9vlP96dJf9PVYHvLTWB3NyCgOOgXtvzwhp855Z\nnkI0mebuDe1oremLpqgOeGmpDtgiuHJuHa11oZz3tHMK08BT6ABaHc/nmMdstNYHtNZ/obVeAXzB\nPDZmNYjW+jat9Uqt9cqmpqYSmiycLFizoh4zJvuu/1nLR36+wRYJJw9s7uRvf7mJvUdRsmfFdWfW\nBo7BWiOsYfHwK4dIprP0mddwcLDwwFcMP35qD998cDtfXrOZZ/f0FvUzw+aglV/xYnsKKUOkrJi+\nM4H5vtuf4/XffAyAu9a384alzaxe3pITN//K7zbzj79/xX6ezWqCXqPtwy/XG8GBZDrLQTNs9Py+\n3D/pWDJDXcgLMCb0AkYJ5gObD3Hp0hnMqDaS0otmhFEKDjvKU8dLvA7FU3zmVy/w7Ye2c9sTuwGo\nDnpZNCMCFCsKRk+j6oBh57vPa+P1i5u45/l2+3usCXpprQ/R3hezBbvPIQrO6qpe8/f47g3t/P2v\nXmDLwUH6YynqQl6aIsY1+jwuWutDtNYbomBNcqzw25RPNAPrgEVKqflKKR9wLbDGeYJSqlEpZdlw\nE3B7Ce0RTnJ+vvZVPnnHRmB0VmTFZK3Zazw1tqTQmp0NJyY/I9912Hjf2uDETc3e/+Pn+J/Hd437\nujNp+ODmQzkDsjXTOxrue+kgZ7bWEvC6+OPLB8e8vmFvL5d+87GcmaX1PeR7CrYomJ7ETjOmX2gG\nengoTudgnNNn19JaF6I/mrJnyIeG4jmhm47+GLFUhtm1QcM7Smbo6DdyAguaqkims/x6Yzurv/ME\nQ/EUsVTGHqDzK3fA8FZ6R5JctbwFl8uI35w2q4aQ120PkE0RP/3RJMOJNFf/55N2SSfkrmuwfm8i\nAQ+tdUF8HhdbO4f4y/9Zy0+ffjXnc7XW/O2dG/nGH7fSNZSgKeInEvDgUsbnLWgKG51TzWuvDnpp\nrQuRzGQ5NDR2Nm+JT23Iy9rdPaz+zhM8uvUwAPt6ovRHk9SGfLYoLGwK43Yp2upDjCQzo5MKK6dw\ngtp9lEwUtNZp4AbgAeAV4C6t9Wal1K1KqbeYp60CtimltgMzgH8qlT3Cyc8fN3fy0JZDaK3tWVH+\nwiVnzDqRzvDM7h67xNKZQNVa8+cd3Wit0Vrz5I6uguGXHYeHzJ8dX1CyWc2j27r4miNpvK1ziIMD\nMbZ2DrKvJ2rPeCMBD3t7o3QPjf4Bd05CFAbjKZ4xE6k7Dg2xq2uEd5w9m0tObeL+lztzriGZzvL2\n769ld/cILztaQQyNKwrWiuZMzrVbCUynF2YNmG0NQbvPjzV77RpK5AiwNfhZoZ72vqg92C+baeQE\nfvL0XrZ2DvH1+7cSTWaYUxukyuce4ymMJNJ8ec1mFjRW8folzbzjnDl88U1L+fDFCwj5PfbvQ2td\nkL5ois0dA7zcMcjj2w7b7+H8nbEG1kjAi8ft4tx5dfx6YwfP7ukdkye5e0M7v9l0gMe3d9E9nKAp\nbIhCU8SP1+2ivspHLJXh0GCCiN9jD+DGdxPL+bzGsB/rVi02BXBr5xCPbjPC3/v7ovRHU9SGvDSZ\nJbpWeKvVfM87nttHf9QQoYDXxWA8XfQ6i2OhpDkFrfV9WutTtdYLtdb/ZB67WWu9xnx8t9Z6kXnO\nh7TWJ37ponDSsPPwMLFUhpFkZtRTGM79lXAuvLrp3pe49rZn7NLEEcfA/ued3fzVj55l0/5+frvp\nAH/9o+e4e8PYvIE1oE1Uu+9cCLbj0BBaa957+7N8+beb+cjPN/CeHz7Dtk7DhrNaa+kdSdA1PCoE\nxXoKWmsu+9bjXHvbMwzGU9z3UidKwRtPa+ENS2dweCjBTkdIwhIPGBUCGK0+6oumckTEXryWypDN\njpasWqG6YUePn188aySfW+tC9iC1vzfKcCJNPJXNSdZa4vL6JUZod1/vqEieNsso2/SYM/47ntvH\nYCxF0Oc2Qy+5onDfSwdp74vxj29bTsDrxu9x86GLF+DzuKjyue2Bdo7pvVjfh/N7sX5nnC2tIwEj\nN3Dl8pm2WJ5iDsJgrO2w2mHv6R4hmszQFPFzSnOY5eY1NJjvt6trmBoz/GV9N9b1DpgL1+Y2GMfd\nLsVCx+dY7OuN0hdNUufwFCxRWNISwe1S/OsD2/jBk7vNY4a4Do7TtfV4Uu5Es3AceGZ3D5/+5abj\nWuWSTGf5m59vOGIt+Od//dJxWWk5EEvZrQAOD8btgSp/VupMUj5mzrqsmbhTMJx9c6xYfHtflA/8\nZJ09ix2Mp+wBe2Sc0NNN977Evc+PpsLuf7mT9r4YhwYTPL2rh709Udr7YnznT9vxuV0snVlN70iS\nw+a1uF3KrjO/e0M7X/39Fv7w4kFu/u3LYz7rwS2H7Fnuvp4o9798kJVz62iuDtgDmHO1qzNk1R9N\n0juS5LofP8dWU6CsRVcWg47H8XRmNKdQIPxmzXjb6p2iELPtG06k+fkze/mvx3ay49AwjWE/p8+u\nNc+Lsr8vis/tsu22Bv+sNga2oNdtxs5z8y3P7+unOuDhgvljixBDPmNgVwpm1QbpjybtslZneatl\n46kzwvb5YfNnrzhthl1RZLW/1lrzhd+8TCKd5W0rZtueVFPEzz+99XT+56/PAUZF5pUDg7RUGzmo\n2bVBlDKu+dFth/ncPcZ6irnmd3bhggZ7dm8Jhdet2N8bYyCaoiY4mlNYNGPUU1j/hTfgdSv+8KIR\nMjx3Xp15X0ofQhJRmAY8/Moh7t3YkZPkOlYODcb54+bOnNloPrFkhl88u4/7XxpbFdQ5kBt37hlO\nTBgTdSb/XnVUbXQNJXLizk5RsGZ8Vpw5msygtTEDtgbigVjKHpA27Ovjka2HeWKHISZPbB+tZLMG\nxH09UTtpnMpkueO5ffzCUbL5as+I3bvH+plz5tYxkswwuy5IY9hHKqPtWPYiswxSa813H9nBT55+\nlR/9eTc/W7t3TEmls8LniR1dbO0c4srlM4GxM1IwPBBrgOuPpvjuIzt5bFtXTnzdqnrJZDVDibQ9\n293TPcJQPE1j2M9wIk0qk7XLN8+fXw8Y1UNNET81QS/VAQ/7eqP2YqyshrvW7eeeDe10Dsbtaw96\n3ezrjdE9lLR/1rhHuR5fyOemtS7Evt4oqUyWp3d1s7VzkI37+jirrc7OJTip8huDeNjnob7KSzqr\n7S6le7pH7A1xuocTeFyKhU3GIBv2e+z3a44E+OKbltEU8dvf/+9fPMhDWw7xmStO5cKFo2LUFPHj\ncik8bmOYbAgb391QIk1LjSEKPo+LmdUB9vdGuff5DiJ+Dx9btdCe0a9e3sInLl3ER1ct5H8/eD4f\numg+qxY3G99/Ik1dyMcFCxr44EXzuXjRaBFNXZWP+Y1VvNoTRSnjd8y4zyIKQhFYM6ODA0df5ZKP\nFV+eKHlrVaR05i1O6h5OcMHXHraTxgB/+8tNfPbuF8d9r52HRz2S7XmzPqcoxAuEeQ7ankKap3b2\n8IZvP85j5oA/EEvZi59eMlezWgPr/S910hj2s2pxEyOJNNFkmiu+8zg/W/sqMJrY2+0ITQzGUjmD\nt8el+MF7V9JaH+TUGWHqq4xZ39bOIaoDHuY3VrG/N8rWziFe7YmSyWq7Gid/g5bekYQ9e/3Z03sB\neOPyFsAIXYR87pwqoM6BOE1hP41hH33RVM4+CZZYWMJp/T/L7Dv0wn7ju7BmoP3RlH2vX3eqMTi1\n1ofsBWILm8P8eWd3TrinvS9K70iSnuEkDVU+lFJGiWZflIFYikjAY4dtAKrMawMI+NzMbwwRS2X4\n+v1bec8PnmX1d55ka+cQK1prKYTlKYQDHmpDPvs7DPncpLPabsfdNZSgMey322lYFUQWH7xoPqtP\na2EgZizK+5c/buX02TV84LXzmVkzWoVmzeAtrHsL5JzXWh9if1+Ujfv6uPjURv5h9RJOM1tgvGHp\nDFrrQ3xu9RJa60N88eplLGiqsn8H66q8VPk9fOnqZVT5czfCXNRs5CLm1AVpqQna96nUiChMA6wY\n6mQSmkfCqvIZL6wCo5Uj+THzf/qDUa641bFUv6M/xu7u8UsBdxwaxpocOkMB3cOJnNlx3BSrHkeu\nIWHO7KPJjF3HbrUz2Hxg0B4Qrdlbe2+MeCrDo9sO88bTZhAJeBlJpDk4ECeeytpJW2uWbcWx5zdW\nMRBLsXF/HyvajIqgZbOqqa/y8bsbLuKb7zzTnolv6xyiKeJn+ewaXu2Jcsdz+8aukN2XLwpJ5jZU\nURP00jkYp7U+aDePU0qZC6UcnsJgnJk1AWpDPnZ1DdtrLgBmmuENq72CVR5pDVYvdRjnnjvP8Ar6\no0k7p3D+/Hq8bkVrXdB+v09dtog93SM5G9n3RVP0x1J0DSfs0Ipl42DcCI04RaGtocp+HPS6WbXY\nSEzf/tSenPj+WW2FRcH2FPweaoOjA/3rzQT3KweNiYVVOWTdC6cNFjVBL4OxFC+2D9DeF+O9F87F\n43blDPaN4XxRGM1RWIM0GKKw7tU+2vtirGg1RPaGS0/h2c9fZnsUTlrrQjl2jIf1nSxqjtglvMcz\nGjAeIgrTgFFPYWJRKGYlp4U10I4kxk/AWoN1vhg9a4acnH9Ug7E0nROs7u3ojzGvsQq3S9mJy9m1\nQbqGEznvH0uO3zcnmsyMqVayQj0+z+iv+r7eKJv29xNNZrh0STNhv5vhRMb+HOu986t35jWEGIyl\n2dsTZenMaj6+6hTe/9p5ANSGfEQCXnvg6ByMM6s2yApzgPvFs/s4f369HVeuDnjYZLZNyJitJHpG\njBl3a70x4JxlDjAWc+pyY/CdAzFaagLUBr1s2NuH86ud12gMwN1mCaW1iOuMOYY9L+wfoCbotePY\n/bFRT6Eh7Oejq07hHeeMLjNatbiZNyydMSbkpbXx+2cNwC01AQ4NxhmMWaIwOuhZcXYww0f1IZbP\nrkZrePMZs7jvkxez+rQWO3yVj9NTqHMM0B+6yJjh/+cjO0ikM3QNG6JQfwRRyGojz+NxKS5fNgOA\nGaaYul2KulBumXJ1wIPXbSi7UzwuOqXRfmzdb6/bZb9XPhcsGL2+/M9wYt2bRc1h6qp8nD67Jsfb\nKhUiCtMAayCcyFO4e0M7p9/yYE4oZCKKCh+ZA9RwIp0jOFbc33J1rf7z0WRm3OqJnpGk4fJX+WxP\nYdGMMIcHEzkLlqz3LtR2OZpMj6lWGq2CGW2XYLj6hlisaKujyuexPQUw8huZrM7pVxP2e2gM++mL\nJumLGoP3Jy5bxNtW5Db2dc4mT50R4Yw5tSgF6azmqtNnsnJuPQ1VPlYvb2Hj/n601vzD3S9y+i0P\n0m3OuK0yx/wwSlu9EYO3hPXgQJyZNUFqQz4ypjuzfLZxnXMbQihlNI078ysP8qXfbibs97CgyRCL\nLQcHjcHGHJT6RpJ2BVPY7+HTl5/Km86YmfP515w1a8x3nn/d9VV++mMpekeSVAe9RPwe20OyBBEg\nYC52e9PpxntedXoLy2ZV899/fY49+OdjDYhhv4dmM7RzyalNrGir45/fdjrbDw1z7/MdZvjIZwtV\nfvgIoDpofMZvN3Vw4cIGOxwVCXgJ+z00VPlw5+U1lBoVCqcHcM1Zs+xQk9UkbyJOaY7Y321ztX/c\n85aa5bxLZ1ZTHfDyu09cxJWnzxz3/OOFiMIUJ2fl7ASiYHWWLLY8cjQkc+ScApDT9Gy094sxqFr9\n52F84eo1Z8lNEb898C9uiRBLZdjdPWwPlNZrhwYTKAXOv9toMkN3nqdgDZbWH5hhV4ond3QxryFE\nfZWPKr+HWCpDh9lTJ5HO0tEXo9chMPVVPmqCXg4PJdA6d/B3YiUjwXD/w34Pi2dE7NLSL7xpKb/8\nyIWc3VZHfzTFnu4R7nneKJVt74tRX+Wzwwsr8sIorfVBYqkMX/39K8YgHjcSnlZowedx2RVAtSEf\n9SFfTtJ5YXOYKseAu2hG2A5f9MdSdvio0MwaRsM0hbC+j4YqH1obK49rgl5cLmVX/syqDdqlqdbA\n/4GL5vHrj73GXsw2ESEz5h4JeJjbUMU9H72QH75vJQCrFjfRHPHz7O4euoeNJHd9eGJPAYyQ4rKZ\nuf2VWmoCY/IJ+dfp9BSUUjz8mUv4/ScussXuSPzHtSv41d9cOKa3k5OFTWF+/bHX8OYzxxfjUlD4\n7gtTBufK2fxulE4sMUhnNfFUBqXA7xn/F9hqbzw8TvgoncmyrydKTdDLQMwo7TylOUI2q0mks3hc\nipFkhmQ6mxNyODgQY3HL2AGgdyRpLw6yWGr+wWzrHOL8+Q3s643aiWYrZKH1aOw/lsyM8RTAmFla\noQvL3qd39fC2FbPt1yHX+9hxeCgnfGSJgvN5IUI+DwGvi3gqa9edv3NlKzsPD9vhhPqq0Zn9netG\ne0ZqbQyqK+fV8/KBAbvG3+L8+Q3Mqglw+1N77BDTzJqAnV9prQsy22zXEfYbi66cJcWLmsMEHeGH\n5bNr7DCMsTrY2E3M7yk8Vwz7PXzwovkMxFJj1nxYYuj8XqzvKxLwMJRIUxP0Uhvy0T2csNti+D1u\nVrTlhsnGw+kpAJwzdzQMo5TirNZaHth8iExWM78x7AgfFfIURo+1OsJaAG86faYtXvnUV/mMFc55\n+YbqgLcoL8HC7VJ2Pmciiv1ujifiKUxxrNBRwOuasPrIqhB6dncPy7/8AIu/+Ee2dg7S0R/jrFsf\nZOfhIfZ0jzDvxj/w9M5uO3xUKNGczWpWffMxth0asn+x7Qogc1C3qlz6o8mcxV+Wp/Cl37zM3/x8\nA2DM5q1eMzMixqAW8XvsgS+V0fbj32zq4LVff4SO/hiN5opTCyunEDEHDSun0RTx2zM/5x+iNROv\ncoiCldh9dk9vTvioocpnL1gyno/v9luvWYnCD140n6/9xek555zSHKbK57Z781jUh31cuLCB//vQ\nBTl5EIBls6p54O9eh1LwW7P3fkt1wA59tNaH7ASoJQqpzGii4dQZYbu6CeDyZTOo8rnxeVzsPDzM\ncDxNOOCZsCX1l65exuevWjrmuFWZ01BQFLz2c8urCR5FbNzOKfgLJ2dXtNURS2XwuBRvWNpMXciH\nx6Vy8g/5tsFYUfi7y0/lE5ctKvgZzRE/LdUBu0x1OiKewhTHEoXTZtXwcscA92xo5+3n5Ma5M1lt\nz0xfbB+w1wD8dtMBLlzQQH80xa6uETsZ+adXDrNkpjGbLyQKXcMJ2vtiXH3GTD77xsX86ZVDowvI\nTFGYWRNgX2+U/rw9bC1xemjLIUYSabTWZmmgMQt75zmtnNIc5tSWSE6Fx8yaIF63sstLB+Mpzmqt\nzSnRG0mm6RpK8FcXzGVFWy2Pb+viVxvaaQqPisJlS5u5YtkMosk0f3G28T1ZVS27uoZ5zcJGVs6r\n48dP7aG1PsTs2iAd/bGc5mgwvqdgvZbMZO3BuhBul6LK72EkmeGd58zh7ufbbU9hIiIBL6c2R9i0\nv5+g180Zc2rt1bytdSE7rBH2e+zZrM/t4j/fs4ILFzbkrHBuNgX4L1e28r/P7mV+Q5U9C58I6/ty\nYtld7wifWXF7S7irgx5qLVEoMsxS6HPD44S3LJF35gh++oHzCnqmznvprLI6Ep++fDHdBTbMmU5M\nX7mrECxRuHzZDBLpLJ/51Qs57Xsht+lYh7kKdklLhPtfOmgvWIqnMuwy2/e21PjtnEKhRLP1fm8/\nZw5zG6poivjthK61qtjyFPpGkjn2dA7EOTgQo3MwzlAizaHBhF02WV/lo60hxIdft4BLTm3K6Wff\nFPHnxGuH4mmaIn574AE4PJggkc7SUh3g6jNm2QNQU8TP4hkRZtYEWNFWy7vObeW61863B0Arzh5P\nZWmpCfDlN59GwONmd9cIs2uDnDuvjhVtdTmzS2fuIJ9z59Vz6eLx4+8WH1u1kHkNIW5+8zK7hLR+\nAg/Ewhr8Xr+kiaDPbSc/2+pDLG6JMLs2yJKZEVsIZ9T4eeNpLVSbSdTakJcvXT26CeLnrlxCTdDL\n7u6RokTB73HbXox1f0YTzWM9BStUY4WPgByPpVgsTyEyjo1nzKlhbkOI95zXZh977SmNY0pLAdvr\nUwpmT0IU2hpCnF2GkM6JRDyFMvKV322mKeLnY6tOAeBbD24jk9X8w+olRb+HtYnHda+ZR3PEz6fv\neoHukUROqMPaMQpGReHac1u55Xdb7AVP8VRmtIdQIoPL/Gsv5ClYAmAlf09pCtsrkp2eAsAn7tho\nDzTVAQ/tfbGc+vwdh4fwWStG8wZEr9tFY9hv1J2H/QS9blvEwBjso46cx97eEfs4jA5KTRE/zdUB\n1t50WcHv0LloaGZ1gPoqH5cvm8G9GzuoDnrtZKYzaTtRKeHNb87fdbYw1712Pte9dj4Ac+pDHBiI\nU181ft26xYq2Wu5ct5/V5mrnWrsPT5DGsJ+nbrwUwN7nd2b16KDncbvYdPMVOe8X9ntYObeeP71y\naNwkcz6DXGzWAAAgAElEQVQRv4eedJJZNUG6h0cX3Tm/F2dOAQxxsNYXHE346EieQsjn4fHPvr6o\n9wr7jO6nM6oDE+bWKhHxFMrI49u6eGzraKuFR7Ye5skd3RP8xFh6RpKE/R4CXrc9I3ImSIfiKb7x\nx22cOiOMz+0imc4S8Lo4y5ztbD5gLNTqi6bspm5D8fSE6xSsUlQr/r5oRphdh4fRWtuVRzPN1w4P\nJdhttnxYtbiZda/28ued3Xa5345Dw7a9hUIylrjkewqA3cUSjH4y1oK7QqIwEc7ZsZXYW22uJHYm\nap2DXH68/1ixBLYYT+HqM2bx+auWsPo0w8Zz59XzhauW2ovBLKzrLrSAKh/L+0hni+ufFQ54CPnc\ntNQYImrlIbxuF9XmPRkjCgGvHd8/tpzCsc9lXS5ltL7OyycIIgplZTiRtmf6YIRaJlpBXAirqRaM\nDqrWxjQAX7t/K4cG43zjHWfaM8rqwGjCzxrgNx8YtAeEwXjKFoVkJpuzgQwYpagt1QF7kF7UHLZD\nQXb4qMBA9JfntpJIZ7lr3X7OnVdHbcjLzq5hO6FbKCRjNR5rivjHxKGNfvfmwO8IEVgbs1hhi/xK\nkXycMXJrYZHV6uGtjtp8K1R1pLj/0XBWay3NEX/OSt3xqPJ7uP51C21h8rpdfPh1CwqKJuSWT46H\nJQq7itzDOOz3UBfysbglwql55aT57SUWNoWZUxck4HWzqDlMY9hP6ChyCnPqjLzS/MaqI59cBAub\nwpw1TkuNSkbCR2VkJJG2B1GtjcVS+cve732+nVe7R1jcUs1LHQPceGVuaKkvmqSuKlcUntzRxZoX\nOrjuNfP5xbP7+PDF8zmrtZZwwGPXj1ubylgVS876/qF4ikR6dOAbSaTxeUaf7+uN2tVAYCzGASMU\nZDUlaygwEF+woIHGsI/ekSQ3XbmUf/zDFnYcGrIH/kIhGWtAawj7CPjG9xSaIn4ODMTxuBRt9cag\nYYlCY2TiQdyaeQa9bruqJOB1s/Wrq3PKM/PF93jynvPaeNfK1oKN4I6WyXgK1kpnZ0nwRIT9HrSG\nr16zfMwq9foqH3u6R+zv/30XzuOvLpgLwDvOmcNbV8w+quqdOXUhNn9l9XHz0n55/QUTVlpVKiIK\nZSKb1XYP/1gyQ1Yb9f35nsI9z7ezrXOYc+fVsXZ3TwFRSNmDqTVY3fN8e84OZZ80y+usBJ3Vk8al\nRvv6WOsdPC5lhI8cPz+cSOeU9bX3RrnA0U3SWo6/49BoLX6hRKLbpfjS1cuIpzKc2VrL/MYqHtvW\nxWmzkuOGZN65spWWmiB+j5ugN/d1a2cs6zEYq2at9zlvXj0feO18zi/Qhjn/fT66aiHvzKvayp95\n+z1uAl5XUSGeyeJyKXzHURAAFjSF+cglC+xQ2ESE/R6++KalYxbMjccHL5pPMpM1w4C5dtdX+fB7\nXPb353IpXOY5Sim7VcTRcDzDdtO5rPRYEFEoE1HHjMwZQhpJGhugWDPG/b0xekcSdA4aragzWZ2z\n/H4glmKOWT0R8Lqp8rltsXlk62Hm1AXtEEvYEet1uRS1Id+YLpozqgM54SPDplGhiibTHByMM7d+\n1IVvqDJaCrzUMWAP0laox+d22auZAa45a7b9uLUuxOEho+FdoQoRMBZYWYuCrPecU2ds+9hSE2BO\nXcho3mbGhuc5mq5V+T1FJX2VUnyuyOR+a12IeQ1TIw7tdiluunLsmoLx+NDFC4o+94rTxheaeQ0h\ndk+iokc4uRCpLBNOj+DwUCJnoZTlwqczWTr6Y2S1MQvX2qjvd+4VbO3eZOGsE3euqoXRMInl1jvj\n15YotNQEzETzqGg5bX2pfQCtjfI/C6UUqxY38/Arh+zeRgGvm003X87zN18+7nfQZg6ua3f1sKCI\nOLGVnPzQRfN57LOriAS8XLFsBo9/9vV2zf1kyguPhjuvv4DPXLG4pJ8x1fn05Yu56yMXltsM4SgR\nT+EYGEmkCfncOXHJWDKD3+M6YmzYWf/fNZTA5xk9fySRpsrv4eBA3F50Zp3/gyd382J7P+9/reG+\nD8RSdtIYjOoVZydNZ08ZayWoFRuvdfycFUZqqQmwp3sk11MwK5AG4yl7L4Az8xJ0V53ewj3Pt/Pw\nK8YubCGf2y71/OKblhZsSjanbrSf0Skzxm5ZmI8VjmgI+5lpLmxzuRSzaoN2iw9rfUSpKJQrEXIJ\n+txHVV0knByIKBwlA7EUF37tYb7zl2fZrnQ6k+XibzzCpy9fzHvOb5vw552z7+7hRE5CcziRppnc\nRWcW2w8NkcpovvK7zfx6YwdaQ43DU8ivjHH2qXfWiwMFV9y2VAeMRHMqg9ulyGQ1I4k0w4k0Z9zy\nIIDdSM7JRYsaqfK5edqsjXfG48cLS7Q5ygGtDUUmwnpPp5hZWAlpqSYRhGNDwkdHycGBGNFkxq7B\nByPp2z2czGlPfe/z7Xb9v5N8T8G5tsCame/vGysK1uKt371wwF4TkOspGIO1vYbAIQpW6aVVR54/\nuHpcigZzO0nDA/HZtr7o2CWsUDWL3+O2e/j7Pa4xbYcLYW3fmG/neFjnFqpS+uBF87n3Y6/hggUT\nJ5UFQZgYEYWjpNdcC+Ds62MN7NaxZDrLp+96gbf911Njft65KKxrOEGvY+/VYbMn0J7uKOONrc59\nCXJyCqYoXPeaeZw7ry6nNW9++Ch/cA353HZSuns4afes74sm7b1wF8+I8KGLCs/8rdl6saEDpZSd\nJF84CVEo5Cl43K5p335AEE4EJQ0fKaVWA/8OuIEfaq2/nvd6G/BToNY850at9X2ltOl4YSWGB52b\n05tlnVZXUGtryKhjX+H/98NnOLutzt5U3O9xcXgwQSonhp/mljWb+enavcxtCNE9lLArigrhbGlh\nDeTXrJjFh1+XO3iH81aa5i+UqvJ7bC+ieyjBirl1hHrcdA4YFUILmqp44O9eN64dlgcxmYVJcxuq\niCYzRa1SjQSMDVsmajEhCMKxUTJRUEq5ge8BlwPtwDql1Bqt9RbHaV8E7tJaf18ptQy4D5hXKpuO\nJ06vYM0LB3jNwgb6RlL2MRjda9gqtxyIpnhqZw/dQ0n++kJjAG2rD9E7kkBrbffhH0mmWbu7hyUt\nEb7yltP43D0vMtIzNpRk4Rwk37mylfmNVXY1jhPnOgWA2ry8QNDnthPCQ4k0AY+LlpoAnYMxNu3v\n45JTJ27yZiV/J7Mg6KarluQI60S8a2Uri1siYzY4FwTh+FHK8NF5wE6t9W6tdRK4E7gm7xwNWPGN\nGuBACe05rliewp7uET55x0Ze87VHbE9hIGaEdqy9hhvNMlFrY/Xth4c4bFbLtNaH6B1J0j2StBOv\ngzFjR65Vi5s5f0EDTRE/bpeyZ/H5OGf8NUEvly2dUfC8JTMjtFQHmG9uyViX18a4yufJaYjm97qZ\nWRPghf0DdA8nc8pQC2GtTC52kAej1UCxG4nUVfnG9PcRBOH4UkpRmA3sdzxvN485uQX4K6VUO4aX\n8IkS2nNcsdo9W91Bk5ks//vMXmB0ULQ2gE+ks3z8/57n/T9ZBxg7bK01N7efUxekZyRJ91DCXni1\n5eAgqYy2k6/NkYCxQCw8unmLx6XsfEN1Ef1yAJa0VPPM5y+zvQjLw7B6BTlzCmCEtlqqg3Zn1fwe\nN/lY4aOhSfZvEgTh5KHcfvi7gZ9orb+llLoQ+LlSarnWOqcDm1LqeuB6gLa2iUs9TxRW+MhZz7/d\nDBdZomA1FxuIpfjDSwdzfn7jvn5CPqOz6VA8TTyV4U2NM+3XYLR9xEdXLeSa/ll831y0dtOVS4wS\n0Tm1bNzXV1SlTyHOm1/P569awsGBOD9+6lWq/B57tg+GKDhDU4uOsJagmB47giCc3JTSU+gAWh3P\n55jHnHwQuAtAa70WCACN+W+ktb5Na71Sa72yqampROZODmcnUshtkjZkNrqz9vx1lpu+78K5LGyq\nIp3VVPk99s+lMprmiJ+Qz81Ws4TVSkYvn13DFae12GGiCxc2cM1Zs5nfWGXvHnY0eN0urn/dQjvH\nEPK5qQl57RCS3+O2B/q6kPeI3UGdgiIIwtSklKKwDliklJqvlPIB1wJr8s7ZB1wGoJRaiiEKXUwB\nnAM9wIV59fHP7O4hldE5Dcb+5e2nc/ObT7Nj6GG/J2egbYr47STq7NrgmIRqXchHlc9t95U/XgQc\nOQUYXVTm97rsMtNFzZEjJpAlASwIU5+SiYLWOg3cADwAvIJRZbRZKXWrUuot5mmfAT6slHoBuAO4\nTuf34T3J2NcTRWudIwpVPjdntuYmYR/bdhiAS04d9Wxa60K4XcpedVvld+d0HzU2kjFuyTlzxyZf\nrz2vjc++8fj33bESzSFzcZu18M3vcdldT4tpQwHw1bcu5xcfPv+42ygIwomhpFM7c83BfXnHbnY8\n3gK8tpQ2HE8ODsRY9c1H+cF7V9IXTdIUMbaKbIz47XYS1rHHtncxqyaQ077Bau9seQ9VvjxPITza\nt+jKAu2Oz5tfz3nz64/7dQXzPAUrZOR1u2itM1pRnzF74soji782++YLgjA1kRXNk6BrKEFWw9bO\nIbIaeweoprDfHvytY3t7omM2e7dEYfGMCEGvOyen4HwdOKGll9bmNZanYC2AG4ynqAl5eeQzl/DO\nla3j/rwgCNMHCQJPAmtl8gGzRHNufYjn9vTSFPEzuzbIO8+Zwzlz63huTy8A5y+ot0XB61b2Y4/b\nxcdWLWR2XZDakA+ljL5DNUEv//y20xmMp05ol8l8T6HZDBkdHjTKbq1upoIgTH9EFCaBtXVm54Cx\n8Mzq3d8U8eNyKf71nWfaLZwB3nhai72DWVPYn5Oo/YS5GxoYCeSAx4VS6ojdVUuBnVMwhWip2S9p\njmyUIggVh4jCJLA8hc683v3OjeGd+wbMqA7Qbza6c4aG8qmvMqqKykXQZ0QRreqh0+fUcO/HXsPy\nWcXlEQRBmD6IKEyCqLkt5SEzrHJKc5gLFtRzoWO/4oDXxcWLGu1tJ60VwhOJwhXLZtiz9XKwoDHM\nma21OSIgHUcFoTIRUZgE1jaZVo+j+pCPO6/P3XZQKcXPPzhakul2KepCXjtOX4h/KHJ/4FJRV+Xj\ntx+fMkVggiCUEBGFSWDtgWCtpAiP06Aun//6f+fY+xELgiCczIgoTIJYMrfRWzF7AAA54SVBEIST\nGVmnMAmcm+V43SpnX2VBEITpgIxqkyCaGhWFsN8zqc1kBEEQpgIiCpMg5vAUis0nCIIgTCVEFCZB\n1JFTCPuL29hGEARhKiGiMAmcOYWItIkWBGEaIqIwCaISPhIEYZojojAJckRBPAVBEKYhIgqTIJZM\nY22HLJ6CIAjTERGFSRBNZqivMnoYSU5BEITpiIjCJIglM/YGNBI+EgRhOiKiUCRaa0aSabvbqYSP\nBEGYjogoFEkinSWrocXsdurcZlMQBGG6INPdIrFWMy9uifDd96zgsiUzymyRIAjC8UdEoUisvkdV\nfjdXnzGrzNYIgiCUhpKGj5RSq5VS25RSO5VSNxZ4/d+UUpvMf9uVUv2ltOdYiCaMFhdBn+ioIAjT\nl5KNcEopN/A94HKgHVinlFqjtd5inaO1/jvH+Z8AVpTKnmNl26EhAObJZjmCIExjSukpnAfs1Frv\n1longTuBayY4/93AHSW055jYtK8fv8fFkpbqcpsiCIJQMkopCrOB/Y7n7eaxMSil5gLzgUdKaM8x\nsXF/P6fPrsEnG+sIgjCNOVlGuGuBu7XWmUIvKqWuV0qtV0qt7+rqOsGmQTKd5aWOAVa01Z7wzxYE\nQTiRlFIUOoBWx/M55rFCXMsEoSOt9W1a65Va65VNTU3H0cTi2N09TDKdZfnsmhP+2YIgCCeSUorC\nOmCRUmq+UsqHMfCvyT9JKbUEqAPWltCWoyKWzHDr77bQORAHoDbkK7NFgiAIpaVk1Uda67RS6gbg\nAcAN3K613qyUuhVYr7W2BOJa4E6ttS6VLUfLT55+lduf2sPLBwYACHrdZbZIEAShtBQlCkqpe4Ef\nAfdrrbPFvrnW+j7gvrxjN+c9v6XY9zvRdA0ljAemXAW8J0sKRhAEoTQUO8r9F/AeYIdS6utKqcUl\ntOmkoT+WBLArjgLiKQiCMM0pShS01n/SWv8/4GzgVeBPSqmnlVLvV0pN285wA9EUAMPWamYRBUEQ\npjlFx0OUUg3AdcCHgI3Av2OIxEMlsewkoD9miEJ/1PAY/BI+EgRhmlNsTuHXwGLg58CbtdYHzZd+\nqZRaXyrjyk2fKQZ9pscg4SNBEKY7xVYf/YfW+tFCL2itVx5He04qus1E82DcEAUJHwmCMN0pNh6y\nTCllL+dVStUppT5WIpvKTkd/jMu+9RiDcSOXoDW4XQqvW8JHgiBMb4od5T6stbbbWmut+4APl8ak\n8vPC/n52dY3kHAtIzyNBECqAYkc6t1JKWU/MttjTdnnvsOkhOJF8giAIlUCxovBHjKTyZUqpyzD6\nFP2xdGaVFyvBvPamS1k8IwKIKAiCUBkUm2j+HPAR4KPm84eAH5bEojKitebgQJz+WAqvW9FSHbBX\nMctqZkEQKoGiRMFsbfF989+0Zf3ePt71P2s5d249tSEfSinbQxBPQRCESqDYdQqLgK8By4CAdVxr\nvaBEdpWFA/0xtIZN+/uZa267GfQZYiDlqIIgVALFxkR+jOElpIHXAz8D/rdURpWLITPBnMxkqTPb\nZAfFUxAEoYIoVhSCWuuHAaW13mt2Nn1T6cwqD1aPI4DakNHSaVQUJKcgCML0p9hEc0Ip5cLoknoD\nxg5q4dKZVR6cpaiWKAR84ikIglA5FDv9/RQQAj4JnAP8FfC+UhlVLpyeghU+CnhEFARBqByO6CmY\nC9X+Umv998Aw8P6SW1UmhnI8BTOn4JOSVEEQKocjjnRa6wxw0QmwpewMJ1L24/ycglQfCYJQCRSb\nU9iolFoD/AqwmwJpre8tiVVlYiSRsR/XWTkFqT4SBKGCKFYUAkAPcKnjmAamlSgMOXIKNUErfCSi\nIAhC5VDsiuZpm0dwMhxPcemSZs5qreWcuXXAaNjIL11SBUGoAIpd0fxjDM8gB631B467RWVkOJGm\nOeLnk5ctso9ZHoLlMQiCIExnip3+/h74g/nvYaAaoxJpQpRSq5VS25RSO5VSN45zzruUUluUUpuV\nUr8o1vBSMBxPE/bn6qS9eM0joiAIwvSn2PDRPc7nSqk7gD9P9DNmKev3gMuBdmCdUmqN1nqL45xF\nwE3Aa7XWfUqp5knaf9zIZDUjyQzhQO5XIolmQRAqiaMNlC8CjjSAnwfs1Frv1longTuBa/LO+TDw\nPXMnN7TWh4/SnmNmJGkkmfM9haaIP+d/QRCE6UyxOYUhcnMKnRh7LEzEbGC/43k7cH7eOaea7/8U\n4AZu0VqXZfMeq8VFJM9TOKU5zJ8+fQkLm6rKYZYgCMIJpdjwUaSEn78IWAXMAZ5QSp3u3A8aQCl1\nPXA9QFtbW0kMsVpchP3eMa+d0jzt2jwJgiAUpKjwkVLqbUqpGsfzWqXUW4/wYx1Aq+P5HPOYk3Zg\njdY6pbXeA2zHEIkctNa3aa1Xaq1XNjU1FWPypLFaXOTnFARBECqJYnMKX9ZaD1hPzJn8l4/wM+uA\nRUqp+UopH3AtsCbvnN9geAkopRoxwkm7i7TpuDLqKYgoCIJQuRQrCoXOm3D01FqngRuAB4BXgLu0\n1puVUrcqpd5invYA0KOU2gI8CnxWa91TpE3HlfFyCoIgCJVEsSPgeqXUtzFKTAE+Dmw40g9pre8D\n7ss7drPjsQY+bf4rK1YzPPEUBEGoZIr1FD4BJIFfYpSWxjGEYdpg5RSqRBQEQahgiq0+GgEKrkie\nLkhOQRAEofjqo4eUUrWO53VKqQdKZ9aJZzieJuRz43apcpsiCIJQNooNHzU61w6YK5DL1pKiFAwn\nxvY9EgRBqDSKFYWsUspeNaaUmkeBrqlTmaFEWtYoCIJQ8RQ7Cn4B+LNS6nFAARdjrjCeLgzH00TE\nUxAEocIpNtH8R6XUSgwh2Iix6CxWSsNONMPiKQiCIBTdEO9DwKcwWlVsAi4A1pK7PeeUZjiepjEc\nKrcZgiAIZaXYnMKngHOBvVrr1wMrgP6Jf2RqYSSaxzbDEwRBqCSKFYW41joOoJTya623AotLZ9aJ\nZyiekhYXgiBUPMWOgu3mOoXfAA8ppfqAvaUz68SitZaSVEEQBIpPNL/NfHiLUupRoAYoy2Y4pSCe\nypLV0jZbEARh0qOg1vrxUhhSToakGZ4gCAJw9Hs0TyusttkiCoIgVDoiCkgzPEEQBAsRBRyeguQU\nBEGocEQUgEEJHwmCIAAiCgAcGowD0FztL7MlgiAI5UVEAdjXGyXgddEUFlEQBKGyEVEA9vdGaa0L\noZRssCMIQmVT0UH0bFbTPZJgX2+UtnpphicIglDRonDP8+189u4XAbhgQUOZrREEQSg/FR0+2tsT\ntR+3iqcgCIJQWlFQSq1WSm1TSu1USt1Y4PXrlFJdSqlN5r8PldKefJxdUVvrgifyowVBEE5KShY+\nUkq5ge8BlwPtwDql1Bqt9Za8U3+ptb6hVHZMRDSZsR/PqRNPQRAEoZSewnnATq31bq11ErgTuKaE\nnzdpYqkMbpfix+8/l2WzqsttjiAIQtkppSjMBvY7nrebx/J5u1LqRaXU3Uqp1kJvpJS6Xim1Xim1\nvqur67gZOJJIUxP08vrFzcftPQVBEKYy5U40/w6Yp7U+A3gI+Gmhk7TWt2mtV2qtVzY1NR23D48l\nMwS97uP2foIgCFOdUopCB+Cc+c8xj9lorXu01gnz6Q+Bc0pozxiiyQwhn4iCIAiCRSlFYR2wSCk1\nXynlA64F1jhPUErNdDx9C/BKCe0ZQzSVISRN8ARBEGxKNiJqrdNKqRuABwA3cLvWerNS6lZgvdZ6\nDfBJpdRbgDTQC1xXKnsKEUumCUn4SBAEwaak02St9X3AfXnHbnY8vgm4qZQ2TEQ0maGl2luujxcE\nQTjpKHeiuaxEkxmCklMQBEGwqXBRSEuiWRAEwUGFi0KGkE8SzYIgCBYVLQoxKUkVBEHIoWJFIZnO\nks5qEQVBEAQHFSsKMbMZXlDCR4IgCDYVKwojyTSAeAqCIAgOKlYUrLbZIgqCIAijVKwoxGxRkPCR\nIAiCRcWKQlTCR4IgCGOoXFFIWYlmEQVBEASLihWFmOQUBEEQxlCxojCcMMNHXskpCIIgWFSsKOw6\nPIzP7WJGjb/cpgiCIJw0VKwobNzXz7JZ1fg9Ej4SBEGwqEhRSGWyvNjRz4q22nKbIgiCcFJRkaKw\nrXOIeCrLira6cpsiCIJwUlGRovDcnl4AVrSKpyAIguCkIkXhj5s7WTwjQmt9qNymCIIgnFRUnCh0\nDSVY92ovq5e3lNsUQRCEk46KE4VHtx5Ga7jydBEFQRCEfCpOFHZ1DePzuDi1OVJuUwRBEE46SioK\nSqnVSqltSqmdSqkbJzjv7UoprZRaWUp7APb3RZlTF8TlUqX+KEEQhClHyURBKeUGvgdcCSwD3q2U\nWlbgvAjwKeDZUtniZF9vlNY6STALgiAUopSewnnATq31bq11ErgTuKbAeV8F/gWIl9AWm/29Mdqk\n6kgQBKEgpRSF2cB+x/N285iNUupsoFVr/YeJ3kgpdb1Sar1San1XV9dRGzQQSzEQS9FaHzzq9xAE\nQZjOlC3RrJRyAd8GPnOkc7XWt2mtV2qtVzY1NR31Z+7vjQKIpyAIgjAOpRSFDqDV8XyOecwiAiwH\nHlNKvQpcAKwpZbLZEoU5klMQBEEoSClFYR2wSCk1XynlA64F1lgvaq0HtNaNWut5Wut5wDPAW7TW\n60tl0IEBI20xu1bCR4IgCIUomShordPADcADwCvAXVrrzUqpW5VSbynV505EzNyXORyQjXUEQRAK\nUdLRUWt9H3Bf3rGbxzl3VSltAYilMnhcCq+74tbsCYIgFEVFjY6xZJagVzbVEQRBGI/KEoVUhoBP\nREEQBGE8KkoU4qkMAW9FXbIgCMKkqKgRMp7KSPhIEARhAipKFGIiCoIgCBNSWaKQzBAQURAEQRiX\nihKFeCpDUBLNgiAI41JRohBLZQh4RBQEQRDGo+JEQTwFQRCE8akoUYinspJTEARBmIDKEoWkVB8J\ngiBMREWJghE+qqhLFgRBmBQVM0KmMlnSWS2egiAIwgRUjCjEUhkAySkIgiBMQMWIQjwpoiAIgnAk\nKkYULE9BwkeCIAjjUzGiEE9lAWSdgiAIwgRUjCiIpyAIgnBkKkcUJKcgCIJwRCpGFOJ29VHFXLIg\nCMKkqZgR0g4fSU5BEARhXCpHFJKSUxAEQTgSJRUFpdRqpdQ2pdROpdSNBV7/G6XUS0qpTUqpPyul\nlpXKFkk0C4IgHJmSiYJSyg18D7gSWAa8u8Cg/wut9ela67OAbwDfLpU9dk5BwkeCIAjjUkpP4Txg\np9Z6t9Y6CdwJXOM8QWs96HhaBehSGdNWH+LK5S3iKQiCIEyAp4TvPRvY73jeDpyff5JS6uPApwEf\ncGmhN1JKXQ9cD9DW1nZUxlxxWgtXnNZyVD8rCIJQKZQ90ay1/p7WeiHwOeCL45xzm9Z6pdZ6ZVNT\n04k1UBAEoYIopSh0AK2O53PMY+NxJ/DWEtojCIIgHIFSisI6YJFSar5SygdcC6xxnqCUWuR4+iZg\nRwntEQRBEI5AyXIKWuu0UuoG4AHADdyutd6slLoVWK+1XgPcoJR6A5AC+oD3lcoeQRAE4ciUMtGM\n1vo+4L68Yzc7Hn+qlJ8vCIIgTI6yJ5oFQRCEkwcRBUEQBMFGREEQBEGwUVqXbBFxSVBKdQF7j/LH\nG4Hu42hOOZFrOTmRazk5kWuBuVrrIy70mnKicCwopdZrrVeW247jgVzLyYlcy8mJXEvxSPhIEARB\nsBFREARBEGwqTRRuK7cBxxG5lpMTuZaTE7mWIqmonIIgCIIwMZXmKQiCIAgTUDGicKStQU92lFKv\nOojpDqoAAAT8SURBVLYuXW8eq1dKPaSU2mH+X1duOwuhlLpdKXVYKfWy41hB25XBf5j36UWl1Nnl\ns3ws41zLLUqpDvPebFJKXeV47SbzWrYppd5YHqvHopRqVUo9qpTaopTarJT6lHl8yt2XCa5lKt6X\ngFLqOaXUC+a1fMU8Pl8p9axp8y/NJqMopfzm853m6/OO2Qit9bT/h9GQbxewAGMznxeAZeW2a5LX\n8CrQmHfsG8CN5uMbgX8pt53j2P464Gzg5SPZDlwF3A8o4ALg2XLbX8S13AL8fYFzl5m/a35gvvk7\n6C73NZi2zQTONh9HgO2mvVPuvkxwLVPxviggbD72As+a3/ddwLXm8f8GPmo+/hjw3+bja4FfHqsN\nleIpHHFr0CnKNcBPzcc/5STdj0Jr/QTQm3d4PNuvAX6mDZ4BapVSM0+MpUdmnGsZj2uAO7XWCa31\nHmAnxu9i2dFaH9RaP28+HgJewdgtccrdlwmuZTxO5vuitdbD5lOv+U9j7Ep5t3k8/75Y9+tu4DKl\nlDoWGypFFAptDTrRL83JiAYeVEptMLcnBZihtT5oPu4EZpTHtKNiPNun6r26wQyr3O4I402JazFD\nDiswZqVT+r7kXQtMwfuilHIrpTYBh4GHMDyZfq112jzFaa99LebrA0DDsXx+pYjCdOAirfXZwJXA\nx5VSr3O+qA3/cUqWkk1l202+DywEzgIOAt8qrznFo5QKA/cAf6u1HnS+NtXuS4FrmZL3RWud0Vqf\nhbFb5XnAkhP5+ZUiCpPdGvSkQ2vdYf5/GPg1xi/LIcuFN/8/XD4LJ814tk+5e6W1PmT+IWeBHzAa\nijipr0Up5cUYRP9Pa32veXhK3pdC1zJV74uF1rofeBS4ECNcZ+1/47TXvhbz9Rqg51g+t1JE4Yhb\ng57MKKWqlFIR6zFwBfAyxjVYu9W9D/hteSw8KsazfQ3wXrPa5QJgwBHOOCnJi62/DePegHEt15oV\nIvOBRcBzJ9q+Qphx5x8Br2itv+14acrdl/GuZYrelyalVK35OAhcjpEjeRR4h3la/n2x7tc7gEdM\nD+/oKXe2/UT9w6ie2I4Rn/tCue2ZpO0LMKolXgA2W/ZjxA4fxtjb+k9AfbltHcf+OzDc9xRGPPSD\n49mOUX3xPfM+vQSsLLf9RVzLz01bXzT/SGc6zv+CeS3bgCvLbb/DroswQkMvApvMf1dNxfsywbVM\nxftyBrDRtPll4Gbz+AIM4doJ/Arwm8cD5vOd5usLjtUGWdEsCIIg2FRK+EgQBEEoAhEFQRAEwUZE\nQRAEQbARURAEQRBsRBQEQRAEGxEFQZgESqmnzf/nKaXeU257BOF4I6IgCJNAa/0a8+E8YFKi4FiR\nKggnLSIKgjAJlFJWB8uvAxebffr/zmxi9q9KqXVmA7aPmOevUko9qZRaA2wpm+GCUCQycxGEo+NG\njF79VwOYnWsHtNbnKqX8wFNKqQfNc88GlmujTbMgnNSIKAjC8eEK4AyllNWfpgajp04SeE4EQZgq\niCgIwvFBAZ/QWj+Qc1CpVcBIWSwShKNAcgqCcHQMYWz9aPEA8FGzhTNKqVPNjraCMKUQT0EQjo4X\ngYxS6gXgJ8C/Y1QkPW+2cu7iJN0eVRAmQrqkCoIgCDYSPhIEQRBsRBQEQRAEGxEFQRAEwUZEQRAE\nQbARURAEQRBsRBQEQRAEGxEFQRAEwUZEQRAEQbD5/0+SlQBkCc5SAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7effdaa84518>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(test_accs)\n",
    "plt.xlabel('iter')\n",
    "plt.ylabel('accuracy')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Is it the perfect way to build a tf-solution? No!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* We implicitly base the whole process on the way Jup Notebook stores local (for specific cells) variables\n",
    "* E.g., it is only possible for us to train the model in item 4\n",
    "* Because Jup Notebook stores in memory refs on placeholder defined in item 1\n",
    "* Important idea: \n",
    "* When we create an op (as in item 1) we actually create $\\textbf{two}$ objects:\n",
    "\n",
    "1) python-ref on an op\n",
    "\n",
    "2) tf-op stored in graph\n",
    "\n",
    "* While the first are temporary, the second live forever!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Building the tf-model in a better way: structuring the whole thing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* OK. Where are the tf-ops stored?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# fetch the link on default graph\n",
    "grph = tf.get_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# get all ops created in the graph\n",
    "ops = grph.get_operations()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'Placeholder' type=Placeholder>,\n",
       " <tf.Operation 'zeros' type=Const>,\n",
       " <tf.Operation 'Variable' type=VariableV2>,\n",
       " <tf.Operation 'Variable/Assign' type=Assign>,\n",
       " <tf.Operation 'Variable/read' type=Identity>,\n",
       " <tf.Operation 'zeros_1' type=Const>,\n",
       " <tf.Operation 'Variable_1' type=VariableV2>,\n",
       " <tf.Operation 'Variable_1/Assign' type=Assign>,\n",
       " <tf.Operation 'Variable_1/read' type=Identity>,\n",
       " <tf.Operation 'MatMul' type=MatMul>,\n",
       " <tf.Operation 'add' type=Add>,\n",
       " <tf.Operation 'Placeholder_1' type=Placeholder>,\n",
       " <tf.Operation 'ArgMax/dimension' type=Const>,\n",
       " <tf.Operation 'ArgMax' type=ArgMax>,\n",
       " <tf.Operation 'Cast' type=Cast>,\n",
       " <tf.Operation 'ArgMax_1/dimension' type=Const>,\n",
       " <tf.Operation 'ArgMax_1' type=ArgMax>,\n",
       " <tf.Operation 'Cast_1' type=Cast>,\n",
       " <tf.Operation 'Equal' type=Equal>,\n",
       " <tf.Operation 'Cast_2' type=Cast>,\n",
       " <tf.Operation 'Const' type=Const>,\n",
       " <tf.Operation 'Mean' type=Mean>,\n",
       " <tf.Operation 'Rank' type=Const>,\n",
       " <tf.Operation 'Shape' type=Shape>,\n",
       " <tf.Operation 'Rank_1' type=Const>,\n",
       " <tf.Operation 'Shape_1' type=Shape>,\n",
       " <tf.Operation 'Sub/y' type=Const>,\n",
       " <tf.Operation 'Sub' type=Sub>,\n",
       " <tf.Operation 'Slice/begin' type=Pack>,\n",
       " <tf.Operation 'Slice/size' type=Const>,\n",
       " <tf.Operation 'Slice' type=Slice>,\n",
       " <tf.Operation 'concat/values_0' type=Const>,\n",
       " <tf.Operation 'concat/axis' type=Const>,\n",
       " <tf.Operation 'concat' type=ConcatV2>,\n",
       " <tf.Operation 'Reshape' type=Reshape>,\n",
       " <tf.Operation 'Rank_2' type=Const>,\n",
       " <tf.Operation 'Shape_2' type=Shape>,\n",
       " <tf.Operation 'Sub_1/y' type=Const>,\n",
       " <tf.Operation 'Sub_1' type=Sub>,\n",
       " <tf.Operation 'Slice_1/begin' type=Pack>,\n",
       " <tf.Operation 'Slice_1/size' type=Const>,\n",
       " <tf.Operation 'Slice_1' type=Slice>,\n",
       " <tf.Operation 'concat_1/values_0' type=Const>,\n",
       " <tf.Operation 'concat_1/axis' type=Const>,\n",
       " <tf.Operation 'concat_1' type=ConcatV2>,\n",
       " <tf.Operation 'Reshape_1' type=Reshape>,\n",
       " <tf.Operation 'SoftmaxCrossEntropyWithLogits' type=SoftmaxCrossEntropyWithLogits>,\n",
       " <tf.Operation 'Sub_2/y' type=Const>,\n",
       " <tf.Operation 'Sub_2' type=Sub>,\n",
       " <tf.Operation 'Slice_2/begin' type=Const>,\n",
       " <tf.Operation 'Slice_2/size' type=Pack>,\n",
       " <tf.Operation 'Slice_2' type=Slice>,\n",
       " <tf.Operation 'Reshape_2' type=Reshape>,\n",
       " <tf.Operation 'gradients/Shape' type=Shape>,\n",
       " <tf.Operation 'gradients/Const' type=Const>,\n",
       " <tf.Operation 'gradients/Fill' type=Fill>,\n",
       " <tf.Operation 'gradients/Reshape_2_grad/Shape' type=Shape>,\n",
       " <tf.Operation 'gradients/Reshape_2_grad/Reshape' type=Reshape>,\n",
       " <tf.Operation 'gradients/zeros_like' type=ZerosLike>,\n",
       " <tf.Operation 'gradients/SoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim' type=Const>,\n",
       " <tf.Operation 'gradients/SoftmaxCrossEntropyWithLogits_grad/ExpandDims' type=ExpandDims>,\n",
       " <tf.Operation 'gradients/SoftmaxCrossEntropyWithLogits_grad/mul' type=Mul>,\n",
       " <tf.Operation 'gradients/Reshape_grad/Shape' type=Shape>,\n",
       " <tf.Operation 'gradients/Reshape_grad/Reshape' type=Reshape>,\n",
       " <tf.Operation 'gradients/add_grad/Shape' type=Shape>,\n",
       " <tf.Operation 'gradients/add_grad/Shape_1' type=Const>,\n",
       " <tf.Operation 'gradients/add_grad/BroadcastGradientArgs' type=BroadcastGradientArgs>,\n",
       " <tf.Operation 'gradients/add_grad/Sum' type=Sum>,\n",
       " <tf.Operation 'gradients/add_grad/Reshape' type=Reshape>,\n",
       " <tf.Operation 'gradients/add_grad/Sum_1' type=Sum>,\n",
       " <tf.Operation 'gradients/add_grad/Reshape_1' type=Reshape>,\n",
       " <tf.Operation 'gradients/add_grad/tuple/group_deps' type=NoOp>,\n",
       " <tf.Operation 'gradients/add_grad/tuple/control_dependency' type=Identity>,\n",
       " <tf.Operation 'gradients/add_grad/tuple/control_dependency_1' type=Identity>,\n",
       " <tf.Operation 'gradients/MatMul_grad/MatMul' type=MatMul>,\n",
       " <tf.Operation 'gradients/MatMul_grad/MatMul_1' type=MatMul>,\n",
       " <tf.Operation 'gradients/MatMul_grad/tuple/group_deps' type=NoOp>,\n",
       " <tf.Operation 'gradients/MatMul_grad/tuple/control_dependency' type=Identity>,\n",
       " <tf.Operation 'gradients/MatMul_grad/tuple/control_dependency_1' type=Identity>,\n",
       " <tf.Operation 'beta1_power/initial_value' type=Const>,\n",
       " <tf.Operation 'beta1_power' type=VariableV2>,\n",
       " <tf.Operation 'beta1_power/Assign' type=Assign>,\n",
       " <tf.Operation 'beta1_power/read' type=Identity>,\n",
       " <tf.Operation 'beta2_power/initial_value' type=Const>,\n",
       " <tf.Operation 'beta2_power' type=VariableV2>,\n",
       " <tf.Operation 'beta2_power/Assign' type=Assign>,\n",
       " <tf.Operation 'beta2_power/read' type=Identity>,\n",
       " <tf.Operation 'Variable/Adam/Initializer/zeros' type=Const>,\n",
       " <tf.Operation 'Variable/Adam' type=VariableV2>,\n",
       " <tf.Operation 'Variable/Adam/Assign' type=Assign>,\n",
       " <tf.Operation 'Variable/Adam/read' type=Identity>,\n",
       " <tf.Operation 'Variable/Adam_1/Initializer/zeros' type=Const>,\n",
       " <tf.Operation 'Variable/Adam_1' type=VariableV2>,\n",
       " <tf.Operation 'Variable/Adam_1/Assign' type=Assign>,\n",
       " <tf.Operation 'Variable/Adam_1/read' type=Identity>,\n",
       " <tf.Operation 'Variable_1/Adam/Initializer/zeros' type=Const>,\n",
       " <tf.Operation 'Variable_1/Adam' type=VariableV2>,\n",
       " <tf.Operation 'Variable_1/Adam/Assign' type=Assign>,\n",
       " <tf.Operation 'Variable_1/Adam/read' type=Identity>,\n",
       " <tf.Operation 'Variable_1/Adam_1/Initializer/zeros' type=Const>,\n",
       " <tf.Operation 'Variable_1/Adam_1' type=VariableV2>,\n",
       " <tf.Operation 'Variable_1/Adam_1/Assign' type=Assign>,\n",
       " <tf.Operation 'Variable_1/Adam_1/read' type=Identity>,\n",
       " <tf.Operation 'Adam/learning_rate' type=Const>,\n",
       " <tf.Operation 'Adam/beta1' type=Const>,\n",
       " <tf.Operation 'Adam/beta2' type=Const>,\n",
       " <tf.Operation 'Adam/epsilon' type=Const>,\n",
       " <tf.Operation 'Adam/update_Variable/ApplyAdam' type=ApplyAdam>,\n",
       " <tf.Operation 'Adam/update_Variable_1/ApplyAdam' type=ApplyAdam>,\n",
       " <tf.Operation 'Adam/mul' type=Mul>,\n",
       " <tf.Operation 'Adam/Assign' type=Assign>,\n",
       " <tf.Operation 'Adam/mul_1' type=Mul>,\n",
       " <tf.Operation 'Adam/Assign_1' type=Assign>,\n",
       " <tf.Operation 'Adam' type=NoOp>,\n",
       " <tf.Operation 'init' type=NoOp>]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# (what a mess!)\n",
    "ops"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* That is, when we create ops\n",
    "* ... we place them in the default graph.\n",
    "* Obviously, not the best way to do things\n",
    "* If we want to build, say, several models in one nb\n",
    "* graph contexts help!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "grapphy = tf.Graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with grapphy.as_default():\n",
    "    xx = tf.placeholder(tf.float32, [None, 784])\n",
    "    summy = tf.reduce_mean(xx, axis=1)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.17054823  0.10235595  0.08560925  0.15968388  0.12482995  0.18528412\n",
      "  0.17763607  0.12184375  0.229967    0.14185175]\n"
     ]
    }
   ],
   "source": [
    "with grapphy.as_default():\n",
    "    sess = tf.Session()\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(10)\n",
    "    print(sess.run(summy, feed_dict={xx: batch_xs}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'Placeholder' type=Placeholder>,\n",
       " <tf.Operation 'Mean/reduction_indices' type=Const>,\n",
       " <tf.Operation 'Mean' type=Mean>]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grapphy.get_operations()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Still, hard to trace the variables\n",
    "* They're very miserable, let's give them a name!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "grapphy = tf.Graph()\n",
    "with grapphy.as_default():\n",
    "    x = tf.placeholder(tf.float32, [None, 784], name='input_pics')\n",
    "    W = tf.Variable(tf.random_normal([784, 10]), name='weights')\n",
    "    b = tf.Variable(tf.zeros([10]), name='bias')\n",
    "    logits = tf.matmul(x, W) + b\n",
    "    logits = tf.identity(logits, 'logits')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'input_pics' type=Placeholder>,\n",
       " <tf.Operation 'random_normal/shape' type=Const>,\n",
       " <tf.Operation 'random_normal/mean' type=Const>,\n",
       " <tf.Operation 'random_normal/stddev' type=Const>,\n",
       " <tf.Operation 'random_normal/RandomStandardNormal' type=RandomStandardNormal>,\n",
       " <tf.Operation 'random_normal/mul' type=Mul>,\n",
       " <tf.Operation 'random_normal' type=Add>,\n",
       " <tf.Operation 'weights' type=VariableV2>,\n",
       " <tf.Operation 'weights/Assign' type=Assign>,\n",
       " <tf.Operation 'weights/read' type=Identity>,\n",
       " <tf.Operation 'zeros' type=Const>,\n",
       " <tf.Operation 'bias' type=VariableV2>,\n",
       " <tf.Operation 'bias/Assign' type=Assign>,\n",
       " <tf.Operation 'bias/read' type=Identity>,\n",
       " <tf.Operation 'MatMul' type=MatMul>,\n",
       " <tf.Operation 'add' type=Add>,\n",
       " <tf.Operation 'logits' type=Identity>]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grapphy.get_operations()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ -2.82080245  -4.68091917 -17.47230339  -1.72977304  -3.8131094\n",
      "    7.76443148   1.36736715   8.98284054 -12.5115633   -7.68815804]]\n"
     ]
    }
   ],
   "source": [
    "with grapphy.as_default():\n",
    "    sess = tf.Session()\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    batch_xs, _ = mnist.train.next_batch(1)\n",
    "    llogits = grapphy.get_tensor_by_name('logits:0')\n",
    "    xx = grapphy.get_tensor_by_name('input_pics:0')\n",
    "    print(sess.run(llogits, feed_dict={xx: batch_xs}))\n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* what if we need to create several layers (e.g. fully-connected)? \n",
    "* we don't want to name vars 'weights_0', 'weights_1'...\n",
    "* variable scopes can be of help!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def fc_layer(scope, input_layer, n_outs):\n",
    "    \"\"\" Build fully-connected layer with n_outs outputs\n",
    "    \n",
    "    Args:\n",
    "        input_layer: input layer\n",
    "        n_outs: dim of output tensor\n",
    "    Return:\n",
    "        output layer (tf-tensor)\n",
    "    \"\"\"\n",
    "    with tf.variable_scope(scope):\n",
    "        W = tf.Variable(tf.random_normal([784, 10]), name='weights')\n",
    "        b = tf.Variable(tf.zeros([10]), name='bias')\n",
    "        output = tf.matmul(x, W) + b\n",
    "        return output\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Structuring the whole thing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "NUM_ITER = 1000\n",
    "BATCH_SIZE = 500\n",
    "grapphy = tf.Graph()\n",
    "test_accs = []\n",
    "with grapphy.as_default():\n",
    "    # ph for feature-vector\n",
    "    x = tf.placeholder(tf.float32, [None, 784], name='pics')\n",
    "    # placeholder for correct labels\n",
    "    y_ = tf.placeholder(tf.float32, [None, 10], name='labels')\n",
    "    \n",
    "    # build the net\n",
    "    net = fc_layer('fc1', x, 300)\n",
    "    net = fc_layer('fc2', net, 10)\n",
    "    logits = tf.identity(net, 'logits')\n",
    "    \n",
    "    # loss\n",
    "    loss = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y_, name='loss')\n",
    "    train_step = tf.train.AdamOptimizer().minimize(loss)\n",
    "    \n",
    "    # stats\n",
    "    labels_hat = tf.cast(tf.argmax(logits, axis=1), tf.float32, name='labels_hat')\n",
    "    labels = tf.cast(tf.argmax(y_, axis=1), tf.float32, name='labels')\n",
    "    accuracy = tf.reduce_mean(tf.cast(tf.equal(labels_hat, labels), tf.float32), name='acc')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'pics' type=Placeholder>,\n",
       " <tf.Operation 'labels' type=Placeholder>,\n",
       " <tf.Operation 'fc1/random_normal/shape' type=Const>,\n",
       " <tf.Operation 'fc1/random_normal/mean' type=Const>,\n",
       " <tf.Operation 'fc1/random_normal/stddev' type=Const>,\n",
       " <tf.Operation 'fc1/random_normal/RandomStandardNormal' type=RandomStandardNormal>,\n",
       " <tf.Operation 'fc1/random_normal/mul' type=Mul>,\n",
       " <tf.Operation 'fc1/random_normal' type=Add>,\n",
       " <tf.Operation 'fc1/weights' type=VariableV2>,\n",
       " <tf.Operation 'fc1/weights/Assign' type=Assign>,\n",
       " <tf.Operation 'fc1/weights/read' type=Identity>,\n",
       " <tf.Operation 'fc1/zeros' type=Const>,\n",
       " <tf.Operation 'fc1/bias' type=VariableV2>,\n",
       " <tf.Operation 'fc1/bias/Assign' type=Assign>,\n",
       " <tf.Operation 'fc1/bias/read' type=Identity>,\n",
       " <tf.Operation 'fc1/MatMul' type=MatMul>,\n",
       " <tf.Operation 'fc1/add' type=Add>,\n",
       " <tf.Operation 'fc2/random_normal/shape' type=Const>,\n",
       " <tf.Operation 'fc2/random_normal/mean' type=Const>,\n",
       " <tf.Operation 'fc2/random_normal/stddev' type=Const>,\n",
       " <tf.Operation 'fc2/random_normal/RandomStandardNormal' type=RandomStandardNormal>,\n",
       " <tf.Operation 'fc2/random_normal/mul' type=Mul>,\n",
       " <tf.Operation 'fc2/random_normal' type=Add>,\n",
       " <tf.Operation 'fc2/weights' type=VariableV2>,\n",
       " <tf.Operation 'fc2/weights/Assign' type=Assign>,\n",
       " <tf.Operation 'fc2/weights/read' type=Identity>,\n",
       " <tf.Operation 'fc2/zeros' type=Const>,\n",
       " <tf.Operation 'fc2/bias' type=VariableV2>,\n",
       " <tf.Operation 'fc2/bias/Assign' type=Assign>,\n",
       " <tf.Operation 'fc2/bias/read' type=Identity>,\n",
       " <tf.Operation 'fc2/MatMul' type=MatMul>,\n",
       " <tf.Operation 'fc2/add' type=Add>,\n",
       " <tf.Operation 'logits' type=Identity>,\n",
       " <tf.Operation 'Rank' type=Const>,\n",
       " <tf.Operation 'Shape' type=Shape>,\n",
       " <tf.Operation 'Rank_1' type=Const>,\n",
       " <tf.Operation 'Shape_1' type=Shape>,\n",
       " <tf.Operation 'Sub/y' type=Const>,\n",
       " <tf.Operation 'Sub' type=Sub>,\n",
       " <tf.Operation 'Slice/begin' type=Pack>,\n",
       " <tf.Operation 'Slice/size' type=Const>,\n",
       " <tf.Operation 'Slice' type=Slice>,\n",
       " <tf.Operation 'concat/values_0' type=Const>,\n",
       " <tf.Operation 'concat/axis' type=Const>,\n",
       " <tf.Operation 'concat' type=ConcatV2>,\n",
       " <tf.Operation 'Reshape' type=Reshape>,\n",
       " <tf.Operation 'Rank_2' type=Const>,\n",
       " <tf.Operation 'Shape_2' type=Shape>,\n",
       " <tf.Operation 'Sub_1/y' type=Const>,\n",
       " <tf.Operation 'Sub_1' type=Sub>,\n",
       " <tf.Operation 'Slice_1/begin' type=Pack>,\n",
       " <tf.Operation 'Slice_1/size' type=Const>,\n",
       " <tf.Operation 'Slice_1' type=Slice>,\n",
       " <tf.Operation 'concat_1/values_0' type=Const>,\n",
       " <tf.Operation 'concat_1/axis' type=Const>,\n",
       " <tf.Operation 'concat_1' type=ConcatV2>,\n",
       " <tf.Operation 'Reshape_1' type=Reshape>,\n",
       " <tf.Operation 'loss' type=SoftmaxCrossEntropyWithLogits>,\n",
       " <tf.Operation 'Sub_2/y' type=Const>,\n",
       " <tf.Operation 'Sub_2' type=Sub>,\n",
       " <tf.Operation 'Slice_2/begin' type=Const>,\n",
       " <tf.Operation 'Slice_2/size' type=Pack>,\n",
       " <tf.Operation 'Slice_2' type=Slice>,\n",
       " <tf.Operation 'Reshape_2' type=Reshape>,\n",
       " <tf.Operation 'gradients/Shape' type=Shape>,\n",
       " <tf.Operation 'gradients/Const' type=Const>,\n",
       " <tf.Operation 'gradients/Fill' type=Fill>,\n",
       " <tf.Operation 'gradients/Reshape_2_grad/Shape' type=Shape>,\n",
       " <tf.Operation 'gradients/Reshape_2_grad/Reshape' type=Reshape>,\n",
       " <tf.Operation 'gradients/zeros_like' type=ZerosLike>,\n",
       " <tf.Operation 'gradients/loss_grad/ExpandDims/dim' type=Const>,\n",
       " <tf.Operation 'gradients/loss_grad/ExpandDims' type=ExpandDims>,\n",
       " <tf.Operation 'gradients/loss_grad/mul' type=Mul>,\n",
       " <tf.Operation 'gradients/Reshape_grad/Shape' type=Shape>,\n",
       " <tf.Operation 'gradients/Reshape_grad/Reshape' type=Reshape>,\n",
       " <tf.Operation 'gradients/fc2/add_grad/Shape' type=Shape>,\n",
       " <tf.Operation 'gradients/fc2/add_grad/Shape_1' type=Const>,\n",
       " <tf.Operation 'gradients/fc2/add_grad/BroadcastGradientArgs' type=BroadcastGradientArgs>,\n",
       " <tf.Operation 'gradients/fc2/add_grad/Sum' type=Sum>,\n",
       " <tf.Operation 'gradients/fc2/add_grad/Reshape' type=Reshape>,\n",
       " <tf.Operation 'gradients/fc2/add_grad/Sum_1' type=Sum>,\n",
       " <tf.Operation 'gradients/fc2/add_grad/Reshape_1' type=Reshape>,\n",
       " <tf.Operation 'gradients/fc2/add_grad/tuple/group_deps' type=NoOp>,\n",
       " <tf.Operation 'gradients/fc2/add_grad/tuple/control_dependency' type=Identity>,\n",
       " <tf.Operation 'gradients/fc2/add_grad/tuple/control_dependency_1' type=Identity>,\n",
       " <tf.Operation 'gradients/fc2/MatMul_grad/MatMul' type=MatMul>,\n",
       " <tf.Operation 'gradients/fc2/MatMul_grad/MatMul_1' type=MatMul>,\n",
       " <tf.Operation 'gradients/fc2/MatMul_grad/tuple/group_deps' type=NoOp>,\n",
       " <tf.Operation 'gradients/fc2/MatMul_grad/tuple/control_dependency' type=Identity>,\n",
       " <tf.Operation 'gradients/fc2/MatMul_grad/tuple/control_dependency_1' type=Identity>,\n",
       " <tf.Operation 'beta1_power/initial_value' type=Const>,\n",
       " <tf.Operation 'beta1_power' type=VariableV2>,\n",
       " <tf.Operation 'beta1_power/Assign' type=Assign>,\n",
       " <tf.Operation 'beta1_power/read' type=Identity>,\n",
       " <tf.Operation 'beta2_power/initial_value' type=Const>,\n",
       " <tf.Operation 'beta2_power' type=VariableV2>,\n",
       " <tf.Operation 'beta2_power/Assign' type=Assign>,\n",
       " <tf.Operation 'beta2_power/read' type=Identity>,\n",
       " <tf.Operation 'fc2/weights/Adam/Initializer/zeros' type=Const>,\n",
       " <tf.Operation 'fc2/weights/Adam' type=VariableV2>,\n",
       " <tf.Operation 'fc2/weights/Adam/Assign' type=Assign>,\n",
       " <tf.Operation 'fc2/weights/Adam/read' type=Identity>,\n",
       " <tf.Operation 'fc2/weights/Adam_1/Initializer/zeros' type=Const>,\n",
       " <tf.Operation 'fc2/weights/Adam_1' type=VariableV2>,\n",
       " <tf.Operation 'fc2/weights/Adam_1/Assign' type=Assign>,\n",
       " <tf.Operation 'fc2/weights/Adam_1/read' type=Identity>,\n",
       " <tf.Operation 'fc2/bias/Adam/Initializer/zeros' type=Const>,\n",
       " <tf.Operation 'fc2/bias/Adam' type=VariableV2>,\n",
       " <tf.Operation 'fc2/bias/Adam/Assign' type=Assign>,\n",
       " <tf.Operation 'fc2/bias/Adam/read' type=Identity>,\n",
       " <tf.Operation 'fc2/bias/Adam_1/Initializer/zeros' type=Const>,\n",
       " <tf.Operation 'fc2/bias/Adam_1' type=VariableV2>,\n",
       " <tf.Operation 'fc2/bias/Adam_1/Assign' type=Assign>,\n",
       " <tf.Operation 'fc2/bias/Adam_1/read' type=Identity>,\n",
       " <tf.Operation 'Adam/learning_rate' type=Const>,\n",
       " <tf.Operation 'Adam/beta1' type=Const>,\n",
       " <tf.Operation 'Adam/beta2' type=Const>,\n",
       " <tf.Operation 'Adam/epsilon' type=Const>,\n",
       " <tf.Operation 'Adam/update_fc2/weights/ApplyAdam' type=ApplyAdam>,\n",
       " <tf.Operation 'Adam/update_fc2/bias/ApplyAdam' type=ApplyAdam>,\n",
       " <tf.Operation 'Adam/mul' type=Mul>,\n",
       " <tf.Operation 'Adam/Assign' type=Assign>,\n",
       " <tf.Operation 'Adam/mul_1' type=Mul>,\n",
       " <tf.Operation 'Adam/Assign_1' type=Assign>,\n",
       " <tf.Operation 'Adam' type=NoOp>,\n",
       " <tf.Operation 'ArgMax/dimension' type=Const>,\n",
       " <tf.Operation 'ArgMax' type=ArgMax>,\n",
       " <tf.Operation 'labels_hat' type=Cast>,\n",
       " <tf.Operation 'ArgMax_1/dimension' type=Const>,\n",
       " <tf.Operation 'ArgMax_1' type=ArgMax>,\n",
       " <tf.Operation 'labels_1' type=Cast>,\n",
       " <tf.Operation 'Equal' type=Equal>,\n",
       " <tf.Operation 'Cast_1' type=Cast>,\n",
       " <tf.Operation 'Const' type=Const>,\n",
       " <tf.Operation 'acc' type=Mean>]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grapphy.get_operations()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with grapphy.as_default():\n",
    "    # train the net\n",
    "    sess = tf.Session()\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    for i in range(NUM_ITER):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(BATCH_SIZE)\n",
    "        sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "        test_xs, test_ys = mnist.test.next_batch(BATCH_SIZE)\n",
    "        test_accs.append(sess.run(accuracy, feed_dict={x: test_xs, y_: test_ys}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with grapphy.as_default():\n",
    "    saver = tf.train.Saver()\n",
    "    saver.save(sess, 'trained/model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['model.index', 'model.data-00000-of-00001', 'checkpoint', 'model.meta']"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "os.listdir('trained/')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Restore the model in different graph; running some ops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from trained/model\n",
      "0.7906\n",
      "(784, 10)\n"
     ]
    }
   ],
   "source": [
    "grapphy_2 = tf.Graph()\n",
    "with grapphy_2.as_default():\n",
    "    new_sess = tf.Session()\n",
    "    new_saver = tf.train.import_meta_graph('trained/model.meta')\n",
    "    new_saver.restore(new_sess, tf.train.latest_checkpoint('trained/'))\n",
    "\n",
    "    # ph for feature-vector\n",
    "    x = grapphy_2.get_tensor_by_name('pics:0')\n",
    "    # placeholder for correct labels\n",
    "    y_ = grapphy_2.get_tensor_by_name('labels:0')\n",
    "    wghts = grapphy_2.get_tensor_by_name('fc1/weights:0')\n",
    "    \n",
    "    # acc\n",
    "    accuracy = grapphy_2.get_tensor_by_name('acc:0')\n",
    "    \n",
    "    # run acc\n",
    "    test_xs, test_ys = mnist.test.next_batch(10000)\n",
    "    print(new_sess.run(accuracy, feed_dict={x: test_xs, y_: test_ys}))\n",
    "    print(new_sess.run(wghts).shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'pics' type=Placeholder>,\n",
       " <tf.Operation 'labels' type=Placeholder>,\n",
       " <tf.Operation 'fc1/random_normal/shape' type=Const>,\n",
       " <tf.Operation 'fc1/random_normal/mean' type=Const>,\n",
       " <tf.Operation 'fc1/random_normal/stddev' type=Const>,\n",
       " <tf.Operation 'fc1/random_normal/RandomStandardNormal' type=RandomStandardNormal>,\n",
       " <tf.Operation 'fc1/random_normal/mul' type=Mul>,\n",
       " <tf.Operation 'fc1/random_normal' type=Add>,\n",
       " <tf.Operation 'fc1/weights' type=VariableV2>,\n",
       " <tf.Operation 'fc1/weights/Assign' type=Assign>,\n",
       " <tf.Operation 'fc1/weights/read' type=Identity>,\n",
       " <tf.Operation 'fc1/zeros' type=Const>,\n",
       " <tf.Operation 'fc1/bias' type=VariableV2>,\n",
       " <tf.Operation 'fc1/bias/Assign' type=Assign>,\n",
       " <tf.Operation 'fc1/bias/read' type=Identity>,\n",
       " <tf.Operation 'fc1/MatMul' type=MatMul>,\n",
       " <tf.Operation 'fc1/add' type=Add>,\n",
       " <tf.Operation 'fc2/random_normal/shape' type=Const>,\n",
       " <tf.Operation 'fc2/random_normal/mean' type=Const>,\n",
       " <tf.Operation 'fc2/random_normal/stddev' type=Const>,\n",
       " <tf.Operation 'fc2/random_normal/RandomStandardNormal' type=RandomStandardNormal>,\n",
       " <tf.Operation 'fc2/random_normal/mul' type=Mul>,\n",
       " <tf.Operation 'fc2/random_normal' type=Add>,\n",
       " <tf.Operation 'fc2/weights' type=VariableV2>,\n",
       " <tf.Operation 'fc2/weights/Assign' type=Assign>,\n",
       " <tf.Operation 'fc2/weights/read' type=Identity>,\n",
       " <tf.Operation 'fc2/zeros' type=Const>,\n",
       " <tf.Operation 'fc2/bias' type=VariableV2>,\n",
       " <tf.Operation 'fc2/bias/Assign' type=Assign>,\n",
       " <tf.Operation 'fc2/bias/read' type=Identity>,\n",
       " <tf.Operation 'fc2/MatMul' type=MatMul>,\n",
       " <tf.Operation 'fc2/add' type=Add>,\n",
       " <tf.Operation 'logits' type=Identity>,\n",
       " <tf.Operation 'Rank' type=Const>,\n",
       " <tf.Operation 'Shape' type=Shape>,\n",
       " <tf.Operation 'Rank_1' type=Const>,\n",
       " <tf.Operation 'Shape_1' type=Shape>,\n",
       " <tf.Operation 'Sub/y' type=Const>,\n",
       " <tf.Operation 'Sub' type=Sub>,\n",
       " <tf.Operation 'Slice/begin' type=Pack>,\n",
       " <tf.Operation 'Slice/size' type=Const>,\n",
       " <tf.Operation 'Slice' type=Slice>,\n",
       " <tf.Operation 'concat/values_0' type=Const>,\n",
       " <tf.Operation 'concat/axis' type=Const>,\n",
       " <tf.Operation 'concat' type=ConcatV2>,\n",
       " <tf.Operation 'Reshape' type=Reshape>,\n",
       " <tf.Operation 'Rank_2' type=Const>,\n",
       " <tf.Operation 'Shape_2' type=Shape>,\n",
       " <tf.Operation 'Sub_1/y' type=Const>,\n",
       " <tf.Operation 'Sub_1' type=Sub>,\n",
       " <tf.Operation 'Slice_1/begin' type=Pack>,\n",
       " <tf.Operation 'Slice_1/size' type=Const>,\n",
       " <tf.Operation 'Slice_1' type=Slice>,\n",
       " <tf.Operation 'concat_1/values_0' type=Const>,\n",
       " <tf.Operation 'concat_1/axis' type=Const>,\n",
       " <tf.Operation 'concat_1' type=ConcatV2>,\n",
       " <tf.Operation 'Reshape_1' type=Reshape>,\n",
       " <tf.Operation 'loss' type=SoftmaxCrossEntropyWithLogits>,\n",
       " <tf.Operation 'Sub_2/y' type=Const>,\n",
       " <tf.Operation 'Sub_2' type=Sub>,\n",
       " <tf.Operation 'Slice_2/begin' type=Const>,\n",
       " <tf.Operation 'Slice_2/size' type=Pack>,\n",
       " <tf.Operation 'Slice_2' type=Slice>,\n",
       " <tf.Operation 'Reshape_2' type=Reshape>,\n",
       " <tf.Operation 'gradients/Shape' type=Shape>,\n",
       " <tf.Operation 'gradients/Const' type=Const>,\n",
       " <tf.Operation 'gradients/Fill' type=Fill>,\n",
       " <tf.Operation 'gradients/Reshape_2_grad/Shape' type=Shape>,\n",
       " <tf.Operation 'gradients/Reshape_2_grad/Reshape' type=Reshape>,\n",
       " <tf.Operation 'gradients/zeros_like' type=ZerosLike>,\n",
       " <tf.Operation 'gradients/loss_grad/ExpandDims/dim' type=Const>,\n",
       " <tf.Operation 'gradients/loss_grad/ExpandDims' type=ExpandDims>,\n",
       " <tf.Operation 'gradients/loss_grad/mul' type=Mul>,\n",
       " <tf.Operation 'gradients/Reshape_grad/Shape' type=Shape>,\n",
       " <tf.Operation 'gradients/Reshape_grad/Reshape' type=Reshape>,\n",
       " <tf.Operation 'gradients/fc2/add_grad/Shape' type=Shape>,\n",
       " <tf.Operation 'gradients/fc2/add_grad/Shape_1' type=Const>,\n",
       " <tf.Operation 'gradients/fc2/add_grad/BroadcastGradientArgs' type=BroadcastGradientArgs>,\n",
       " <tf.Operation 'gradients/fc2/add_grad/Sum' type=Sum>,\n",
       " <tf.Operation 'gradients/fc2/add_grad/Reshape' type=Reshape>,\n",
       " <tf.Operation 'gradients/fc2/add_grad/Sum_1' type=Sum>,\n",
       " <tf.Operation 'gradients/fc2/add_grad/Reshape_1' type=Reshape>,\n",
       " <tf.Operation 'gradients/fc2/add_grad/tuple/group_deps' type=NoOp>,\n",
       " <tf.Operation 'gradients/fc2/add_grad/tuple/control_dependency' type=Identity>,\n",
       " <tf.Operation 'gradients/fc2/add_grad/tuple/control_dependency_1' type=Identity>,\n",
       " <tf.Operation 'gradients/fc2/MatMul_grad/MatMul' type=MatMul>,\n",
       " <tf.Operation 'gradients/fc2/MatMul_grad/MatMul_1' type=MatMul>,\n",
       " <tf.Operation 'gradients/fc2/MatMul_grad/tuple/group_deps' type=NoOp>,\n",
       " <tf.Operation 'gradients/fc2/MatMul_grad/tuple/control_dependency' type=Identity>,\n",
       " <tf.Operation 'gradients/fc2/MatMul_grad/tuple/control_dependency_1' type=Identity>,\n",
       " <tf.Operation 'beta1_power/initial_value' type=Const>,\n",
       " <tf.Operation 'beta1_power' type=VariableV2>,\n",
       " <tf.Operation 'beta1_power/Assign' type=Assign>,\n",
       " <tf.Operation 'beta1_power/read' type=Identity>,\n",
       " <tf.Operation 'beta2_power/initial_value' type=Const>,\n",
       " <tf.Operation 'beta2_power' type=VariableV2>,\n",
       " <tf.Operation 'beta2_power/Assign' type=Assign>,\n",
       " <tf.Operation 'beta2_power/read' type=Identity>,\n",
       " <tf.Operation 'fc2/weights/Adam/Initializer/zeros' type=Const>,\n",
       " <tf.Operation 'fc2/weights/Adam' type=VariableV2>,\n",
       " <tf.Operation 'fc2/weights/Adam/Assign' type=Assign>,\n",
       " <tf.Operation 'fc2/weights/Adam/read' type=Identity>,\n",
       " <tf.Operation 'fc2/weights/Adam_1/Initializer/zeros' type=Const>,\n",
       " <tf.Operation 'fc2/weights/Adam_1' type=VariableV2>,\n",
       " <tf.Operation 'fc2/weights/Adam_1/Assign' type=Assign>,\n",
       " <tf.Operation 'fc2/weights/Adam_1/read' type=Identity>,\n",
       " <tf.Operation 'fc2/bias/Adam/Initializer/zeros' type=Const>,\n",
       " <tf.Operation 'fc2/bias/Adam' type=VariableV2>,\n",
       " <tf.Operation 'fc2/bias/Adam/Assign' type=Assign>,\n",
       " <tf.Operation 'fc2/bias/Adam/read' type=Identity>,\n",
       " <tf.Operation 'fc2/bias/Adam_1/Initializer/zeros' type=Const>,\n",
       " <tf.Operation 'fc2/bias/Adam_1' type=VariableV2>,\n",
       " <tf.Operation 'fc2/bias/Adam_1/Assign' type=Assign>,\n",
       " <tf.Operation 'fc2/bias/Adam_1/read' type=Identity>,\n",
       " <tf.Operation 'Adam/learning_rate' type=Const>,\n",
       " <tf.Operation 'Adam/beta1' type=Const>,\n",
       " <tf.Operation 'Adam/beta2' type=Const>,\n",
       " <tf.Operation 'Adam/epsilon' type=Const>,\n",
       " <tf.Operation 'Adam/update_fc2/weights/ApplyAdam' type=ApplyAdam>,\n",
       " <tf.Operation 'Adam/update_fc2/bias/ApplyAdam' type=ApplyAdam>,\n",
       " <tf.Operation 'Adam/mul' type=Mul>,\n",
       " <tf.Operation 'Adam/Assign' type=Assign>,\n",
       " <tf.Operation 'Adam/mul_1' type=Mul>,\n",
       " <tf.Operation 'Adam/Assign_1' type=Assign>,\n",
       " <tf.Operation 'Adam' type=NoOp>,\n",
       " <tf.Operation 'ArgMax/dimension' type=Const>,\n",
       " <tf.Operation 'ArgMax' type=ArgMax>,\n",
       " <tf.Operation 'labels_hat' type=Cast>,\n",
       " <tf.Operation 'ArgMax_1/dimension' type=Const>,\n",
       " <tf.Operation 'ArgMax_1' type=ArgMax>,\n",
       " <tf.Operation 'labels_1' type=Cast>,\n",
       " <tf.Operation 'Equal' type=Equal>,\n",
       " <tf.Operation 'Cast_1' type=Cast>,\n",
       " <tf.Operation 'Const' type=Const>,\n",
       " <tf.Operation 'acc' type=Mean>]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# not a mess anymore!\n",
    "grapphy.get_operations()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}