GANWithBetterParams.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 65
    },
    "colab_type": "code",
    "id": "_iaQa45ebnmi",
    "outputId": "cb1ce274-5607-416a-f5e8-84de3a2b01f2"
   },
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import collections\n",
    "import gzip\n",
    "import os\n",
    "\n",
    "import numpy\n",
    "from six.moves import urllib\n",
    "from six.moves import xrange  # pylint: disable=redefined-builtin\n",
    "\n",
    "from tensorflow.python.framework import dtypes\n",
    "from tensorflow.python.framework import random_seed\n",
    "from tensorflow.python.platform import gfile\n",
    "from tensorflow.python.util.deprecation import deprecated\n",
    "\n",
    "_Datasets = collections.namedtuple('_Datasets', ['train', 'validation', 'test'])\n",
    "\n",
    "# CVDF mirror of http://yann.lecun.com/exdb/mnist/\n",
    "DEFAULT_SOURCE_URL = 'https://storage.googleapis.com/cvdf-datasets/mnist/'\n",
    "\n",
    "\n",
    "def _read32(bytestream):\n",
    "  dt = numpy.dtype(numpy.uint32).newbyteorder('>')\n",
    "  return numpy.frombuffer(bytestream.read(4), dtype=dt)[0]\n",
    "\n",
    "\n",
    "@deprecated(None, 'Please use tf.data to implement this functionality.')\n",
    "def _extract_images(f):\n",
    "  \"\"\"Extract the images into a 4D uint8 numpy array [index, y, x, depth].\n",
    "  Args:\n",
    "    f: A file object that can be passed into a gzip reader.\n",
    "  Returns:\n",
    "    data: A 4D uint8 numpy array [index, y, x, depth].\n",
    "  Raises:\n",
    "    ValueError: If the bytestream does not start with 2051.\n",
    "  \"\"\"\n",
    "  print('Extracting', f.name)\n",
    "  with gzip.GzipFile(fileobj=f) as bytestream:\n",
    "    magic = _read32(bytestream)\n",
    "    if magic != 2051:\n",
    "      raise ValueError('Invalid magic number %d in MNIST image file: %s' %\n",
    "                       (magic, f.name))\n",
    "    num_images = _read32(bytestream)\n",
    "    rows = _read32(bytestream)\n",
    "    cols = _read32(bytestream)\n",
    "    buf = bytestream.read(rows * cols * num_images)\n",
    "    data = numpy.frombuffer(buf, dtype=numpy.uint8)\n",
    "    data = data.reshape(num_images, rows, cols, 1)\n",
    "    return data\n",
    "\n",
    "\n",
    "@deprecated(None, 'Please use tf.one_hot on tensors.')\n",
    "def _dense_to_one_hot(labels_dense, num_classes):\n",
    "  \"\"\"Convert class labels from scalars to one-hot vectors.\"\"\"\n",
    "  num_labels = labels_dense.shape[0]\n",
    "  index_offset = numpy.arange(num_labels) * num_classes\n",
    "  labels_one_hot = numpy.zeros((num_labels, num_classes))\n",
    "  labels_one_hot.flat[index_offset + labels_dense.ravel()] = 1\n",
    "  return labels_one_hot\n",
    "\n",
    "\n",
    "@deprecated(None, 'Please use tf.data to implement this functionality.')\n",
    "def _extract_labels(f, one_hot=False, num_classes=10):\n",
    "  \"\"\"Extract the labels into a 1D uint8 numpy array [index].\n",
    "  Args:\n",
    "    f: A file object that can be passed into a gzip reader.\n",
    "    one_hot: Does one hot encoding for the result.\n",
    "    num_classes: Number of classes for the one hot encoding.\n",
    "  Returns:\n",
    "    labels: a 1D uint8 numpy array.\n",
    "  Raises:\n",
    "    ValueError: If the bystream doesn't start with 2049.\n",
    "  \"\"\"\n",
    "  print('Extracting', f.name)\n",
    "  with gzip.GzipFile(fileobj=f) as bytestream:\n",
    "    magic = _read32(bytestream)\n",
    "    if magic != 2049:\n",
    "      raise ValueError('Invalid magic number %d in MNIST label file: %s' %\n",
    "                       (magic, f.name))\n",
    "    num_items = _read32(bytestream)\n",
    "    buf = bytestream.read(num_items)\n",
    "    labels = numpy.frombuffer(buf, dtype=numpy.uint8)\n",
    "    if one_hot:\n",
    "      return _dense_to_one_hot(labels, num_classes)\n",
    "    return labels\n",
    "\n",
    "\n",
    "class _DataSet(object):\n",
    "  \"\"\"Container class for a _DataSet (deprecated).\n",
    "  THIS CLASS IS DEPRECATED.\n",
    "  \"\"\"\n",
    "\n",
    "  @deprecated(None, 'Please use alternatives such as official/mnist/_DataSet.py'\n",
    "              ' from tensorflow/models.')\n",
    "  def __init__(self,\n",
    "               images,\n",
    "               labels,\n",
    "               fake_data=False,\n",
    "               one_hot=False,\n",
    "               dtype=dtypes.float32,\n",
    "               reshape=True,\n",
    "               seed=None):\n",
    "    \"\"\"Construct a _DataSet.\n",
    "    one_hot arg is used only if fake_data is true.  `dtype` can be either\n",
    "    `uint8` to leave the input as `[0, 255]`, or `float32` to rescale into\n",
    "    `[0, 1]`.  Seed arg provides for convenient deterministic testing.\n",
    "    Args:\n",
    "      images: The images\n",
    "      labels: The labels\n",
    "      fake_data: Ignore inages and labels, use fake data.\n",
    "      one_hot: Bool, return the labels as one hot vectors (if True) or ints (if\n",
    "        False).\n",
    "      dtype: Output image dtype. One of [uint8, float32]. `uint8` output has\n",
    "        range [0,255]. float32 output has range [0,1].\n",
    "      reshape: Bool. If True returned images are returned flattened to vectors.\n",
    "      seed: The random seed to use.\n",
    "    \"\"\"\n",
    "    seed1, seed2 = random_seed.get_seed(seed)\n",
    "    # If op level seed is not set, use whatever graph level seed is returned\n",
    "    numpy.random.seed(seed1 if seed is None else seed2)\n",
    "    dtype = dtypes.as_dtype(dtype).base_dtype\n",
    "    if dtype not in (dtypes.uint8, dtypes.float32):\n",
    "      raise TypeError('Invalid image dtype %r, expected uint8 or float32' %\n",
    "                      dtype)\n",
    "    if fake_data:\n",
    "      self._num_examples = 10000\n",
    "      self.one_hot = one_hot\n",
    "    else:\n",
    "      assert images.shape[0] == labels.shape[0], (\n",
    "          'images.shape: %s labels.shape: %s' % (images.shape, labels.shape))\n",
    "      self._num_examples = images.shape[0]\n",
    "\n",
    "      # Convert shape from [num examples, rows, columns, depth]\n",
    "      # to [num examples, rows*columns] (assuming depth == 1)\n",
    "      if reshape:\n",
    "        assert images.shape[3] == 1\n",
    "        images = images.reshape(images.shape[0],\n",
    "                                images.shape[1] * images.shape[2])\n",
    "      if dtype == dtypes.float32:\n",
    "        # Convert from [0, 255] -> [0.0, 1.0].\n",
    "        images = images.astype(numpy.float32)\n",
    "        images = numpy.multiply(images, 1.0 / 255.0)\n",
    "    self._images = images\n",
    "    self._labels = labels\n",
    "    self._epochs_completed = 0\n",
    "    self._index_in_epoch = 0\n",
    "\n",
    "  @property\n",
    "  def images(self):\n",
    "    return self._images\n",
    "\n",
    "  @property\n",
    "  def labels(self):\n",
    "    return self._labels\n",
    "\n",
    "  @property\n",
    "  def num_examples(self):\n",
    "    return self._num_examples\n",
    "\n",
    "  @property\n",
    "  def epochs_completed(self):\n",
    "    return self._epochs_completed\n",
    "\n",
    "  def next_batch(self, batch_size, fake_data=False, shuffle=True):\n",
    "    \"\"\"Return the next `batch_size` examples from this data set.\"\"\"\n",
    "    if fake_data:\n",
    "      fake_image = [1] * 784\n",
    "      if self.one_hot:\n",
    "        fake_label = [1] + [0] * 9\n",
    "      else:\n",
    "        fake_label = 0\n",
    "      return [fake_image for _ in xrange(batch_size)\n",
    "             ], [fake_label for _ in xrange(batch_size)]\n",
    "    start = self._index_in_epoch\n",
    "    # Shuffle for the first epoch\n",
    "    if self._epochs_completed == 0 and start == 0 and shuffle:\n",
    "      perm0 = numpy.arange(self._num_examples)\n",
    "      numpy.random.shuffle(perm0)\n",
    "      self._images = self.images[perm0]\n",
    "      self._labels = self.labels[perm0]\n",
    "    # Go to the next epoch\n",
    "    if start + batch_size > self._num_examples:\n",
    "      # Finished epoch\n",
    "      self._epochs_completed += 1\n",
    "      # Get the rest examples in this epoch\n",
    "      rest_num_examples = self._num_examples - start\n",
    "      images_rest_part = self._images[start:self._num_examples]\n",
    "      labels_rest_part = self._labels[start:self._num_examples]\n",
    "      # Shuffle the data\n",
    "      if shuffle:\n",
    "        perm = numpy.arange(self._num_examples)\n",
    "        numpy.random.shuffle(perm)\n",
    "        self._images = self.images[perm]\n",
    "        self._labels = self.labels[perm]\n",
    "      # Start next epoch\n",
    "      start = 0\n",
    "      self._index_in_epoch = batch_size - rest_num_examples\n",
    "      end = self._index_in_epoch\n",
    "      images_new_part = self._images[start:end]\n",
    "      labels_new_part = self._labels[start:end]\n",
    "      return numpy.concatenate((images_rest_part, images_new_part),\n",
    "                               axis=0), numpy.concatenate(\n",
    "                                   (labels_rest_part, labels_new_part), axis=0)\n",
    "    else:\n",
    "      self._index_in_epoch += batch_size\n",
    "      end = self._index_in_epoch\n",
    "      return self._images[start:end], self._labels[start:end]\n",
    "\n",
    "\n",
    "@deprecated(None, 'Please write your own downloading logic.')\n",
    "def _maybe_download(filename, work_directory, source_url):\n",
    "  \"\"\"Download the data from source url, unless it's already here.\n",
    "  Args:\n",
    "      filename: string, name of the file in the directory.\n",
    "      work_directory: string, path to working directory.\n",
    "      source_url: url to download from if file doesn't exist.\n",
    "  Returns:\n",
    "      Path to resulting file.\n",
    "  \"\"\"\n",
    "  if not gfile.Exists(work_directory):\n",
    "    gfile.MakeDirs(work_directory)\n",
    "  filepath = os.path.join(work_directory, filename)\n",
    "  if not gfile.Exists(filepath):\n",
    "    urllib.request.urlretrieve(source_url, filepath)\n",
    "    with gfile.GFile(filepath) as f:\n",
    "      size = f.size()\n",
    "    print('Successfully downloaded', filename, size, 'bytes.')\n",
    "  return filepath\n",
    "\n",
    "\n",
    "@deprecated(None, 'Please use alternatives such as:'\n",
    "            ' tensorflow_datasets.load(\\'mnist\\')')\n",
    "def read_data_sets(train_dir,\n",
    "                   fake_data=False,\n",
    "                   one_hot=False,\n",
    "                   dtype=dtypes.float32,\n",
    "                   reshape=True,\n",
    "                   validation_size=5000,\n",
    "                   seed=None,\n",
    "                   source_url=DEFAULT_SOURCE_URL):\n",
    "  if fake_data:\n",
    "\n",
    "    def fake():\n",
    "      return _DataSet([], [],\n",
    "                      fake_data=True,\n",
    "                      one_hot=one_hot,\n",
    "                      dtype=dtype,\n",
    "                      seed=seed)\n",
    "\n",
    "    train = fake()\n",
    "    validation = fake()\n",
    "    test = fake()\n",
    "    return _Datasets(train=train, validation=validation, test=test)\n",
    "\n",
    "  if not source_url:  # empty string check\n",
    "    source_url = DEFAULT_SOURCE_URL\n",
    "\n",
    "  train_images_file = 'train-images-idx3-ubyte.gz'\n",
    "  train_labels_file = 'train-labels-idx1-ubyte.gz'\n",
    "  test_images_file = 't10k-images-idx3-ubyte.gz'\n",
    "  test_labels_file = 't10k-labels-idx1-ubyte.gz'\n",
    "\n",
    "  local_file = _maybe_download(train_images_file, train_dir,\n",
    "                               source_url + train_images_file)\n",
    "  with gfile.Open(local_file, 'rb') as f:\n",
    "    train_images = _extract_images(f)\n",
    "\n",
    "  local_file = _maybe_download(train_labels_file, train_dir,\n",
    "                               source_url + train_labels_file)\n",
    "  with gfile.Open(local_file, 'rb') as f:\n",
    "    train_labels = _extract_labels(f, one_hot=one_hot)\n",
    "\n",
    "  local_file = _maybe_download(test_images_file, train_dir,\n",
    "                               source_url + test_images_file)\n",
    "  with gfile.Open(local_file, 'rb') as f:\n",
    "    test_images = _extract_images(f)\n",
    "\n",
    "  local_file = _maybe_download(test_labels_file, train_dir,\n",
    "                               source_url + test_labels_file)\n",
    "  with gfile.Open(local_file, 'rb') as f:\n",
    "    test_labels = _extract_labels(f, one_hot=one_hot)\n",
    "\n",
    "  if not 0 <= validation_size <= len(train_images):\n",
    "    raise ValueError(\n",
    "        'Validation size should be between 0 and {}. Received: {}.'.format(\n",
    "            len(train_images), validation_size))\n",
    "\n",
    "  validation_images = train_images[:validation_size]\n",
    "  validation_labels = train_labels[:validation_size]\n",
    "  train_images = train_images[validation_size:]\n",
    "  train_labels = train_labels[validation_size:]\n",
    "\n",
    "  options = dict(dtype=dtype, reshape=reshape, seed=seed)\n",
    "\n",
    "  train = _DataSet(train_images, train_labels, **options)\n",
    "  validation = _DataSet(validation_images, validation_labels, **options)\n",
    "  test = _DataSet(test_images, test_labels, **options)\n",
    "\n",
    "  return _Datasets(train=train, validation=validation, test=test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3BshiewAbfPU",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import numpy as np,sys,time\n",
    "from sklearn.utils import shuffle\n",
    "import matplotlib\n",
    "import tensorflow as tf\n",
    "import tensorflow_datasets\n",
    "import os\n",
    "import numpy as np,sys\n",
    "from sklearn.utils import shuffle\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.gridspec as gridspec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZffTWt8ZbfPo"
   },
   "outputs": [],
   "source": [
    "def ReLu(x):\n",
    "    mask = (x>0) * 1.0\n",
    "    return mask *x\n",
    "def d_ReLu(x):\n",
    "    mask = (x>0) * 1.0\n",
    "    return mask \n",
    "\n",
    "def Lrelu(x):\n",
    "#    alpha = 0.0001\n",
    "#    return np.where(x > 0, x, x * alpha) \n",
    "    y1 = ((x > 0) * x)                                                 \n",
    "    y2 = ((x <= 0) * x * 0.01)                                         \n",
    "    return y1 + y2\n",
    "def d_Lrelu(x,alpha = 0.01):\n",
    "  dx = np.ones_like(x)\n",
    "  dx[x < 0] = alpha\n",
    "  return dx\n",
    "def arctan(x):\n",
    "    return np.arctan(x)\n",
    "def d_arctan(x):\n",
    "    return 1 / (1 + x ** 2)\n",
    "\n",
    "def log(x):\n",
    "    return 1 / ( 1+ np.exp(-1*x))\n",
    "def d_log(x):\n",
    "    return log(x) * (1 - log(x))\n",
    "\n",
    "def tanh(x):\n",
    "    return np.tanh(x)\n",
    "def d_tanh(x):\n",
    "    return 1 - np.tanh(x) ** 2\n",
    "\n",
    "def plot(samples, title):\n",
    "    fig = plt.figure(figsize=(4, 4))\n",
    "    gs = gridspec.GridSpec(4, 4)\n",
    "    gs.update(wspace=0.05, hspace=0.05)\n",
    "\n",
    "    for i, sample in enumerate(samples):\n",
    "        ax = plt.subplot(gs[i])\n",
    "        plt.axis('off')\n",
    "        ax.set_xticklabels([])\n",
    "        ax.set_yticklabels([])\n",
    "        ax.set_aspect('equal')\n",
    "        plt.title(title)\n",
    "        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n",
    "\n",
    "    return fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aPAKJeiAbfP4"
   },
   "outputs": [],
   "source": [
    "random_numer = 1222 #int(input(\"Please Input a Random Number to Seed\"))\n",
    "np.random.seed(random_numer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 412
    },
    "colab_type": "code",
    "id": "Ulnl3-qYbfQH",
    "outputId": "77c1acc1-c68d-49bb-cdb1-867abf11a735",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------- Load Data ----------\n",
      "Extracting MNIST_data\\train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data\\train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data\\t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# 1. Load Data and declare hyper\n",
    "print('--------- Load Data ----------')\n",
    "mnist = read_data_sets('MNIST_data', one_hot=False)\n",
    "temp = mnist.test\n",
    "images, labels = temp.images, temp.labels\n",
    "images, labels = shuffle(np.asarray(images),np.asarray(labels))\n",
    "num_epoch = 50\n",
    "learing_rate = 0.000002\n",
    "learing_rate2 = 0.00002\n",
    "G_input = 100\n",
    "hidden_input,hidden_input2,hidden_input3 = 128,256,346\n",
    "hidden_input4,hidden_input5,hidden_input6 = 480,560,686\n",
    "hidden_input7,hidden_input8,hidden_input9 = 800,1020,1400\n",
    "hidden_input10 = 1800"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 36
    },
    "colab_type": "code",
    "id": "nh1txYT4bfQT",
    "outputId": "d6e9e4af-be67-47bb-b53f-da91a9335f89"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------- Declare Hyper Parameters ----------\n"
     ]
    }
   ],
   "source": [
    "print('--------- Declare Hyper Parameters ----------')\n",
    "D_W1 = np.random.normal(size=(784,hidden_input),scale=(1. / np.sqrt(784 / 2.)))   *0.002\n",
    "# D_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))       *0.002\n",
    "D_b1 = np.zeros(hidden_input)\n",
    "\n",
    "D_W2 = np.random.normal(size=(hidden_input,hidden_input2),scale=(1. / np.sqrt(hidden_input / 2.)))   *0.002\n",
    "# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "D_b2 = np.zeros(hidden_input2)\n",
    "\n",
    "D_W3 = np.random.normal(size=(hidden_input2,hidden_input3),scale=(1. / np.sqrt(hidden_input2 / 2.)))   *0.002\n",
    "# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "D_b3 = np.zeros(hidden_input3)\n",
    "\n",
    "D_W4 = np.random.normal(size=(hidden_input3,1),scale=(1. / np.sqrt(hidden_input / 2.)))     *0.002\n",
    "# D_b2 = np.random.normal(size=(1),scale=(1. / np.sqrt(1 / 2.)))           *0.002\n",
    "D_b4 = np.zeros(1)\n",
    "\n",
    "\n",
    "G_W1 = np.random.normal(size=(G_input,hidden_input),scale=(1. / np.sqrt(G_input / 2.)))   *0.002\n",
    "# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "G_b1 = np.zeros(hidden_input)\n",
    "\n",
    "G_W2 = np.random.normal(size=(hidden_input,hidden_input2),scale=(1. / np.sqrt(hidden_input / 2.)))   *0.002\n",
    "# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "G_b2 = np.zeros(hidden_input2)\n",
    "\n",
    "G_W3 = np.random.normal(size=(hidden_input2,hidden_input3),scale=(1. / np.sqrt(hidden_input2 / 2.)))   *0.002\n",
    "# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "G_b3 = np.zeros(hidden_input3)\n",
    "\n",
    "G_W4 = np.random.normal(size=(hidden_input3,hidden_input4),scale=(1. / np.sqrt(hidden_input3 / 2.)))   *0.002\n",
    "# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "G_b4 = np.zeros(hidden_input4)\n",
    "\n",
    "G_W5 = np.random.normal(size=(hidden_input4,hidden_input5),scale=(1. / np.sqrt(hidden_input4 / 2.)))   *0.002\n",
    "# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "G_b5 = np.zeros(hidden_input5)\n",
    "\n",
    "G_W6 = np.random.normal(size=(hidden_input5,hidden_input6),scale=(1. / np.sqrt(hidden_input5 / 2.)))   *0.002\n",
    "# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "G_b6 = np.zeros(hidden_input6)\n",
    "\n",
    "G_W7 = np.random.normal(size=(hidden_input6,hidden_input7),scale=(1. / np.sqrt(hidden_input5 / 2.)))   *0.002\n",
    "# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "G_b7 = np.zeros(hidden_input7)\n",
    "\n",
    "G_W8 = np.random.normal(size=(hidden_input7,hidden_input8),scale=(1. / np.sqrt(hidden_input5 / 2.)))   *0.002\n",
    "# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "G_b8 = np.zeros(hidden_input8)\n",
    "\n",
    "G_W9 = np.random.normal(size=(hidden_input8,hidden_input9),scale=(1. / np.sqrt(hidden_input5 / 2.)))   *0.002\n",
    "# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "G_b9 = np.zeros(hidden_input9)\n",
    "\n",
    "G_W10 = np.random.normal(size=(hidden_input9,hidden_input10),scale=(1. / np.sqrt(hidden_input5 / 2.)))   *0.002\n",
    "# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "G_b10 = np.zeros(hidden_input10)\n",
    "\n",
    "G_W11 = np.random.normal(size=(hidden_input10,784),scale=(1. / np.sqrt(hidden_input6 / 2.)))  *0.002\n",
    "# G_b2 = np.random.normal(size=(784),scale=(1. / np.sqrt(784 / 2.)))      *0.002\n",
    "G_b11 = np.zeros(784)\n",
    "# 2. Declare Weights\n",
    "def ResetWeights():\n",
    "    D_W1 = np.random.normal(size=(784,hidden_input),scale=(1. / np.sqrt(784 / 2.)))   *0.002\n",
    "    # D_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))       *0.002\n",
    "    D_b1 = np.zeros(hidden_input)\n",
    "\n",
    "    D_W2 = np.random.normal(size=(hidden_input,hidden_input2),scale=(1. / np.sqrt(hidden_input / 2.)))   *0.002\n",
    "    # G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "    D_b2 = np.zeros(hidden_input2)\n",
    "\n",
    "    D_W3 = np.random.normal(size=(hidden_input2,hidden_input3),scale=(1. / np.sqrt(hidden_input2 / 2.)))   *0.002\n",
    "    # G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "    D_b3 = np.zeros(hidden_input3)\n",
    "\n",
    "    D_W4 = np.random.normal(size=(hidden_input3,1),scale=(1. / np.sqrt(hidden_input / 2.)))     *0.002\n",
    "    # D_b2 = np.random.normal(size=(1),scale=(1. / np.sqrt(1 / 2.)))           *0.002\n",
    "    D_b4 = np.zeros(1)\n",
    "\n",
    "\n",
    "    G_W1 = np.random.normal(size=(G_input,hidden_input),scale=(1. / np.sqrt(G_input / 2.)))   *0.002\n",
    "    # G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "    G_b1 = np.zeros(hidden_input)\n",
    "\n",
    "    G_W2 = np.random.normal(size=(hidden_input,hidden_input2),scale=(1. / np.sqrt(hidden_input / 2.)))   *0.002\n",
    "    # G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "    G_b2 = np.zeros(hidden_input2)\n",
    "\n",
    "    G_W3 = np.random.normal(size=(hidden_input2,hidden_input3),scale=(1. / np.sqrt(hidden_input2 / 2.)))   *0.002\n",
    "    # G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "    G_b3 = np.zeros(hidden_input3)\n",
    "\n",
    "    G_W4 = np.random.normal(size=(hidden_input3,hidden_input4),scale=(1. / np.sqrt(hidden_input3 / 2.)))   *0.002\n",
    "    # G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "    G_b4 = np.zeros(hidden_input4)\n",
    "\n",
    "    G_W5 = np.random.normal(size=(hidden_input4,hidden_input5),scale=(1. / np.sqrt(hidden_input4 / 2.)))   *0.002\n",
    "    # G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "    G_b5 = np.zeros(hidden_input5)\n",
    "\n",
    "    G_W6 = np.random.normal(size=(hidden_input5,hidden_input6),scale=(1. / np.sqrt(hidden_input5 / 2.)))   *0.002\n",
    "    # G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "    G_b6 = np.zeros(hidden_input6)\n",
    "\n",
    "    G_W7 = np.random.normal(size=(hidden_input6,hidden_input7),scale=(1. / np.sqrt(hidden_input5 / 2.)))   *0.002\n",
    "    # G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "    G_b7 = np.zeros(hidden_input7)\n",
    "\n",
    "    G_W8 = np.random.normal(size=(hidden_input7,hidden_input8),scale=(1. / np.sqrt(hidden_input5 / 2.)))   *0.002\n",
    "    # G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "    G_b8 = np.zeros(hidden_input8)\n",
    "\n",
    "    G_W9 = np.random.normal(size=(hidden_input8,hidden_input9),scale=(1. / np.sqrt(hidden_input5 / 2.)))   *0.002\n",
    "    # G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "    G_b9 = np.zeros(hidden_input9)\n",
    "\n",
    "    G_W10 = np.random.normal(size=(hidden_input9,hidden_input10),scale=(1. / np.sqrt(hidden_input5 / 2.)))   *0.002\n",
    "    # G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "    G_b10 = np.zeros(hidden_input10)\n",
    "\n",
    "    G_W11 = np.random.normal(size=(hidden_input10,784),scale=(1. / np.sqrt(hidden_input6 / 2.)))  *0.002\n",
    "    # G_b2 = np.random.normal(size=(784),scale=(1. / np.sqrt(784 / 2.)))      *0.002\n",
    "    G_b11 = np.zeros(784)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "efhLIYTtbfQn"
   },
   "outputs": [],
   "source": [
    "# 3. For Adam Optimzier\n",
    "m1w1,m1b1 = 0,0\n",
    "m2w2,m2b2 = 0,0\n",
    "m3w3,m3b3 = 0,0\n",
    "m4w4,m4b4 = 0,0\n",
    "\n",
    "v1w1,v1b1 = 0,0\n",
    "v2w2,v2b2 = 0,0\n",
    "v3w3,v3b3 = 0,0\n",
    "v4w4,v4b4 = 0,0\n",
    "\n",
    "v5,m5 = 0,0\n",
    "v6,m6 = 0,0\n",
    "v7,m7 = 0,0\n",
    "v8,m8 = 0,0\n",
    "v9,m9 = 0,0\n",
    "v10,m10 = 0,0\n",
    "v11,m11 = 0,0\n",
    "v12,m12 = 0,0\n",
    "v13,m13 = 0,0\n",
    "v14,m14 = 0,0\n",
    "v15,m15 = 0,0\n",
    "v16,m16 = 0,0\n",
    "v17,m17 = 0,0\n",
    "v18,m18 = 0,0\n",
    "v19,m19 = 0,0\n",
    "v20,m20 = 0,0\n",
    "v21,m21 = 0,0\n",
    "v22,m22 = 0,0\n",
    "v23,m23 = 0,0\n",
    "v24,m24 = 0,0\n",
    "v25,m25 = 0,0\n",
    "v26,m26 = 0,0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "iYhaQBh8bfQy"
   },
   "outputs": [],
   "source": [
    "beta_1,beta_2,eps = 0.5,0.999,0.000001\n",
    "D_list_cost = []\n",
    "G_list_cost = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 954
    },
    "colab_type": "code",
    "id": "sIXkEq74bfQ7",
    "outputId": "b365edb5-0524-46db-fdec-33e4a4d529df"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------- Started Training ----------\n",
      "Current Iter:  1  Current D cost: [[1.38629436]]  Current G cost:  [[0.6931497]]\n",
      "Current Iter:  2  Current D cost: [[1.38629436]]  Current G cost:  [[0.69315206]]\n",
      "Current Iter:  3  Current D cost: [[1.38629436]]  Current G cost:  [[0.6931545]]\n",
      "Current Iter:  4  Current D cost: [[1.38629436]]  Current G cost:  [[0.69315711]]\n",
      "Current Iter:  5  Current D cost: [[1.38629436]]  Current G cost:  [[0.69315991]]\n",
      "Current Iter:  6  Current D cost: [[1.38629436]]  Current G cost:  [[0.69316291]]\n",
      "Current Iter:  7  Current D cost: [[1.38629436]]  Current G cost:  [[0.6931661]]\n",
      "Current Iter:  8  Current D cost: [[1.38629436]]  Current G cost:  [[0.69316947]]\n",
      "Current Iter:  9  Current D cost: [[1.38629436]]  Current G cost:  [[0.69317303]]\n",
      "Current Iter:  10  Current D cost: [[1.38629436]]  Current G cost:  [[0.6931768]]\n",
      "Current Iter:  11  Current D cost: [[1.38629436]]  Current G cost:  [[0.69318073]]\n",
      "Current Iter:  12  Current D cost: [[1.38629436]]  Current G cost:  [[0.69318483]]\n",
      "Current Iter:  13  Current D cost: [[1.38629436]]  Current G cost:  [[0.69318909]]\n",
      "Current Iter:  14  Current D cost: [[1.38629436]]  Current G cost:  [[0.69319352]]\n",
      "Current Iter:  15  Current D cost: [[1.38629436]]  Current G cost:  [[0.69319809]]\n",
      "Current Iter:  16  Current D cost: [[1.38629436]]  Current G cost:  [[0.69320282]]\n",
      "Current Iter:  17  Current D cost: [[1.38629436]]  Current G cost:  [[0.69320771]]\n",
      "Current Iter:  18  Current D cost: [[1.38629437]]  Current G cost:  [[0.69321274]]\n",
      "Current Iter:  19  Current D cost: [[1.38629437]]  Current G cost:  [[0.69321794]]\n",
      "Current Iter:  20  Current D cost: [[1.38629437]]  Current G cost:  [[0.69322328]]\n",
      "Current Iter:  21  Current D cost: [[1.38629437]]  Current G cost:  [[0.69322877]]\n",
      "Current Iter:  22  Current D cost: [[1.38629437]]  Current G cost:  [[0.69323441]]\n",
      "Current Iter:  23  Current D cost: [[1.38629437]]  Current G cost:  [[0.6932402]]\n",
      "Current Iter:  24  Current D cost: [[1.38629437]]  Current G cost:  [[0.69324615]]\n",
      "Current Iter:  25  Current D cost: [[1.38629437]]  Current G cost:  [[0.69325225]]\n",
      "Current Iter:  26  Current D cost: [[1.38629437]]  Current G cost:  [[0.69325849]]\n",
      "Current Iter:  27  Current D cost: [[1.38629438]]  Current G cost:  [[0.69326489]]\n",
      "Current Iter:  28  Current D cost: [[1.38629438]]  Current G cost:  [[0.69327143]]\n",
      "Current Iter:  29  Current D cost: [[1.38629438]]  Current G cost:  [[0.69327812]]\n",
      "Current Iter:  30  Current D cost: [[1.38629438]]  Current G cost:  [[0.69328495]]\n",
      "Current Iter:  31  Current D cost: [[1.38629439]]  Current G cost:  [[0.69329193]]\n",
      "Current Iter:  32  Current D cost: [[1.38629439]]  Current G cost:  [[0.69329906]]\n",
      "Current Iter:  33  Current D cost: [[1.38629439]]  Current G cost:  [[0.69330633]]\n",
      "Current Iter:  34  Current D cost: [[1.3862944]]  Current G cost:  [[0.69331374]]\n",
      "Current Iter:  35  Current D cost: [[1.3862944]]  Current G cost:  [[0.69332129]]\n",
      "Current Iter:  36  Current D cost: [[1.38629441]]  Current G cost:  [[0.69332897]]\n",
      "Current Iter:  37  Current D cost: [[1.38629441]]  Current G cost:  [[0.69333679]]\n",
      "Current Iter:  38  Current D cost: [[1.38629442]]  Current G cost:  [[0.69334475]]\n",
      "Current Iter:  39  Current D cost: [[1.38629442]]  Current G cost:  [[0.69335285]]\n",
      "Current Iter:  40  Current D cost: [[1.38629443]]  Current G cost:  [[0.69336108]]\n",
      "Current Iter:  41  Current D cost: [[1.38629444]]  Current G cost:  [[0.69336944]]\n",
      "Current Iter:  42  Current D cost: [[1.38629445]]  Current G cost:  [[0.69337793]]\n",
      "Current Iter:  43  Current D cost: [[1.38629446]]  Current G cost:  [[0.69338655]]\n",
      "Current Iter:  44  Current D cost: [[1.38629447]]  Current G cost:  [[0.6933953]]\n",
      "Current Iter:  45  Current D cost: [[1.38629447]]  Current G cost:  [[0.69340418]]\n",
      "Current Iter:  46  Current D cost: [[1.38629448]]  Current G cost:  [[0.69341319]]\n",
      "Current Iter:  47  Current D cost: [[1.3862945]]  Current G cost:  [[0.69342232]]\n",
      "Current Iter:  48  Current D cost: [[1.38629452]]  Current G cost:  [[0.69343157]]\n",
      "Current Iter:  49  Current D cost: [[1.38629453]]  Current G cost:  [[0.69344095]]\n",
      "Current Iter:  50  Current D cost: [[1.38629454]]  Current G cost:  [[0.69345045]]\n",
      "Current Iter:  51  Current D cost: [[1.38629455]]  Current G cost:  [[0.69346007]]\n",
      "Current Iter:  52  Current D cost: [[1.38629458]]  Current G cost:  [[0.69346981]]\n",
      "Current Iter:  53  Current D cost: [[1.3862946]]  Current G cost:  [[0.69347967]]\n",
      "Current Iter:  54  Current D cost: [[1.38629458]]  Current G cost:  [[0.69348965]]\n",
      "Current Iter:  55  Current D cost: [[1.38629461]]  Current G cost:  [[0.69349975]]\n",
      "Current Iter:  56  Current D cost: [[1.38629466]]  Current G cost:  [[0.69350996]]\n",
      "Current Iter:  57  Current D cost: [[1.38629466]]  Current G cost:  [[0.69352029]]\n",
      "Current Iter:  58  Current D cost: [[1.38629467]]  Current G cost:  [[0.69353073]]\n",
      "Current Iter:  59  Current D cost: [[1.38629475]]  Current G cost:  [[0.69354129]]\n",
      "Current Iter:  60  Current D cost: [[1.38629478]]  Current G cost:  [[0.69355196]]\n",
      "Current Iter:  61  Current D cost: [[1.38629469]]  Current G cost:  [[0.69356275]]\n",
      "Current Iter:  62  Current D cost: [[1.38629477]]  Current G cost:  [[0.69357365]]\n",
      "Current Iter:  63  Current D cost: [[1.38629486]]  Current G cost:  [[0.69358465]]\n",
      "Current Iter:  64  Current D cost: [[1.38629484]]  Current G cost:  [[0.69359577]]\n",
      "Current Iter:  65  Current D cost: [[1.38629485]]  Current G cost:  [[0.693607]]\n",
      "Current Iter:  66  Current D cost: [[1.38629492]]  Current G cost:  [[0.69361833]]\n",
      "Current Iter:  67  Current D cost: [[1.38629495]]  Current G cost:  [[0.69362977]]\n",
      "Current Iter:  68  Current D cost: [[1.386295]]  Current G cost:  [[0.69364132]]\n",
      "Current Iter:  69  Current D cost: [[1.38629493]]  Current G cost:  [[0.69365296]]\n"
     ]
    }
   ],
   "source": [
    "print('--------- Started Training ----------')\n",
    "ResetWeights()\n",
    "for iter in range(1,70):\n",
    "    random_int = np.random.randint(len(images) - 5)\n",
    "    current_image = np.expand_dims(images[random_int],axis=0)\n",
    "    #fig = plot(current_image,\"REAL\")\n",
    "    # Func: Generate The first Fake Data\n",
    "    Z = np.random.normal(0., 1., size=[1, G_input])\n",
    "    Gl1 = Z.dot(G_W1) + G_b1\n",
    "    Gl1A = Lrelu(Gl1)\n",
    "    Gl2 = Gl1A.dot(G_W2) + G_b2\n",
    "    Gl2A = Lrelu(Gl2)\n",
    "    Gl3 = Gl2A.dot(G_W3) + G_b3\n",
    "    Gl3A = Lrelu(Gl3)\n",
    "    Gl4 = Gl3A.dot(G_W4) + G_b4\n",
    "    Gl4A = Lrelu(Gl4)\n",
    "    Gl5 = Gl4A.dot(G_W5) + G_b5\n",
    "    Gl5A = Lrelu(Gl5)\n",
    "    Gl6 = Gl5A.dot(G_W6) + G_b6\n",
    "    Gl6A = Lrelu(Gl6)\n",
    "    Gl7 = Gl6A.dot(G_W7) + G_b7\n",
    "    Gl7A = Lrelu(Gl7)\n",
    "    Gl8 = Gl7A.dot(G_W8) + G_b8\n",
    "    Gl8A = Lrelu(Gl8)\n",
    "    Gl9 = Gl8A.dot(G_W9) + G_b9\n",
    "    Gl9A = Lrelu(Gl9)\n",
    "    Gl10 = Gl9A.dot(G_W10) + G_b10\n",
    "    Gl10A = Lrelu(Gl10)\n",
    "    Gl11 = Gl10A.dot(G_W11) + G_b11\n",
    "\n",
    "    current_fake_data = log(Gl11)\n",
    "\n",
    "    # Func: Forward Feed for Real data\n",
    "    Dl1_r = current_image.dot(D_W1) + D_b1\n",
    "    Dl1_rA = Lrelu(Dl1_r)\n",
    "    Dl2_r = Dl1_rA.dot(D_W2) + D_b2\n",
    "    Dl2_rA = Lrelu(Dl2_r)\n",
    "    Dl3_r = Dl2_rA.dot(D_W3) + D_b3\n",
    "    Dl3_rA = Lrelu(Dl3_r)\n",
    "    Dl4_r = Dl3_rA.dot(D_W4) + D_b4\n",
    "    Dl4_rA = log(Dl4_r)\n",
    "\n",
    "    # Func: Forward Feed for Fake Data\n",
    "    Dl1_f = current_fake_data.dot(D_W1) + D_b1\n",
    "    Dl1_fA = Lrelu(Dl1_f)\n",
    "    Dl2_f = Dl1_fA.dot(D_W2) + D_b2\n",
    "    Dl2_fA = Lrelu(Dl2_f)\n",
    "    Dl3_f = Dl2_fA.dot(D_W3) + D_b3\n",
    "    Dl3_fA = Lrelu(Dl3_f)\n",
    "    Dl4_f = Dl3_fA.dot(D_W4) + D_b4\n",
    "    Dl4_fA = log(Dl4_f)\n",
    "\n",
    "\n",
    "    # Func: Cost D\n",
    "    D_cost = -( np.log(Dl4_rA) +  np.log(1.0- Dl4_fA))\n",
    "    D_list_cost.append(D_cost[0][0])\n",
    "\n",
    "    # Func: Gradient\n",
    "    grad_f_w4_part_1 =  1/(1-Dl4_fA)\n",
    "    grad_f_w4_part_2 =  d_log(Dl4_f)\n",
    "    grad_f_w4_part_3 =   Dl3_fA\n",
    "    grad_f_w4 =     grad_f_w4_part_3.T.dot(grad_f_w4_part_1 * grad_f_w4_part_2) \n",
    "    grad_f_b4 = grad_f_w4_part_1 * grad_f_w4_part_2\n",
    "\n",
    "    grad_f_w3_part_1 =  (grad_f_w4_part_1 * grad_f_w4_part_2).dot(D_W4.T)\n",
    "    grad_f_w3_part_2 =  d_Lrelu(Dl3_f)\n",
    "    grad_f_w3_part_3 =   Dl2_fA\n",
    "    grad_f_w3 =       grad_f_w3_part_3.T.dot(grad_f_w3_part_1 * grad_f_w3_part_2) \n",
    "    grad_f_b3 =      grad_f_w3_part_1 * grad_f_w3_part_2\n",
    "    \n",
    "    grad_f_w2_part_1 = (grad_f_w3_part_1 * grad_f_w3_part_2).dot(D_W3.T)\n",
    "    grad_f_w2_part_2 = d_Lrelu(Dl2_f)\n",
    "    grad_f_w2_part_3 = Dl1_fA\n",
    "    grad_f_w2 = grad_f_w2_part_3.T.dot(grad_f_w2_part_1 * grad_f_w2_part_2)\n",
    "    grad_f_b2 = (grad_f_w2_part_1 * grad_f_w2_part_2)\n",
    "\n",
    "    grad_f_w1_part_1 = (grad_f_w2_part_1 * grad_f_w2_part_2).dot(D_W2.T)\n",
    "    grad_f_w1_part_2 = d_Lrelu(Dl1_f)\n",
    "    grad_f_w1_part_3 = current_fake_data\n",
    "    grad_f_w1 = grad_f_w1_part_3.T.dot(grad_f_w1_part_1 * grad_f_w1_part_2)\n",
    "    grad_f_b1 = grad_f_w1_part_1 * grad_f_w1_part_2\n",
    "    \n",
    "    #LATER\n",
    "    grad_r_w4_part_1 =  1/(1-Dl4_rA)\n",
    "    grad_r_w4_part_2 =  d_log(Dl4_r)\n",
    "    grad_r_w4_part_3 =   Dl3_rA\n",
    "    grad_r_w4 =     grad_r_w4_part_3.T.dot(grad_r_w4_part_1 * grad_r_w4_part_2) \n",
    "    grad_r_b4 = grad_r_w4_part_1 * grad_r_w4_part_2\n",
    "\n",
    "    grad_r_w3_part_1 =  (grad_r_w4_part_1 * grad_r_w4_part_2).dot(D_W4.T)\n",
    "    grad_r_w3_part_2 =  d_Lrelu(Dl3_r)\n",
    "    grad_r_w3_part_3 =   Dl2_rA\n",
    "    grad_r_w3 =       grad_r_w3_part_3.T.dot(grad_r_w3_part_1 * grad_r_w3_part_2) \n",
    "    grad_r_b3 =      grad_r_w3_part_1 * grad_r_w3_part_2\n",
    "    \n",
    "    grad_r_w2_part_1 = (grad_r_w3_part_1 * grad_r_w3_part_2).dot(D_W3.T)\n",
    "    grad_r_w2_part_2 = d_Lrelu(Dl2_r)\n",
    "    grad_r_w2_part_3 = Dl1_rA\n",
    "    grad_r_w2 = grad_r_w2_part_3.T.dot(grad_r_w2_part_1 * grad_r_w2_part_2)\n",
    "    grad_r_b2 = (grad_r_w2_part_1 * grad_r_w2_part_2)\n",
    "\n",
    "    grad_r_w1_part_1 = (grad_r_w2_part_1 * grad_r_w2_part_2).dot(D_W2.T)\n",
    "    grad_r_w1_part_2 = d_Lrelu(Dl1_r)\n",
    "    grad_r_w1_part_3 = current_image\n",
    "    grad_r_w1 = grad_r_w1_part_3.T.dot(grad_r_w1_part_1 * grad_r_w1_part_2)\n",
    "    grad_r_b1 = grad_r_w1_part_1 * grad_r_w1_part_2\n",
    "\n",
    "    grad_w1 =grad_f_w1 + grad_r_w1\n",
    "    grad_w2 =grad_f_w2 + grad_r_w2\n",
    "    grad_w3 =grad_f_w3 + grad_r_w3\n",
    "    grad_w4 =grad_f_w4 + grad_r_w4\n",
    "\n",
    "    grad_b1 =grad_f_b1 + grad_r_b1\n",
    "    grad_b2 =grad_f_b2 + grad_r_b2\n",
    "    grad_b3 =grad_f_b3 + grad_r_b3\n",
    "    grad_b4 =grad_f_b4 + grad_r_b4\n",
    "\n",
    "    # ---- Update Gradient ----\n",
    "    m1w1 = beta_1 * m1w1 + (1 - beta_1) * grad_w1\n",
    "    v1w1 = beta_2 * v1w1 + (1 - beta_2) * grad_w1 ** 2\n",
    "    \n",
    "    m2w2 = beta_1 * m2w2 + (1 - beta_1) * grad_w2\n",
    "    v2w2 = beta_2 * v2w2 + (1 - beta_2) * grad_w2 ** 2\n",
    "    \n",
    "    m3w3 = beta_1 * m3w3 + (1 - beta_1) * grad_w3\n",
    "    v3w3 = beta_2 * v3w3 + (1 - beta_2) * grad_w3 ** 2\n",
    "    \n",
    "    m4w4 = beta_1 * m4w4 + (1 - beta_1) * grad_w4\n",
    "    v4w4 = beta_2 * v4w4 + (1 - beta_2) * grad_w4 ** 2\n",
    "    \n",
    "    m1b1 = beta_1 * m1b1 + (1 - beta_1) * grad_b1\n",
    "    v1b1 = beta_2 * v1b1 + (1 - beta_2) * grad_b1 ** 2\n",
    "    \n",
    "    m2b2 = beta_1 * m2b2 + (1 - beta_1) * grad_b2\n",
    "    v2b2 = beta_2 * v2b2 + (1 - beta_2) * grad_b2 ** 2\n",
    "    \n",
    "    m3b3 = beta_1 * m3b3 + (1 - beta_1) * grad_b3\n",
    "    v3b3 = beta_2 * v3b3 + (1 - beta_2) * grad_b3 ** 2\n",
    "    \n",
    "    m4b4 = beta_1 * m4b4 + (1 - beta_1) * grad_b4\n",
    "    v4b4 = beta_2 * v4b4 + (1 - beta_2) * grad_b4 ** 2\n",
    "\n",
    "    D_W1 = D_W1 - (learing_rate2 / (np.sqrt(v1w1 /(1-beta_2**iter )) + eps)) * (m1w1/(1-beta_1**iter)  + eps)\n",
    "    D_b1 = D_b1 - (learing_rate2 / (np.sqrt(v1b1 /(1-beta_2**iter )) + eps)) * (m1b1/(1-beta_1**iter)  + eps)\n",
    "        \n",
    "    D_W2 = D_W2 - (learing_rate2 / (np.sqrt(v2w2 /(1-beta_2**iter )) + eps)) * (m2w2/(1-beta_1**iter)  + eps)\n",
    "    D_b2 = D_b2 - (learing_rate2 / (np.sqrt(v2b2 /(1-beta_2**iter )) + eps)) * (m2b2/(1-beta_1**iter)  + eps)\n",
    "    \n",
    "    D_W3 = D_W3 - (learing_rate2 / (np.sqrt(v3w3 /(1-beta_2**iter )) + eps)) * (m3w3/(1-beta_1**iter)  + eps)\n",
    "    D_b3 = D_b3 - (learing_rate2 / (np.sqrt(v3b3 /(1-beta_2**iter )) + eps)) * (m3b3/(1-beta_1**iter)  + eps)\n",
    "    \n",
    "    D_W4 = D_W4 - (learing_rate2 / (np.sqrt(v4w4 /(1-beta_2**iter )) + eps)) * (m4w4/(1-beta_1**iter)  + eps)\n",
    "    D_b4 = D_b4 - (learing_rate2 / (np.sqrt(v4b4 /(1-beta_2**iter )) + eps)) * (m4b4/(1-beta_1**iter)  + eps)\n",
    "    \n",
    "    # Func: Forward Feed for G\n",
    "    Z = np.random.normal(0., 1., size=[1, G_input]) \n",
    "    Gl1 = Z.dot(G_W1) + G_b1\n",
    "    Gl1A = Lrelu(Gl1)\n",
    "    Gl2 = Gl1A.dot(G_W2) + G_b2\n",
    "    Gl2A = Lrelu(Gl2)\n",
    "    Gl3 = Gl2A.dot(G_W3) + G_b3\n",
    "    Gl3A = Lrelu(Gl3)\n",
    "    Gl4 = Gl3A.dot(G_W4) + G_b4\n",
    "    Gl4A = Lrelu(Gl4)\n",
    "    Gl5 = Gl4A.dot(G_W5) + G_b5\n",
    "    Gl5A = Lrelu(Gl5)\n",
    "    Gl6 = Gl5A.dot(G_W6) + G_b6\n",
    "    Gl6A = Lrelu(Gl6)\n",
    "    Gl7 = Gl6A.dot(G_W7) + G_b7\n",
    "    Gl7A = Lrelu(Gl7)\n",
    "    Gl8 = Gl7A.dot(G_W8) + G_b8\n",
    "    Gl8A = Lrelu(Gl8)\n",
    "    Gl9 = Gl8A.dot(G_W9) + G_b9\n",
    "    Gl9A = Lrelu(Gl9)\n",
    "    Gl10 = Gl9A.dot(G_W10) + G_b10\n",
    "    Gl10A = Lrelu(Gl10)\n",
    "    Gl11 = Gl10A.dot(G_W11) + G_b11\n",
    "    \n",
    "    current_fake_data = log(Gl11)\n",
    "    #fig = plot(current_fake_data,\"FAKE\")\n",
    "    Dl1 = current_fake_data.dot(D_W1) + D_b1\n",
    "    Dl1_A = Lrelu(Dl1)\n",
    "    Dl2 = Dl1_A.dot(D_W2) + D_b2\n",
    "    Dl2_A = Lrelu(Dl2)\n",
    "    Dl3 = Dl2_A.dot(D_W3) + D_b3\n",
    "    Dl3_A = Lrelu(Dl3)\n",
    "    Dl4 = Dl3_A.dot(D_W4) + D_b4\n",
    "    Dl4_A = log(Dl4)\n",
    "\n",
    "    # Func: Cost G\n",
    "    G_cost = -np.log(Dl4_A)\n",
    "    G_list_cost.append(G_cost[0][0])\n",
    "    # Func: Gradient\n",
    "    grad_G_w11_part_1 = (((( (-1/(Dl4_A + eps)) * d_log(Dl4).dot(D_W4.T) * (d_Lrelu(Dl3)) ).dot(D_W3.T) * (d_Lrelu(Dl2))).dot(D_W2.T) * (d_Lrelu(Dl1))).dot(D_W1.T))\n",
    "    grad_G_w11_part_2 = d_log(Gl11)\n",
    "    grad_G_w11_part_3 = Gl10A\n",
    "    grad_G_w11 = grad_G_w11_part_3.T.dot(grad_G_w11_part_1 * grad_G_w11_part_1)\n",
    "    grad_G_b11 = grad_G_w11_part_1 * grad_G_w11_part_2\n",
    "\n",
    "    grad_G_w10_part_1 = (grad_G_w11_part_1 * grad_G_w11_part_2).dot(G_W11.T)\n",
    "    grad_G_w10_part_2 = d_Lrelu(Gl10)\n",
    "    grad_G_w10_part_3 = Gl9A\n",
    "    grad_G_w10 = grad_G_w10_part_3.T.dot(grad_G_w10_part_1 * grad_G_w10_part_2)\n",
    "    grad_G_b10 = (grad_G_w10_part_1 * grad_G_w10_part_2)\n",
    "    \n",
    "    grad_G_w9_part_1 = (grad_G_w10_part_1 * grad_G_w10_part_2).dot(G_W10.T)\n",
    "    grad_G_w9_part_2 = d_Lrelu(Gl9)\n",
    "    grad_G_w9_part_3 = Gl8A\n",
    "    grad_G_w9 = grad_G_w9_part_3.T.dot(grad_G_w9_part_1 * grad_G_w9_part_2)\n",
    "    grad_G_b9 = (grad_G_w9_part_1 * grad_G_w9_part_2)\n",
    "    \n",
    "    grad_G_w8_part_1 = (grad_G_w9_part_1 * grad_G_w9_part_2).dot(G_W9.T)\n",
    "    grad_G_w8_part_2 = d_Lrelu(Gl8)\n",
    "    grad_G_w8_part_3 = Gl7A\n",
    "    grad_G_w8 = grad_G_w8_part_3.T.dot(grad_G_w8_part_1 * grad_G_w8_part_2)\n",
    "    grad_G_b8 = (grad_G_w8_part_1 * grad_G_w8_part_2)\n",
    "   \n",
    "    grad_G_w7_part_1 = (grad_G_w8_part_1 * grad_G_w8_part_2).dot(G_W8.T)\n",
    "    grad_G_w7_part_2 = d_Lrelu(Gl7)\n",
    "    grad_G_w7_part_3 = Gl6A\n",
    "    grad_G_w7 = grad_G_w7_part_3.T.dot(grad_G_w7_part_1 * grad_G_w7_part_2)\n",
    "    grad_G_b7 = (grad_G_w7_part_1 * grad_G_w7_part_2)\n",
    "\n",
    "    grad_G_w6_part_1 = (grad_G_w7_part_1 * grad_G_w7_part_2).dot(G_W7.T)\n",
    "    grad_G_w6_part_2 = d_Lrelu(Gl6)\n",
    "    grad_G_w6_part_3 = Gl5A\n",
    "    grad_G_w6 = grad_G_w6_part_3.T.dot(grad_G_w6_part_1 * grad_G_w6_part_2)\n",
    "    grad_G_b6 = (grad_G_w6_part_1 * grad_G_w6_part_2)\n",
    "\n",
    "    grad_G_w5_part_1 = (grad_G_w6_part_1 * grad_G_w6_part_2).dot(G_W6.T)\n",
    "    grad_G_w5_part_2 = d_Lrelu(Gl5)\n",
    "    grad_G_w5_part_3 = Gl4A\n",
    "    grad_G_w5 = grad_G_w5_part_3.T.dot(grad_G_w5_part_1 * grad_G_w5_part_2)\n",
    "    grad_G_b5 = (grad_G_w5_part_1 * grad_G_w5_part_2)\n",
    "\n",
    "    grad_G_w4_part_1 = (grad_G_w5_part_1 * grad_G_w5_part_2).dot(G_W5.T)\n",
    "    grad_G_w4_part_2 = d_Lrelu(Gl4)\n",
    "    grad_G_w4_part_3 = Gl3A\n",
    "    grad_G_w4 = grad_G_w4_part_3.T.dot(grad_G_w4_part_1 * grad_G_w4_part_2)\n",
    "    grad_G_b4 = (grad_G_w4_part_1 * grad_G_w4_part_2)\n",
    "\n",
    "    grad_G_w3_part_1 = (grad_G_w4_part_1 * grad_G_w4_part_2).dot(G_W4.T)\n",
    "    grad_G_w3_part_2 = d_Lrelu(Gl3)\n",
    "    grad_G_w3_part_3 = Gl2A\n",
    "    grad_G_w3 = grad_G_w3_part_3.T.dot(grad_G_w3_part_1 * grad_G_w3_part_2)\n",
    "    grad_G_b3 = (grad_G_w3_part_1 * grad_G_w3_part_2)\n",
    "\n",
    "    grad_G_w2_part_1 = (grad_G_w3_part_1 * grad_G_w3_part_2).dot(G_W3.T)\n",
    "    grad_G_w2_part_2 = d_Lrelu(Gl2)\n",
    "    grad_G_w2_part_3 = Gl1A\n",
    "    grad_G_w2 = grad_G_w2_part_3.T.dot(grad_G_w2_part_1 * grad_G_w2_part_2)\n",
    "    grad_G_b2 = (grad_G_w2_part_1 * grad_G_w2_part_2)\n",
    "\n",
    "    grad_G_w1_part_1 = (grad_G_w2_part_1 * grad_G_w2_part_2).dot(G_W2.T)\n",
    "    grad_G_w1_part_2 = d_Lrelu(Gl1)\n",
    "    grad_G_w1_part_3 = Z\n",
    "    grad_G_w1 = grad_G_w1_part_3.T.dot(grad_G_w1_part_1 * grad_G_w1_part_2)\n",
    "    grad_G_b1 = grad_G_w1_part_1 * grad_G_w1_part_2\n",
    "\n",
    "    # ---- Update Gradient ----\n",
    "    m5 = beta_1 * m5 + (1 - beta_1) * grad_G_w1\n",
    "    v5 = beta_2 * v5 + (1 - beta_2) * grad_G_w1 ** 2\n",
    "\n",
    "    m6 = beta_1 * m6 + (1 - beta_1) * grad_G_b1\n",
    "    v6 = beta_2 * v6 + (1 - beta_2) * grad_G_b1 ** 2\n",
    "\n",
    "    m7 = beta_1 * m7 + (1 - beta_1) * grad_G_w2\n",
    "    v7 = beta_2 * v7 + (1 - beta_2) * grad_G_w2 ** 2\n",
    "\n",
    "    m8 = beta_1 * m8 + (1 - beta_1) * grad_G_b2\n",
    "    v8 = beta_2 * v8 + (1 - beta_2) * grad_G_b2 ** 2\n",
    "\n",
    "    m9 = beta_1 * m9 + (1 - beta_1) * grad_G_w3\n",
    "    v9 = beta_2 * v9 + (1 - beta_2) * grad_G_w3 ** 2\n",
    "\n",
    "    m10 = beta_1 * m10 + (1 - beta_1) * grad_G_b3\n",
    "    v10 = beta_2 * v10 + (1 - beta_2) * grad_G_b3 ** 2\n",
    "\n",
    "    m11 = beta_1 * m11 + (1 - beta_1) * grad_G_w4\n",
    "    v11 = beta_2 * v11 + (1 - beta_2) * grad_G_w4 ** 2\n",
    "\n",
    "    m12 = beta_1 * m12 + (1 - beta_1) * grad_G_b4\n",
    "    v12 = beta_2 * v12 + (1 - beta_2) * grad_G_b4 ** 2\n",
    "\n",
    "    m13 = beta_1 * m13 + (1 - beta_1) * grad_G_w5\n",
    "    v13 = beta_2 * v13 + (1 - beta_2) * grad_G_w5 ** 2\n",
    "\n",
    "    m14 = beta_1 * m14 + (1 - beta_1) * grad_G_b5\n",
    "    v14 = beta_2 * v14 + (1 - beta_2) * grad_G_b5 ** 2\n",
    "\n",
    "    m15 = beta_1 * m15 + (1 - beta_1) * grad_G_w6\n",
    "    v15 = beta_2 * v15 + (1 - beta_2) * grad_G_w6 ** 2\n",
    "\n",
    "    m16 = beta_1 * m16 + (1 - beta_1) * grad_G_b6\n",
    "    v16 = beta_2 * v16 + (1 - beta_2) * grad_G_b6 ** 2\n",
    "\n",
    "    m17 = beta_1 * m17 + (1 - beta_1) * grad_G_w7\n",
    "    v17 = beta_2 * v17 + (1 - beta_2) * grad_G_w7 ** 2\n",
    "\n",
    "    m18 = beta_1 * m18 + (1 - beta_1) * grad_G_b7\n",
    "    v18 = beta_2 * v18 + (1 - beta_2) * grad_G_b7 ** 2\n",
    "    \n",
    "    m19 = beta_1 * m19 + (1 - beta_1) * grad_G_w8\n",
    "    v19 = beta_2 * v19 + (1 - beta_2) * grad_G_w8 ** 2\n",
    "    \n",
    "    m20 = beta_1 * m20 + (1 - beta_1) * grad_G_b8\n",
    "    v20 = beta_2 * v20 + (1 - beta_2) * grad_G_b8 ** 2\n",
    "    \n",
    "    m21 = beta_1 * m21 + (1 - beta_1) * grad_G_w9\n",
    "    v21 = beta_2 * v21 + (1 - beta_2) * grad_G_w9 ** 2\n",
    "    \n",
    "    m22 = beta_1 * m22 + (1 - beta_1) * grad_G_b9\n",
    "    v22 = beta_2 * v22 + (1 - beta_2) * grad_G_b9 ** 2\n",
    "    \n",
    "    m23 = beta_1 * m23 + (1 - beta_1) * grad_G_w10\n",
    "    v23 = beta_2 * v23 + (1 - beta_2) * grad_G_w10 ** 2\n",
    "\n",
    "    m24 = beta_1 * m24 + (1 - beta_1) * grad_G_b10\n",
    "    v24 = beta_2 * v24 + (1 - beta_2) * grad_G_b10 ** 2\n",
    "    \n",
    "    m25 = beta_1 * m25 + (1 - beta_1) * grad_G_w11\n",
    "    v25 = beta_2 * v25 + (1 - beta_2) * grad_G_w11 ** 2\n",
    "    \n",
    "    m26 = beta_1 * m26 + (1 - beta_1) * grad_G_b11\n",
    "    v26 = beta_2 * v26 + (1 - beta_2) * grad_G_b11 ** 2\n",
    "    \n",
    "\n",
    "    G_W1 = G_W1 - (learing_rate / (np.sqrt(v5 /(1-beta_2**iter )) + eps)) * (m5/(1-beta_1**iter)  + eps)\n",
    "    G_b1 = G_b1 - (learing_rate / (np.sqrt(v6 /(1-beta_2**iter )) + eps)) * (m6/(1-beta_1**iter)  + eps)\n",
    "    \n",
    "    G_W2 = G_W2 - (learing_rate / (np.sqrt(v7 /(1-beta_2**iter )) + eps)) * (m7/(1-beta_1**iter)  + eps)\n",
    "    G_b2 = G_b2 - (learing_rate / (np.sqrt(v8 /(1-beta_2**iter )) + eps)) * (m8/(1-beta_1**iter)  + eps)\n",
    "\n",
    "    G_W3 = G_W3 - (learing_rate / (np.sqrt(v9 /(1-beta_2**iter )) + eps)) * (m9/(1-beta_1**iter)  + eps)\n",
    "    G_b3 = G_b3 - (learing_rate / (np.sqrt(v10 /(1-beta_2**iter )) + eps)) * (m10/(1-beta_1**iter)  + eps)\n",
    "\n",
    "    G_W4 = G_W4 - (learing_rate / (np.sqrt(v11 /(1-beta_2**iter )) + eps)) * (m11/(1-beta_1**iter)  + eps)\n",
    "    G_b4 = G_b4 - (learing_rate / (np.sqrt(v12 /(1-beta_2**iter )) + eps)) * (m12/(1-beta_1**iter)  + eps)\n",
    "\n",
    "    G_W5 = G_W5 - (learing_rate / (np.sqrt(v13 /(1-beta_2**iter )) + eps)) * (m13/(1-beta_1**iter)  + eps)\n",
    "    G_b5 = G_b5 - (learing_rate / (np.sqrt(v14 /(1-beta_2**iter )) + eps)) * (m14/(1-beta_1**iter)  + eps)\n",
    "\n",
    "    G_W6 = G_W6 - (learing_rate / (np.sqrt(v15 /(1-beta_2**iter )) + eps)) * (m15/(1-beta_1**iter)  + eps)\n",
    "    G_b6 = G_b6 - (learing_rate / (np.sqrt(v16 /(1-beta_2**iter )) + eps)) * (m16/(1-beta_1**iter)  + eps)\n",
    "\n",
    "    G_W7 = G_W7 - (learing_rate / (np.sqrt(v17 /(1-beta_2**iter ))+ eps)) * (m17/(1-beta_1**iter)  + eps)\n",
    "    G_b7 = G_b7 - (learing_rate / (np.sqrt(v18 /(1-beta_2**iter )) + eps)) * (m18/(1-beta_1**iter)  + eps)\n",
    "    \n",
    "    G_W8 = G_W8 - (learing_rate / (np.sqrt(v19 /(1-beta_2**iter )) + eps)) * (m19/(1-beta_1**iter)  + eps)\n",
    "    G_b8 = G_b8 - (learing_rate / (np.sqrt(v20 /(1-beta_2**iter )) + eps)) * (m20/(1-beta_1**iter)  + eps)\n",
    "    \n",
    "    G_W9 = G_W9 - (learing_rate / (np.sqrt(v21 /(1-beta_2**iter )) + eps)) * (m21/(1-beta_1**iter)  + eps)\n",
    "    G_b9 = G_b9 - (learing_rate / (np.sqrt(v22 /(1-beta_2**iter )) + eps)) * (m22/(1-beta_1**iter)  + eps)\n",
    "    \n",
    "    G_W10 = G_W10 - (learing_rate / (np.sqrt(v23 /(1-beta_2**iter )) + eps)) * (m23/(1-beta_1**iter)  + eps)\n",
    "    G_b10 = G_b10 - (learing_rate / (np.sqrt(v24 /(1-beta_2**iter )) + eps)) * (m24/(1-beta_1**iter)  + eps)\n",
    "    \n",
    "    G_W11 = G_W11 - (learing_rate / (np.sqrt(v25 /(1-beta_2**iter )) + eps)) * (m25/(1-beta_1**iter)  + eps)\n",
    "    G_b11 = G_b11 - (learing_rate / (np.sqrt(v26 /(1-beta_2**iter )) + eps)) * (m26/(1-beta_1**iter)  + eps)\n",
    "\n",
    "    print(\"Current Iter: \",iter, \" Current D cost:\",D_cost, \" Current G cost: \", G_cost)\n",
    "    \n",
    "    if iter == 0:\n",
    "        learing_rate = learing_rate #* 0.01\n",
    "    if iter == 40:\n",
    "        learing_rate = learing_rate #* 0.01\n",
    "\n",
    "    # ---- Print to Out put ----\n",
    "# -- end code --"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 284
    },
    "colab_type": "code",
    "id": "zYry6Fe6eL4m",
    "outputId": "ff02d8fc-befe-4618-90e0-a200bd5506e9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x27cc7ec8c88>]"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFsVJREFUeJzt3X2UXXV97/H3l4QkPCcwAxISCAo+cBEERp6sJT61CbpArF0S4OK9FfOP2kq1FyhW1Np1r0qv2haIKRdjXRQKQm3K5aElxdJVeRqkhMiTERAGEIbwYIMEMsn3/vE7c3Mymck5mTmZc87m/Vprr7P3/v1m72925nzOb35nn5nITCRJ1bJDuwuQJLWe4S5JFWS4S1IFGe6SVEGGuyRVkOEuSRVkuEtSBRnuklRBhrskVdDUdp24p6cn582b167TS1JXuvvuu5/LzN5G/doW7vPmzaO/v79dp5ekrhQRv2imn9MyklRBDcM9Ii6LiGcjYlWDfu+MiA0R8dHWlSdJGo9mRu7LgAVb6xARU4CvATe1oCZJ0gQ1DPfMvBV4vkG3zwDXAM+2oihJ0sRMeM49IvYDTgGWNNF3cUT0R0T/4ODgRE8tSRpDK95Q/RZwTmZuaNQxM5dmZl9m9vX2NryTR5I0Tq24FbIPuDIiAHqAEyNiKDN/2IJjS5LGYcLhnpkHDq9HxDLguu0a7KtWwVVXjd1eXmQmp20yzzXZbZ1Sx3jbOqWOrbVFwA47bHocXuq3R7YNL1OmbFqGt6dO3fQ4vNRvb61t6lTYccex/w3qOg3DPSKuAOYDPRExAFwA7AiQmQ3n2VvugQfgq18dvc2/ByuN3667wt57lxeUl16C116DGTNgp502PdYvM2aUF4WhofI1M2eWZdassrz5zXD44TBtGjz5JLzySmnfbbfyNRs3wr77lhcntVy06w9k9/X1Zcd8QnVr12CstvF8Tbe0dUod423rlDoatW3cWNrrH4eXsbY3bNj0OHJ9aGjz9eGlme3XXoMXXoBnninn2mOPEsrr1pXllVfKMnJ9aKiM+DduhBdfLMdYt27sf/NIe+wB73wnzJ4N06fD/vvD+94HBx0EK1fCY4/BO95RlilTmj9uhUXE3ZnZ16hf2379QEcZ74/bkra0bh2sWVN+yl65sgT/7Nmw887lBWDt2vKCsGFDab/zTvjZz+DVV8uLy5/8yZbH3G036O0tLwBvfjO8//1w5JHlp4dXXinH6O8vPyEMDsKhh8Jpp8GCBeVrXoccuUvqHGvWwC23wOOPw9vfDgccUEL7xz8uLwzr1sE998Ajj2z5tXPnlv6zZsFtt8Fzz5Xpo3e/G44/HvbZB/baC3p6ygvFW97Sle8zNDtyN9wldZ9HHoGHHy6j/SlT4Kijyvz9sPXr4eab4YYbYMUKuP/+LY8xcyYsXFh+Eti4EQ4+uIz2O3z6x3CXpGGvvgrPP19G82vWlOmbm2+G664r+4YdcwwsWVLm+KH8pHDNNWW6CMpPAMceO/n113HOXZKGTZ9eRvb1o/vTT9/8De+//Vs4+2w44ogyn3/88XD11ZuCHcqofskSOOussr1hQ8eO9L0HSdLrV8Sm5fTT4cEH4etfLy8GF11URvA331xuDf3lL+EDH4BPfhI++MEyjTNjRnlB+NWv4Kmn4M//HP7t39r9rwKclpGk0Q0Nlfv4661fD5/9bJmqOfrocivn5ZeXN3FffLHM3e+9d7lTaM89yzEefbS8ELRIs9MyjtwlaTQjgx3K3TUXXVRG8cuXw/e/D7ffDiecAOecAz/8YZnTP/fcMs//4Q+XN2y//e3JL3/SzyhJVXL00XDttZu2zz4bLrwQ7ruvBP9RR5XR/tq18Md/PGmfnXHkLkmt9KUvlU/a3n47XHJJeTzjDPjCF8q9+9/85uZ36GwnjtwlqZV22QVuvBF+8YvyCVmA730P5s+Hv/5r+MM/LPfp/+VfbtcyfENVkibTqlXll7TNmzeuL/c+d0nqRIceOimncc5dkirIcJekCjLcJamCDHdJqiDDXZIqyHCXpApqGO4RcVlEPBsRq8ZoPzkiVkbEf0REf0T8RuvLlCRti2ZG7suABVtpXwEcnpnvAH4PuLQFdUmSJqBhuGfmrcDzW2lfm5s+5roL0J6PvEqS/r+WzLlHxCkR8SDwfymjd0lSG7Uk3DPz7zPzrcCHgT8dq19ELK7Ny/cPDg624tSSpFG09G6Z2hTOmyKiZ4z2pZnZl5l9vb29rTy1JKnOhMM9Ig6KKL99PiKOBKYBayZ6XEnS+DX8rZARcQUwH+iJiAHgAmBHgMxcAvwOcGZErAdeAT6W7fo9wpIkoIlwz8xFDdq/BnytZRVJkibMT6hKUgUZ7pJUQYa7JFWQ4S5JFWS4S1IFGe6SVEGGuyRVkOEuSRVkuEtSBRnuklRBhrskVZDhLkkVZLhLUgUZ7pJUQYa7JFWQ4S5JFWS4S1IFGe6SVEGGuyRVkOEuSRXUMNwj4rKIeDYiVo3RfnpErKwtP46Iw1tfpiRpWzQzcl8GLNhK+6PACZl5GPCnwNIW1CVJmoCpjTpk5q0RMW8r7T+u27wdmDPxsiRJE9HqOfdPADeM1RgRiyOiPyL6BwcHW3xqSdKwloV7RLyHEu7njNUnM5dmZl9m9vX29rbq1JKkERpOyzQjIg4DLgUWZuaaVhxTkjR+Ex65R8T+wLXAf83MhydekiRpohqO3CPiCmA+0BMRA8AFwI4AmbkE+CKwF3BxRAAMZWbf9ipYktRYM3fLLGrQfhZwVssqkiRNmJ9QlaQKMtwlqYIMd0mqIMNdkirIcJekCjLcJamCDHdJqiDDXZIqyHCXpAoy3CWpggx3Saogw12SKshwl6QKMtwlqYIMd0mqIMNdkirIcJekCjLcJamCDHdJqqCG4R4Rl0XEsxGxaoz2t0bEbRHxakR8vvUlSpK2VTMj92XAgq20Pw/8PnBhKwqSJE1cw3DPzFspAT5W+7OZeRewvpWFSZLGzzl3SaqgSQ33iFgcEf0R0T84ODiZp5ak15VJDffMXJqZfZnZ19vbO5mnlqTXFadlJKmCpjbqEBFXAPOBnogYAC4AdgTIzCUR8QagH9gd2BgRnwUOycxfbbeqJUlb1TDcM3NRg/ZfAnNaVpEkacKclpGkCjLcJamCDHdJqiDDXZIqyHCXpAoy3CWpggx3Saogw12SKshwl6QKMtwlqYIMd0mqIMNdkirIcJekCjLcJamCDHdJqiDDXZIqyHCXpAoy3CWpggx3Saogw12SKqhhuEfEZRHxbESsGqM9IuIvImJ1RKyMiCNbX6YkaVs0M3JfBizYSvtC4ODashi4ZOJlSZImomG4Z+atwPNb6XIy8DdZ3A7MjIh9W1WgJGnbtWLOfT/gibrtgdq+LUTE4ojoj4j+wcHBFpxakjSaVoR7jLIvR+uYmUszsy8z+3p7e1twaknSaFoR7gPA3LrtOcBTLTiuJGmcWhHuy4Eza3fNHAu8lJlPt+C4kqRxmtqoQ0RcAcwHeiJiALgA2BEgM5cA1wMnAquBXwP/fXsVK0lqTsNwz8xFDdoT+FTLKpIkTZifUJWkCjLcJamCDHdJqiDDXZIqyHCXpAoy3CWpggx3Saogw12SKshwl6QKMtwlqYIMd0mqIMNdkirIcJekCjLcJamCDHdJqiDDXZIqyHCXpAoy3CWpggx3SaqgpsI9IhZExEMRsToizh2l/YCIWBERKyPiRxExp/WlSpKa1TDcI2IKcBGwEDgEWBQRh4zodiHwN5l5GPAV4H+2ulBJUvOaGbkfDazOzEcy8zXgSuDkEX0OAVbU1m8ZpV2SNImaCff9gCfqtgdq++rdC/xObf0UYLeI2Gvi5UmSxqOZcI9R9uWI7c8DJ0TEPcAJwJPA0BYHilgcEf0R0T84OLjNxUqSmtNMuA8Ac+u25wBP1XfIzKcy8yOZeQRwfm3fSyMPlJlLM7MvM/t6e3snULYkaWuaCfe7gIMj4sCImAacCiyv7xARPRExfKzzgMtaW6YkaVs0DPfMHAI+DdwEPABclZk/jYivRMRJtW7zgYci4mFgH+DPtlO9kqQmRObI6fPJ0dfXl/39/W05tyR1q4i4OzP7GvXzE6qSVEGGuyRVkOEuSRVkuEtSBRnuklRBhrskVZDhLkkVZLhLUgUZ7pJUQYa7JFWQ4S5JFWS4S1IFGe6SVEGGuyRVkOEuSRVkuEtSBRnuklRBhrskVZDhLkkVZLhLUgU1Fe4RsSAiHoqI1RFx7ijt+0fELRFxT0SsjIgTW1+qJKlZDcM9IqYAFwELgUOARRFxyIhuXwCuyswjgFOBi1tdqCSpec2M3I8GVmfmI5n5GnAlcPKIPgnsXlvfA3iqdSVKkrZVM+G+H/BE3fZAbV+9LwFnRMQAcD3wmdEOFBGLI6I/IvoHBwfHUa4kqRnNhHuMsi9HbC8ClmXmHOBE4PsRscWxM3NpZvZlZl9vb++2VytJakoz4T4AzK3bnsOW0y6fAK4CyMzbgBlATysKlCRtu2bC/S7g4Ig4MCKmUd4wXT6iz+PA+wAi4m2UcHfeRZLapGG4Z+YQ8GngJuAByl0xP42Ir0TESbVunwM+GRH3AlcA/y0zR07dSJImydRmOmXm9ZQ3Suv3fbFu/X7gXa0tTZI0Xn5CVZIqyHCXpAoy3CWpggx3Saogw12SKshwl6RJsn49XH45/OQn2/9chrskbWcvvQTf+Aa88Y1wxhmwbNn2P2dT97lLkrbdz38Of/VXcOmlsHYtvPe98J3vwIIF2//chrsktdDQEPzjP8KSJfBP/wRTp8LHPgaf+xwcccTk1WG4S1ILPPIIfO97ZZT+1FMwZw58+ctw1lkwe/bk12O4S9I4rV0L11wD3/0u/Ou/QgT81m/BxRfDBz9YRu3tYrhL0jZ45RW48Ua4+mpYvhxefhkOOgi++lU480yYO7fxMSaD4S5JDfz613DDDSXQr7uuBHpPD5x2Wgn0d72rjNo7ieEuSaN48km46aYS6tdfXwK+t7fcyvjRj8L8+e2ddmmkg0uTpMnzn/8J//7vsGJFCfX77iv7Z88uo/Pf/V34zd/s7ECv1yVlSlJrrV1bwvxHPyrLXXfBhg0wbRq8+93w9a+X+9EPPbTzplyaYbhLqrxMePRRuOOOstx+O9x9d7knfepUOPpoOPfcMtVy3HGwyy7trnjiDHdJlZIJTz8N994L/f0lzO+8EwZrf9V5p53gqKPgj/4I3vMeOP74aoT5SIa7pK718svw8MMlyO+9F1auLI9r1pT2CHjb2+BDHyqj82OOKdMsO+7Y3ronQ1PhHhELgG8DU4BLM/N/jWj/JvCe2ubOwN6ZObOVhUp6/cksI+7HHy+fAF29evPl6ac39d1pJ3j72+GUU+Dww+Gww8rjHnu0r/52ahjuETEFuAj4ADAA3BURy2t/FBuAzDy7rv9ngEn8DQqSutGGDfDcc/DMMzAwAE88sfny+ONl/6uvbv51s2eXDw0tXFgeDzqoBPlBB8GUKe35t3SiZkbuRwOrM/MRgIi4EjgZuH+M/ouAC1pTnqRukFnuPnnhhbK8+OKm9cHBEuAjl+eeg40bNz/OlCklvOfOhb4++MhHyvrcufCmN5VfmVvF+fHtoZlw3w94om57ADhmtI4RcQBwIPAvEy9NUqtt2FBGwiOXdevKh3TWri3z2C+/vPX1l17aFN7DYb5hw9jn3Wkn2Gefshx4IBx77KbtffaB/faD/feHN7yhe+4j73TNXMbR7vDMMfqeCvwgM0f9b46IxcBigP3337+pAtUamVs+jrZvIn1b1We8x8ssI8GNGzdf75TtoaESgENDm6+Ptq+Z9ZH7Xntt9NCu3x4a2vr3yVimTSsj5l13LY977AF77VWmQmbOhFmzylK/Przd01O+rhvvFe9mzYT7AFD/q3DmAE+N0fdU4FNjHSgzlwJLAfr6+sZ6gdiqm26Cs8/uzHDpxHOqe0ydummZMmXL9dH2jVyfNQumT9+0zJix+fZY+6ZP3zy8R66/Hu4uqZpmwv0u4OCIOBB4khLgp43sFBFvAWYBt7W0whF2373cylTOueXjaPvG06fVx+vUc3ZqXeM53g47lPUddti0dMp2xNaDeQf/4KVarGG4Z+ZQRHwauIlyK+RlmfnTiPgK0J+Zy2tdFwFXZm7fMeNxx5VFkjS2pt66yMzrgetH7PviiO0vta4sSdJE+MOgJFWQ4S5JFWS4S1IFGe6SVEGGuyRVkOEuSRVkuEtSBcV2/szR2CeOGAR+Mc4v7wGea2E5k6Hbau62eqH7au62eqH7aq5ivQdkZm+jA7Ut3CciIvozs6/ddWyLbqu52+qF7qu52+qF7qv59Vyv0zKSVEGGuyRVULeG+9J2FzAO3VZzt9UL3Vdzt9UL3Vfz67berpxzlyRtXbeO3CVJW9F14R4RCyLioYhYHRHntruekSJibkTcEhEPRMRPI+IPavv3jIh/joif1R5ntbvWehExJSLuiYjratsHRsQdtXr/LiKmtbvGehExMyJ+EBEP1q71cV1wjc+ufU+siogrImJGJ13niLgsIp6NiFV1+0a9plH8Re15uDIijuygmr9R+75YGRF/HxEz69rOq9X8UET8difUW9f2+YjIiOipbU/oGndVuEfEFOAiYCFwCLAoIg5pb1VbGAI+l5lvA44FPlWr8VxgRWYeDKyobXeSPwAeqNv+GvDNWr0vAJ9oS1Vj+zZwY2a+FTicUnvHXuOI2A/4faAvMw+l/OGbU+ms67wMWDBi31jXdCFwcG1ZDFwySTWOtIwta/5n4NDMPAx4GDgPoPY8PBX4L7WvubiWKZNpGVvWS0TMBT4APF63e2LXODO7ZgGOA26q2z4POK/ddTWo+R9q/2kPAfvW9u0LPNTu2upqnEN54r4XuI7yR9GfA6aOdt3bvQC7A49Se8+obn8nX+P9gCeAPSl/JOc64Lc77ToD84BVja4p8B1g0Wj92l3ziLZTgMtr65vlBeWvyx3XCfUCP6AMUh4Delpxjbtq5M6mJ8iwgdq+jhQR84AjgDuAfTLzaYDa497tq2wL3wL+B7Cxtr0X8GJmDtW2O+06vxEYBL5bm0q6NCJ2oYOvcWY+CVxIGZk9DbwE3E1nX2cY+5p2y3Px94AbausdWXNEnAQ8mZn3jmiaUL3dFu4xyr6OvN0nInYFrgE+m5m/anc9Y4mIDwHPZubd9btH6dpJ13kqcCRwSWYeAbxMB03BjKY2V30ycCAwG9iF8mP3SJ10nbem079HiIjzKdOklw/vGqVbW2uOiJ2B84EvjtY8yr6m6+22cB8A5tZtzwGealMtY4qIHSnBfnlmXlvb/UxE7Ftr3xd4tl31jfAu4KSIeAy4kjI18y1gZkQM/43dTrvOA8BAZt5R2/4BJew79RoDvB94NDMHM3M9cC1wPJ19nWHsa9rRz8WI+DjwIeD0rM1p0Jk1v4nygn9v7Tk4B/hJRLyBCdbbbeF+F3Bw7Q6DaZQ3R5a3uabNREQA/wd4IDP/d13TcuDjtfWPU+bi2y4zz8vMOZk5j3I9/yUzTwduAT5a69Yx9QJk5i+BJyLiLbVd7wPup0Ovcc3jwLERsXPte2S45o69zjVjXdPlwJm1OzqOBV4anr5pt4hYAJwDnJSZv65rWg6cGhHTI+JAyhuVd7ajxmGZeV9m7p2Z82rPwQHgyNr3+MSucTveAJngmxEnUt4B/zlwfrvrGaW+36D86LQS+I/aciJlHnsF8LPa457trnWU2ucD19XW30j5xl8NXA1Mb3d9I2p9B9Bfu84/BGZ1+jUGvgw8CKwCvg9M76TrDFxBeT9gfS1kPjHWNaVMGVxUex7eR7kLqFNqXk2Zqx5+/i2p639+reaHgIWdUO+I9sfY9IbqhK6xn1CVpArqtmkZSVITDHdJqiDDXZIqyHCXpAoy3CWpggx3Saogw12SKshwl6QK+n+OdxibueIQdgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(D_list_cost,color = 'r')\n",
    "plt.plot(G_list_cost,color = 'b')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 209
    },
    "colab_type": "code",
    "id": "Sub4goywbfRI",
    "outputId": "c94ef91e-84fa-48dc-f021-151b415f562b"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPMAAAD7CAYAAAC/paJSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztnV1sldfZpu/XP9v2dvuBbcouboHaUHDBLT9JACEwiUgUBAoTFOmbA6REkTIHJEpyHHHyHUY5mQMUac4yB3OElCEHSDMhfCIQwObHZsClBtcF19BQF9zWxtjG2/aeg51rvfa2s5XXgQJrPbeELNvv373Nez/ruZ9nrRXlcjkZDIZnHyVP+gEMBsOjgb3MBoMnsJfZYPAE9jIbDJ7AXmaDwRPYy2wweAJ7mQ0GT/DYXuYoioan/ZuKomh02vf7oyj6jyiKsgXH/XPa+f8liqL/F0XRUBRF96Io+s8oin4VRdH/mHb8eME1/s/j4hMavxA4escvl8s99n+SeiW9XPCz/5D0v77n+JWSBiXtlBRJ+qmkNyQt+6HX+Ff+851fCBx94FeW/PX/l2C9pJu5XO4/v/v+vqTPn+DzPGr4zk/yn+NTx+9pzZk7JDVFUfTfoyh6KYqinzzpB3rE8J2f5D/Hp47fk36Z/z2Kon9O+3dCknK53A1JL0r6haTDku5FUfQ/n4YPLCF85yf5z/GZ4fekX+bDuVxu4bR/L/GLXC7Xlsvl/j2Xy/1M0nZJLZIOPrEnnR985yf5z/GZ4fekX+YfhFwud0HS/5bU/KSf5XHAd36S/xyfBn5P5cscRdG2KIr+WxRFi7/7vknSXkltT/bJHg185yf5z/Fp5PekX+b/WlDDG/7uw/mn8h9MZxRFw5L+r6Qjkj55kg87D/jOT/Kf4zPDL/quFmYwGJ5xPOnIbDAYHhHsZTYYPIG9zAaDJ7CX2WDwBPYyGwyeINFEi0OHDuUkqbKyUpJUXl4uSfrnP/OzwjKZjPr7+yVJU1NTM46pqqqSJGWzWUnS3//+d0lSTU2NJDHDRNlsVhUVFTPOLSnJa87ExIQk6R//+IckacGCBZKk+vp6SVJvb68kKZVKuWceGRmRJE1OTkqSxsfHZ3A6ePBgFAq/EDj6zq8YEr3MZWX5w//6179KkhobGyVJDx48kJQnyYd4//59SdLPfvYzSdKf//znWSQk6eHDh+5cSSotLdXg4KAkadmyZZKkmzdvSor/QD/5Sb799c6dO5Kk4eFhSVJ1dfWMa01OTroPc2BgYMb9+GOExC8Ejr7zKwYbZhsMniBRZI6ifLRnSPHHP/5RUqw23d3d+uUvfylJGhsbkyR9++23kqTf/e53kqQLFy5IitWQIQzDkq6uLve7W7duSZJGR0dnnFOoijdu3JAUqyKqV15e7tQOxUapUeGQ+IXA0Xd+xWCR2WDwBIkiM2rz05/+VFKch/B9eXm5S+JRxt/85jeSYgNg06ZNkmJFW7RokSTpb3/7mySpublZ3d3dkmLD4d/+7d8kxUqKUUFe8utf/1pSrHZ9fX2SpNraWqd2/O727duS5NQ5JH4hcPSdXzFYZDYYPEGiyExusWrVKkmxstXV1UnKK8rChQsl5UsAkrRu3boZx6JUTU1NkuL8ZO3atZKknp4e/fa3v5UkNTQ0SIrtfZSLn6NuXPvEiROSYpUsKytTaWmppNih5Ovvf//74PiFwNF3fsWQ6GVevHixpNl1OZL7KIpcgr9ixQpJMTl+/otf/EKS9Je//EVS/IFiHFRVVbmywp/+9CdJsfHAMIihFMZEV1eXpHgo09PT446nhkcZAru/uXn2HHLf+YXA0Xd+xWDDbIPBEySKzHSmUADne7peFi9erF/96leSZnfgYN1jPlAyQJXoyunp6XFGw9DQkKS8SSDF6kfnDgZBOp2WFA+tUMf+/n53fZ4HdaUjKCR+IXD0nV8xWGQ2GDxBosjM+B8rHVVCBSsrK13v6Zo1a2acS9GeXOH48eOS4tyCaw4MDDhFojSwe/duSbGCYhBQBli5cqWkuA0OZUun0071yH9Q1Lla5XznFwJH3/kVg0Vmg8ETJIrM2PsUynHfyEFqamq0fPlySbPzEYr3586dkyQtWbIk/wDfqR3fnzlzxp2zdetWSbHtzzHkMpyLspGHUFoYGRnR3bt3JcVKyDOjkCHxC4Gj7/yKwSKzweAJEkVmFIP8A4VDDTOZjHMCyV04tr29XVKcQ5Cf4PKBNWvWuAI7uQpqx/xSvnIvXEja7lDDdDrt3ETa7/gd7mZI/ELg6Du/YrDIbDB4gkSRmTob7W+0qqEsk5OTruNmuvPH76R4ihouICpIXa6mpsZdg/uQbxw5ckRSXH+jUwfVxf2jqf7bb791biZdPlw7RH4hcPSdXzFYZDYYPEGiyLxx40ZJcZ8pucQf/vAHdwwN5igVakbOgGJRy9uzZ4+k2FkcHh7Wli1bJMWTs+l/xd1DKenDBYXT0LLZrDuH63P/wqVhQuAXAkff+RWDRWaDwRMkisyoDzUywBSuwcFBlxNwzL179yTFqxKiZHS7kONwXnt7u1sE7fTp05KkS5cuSYqdyv3790uK+2FxCFE76nb37993Sn348GFJseuIUxkSvxA4+s6vGCwyGwye4Ed1gOEUkgcsX77c1ddwBFEkFIochg6ao0ePzrhHU1OTzp49K0m6fPmypNhlXL9+ff6hv3MhcQqpD/785z+XFM9cmX4sc1JxLueq4fnOLwSOvvMrhkQvMwYBVjrDDz6U/v5+7dq1S1K84gPDG6z4DRs2SIrXE6apHFy7ds1Z8xgTFP6ZTI6pwAeFUcFzsZJEOp12RgjXou2O1r2Q+IXA0Xd+xWDDbIPBEySKzCgUCoIK0dK2c+dOpyqFwxyGNhcvXpQUN6ifOnVKUl7tpHwjOkMlJnIz/KH9bunSpZLiIRVN7VevXpUktbS0SMqrMc+B8cA5DK1C4hcCR9/5FYNFZoPBEySKzFjmJO+oD+P/hw8fugI4U70Klz7Zu3evpFj9sOVRo6qqKncNcheutXr1akmxQYCpgLmBwtEEPzY2NmuCN7kOuVRI/ELg6Du/YrDIbDB4gnntaFHYkM7yoWNjY86Bw25n3E+jO0ueMlkchWMd4q6urll5z+bNmyXF+RC5Bc4li7axMwH3Li8vd44kNj/T3Sjah8QvBI6+8ysGi8wGgydIFJkBSsaeONPdQXIDiuPsW7tjxw5J8TQvFIzchvNSqZRrs8MZRKE4FteRNrxt27bNeD7a8Do7O10+Qu5Eu12xBcZ95xcCR9/5zQWLzAaDJ0gUmWn8Ji9BSVCyRYsWOSeOnIU8gM4XHMHCpUjJKerr6109D8U6efKkpHj/XJSLrhsmiKOK1AXr6+td8zzPwfNxjZD4hcDRd37FYJHZYPAE8+oAIx8hT6DvdHR0dNZi37iJ5Bj0rJLLoFTU31pbW93ud2zOhVLSPcP9WcS8sHMGNe7o6HBN8zwHik3dMSR+IXD0nV8xWGQ2GDxBosiMmuDqoUosd3Lnzp1ZG2nh9lG7Q+VwEOm+oV734MEDNzmcnlmWbUEZqd2R05BroKw8TyaTcdPYcBU5h/wnJH4hcPSdXzHMqzT1Q8Dq/FJ+raVUKqWSkhLlcjm9//77OnPmjNra2lxbWxRFiqJIr7zyiqQ8+YmJCU1NTemLL77QwoUL9eKLL+rKlStuNYlPP/1UuVzOkX/hhRf02WefPS5KQfELgaNv/OblZpNrMNZHjRYsWOBIUEebmJjQhg0b9Mknn6ilpcXlNN98841WrVrl3D/yhWPHjqm0tFQjIyNat26dFi5cqO3bt6unp0evv/66Dhw44O73+eefq6+vT++++66kfG2vra3NKdrY2JhbtI2J3ziYKGNI/ELg6Du/YnhskfnHYHJyUiUlJW74U1FRobVr1zqyzzp85yf5z/Fp5JfoZSZnIIegxxX16+vrc502ra2tkvJu4ujoqO7evatbt245xaypqdHExIRefvllSfHi4ZlMRlEU6dq1axoeHtbKlSu1fft2pdNp5wReuXJFUj7HGBoacis14CDS69rc3OzcTJ610OUMiV8IHH3nVwyJXmZa1CjAYyBQ3K6rq3NDGIrpvb29Gh8fV0dHhwYHB51BUVpaqkuXLunDDz+UlB/CNDQ06MCBA0qlUrpx44YuX76sL7/8UkeOHNEbb7yh9957T+l02pUZ7t27p8HBQde0zrCJont7e7tbtoWlYeAw15rEvvMLgaPv/IrhiQ6zn3vuOZdLsP4Stb3Gxka9+eabkvIfyNtvv62Kigp98MEHT+Zh5wHf+Un+c3yW+CV6mVE7TAaGAQwtpFgJmVi9ZMkSlZaWqqysTKlUyg1xMpmMstmsXnrpJUmxLV9SUuKGJOfPn5ckPf/882ppaVF3d7dGRkacyk1NTWlqaspNGUNpMRDq6urcfkHcl+HXXAV53/mFwNF3fsXwVDaN9Pb26sKFC64eePPmTX399ddau3btE36yRwPf+Un+c3wa+c0rMlMAxzrn68DAwIxpYlK+SF5SUqJUKqXKykpdv37dHXv8+HG3IBq2/0cffaSqqip1dXXpq6++0vj4uKqrq7Vu3Tq99tprGhoaciZHRUWFUqmUUzAK8uREdXV1bpobHzrPSu0wJH4hcPSdXzH8S3Lmjz/+2NXSwL59+7Rv3z437GAY1NnZqVQqpbfeest92HwwDF3A7t27H/ej/yD4zk/yn6MP/BK9zKgKCkUOQWN6bW2tW1IU4rTGURrgK0Vz8gV26WtoaJi16DhN6p2dnZJm7m0rxQqLY8lzVldXu7Y7npFrFf7hQuAXAkff+RXDU5kzGwyG5IhoOzMYDM82LDIbDJ7AXmaDwRMkMsAOHTqUk2LrnHmYFLkzmYwrxOPucQzta5QOaFmjUZ3hfjabde4h52LRU27AsKDIzm4CrKU0vQ0OE4EifeEc0YMHD0ah8AuBo+/8iiHRy4ztzoTvxsZGSXGDeDabdR8irh1zRnHsCvtNaWrnAywtLXXLpixbtkxSPEmca+ME0qxOdw1dP1xrcnLSfZhsF8L9CrcECYFfCBx951cMNsw2GDxBoshM7Y4hBfU61Ka7u9tt8UGtjDobE7wvXLggKVZDhjAMS7q6utzvqO+xBAs/L1RFlntBFVG98vJyp3YoNkqNCofELwSOvvMrBovMBoMnmNfGcXS3kIfwfXl5uUviUUY2ysIA2LRpk6RY0ZiITUdNc3Ozuru7JcWGA720KClGBXkJm4KhdiyyVltb69SO3zF9DXUOiV8IHH3nVwwWmQ0GT5AoMpNbMIsEZaPH9fbt2252CGshsS0Hx6JUTU1NkuL8hKljPT09buvMhoYGSbG9j3Lxc9SNa584cUJSrJJlZWVu9QYcSr6yiHlI/ELg6Du/Ykj0MtMIXliXI7mPosgl+CtWrJAUk+PnLJHC8il8oBgHVVVVrqzANDGMB4ZBDKUwJpgozlCGJVvq6+tdDY8yBHb/XDvs+c4vBI6+8ysGG2YbDJ4gUWSmM4UCON/T9bJ48WK323xhBw7WPeYDJQNUia6cnp4eZzQU7jyA+tG5g0HAlDWGVqhjf3+/uz7Pg7rSERQSvxA4+s6vGCwyGwyeIFFkZvyPlY4qoYKVlZWu95Td7wBFe3KF48ePS4pzC645MDDgFInSAKs1oKCF++YW7uqHsqXTaad65D8o6lytcr7zC4Gj7/yKwSKzweAJEkVm7H0K5bhv5CA1NTVup/jCfITi/blz5yTllzeVYrXj+zNnzrhztm7dKim2/TmGXIZzUTbyEEoLIyMjbqlVlJBnRiFD4hcCR9/5FYNFZoPBEySKzCgG+QcKhxpmMhnnBJK7cGx7e7ukOIcgP8HlA2vWrHEFdnIV1I75pXzlXriQtN2hhul02rmJtN/xO9zNkPiFwNF3fsVgkdlg8ASJIjN1NtrfaFVDWSYnJ13HzXTnj99J8RQ1XEBUkLpcTU2Nuwb3Id9gFz6cRDp1UF3cv+nLnOJm0uXDtUPkFwJH3/kVg0Vmg8ETJIrMGzdulBT3mZJLsDi4FDeYo1SoGTkDikUtb8+ePZJiZ3F4eFhbtmyRFE/Opv8Vdw+lpA8XFE5Dy2az7hyuz/3n2i7Td34hcPSdXzFYZDYYPEGiyIz6UCMDTOEaHBx0OQHHsD0IXTcoGd0u5Dic197e7hZBO336tCTp0qVLkmKncv/+/ZLiflgcQtSOut39+/edUh8+fFhS7DriVIbELwSOvvMrBovMBoMn+FEdYDiF5AHLly939TUcQRQJhSKHoYPm6NGjM+7R1NSks2fPSpIuX74sKXYZ169fn3/o71xInELqg2yNycyV6ccyJxXncq4anu/8QuDoO79iSPQyYxBgpTP84EPp7+/Xrl27JMUrPjC8wYrfsGGDpHg94cLd9K5du+aseYwJCv9MJsdU4IPCqOC5WEkinU47I4Rr0XZH615I/ELg6Du/YrBhtsHgCRJFZhQKBUGFaGnbuXOnU5XCYQ5Dm4sXL0qKG9RPnTolKa92Ur4RnaESE7kZ/tB+t3TpUknxkIqm9qtXr0qSWlpaJOXVmOfAeOAchlYh8QuBo+/8isEis8HgCRJFZixzknfUh/H/w4cPXQGcqV6FS5/s3btXUqx+2PKoUVVVlbsGuQvXWr16taTYIMBUwNxA4WiCHxsbmzXBm1yHXCokfiFw9J1fMVhkNhg8wbx2tChsSGf50LGxMefAYbcz7qfRnSVPmSyOwrEOcVdX16y8Z/PmzZLifIjcAueSRdvYmYB7l5eXO0cSm5/pbhTtQ+IXAkff+RWDRWaDwRMkiswAJWNPnOnuILkBxXH2rd2xY4ekeJoXCkZuw3mpVMq12eEMolAci+tIG962bdtmPB9teJ2dnS4fIXei3a7YAuO+8wuBo+/85oJFZoPBEySKzDR+k5egJCjZokWLnBNHzkIeQOcLjmDhUqTkFPX19a6eh2KdPHlSUrx/LspF1w0TxFFF6oL19fWueZ7n4Pm4Rkj8QuDoO79isMhsMHiCeXWAkY+QJ9B3Ojo6Omuxb9xEcgx6VsllUCrqb62trW73OzbnQinpnuH+LGJe2DmDGnd0dLimeZ4DxabuGBK/EDj6zq8YLDIbDJ4gUWRGTXD1UCWWO7lz586sjbRw+6jdoXI4iHTfUK978OCBmxxOzyzLtqCM1O7Iacg1UFaeJ5PJuGlsuIqcQ/4TEr8QOPrOrxjmVZr6IWB1fim/1lIqlVJJSYlyuZzef/99nTlzRm1tba6tLYoiRVGkV155RVKe/MTEhKampvTFF19o4cKFevHFF3XlyhW3msSnn36qXC7nyL/wwgv67LPPHheloPiFwNE3fvNys8k1GOujRgsWLHAkqKNNTExow4YN+uSTT9TS0uJymm+++UarVq1y7h/5wrFjx1RaWqqRkRGtW7dOCxcu1Pbt29XT06PXX39dBw4ccPf7/PPP1dfXp3fffVdSvrbX1tbmFG1sbMwt2sbEbxxMlDEkfiFw9J1fMTy2yPxjMDk5qZKSEjf8qaio0Nq1ax3ZZx2+85P85/g08kv0MpMzkEPQ44r69fX1uU6b1tZWSXk3cXR0VHfv3tWtW7ecYtbU1GhiYkIvv/yypHjx8EwmoyiKdO3aNQ0PD2vlypXavn270um0cwKvXLkiKZ9jDA0NuZUacBDpdW1ubnZuJs9a6HKGxC8Ejr7zK4ZELzMtahTgMRAobtfV1bkhDMX03t5ejY+Pq6OjQ4ODg86gKC0t1aVLl/Thhx9Kyg9hGhoadODAAaVSKd24cUOXL1/Wl19+qSNHjuiNN97Qe++9p3Q67coM9+7d0+DgoGtaZ9hE0b29vd0t28LSMHCYa01i3/mFwNF3fsXwRIfZzz33nMslWH+J2l5jY6PefPNNSfkP5O2331ZFRYU++OCDJ/Ow84Dv/CT/OT5L/BK9zKgdJgPDAIYWUqyETKxesmSJSktLVVZWplQq5YY4mUxG2WxWL730kqTYli8pKXFDkvPnz0uSnn/+ebW0tKi7u1sjIyNO5aampjQ1NeWmjKG0GAh1dXVuvyDuy/BrroK87/xC4Og7v2J4KptGent7deHCBVcPvHnzpr7++mutXbv2CT/Zo4Hv/CT/OT6N/OYVmSmAY53zdWBgYMY0MSlfJC8pKVEqlVJlZaWuX7/ujj1+/LhbEA3b/6OPPlJVVZW6urr01VdfaXx8XNXV1Vq3bp1ee+01DQ0NOZOjoqJCqVTKKRgFeXKiuro6N82ND51npXYYEr8QOPrOrxj+JTnzxx9/7GppYN++fdq3b58bdjAM6uzsVCqV0ltvveU+bD4Yhi5g9+7dj/vRfxB85yf5z9EHfoleZlQFhSKHoDG9trbWLSkKcVrjKA3wlaI5+QK79DU0NMxadJwm9c7OTkkz97aVYoXFseQ5q6urXdsdz8i1Cv9wIfALgaPv/IrhqcyZDQZDckS0nRkMhmcbFpkNBk9gL7PB4AkSGWCHDh3KSbF1zjxMityZTMYV4nH3OIb2NUoHtKzRqM5wP5vNOveQc7HoKTdgWFBkZzcB1lKa3gaHiUCRvnCO6MGDB6NQ+IXA0Xd+xZDoZcZ2Z8J3Y2OjpLhBPJvNug8R1445ozh2hf2mNLXzAZaWlrplU5YtWyYpniTOtXECaVanu4auH641OTnpPky2C+F+hVuChMAvBI6+8ysGG2YbDJ4gUWSmdseQgnodatPd3e22+KBWRp2NCd4XLlyQFKshQxiGJV1dXe531PdYgoWfF6oiy72giqheeXm5UzsUG6VGhUPiFwJH3/kVg0Vmg8ETzGvjOLpbyEP4vry83CXxKCMbZWEAbNq0SVKsaEzEpqOmublZ3d3dkmLDgV5alBSjgryETcFQOxZZq62tdWrH75i+hjqHxC8Ejr7zKwaLzAaDJ0gUmcktmEWCstHjevv2bTc7hLWQ2JaDY1GqpqYmSXF+wtSxnp4et3VmQ0ODpNjeR7n4OerGtU+cOCEpVsmysjK3egMOJV9ZxDwkfiFw9J1fMSR6mWkEL6zLkdxHUeQS/BUrVkiKyfFzlkhh+RQ+UIyDqqoqV1ZgmhjGA8MghlIYE0wUZyjDki319fWuhkcZArt/rh32fOcXAkff+RWDDbMNBk+QKDLTmUIBnO/pelm8eLHbbb6wAwfrHvOBkgGqRFdOT0+PMxoKdx5A/ejcwSBgyhpDK9Sxv7/fXZ/nQV3pCAqJXwgcfedXDBaZDQZPkCgyM/7HSkeVUMHKykrXe8rud4CiPbnC8ePHJcW5BdccGBhwikRpgNUaUNDCfXMLd/VD2dLptFM98h8Uda5WOd/5hcDRd37FYJHZYPAEiSIz9j6Fctw3cpCamhq3U3xhPkLx/ty5c5Lyy5tKsdrx/ZkzZ9w5W7dulRTb/hxDLsO5KBt5CKWFkZERt9QqSsgzo5Ah8QuBo+/8isEis8HgCRJFZhSD/AOFQw0zmYxzAsldOLa9vV1SnEOQn+DygTVr1rgCO7kKasf8Ur5yL1xI2u5Qw3Q67dxE2u/4He5mSPxC4Og7v2KwyGwweIJEkZk6G+1vtKqhLJOTk67jZrrzx++keIoaLiAqSF2upqbGXYP7kG+wCx9OIp06qC7u3/RlTnEz6fLh2iHyC4Gj7/yKwSKzweAJEkXmjRs3Sor7TMklWBxcihvMUSrUjJwBxaKWt2fPHkmxszg8PKwtW7ZIiidn0/+Ku4dS0ocLCqehZbNZdw7X5/5zbZfpO78QOPrOrxgsMhsMniBRZEZ9qJEBpnANDg66nIBj2B6ErhuUjG4XchzOa29vd4ugnT59WpJ06dIlSbFTuX//fklxPywOIWpH3e7+/ftOqQ8fPiwpdh1xKkPiFwJH3/kVg0Vmg8ET/KgOMJxC8oDly5e7+hqOIIqEQpHD0EFz9OjRGfdoamrS2bNnJUmXL1+WFLuM69evzz/0dy4kTiH1QbbGZObK9GOZk4pzOVcNz3d+IXD0nV8xJHqZMQiw0hl+8KH09/dr165dkuIVHxjeYMVv2LBBUryecOFueteuXXPWPMYEhX8mk2Mq8EFhVPBcrCSRTqedEcK1aLujdS8kfiFw9J1fMdgw22DwBIkiMwqFgqBCtLTt3LnTqUrhMIehzcWLFyXFDeqnTp2SlFc7Kd+IzlCJidwMf2i/W7p0qaR4SEVT+9WrVyVJLS0tkvJqzHNgPHAOQ6uQ+IXA0Xd+xWCR2WDwBIkiM5Y5yTvqw/j/4cOHrgDOVK/CpU/27t0rKVY/bHnUqKqqyl2D3IVrrV69WlJsEGAqYG6gcDTBj42NzZrgTa5DLhUSvxA4+s6vGCwyGwyeYF47WhQ2pLN86NjYmHPgsNsZ99PozpKnTBZH4ViHuKura1bes3nzZklxPkRugXPJom3sTMC9y8vLnSOJzc90N4r2IfELgaPv/IrBIrPB4AkSRWaAkrEnznR3kNyA4jj71u7YsUNSPM0LBSO34bxUKuXa7HAGUSiOxXWkDW/btm0zno82vM7OTpePkDvRbldsgXHf+YXA0Xd+c8Eis8HgCRJFZhq/yUtQEpRs0aJFzokjZyEPoPMFR7BwKVJyivr6elfPQ7FOnjwpKd4/F+Wi64YJ4qgidcH6+nrXPM9z8HxcIyR+IXD0nV8xWGQ2GDzBvDrAyEfIE+g7HR0dnbXYN24iOQY9q+QyKBX1t9bWVrf7HZtzoZR0z3B/FjEv7JxBjTs6OlzTPM+BYlN3DIlfCBx951cMFpkNBk+QKDKjJrh6qBLLndy5c2fWRlq4fdTuUDkcRLpvqNc9ePDATQ6nZ5ZlW1BGanfkNOQaKCvPk8lk3DQ2XEXOIf8JiV8IHH3nVwzzKk39ELA6v5RfaymVSqmkpES5XE7vv/++zpw5o7a2NtfWFkWRoijSK6+8IilPfmJiQlNTU/riiy+0cOFCvfjii7py5YpbTeLTTz9VLpdz5F944QV99tlnj4tSUPxC4egT5uVmk2sw1keNFixY4P5I1NEmJia0YcMGffLJJ2ppaXE5zTfffKNVq1Y594984dixYyotLdXIyIjWrVunhQsXavv27erp6dHrr7+uAwcOuPt9/vnn6uvr07vvvispX9tra2tzijY2NuYWbWPmQ7NvAAAMUElEQVTiNw4myhgSvxA4JuFXuLQQc4+JlERT7k+0PXv2rBMfFiOgO4x6MvejVxzHuq2tTZJ+1N/w+/DYIvOPweTkpEpKStzwp6KiQmvXrnVkn3X4zk8Kg+PThkQvMzkDOQQ9rqhfX1+f67RpbW2VlHcTR0dHdffuXd26dcspZk1NjSYmJvTyyy9LihcPz2QyiqJI165d0/DwsFauXKnt27crnU47J/DKlSuS8jnG0NCQW6kBB5Fe1+bmZudm8qyFLmdI/ELgOF9+Ujw3GX6NjY2S4n7q6fyImvRkv/rqq5I0ix/5Ofn4o/gbfh8Svcy0qFGAx0CguF1XV+eGMBTTe3t7NT4+ro6ODg0ODjqDorS0VJcuXdKHH34oKT9Ea2ho0IEDB5RKpXTjxg1dvnxZX375pY4cOaI33nhD7733ntLptCsz3Lt3T4ODg65pnWEhRff29nY3dGJpGDjMtSax7/xC4DhfflK8Kif8uD5f33nnHUn5khU/Y1hNcwjXhh9DZsTkUfwNvw9PdJj93HPPuVyJ9Zeo7TU2NurNN9+UlP+Dv/3226qoqNAHH3zwZB52HvCdnxQGx2cFiV5m1A6TgWEAQwspVkImVi9ZskSlpaUqKytTKpVyQ5xMJqNsNquXXnpJUmzLl5SUuCHX+fPnJUnPP/+8Wlpa1N3drZGREafiU1NTmpqaclPGUFrUsK6uzu0XxH0Zfs1VkPedXwgc58tv+rnch4kYLDxQjB/De8ws+NEs8ij/ht+Hp7JppLe3VxcuXHD5xs2bN/X1119r7dq1T/jJHg185yeFwfFpw7wiM5Y9JgBfBwYGZkwTk/L2f0lJiVKplCorK3X9+nV37PHjx53pQFnjo48+UlVVlbq6uvTVV19pfHxc1dXVWrdunV577TUNDQ05FayoqFAqlXIKRqmBvKWurs6pK/+peFZqoyHxC4HjfPlJcVSFH/cjlya637x503Elih47dkxSPG2xcNlcIvWj+Bt+H/4lOfPHH3/sPiiwb98+7du3zw07+KA6OzuVSqX01ltvuQ+bD4YPBOzevftxP/oPgu/8pDA4PutI9DKjKqgSOQSN6bW1tW5JUf6wtMbh5vGVojn5Arv0NTQ0zFp0nLyjs7NT0sy9baVYYXEsec7q6mqnqjwj1yr8jxkCvxA4+s6vGJ7KnNlgMCRHRNuZwWB4tmGR2WDwBPYyGwyeIJEBdujQoZwUW+fMLsGez2QyrhCPe8kxtK9ROqBljUZ8hvvZbNa5o5yLRU+5AcMC25+iPm1509vgMBEo0hfOET148GAUCr9QOIaKRC8zZQUmfNOIToN4Npt1/0lw7ZgTi2NX2G9KUzv/QUpLS92yKcuWLZMUTxLn2jiBNKvTXUPXD9eanJx0/1nYLoT7FW4JEgK/UDiGChtmGwyeIFFkpnbHkIl6HWra3d3ttvigVkadjQnsFy5ckBSrPUM0hl1dXV3ud9T3mBTOzwtVn+llqD6qXl5e7tSciEQkIsqExC8UjqHCIrPB4AnmtXEc3S3kWXxfXl7uTAqUn4ndGBybNm2SFCs2E7HpqGlublZ3d7ek2FChl5ZIgRFD3sWmYKg5i6zV1tY6Ned3TM8j+oTELxSOocIis8HgCRJFZnInZsmg3PS43r59281+Ya0ntuXgWJS4qalJUpx/MTWup6fHLY7W0NAgKS5foMz8HPXm2idOnJAUR4GysjK3ekPhqhEsYh4Sv1A4hopELzON4IV1R8yLKIqcgbFixQpJ8R+Pn7NECsun8B8GY6SqqsqVTZgmhrHCMI+hIsYLE8UZqrFkS319vatRUmahnDHXDnu+8wuFY6iwYbbB4AkSRWY6byjw8z1dPYsXL3a7zRd2GFGawFyhJILq0nXU09PjjJTCnQdQdzqTMECYssbQEfXv7+931+d5iB50PIXELxSOocIis8HgCRJFZvIbSgWoLipfWVnpemvZ/Q7QlEAudPz4cUlx7sQ1BwYGnOJS+mA1CiJE4b65hbv6odzpdNqpOvkdEWOuVkDf+YXCMVRYZDYYPEGiyEz5gkYA3EVyrJqaGrcYeGG+RXPCuXPnJMXLm6LmfH/mzBl3ztatWyXFZQ2OIVfjXJSbPGv6Am0stYrS88xEgJD4hcIxVFhkNhg8QaLIjCKSX6HgqH0mk3FOJ7kZx7a3t0uKcyTyL1xMsGbNGtdAQC6GmjN/lq/cC5eVtkLUPp1OO7eU9kJ+h3sbEr9QOIYKi8wGgydIFJmpI9LeRyseyjk5Oek6iqY7m/xOiqfg4XKi8tQda2pq3DW4D/kUu/DhlNKJRFTB3Zy+zCluLV1MXDtEfqFwDBUWmQ0GT5AoMm/cuFFS3EdLrsTi4FLcQI8So9bkRCgytco9e/ZIip3T4eFhbdmyRVI8+Zz+XtxLIgF9xqBwml02m3XncH3uP9d2mb7zC4VjqLDIbDB4gkSRGXWlBgiYojY4OOhyHo5he5DCDa3p5iGH47z29na3yNvp06clSZcuXZIUO7H79++XFPf74oCi5tQl79+/7yLR4cOHJcWuKk5sSPxC4RgqLDIbDJ7gR3WA4YSS5yxfvtzVD3E8UVwUmByNDqGjR4/OuEdTU5POnj0rSbp8+bKk2EVdv359/qG/c1lxQql/sjUmM3OmH8ucW5zZuWqUvvMLhWOoSPQyY4BQKmB4xR+9v79fu3btkhSvaMHwjVLDhg0bJMXrJRfupnft2jVXesB4obGByfKYJvxHwIjhuVgpI51OO6OncL9cWhND4hcKx1Bhw2yDwRMkiswoMAqJytKyt3PnTqeahcM4hm4XL16UFDfgnzp1SlJezaV8oz1DQSaqM7yjvXDp0qWS4iEjTftXr16VJLW0tEjKRxueA2OFcxg6hsQvFI6hwiKzweAJEkVmSgKYE6gr+c3Dhw9dgZ+pbIVLu+zdu1dSrO6UHVDbqqoqdw1yM661evVqSbEBgmmCeYOC0+Q/NjY2awI7uRy5Ykj8QuEYKiwyGwyeYF47WhQ23LM86tjYmHMYKSeQ19DIz5KuTIZHwVlnuaura1Zet3nzZklxvkfuhDPLonTsvMC9y8vLneNKGYPpfDQlhMQvFI6hwiKzweAJEkVmgFKz589095Pch+I/+/Lu2LFDUjyNDYUmd+O8VCrl2ghxPlFgjsVVpc1w27ZtM56PNsPOzk6Xb5Eb0k5YbAF13/mFwjE0WGQ2GDxBoshMYzt5F0qJUi9atMg5jeRk5Dl09uB4Fi61Ss5UX1/v6pUo8smTJyXF+wOjzHQVMQEe1afuWV9f7yYH8Bw8H9cIiV8oHEOFRWaDwRPMqwOMfIs8iL7a0dHRWYuZ45aSQ9GTS66GElNfbG1tdbv7sfkYkYDuIO7PIu2FnUFEm46ODjcpgOcgIlFXDYlfKBxDhUVmg8ETJIrMqCWuJarLci537tyZtVEYbia1SVQch5TuIuqRDx48cJPf6QlmWRqUn9okORu5FJGD58lkMm6aHq4p55DfhcQvFI6hwiKzweAJ5uVmk0uRy6C2CxYs+N5laZi3isqixCwah1KfPXvWKS8T2eksohbJ/egzxu1sa2uTFCv22NiYuz4T23FoUf6Q+IXCMVRYZDYYPEGiyExORI5EDy/q3tfX5zqJWltbJcVuKfNaiQiNjY2S4l5cFkfPZDJOcennffXVVyXFTueVK1ckxXVPcjkcUnp5m5ub3f151kIXNyR+oXAMFYleZlrwaDDAIKF4X1dX54ZoNAsUruiIAcMwjK/vvPOOpHy5g58xJKOxgGtTRmG4xX9EWgBpKmhvb3dDQ5a+gcNcay77zi8UjqHChtkGgydIFJlRc0wUhjkMnaRY6Zk4jsHBuQzhaOJn0jplh5KSEreA2/nz5yXFQ0OMEKbH0WjAlDgiCWpfV1fn9kPivgwv52o48J1fKBxDhUVmg8ETzCsyU5LA5ODrwMDAjGlwUlzeQJGvX78uKTY+yMOIDDdv3nRteyjwsWPHJMVT3gqXXEXluRd5WV1dnYse3I9nZVmakPiFwjFU2KdhMHiCRJEZ1UR1yZFovK+trXVLpqLStP7hVvKVpgDyIXYhbGhomLWoOnlVZ2enpJl790pxBMGR5Tmrq6td1OAZuRZRJiR+oXAMFRaZDQZPENFWZzAYnm1YZDYYPIG9zAaDJ7CX2WDwBPYyGwyewF5mg8ET2MtsMHgCe5kNBk9gL7PB4AnsZTYYPIG9zAaDJ/j/m1kSfOIMfVIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 15 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "l=[]\n",
    "for i in range(15):\n",
    "    Z = np.random.normal(0., 1., size=[1, G_input])\n",
    "    Gl1 = Z.dot(G_W1) + G_b1\n",
    "    Gl1A = Lrelu(Gl1)\n",
    "    Gl2 = Gl1A.dot(G_W2) + G_b2\n",
    "    Gl2A = Lrelu(Gl2)\n",
    "    Gl3 = Gl2A.dot(G_W3) + G_b3\n",
    "    Gl3A = Lrelu(Gl3)\n",
    "    Gl4 = Gl3A.dot(G_W4) + G_b4\n",
    "    Gl4A = Lrelu(Gl4)\n",
    "    Gl5 = Gl4A.dot(G_W5) + G_b5\n",
    "    Gl5A = Lrelu(Gl5)\n",
    "    Gl6 = Gl5A.dot(G_W6) + G_b6\n",
    "    Gl6A = Lrelu(Gl6)\n",
    "    Gl7 = Gl6A.dot(G_W7) + G_b7\n",
    "    Gl7A = Lrelu(Gl7)\n",
    "    Gl8 = Gl7A.dot(G_W8) + G_b8\n",
    "    Gl8A = Lrelu(Gl8)\n",
    "    Gl9 = Gl8A.dot(G_W9) + G_b9\n",
    "    Gl9A = Lrelu(Gl9)\n",
    "    Gl10 = Gl9A.dot(G_W10) + G_b10\n",
    "    Gl10A = Lrelu(Gl10)\n",
    "    Gl11 = Gl10A.dot(G_W11) + G_b11\n",
    "    \n",
    "    current_fake_data = log(Gl11)\n",
    "    l.append(current_fake_data)\n",
    "fig = plot(l,\"TEST\")\n",
    "fig.savefig('Click_Me_{}.png'.format(str(iter).zfill(3)+\"_Ginput_\"+str(G_input)+ \"_hiddenone\"+str(hidden_input) + \"_hiddentwo\"+str(hidden_input2) + \"_LR_\" + str(learing_rate)), bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 36
    },
    "colab_type": "code",
    "id": "FnpFQdTpbfRS",
    "outputId": "30ca1f5c-f62d-435c-e4b8-9dfa2ea92a54"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEkAAABWCAYAAACD87uSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAAfhJREFUeJzt3DFLW1EchvHnlY4VXOxSKA4OGRz6AdpSqbMdBB0691O4OLt07ejQSWg76iJdhYII7rYfwCHaQKEF/w5GuASqr9hwjLwPnOEkJzcnPy5JboaoqkjXN9V6A5NQkIyCZBQkoyAZBckoSEZjQ5I06IxzSb8783eSNiT9HVnX7zz+raRDSWeSTiTtSZqT9LGz/s/IMXbG8mKqauwD+Aksjdy2AXz6x/p54BR4AwiYBlaAZ+4x/ud4NBb5u/cc+FFVe8P5L+Bzq83c1/ekA6An6YOkRUmPW26mNdKqpH5nfAOoqmPgNfAU2AZOJG21wmqNtF1VM52xeHVHVe1X1WpVzQIvgVfAeotNtkayqqrvwBdgocXz30skSS8kvZf0ZDjvAcvAfov9tEZaG/meNBjC9LlEOZI0AHaBr8Bmi00qP7rdXOszaSIKklGQjIJkFCSjW13gSnpwH4VVpZvW5EwyCpJRkIyCZBQkoyAZBckoSEZBMgqSUZCMgmQUJKMgGQXJKEhGQTIKklGQjIJkFCSjIBkFyShIRkEyCpJRkIyCZBQkoyAZBckoSEZBMgqSUZCMgmQUJKMgGQXJKEhGQTIKklGQjIJkFCSjIBkFyShIRkEyCpJRkIzyDxNGOZOMgmQUJKMgGQXJKEhGQTIKklGQjIJkdAFRbBioA9j9WQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "Z = np.random.normal(0., 1., size=[1, G_input])\n",
    "Gl1 = Z.dot(G_W1) + G_b1\n",
    "Gl1A = Lrelu(Gl1)\n",
    "Gl2 = Gl1A.dot(G_W2) + G_b2\n",
    "Gl2A = Lrelu(Gl2)\n",
    "Gl3 = Gl2A.dot(G_W3) + G_b3\n",
    "Gl3A = Lrelu(Gl3)\n",
    "Gl4 = Gl3A.dot(G_W4) + G_b4\n",
    "Gl4A = Lrelu(Gl4)\n",
    "Gl5 = Gl4A.dot(G_W5) + G_b5\n",
    "Gl5A = Lrelu(Gl5)\n",
    "Gl6 = Gl5A.dot(G_W6) + G_b6\n",
    "Gl6A = Lrelu(Gl6)\n",
    "Gl7 = Gl6A.dot(G_W7) + G_b7\n",
    "Gl7A = Lrelu(Gl7)\n",
    "Gl8 = Gl7A.dot(G_W8) + G_b8\n",
    "Gl8A = Lrelu(Gl8)\n",
    "Gl9 = Gl8A.dot(G_W9) + G_b9\n",
    "Gl9A = Lrelu(Gl9)\n",
    "Gl10 = Gl9A.dot(G_W10) + G_b10\n",
    "Gl10A = Lrelu(Gl10)\n",
    "Gl11 = Gl10A.dot(G_W11) + G_b11\n",
    "\n",
    "current_fake_data = log(Gl11)\n",
    "\n",
    "\n",
    "Y = np.random.normal(0., 1., size=[1, G_input])\n",
    "Gl1 = Y.dot(G_W1) + G_b1\n",
    "Gl1A = Lrelu(Gl1)\n",
    "Gl2 = Gl1A.dot(G_W2) + G_b2\n",
    "Gl2A = Lrelu(Gl2)\n",
    "Gl3 = Gl2A.dot(G_W3) + G_b3\n",
    "Gl3A = Lrelu(Gl3)\n",
    "Gl4 = Gl3A.dot(G_W4) + G_b4\n",
    "Gl4A = Lrelu(Gl4)\n",
    "Gl5 = Gl4A.dot(G_W5) + G_b5\n",
    "Gl5A = Lrelu(Gl5)\n",
    "Gl6 = Gl5A.dot(G_W6) + G_b6\n",
    "Gl6A = Lrelu(Gl6)\n",
    "Gl7 = Gl6A.dot(G_W7) + G_b7\n",
    "Gl7A = Lrelu(Gl7)\n",
    "Gl8 = Gl7A.dot(G_W8) + G_b8\n",
    "Gl8A = Lrelu(Gl8)\n",
    "Gl9 = Gl8A.dot(G_W9) + G_b9\n",
    "Gl9A = Lrelu(Gl9)\n",
    "Gl10 = Gl9A.dot(G_W10) + G_b10\n",
    "Gl10A = Lrelu(Gl10)\n",
    "Gl11 = Gl10A.dot(G_W11) + G_b11\n",
    "\n",
    "s_data = log(Gl11)\n",
    "\n",
    "fig = plot(s_data - current_fake_data,\"TEST\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "RfjgFlc9bfRf",
    "outputId": "0905a244-2393-4be7-a7e2-455bbb82ea80"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "sXhuAW8zbfRs",
    "outputId": "46292ee1-8885-480b-e0df-961713409251"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZA9tB9_2bfR2"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "56R3Qb35bfSC"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4twoZKgvbfSk",
    "outputId": "4befe795-f4fa-4f14-b52c-1d140ff1f06c"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0b_-c-6TbfTC"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lGK_P4habfTp"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "GANTRIQALSend.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}