Assignment/15_Reinforcement-Learning/hw15_reinforcement_learning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "hw15_reinforcement_learning.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "768c24e7c9734fe6a649874afdc9b581": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_139598a4e42e412587aac7b2476e27ac",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2e0caf29163745148a8f19db071c93ce",
              "IPY_MODEL_8b6960000f7d4f5aa56a777ec424d23f"
            ]
          }
        },
        "139598a4e42e412587aac7b2476e27ac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2e0caf29163745148a8f19db071c93ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_83437c368b3f4412b3342aa66db98fac",
            "_dom_classes": [],
            "description": "Total: -98.6, Final: -100.0:   5%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 400,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 21,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d0d7cf2908c0443f86f74a4505d68fa2"
          }
        },
        "8b6960000f7d4f5aa56a777ec424d23f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_75979e278c40405da4e546df1137c3d1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 21/400 [00:40&lt;11:36,  1.84s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bb8b16d9fc324bc2a4bb60d1f09c70c5"
          }
        },
        "83437c368b3f4412b3342aa66db98fac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d0d7cf2908c0443f86f74a4505d68fa2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "75979e278c40405da4e546df1137c3d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bb8b16d9fc324bc2a4bb60d1f09c70c5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Fp30SB4bxeQb",
        "colab_type": "text"
      },
      "source": [
        "# **Homework 15 - Reinforcement Learning**\n",
        "\n",
        "若有任何問題，歡迎來信至助教信箱 ntu-ml-2020spring-ta@googlegroups.com\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yXsnCWPtWSNk",
        "colab_type": "text"
      },
      "source": [
        "## 前置作業\n",
        "\n",
        "首先我們需要安裝必要的系統套件及 PyPi 套件。\n",
        "gym 這個套件由 OpenAI 所提供，是一套用來開發與比較 Reinforcement Learning 演算法的工具包（toolkit）。\n",
        "而其餘套件則是為了在 Notebook 中繪圖所需要的套件。"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5e2bScpnkVbv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%capture\n",
        "sudo apt update\n",
        "sudo apt install python-opengl xvfb -y\n",
        "pip install gym[box2d] pyvirtualdisplay piglet tqdm"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M_-i3cdoYsks",
        "colab_type": "text"
      },
      "source": [
        "接下來，設置好 virtual display，並引入所有必要的套件。"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nl2nREINDLiw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%capture\n",
        "from pyvirtualdisplay import Display\n",
        "virtual_display = Display(visible=0, size=(1400, 900))\n",
        "virtual_display.start()\n",
        "\n",
        "%matplotlib inline\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from IPython import display\n",
        "\n",
        "import numpy as np\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torch.nn.functional as F\n",
        "from torch.distributions import Categorical\n",
        "from tqdm import tqdm_notebook"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "He0XDx6bzjgC",
        "colab_type": "text"
      },
      "source": [
        "最後，引入 OpenAI 的 gym，並建立一個 [Lunar Lander](https://gym.openai.com/envs/LunarLander-v2/) 環境。"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N_4-xJcbBt09",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%capture\n",
        "import gym\n",
        "env = gym.make('LunarLander-v2')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NrkVvTrvWZ5H",
        "colab_type": "text"
      },
      "source": [
        "## 什麼是 Lunar Lander？\n",
        "\n",
        "“LunarLander-v2” 這個環境是在模擬登月小艇降落在月球表面時的情形。\n",
        "這個任務的目標是讓登月小艇「安全地」降落在兩個黃色旗幟間的平地上。\n",
        "> Landing pad is always at coordinates (0,0).\n",
        "> Coordinates are the first two numbers in state vector.\n",
        "\n",
        "![](https://gym.openai.com/assets/docs/aeloop-138c89d44114492fd02822303e6b4b07213010bb14ca5856d2d49d6b62d88e53.svg)\n",
        "\n",
        "所謂的「環境」其實同時包括了 agent 和 environment。\n",
        "我們利用 `step()` 這個函式讓 agent 行動，而後函式便會回傳 environment 給予的 observation/state（以下這兩個名詞代表同樣的意思）和 reward。"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bIbp82sljvAt",
        "colab_type": "text"
      },
      "source": [
        "### Observation / State\n",
        "\n",
        "首先，我們可以看看 environment 回傳給 agent 的 observation 究竟是長什麼樣子的資料："
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rsXZra3N9R5T",
        "colab_type": "code",
        "outputId": "af4cd955-c483-49d0-f368-b29614b52107",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print(env.observation_space)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Box(8,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ezdfoThbAQ49",
        "colab_type": "text"
      },
      "source": [
        "`Box(8,)` 說明我們會拿到 8 維的向量作為 observation，其中包含：垂直及水平座標、速度、角度、加速度等等，這部分我們就不細說。\n",
        "\n",
        "### Action\n",
        "\n",
        "而在 agent 得到 observation 和 reward 以後，能夠採取的動作有："
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p1k4dIrBAaKi",
        "colab_type": "code",
        "outputId": "106f25a1-af03-469a-c6ea-cce8345460e4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print(env.action_space)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Discrete(4)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dejXT6PHBrPn",
        "colab_type": "text"
      },
      "source": [
        "`Discrete(4)` 說明 agent 可以採取四種離散的行動：\n",
        "- 0 代表不採取任何行動\n",
        "- 2 代表主引擎向下噴射\n",
        "- 1, 3 則是向左右噴射\n",
        "\n",
        "接下來，我們嘗試讓 agent 與 environment 互動。\n",
        "在進行任何操作前，建議先呼叫 `reset()` 函式讓整個「環境」重置。\n",
        "而這個函式同時會回傳「環境」最初始的狀態。"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pi4OmrmZgnWA",
        "colab_type": "code",
        "outputId": "29867f7c-aa4e-4e8f-ca39-e16f13c51135",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "initial_state = env.reset()\n",
        "print(initial_state)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[-0.00611763  1.4042023  -0.6196678  -0.29858398  0.00709562  0.14036398\n",
            "  0.          0.        ]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uBx0mEqqgxJ9",
        "colab_type": "text"
      },
      "source": [
        "接著，我們試著從 agent 的四種行動空間中，隨機採取一個行動"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vxkOEXRKgizt",
        "colab_type": "code",
        "outputId": "f621e59f-1c2b-40ef-a650-f370cd4d68ff",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "random_action = env.action_space.sample()\n",
        "print(random_action)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mns-bO01g0-J",
        "colab_type": "text"
      },
      "source": [
        "再利用 `step()` 函式讓 agent 根據我們隨機抽樣出來的 `random_action` 動作。\n",
        "而這個函式會回傳四項資訊：\n",
        "- observation / state\n",
        "- reward\n",
        "- 完成與否\n",
        "- 其餘資訊"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E_WViSxGgIk9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "observation, reward, done, info = env.step(random_action)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FdieGq7NuBIm",
        "colab_type": "text"
      },
      "source": [
        "第一項資訊 `observation` 即為 agent 採取行動之後，agent 對於環境的 observation 或者說環境的 state 為何。\n",
        "而第三項資訊 `done` 則是 `True` 或 `False` 的布林值，當登月小艇成功著陸或是不幸墜毀時，代表這個回合（episode）也就跟著結束了，此時 `step()` 函式便會回傳 `done = True`，而在那之前，`done` 則保持 `False`。"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yK7r126kuCNp",
        "colab_type": "code",
        "outputId": "e28661e1-aadc-451a-e3b7-8c39235e6e32",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print(done)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "False\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GKdS8vOihxhc",
        "colab_type": "text"
      },
      "source": [
        "### Reward\n",
        "\n",
        "而「環境」給予的 reward 大致是這樣計算：\n",
        "- 小艇墜毀得到 -100 分\n",
        "- 小艇在黃旗幟之間成功著地則得 100~140 分\n",
        "- 噴射主引擎（向下噴火）每次 -0.3 分\n",
        "- 小艇最終完全靜止則再得 100 分\n",
        "\n",
        "> Reward for moving from the top of the screen to landing pad and zero speed is about 100..140 points.\n",
        "> If lander moves away from landing pad it loses reward back.\n",
        "> Episode finishes if the lander crashes or comes to rest, receiving additional -100 or +100 points.\n",
        "> Each leg ground contact is +10.\n",
        "> Firing main engine is -0.3 points each frame.\n",
        "> Solved is 200 points."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vxQNs77hi0_7",
        "colab_type": "code",
        "outputId": "fcd26c3e-0f72-4ec0-caa3-f723ec626d7c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print(reward)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-1.0455870056177616\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Mhqp6D-XgHpe",
        "colab_type": "text"
      },
      "source": [
        "### Random Agent\n",
        "\n",
        "最後，在進入實做之前，我們就來看看這樣一個 random agent 能否成功登陸月球："
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y3G0bxoccelv",
        "colab_type": "code",
        "outputId": "f3700954-ffcc-4c49-9f48-3d7eef03d0e7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        }
      },
      "source": [
        "env.reset()\n",
        "\n",
        "img = plt.imshow(env.render(mode='rgb_array'))\n",
        "\n",
        "done = False\n",
        "while not done:\n",
        "    action = env.action_space.sample()\n",
        "    observation, reward, done, _ = env.step(action)\n",
        "\n",
        "    img.set_data(env.render(mode='rgb_array'))\n",
        "    display.display(plt.gcf())\n",
        "    display.clear_output(wait=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAeiElEQVR4nO3deZRU5bnv8e/TczP2SE/MgkQGmVrUBV7BXOKwYojJ0au5RlcGSa4aNYOJnnuu8ZyMGk0coihGj2hM4oBE1BCQIYk4YasEASU2gzYtkzaDDd00Xf3cP2qDxdhjUbW7f5+1avXe79679vO21T+2b71V29wdEREJj5REFyAiIq2j4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZCJW3Cb2TlmtsbMKs3shnidR0Skq7F4zOM2s1TgX8BUYCPwOnCJu6/u8JOJiHQx8bringBUuvs6d28A/gRMi9O5RES6lLQ4PW8ZUBWzvhE49Wg7m5k+vikicgh3tyO1xyu4m2Vm04HpiTq/iEhYxSu4q4F+Met9g7YD3H0mMBN0xS0i0hrxGuN+HRhqZoPMLAO4GJgbp3OJiHQpcbnidvdGM7samA+kAg+5+6p4nEtEpKuJy3TAVhehoRIRkcMc7c1JfXJSRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIdOue06a2QbgEyACNLp7uZnlAY8DA4ENwEXuvr19ZYqIyH4dccU9xd3HuHt5sH4DsMjdhwKLgnUREekg8RgqmQbMCpZnAV+MwzlERLqs9ga3AwvM7A0zmx60Fbn7pmB5M1DUznOIiEiMdo1xA5PcvdrM+gAvmNm7sRvd3c3Mj3RgEPTTj7RNRESOztyPmKutfyKzm4Fa4ApgsrtvMrMS4G/uPqyZYzumCBGRTsTd7UjtbR4qMbPuZtZz/zLwOWAlMBe4PNjtcuCZtp5DREQO1+YrbjMbDMwJVtOAP7j7z8wsH3gC6A+8T3Q6YE0zz6UrbhGRQxztirvDhkraQ8EtInK4Dh8qERGRxFBwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGSaDW4ze8jMtprZypi2PDN7wczeC37mBu1mZneZWaWZrTCzcfEsXkSkK2rJFffDwDmHtN0ALHL3ocCiYB3gXGBo8JgOzOiYMkVEZL9mg9vd/wHUHNI8DZgVLM8CvhjT/ohHvQrkmFlJRxUrIiJtH+MucvdNwfJmoChYLgOqYvbbGLQdxsymm1mFmVW0sQYRkS4prb1P4O5uZt6G42YCMwHacryISFfV1ivuLfuHQIKfW4P2aqBfzH59gzYREekgbQ3uucDlwfLlwDMx7ZcFs0tOA3bGDKmIiEgHMPdjj1KY2R+ByUABsAX4MfBn4AmgP/A+cJG715iZAb8lOgtlD/A1d292DFtDJSIih3N3O1J7s8F9PCi4RUQOd7Tg1icnRURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjLNBreZPWRmW81sZUzbzWZWbWbLg8d5MdtuNLNKM1tjZmfHq3ARka6qJTcL/h9ALfCIu48M2m4Gat39tkP2HQ78EZgAlAILgRPdPdLMOXTPSRGRQ7T5npPu/g+gpoXnmQb8yd33uvt6oJJoiIuISAdpzxj31Wa2IhhKyQ3ayoCqmH02Bm2HMbPpZlZhZhXtqEFEpMtpa3DPAE4AxgCbgNtb+wTuPtPdy929vI01iIh0SW0Kbnff4u4Rd28CHuDT4ZBqoF/Mrn2DNhER6SBtCm4zK4lZvQDYP+NkLnCxmWWa2SBgKLCsfSWKiEistOZ2MLM/ApOBAjPbCPwYmGxmYwAHNgDfAnD3VWb2BLAaaASuam5GiYiItE6z0wGPSxGaDigicpg2TwcUEZHkouAWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCZlmg9vM+pnZEjNbbWarzOzaoD3PzF4ws/eCn7lBu5nZXWZWaWYrzGxcvDshItKVtOSKuxH4vrsPB04DrjKz4cANwCJ3HwosCtYBziV6d/ehwHRgRodXLSLShTUb3O6+yd3fDJY/Ad4ByoBpwKxgt1nAF4PlacAjHvUqkGNmJR1euYhIF9WqMW4zGwiMBV4Ditx9U7BpM1AULJcBVTGHbQzaDn2u6WZWYWYVraxZRKRLa3Fwm1kPYDZwnbvvit3m7g54a07s7jPdvdzdy1tznIhIV9ei4DazdKKh/Zi7Px00b9k/BBL83Bq0VwP9Yg7vG7SJiEgHaMmsEgMeBN5x91/HbJoLXB4sXw48E9N+WTC75DRgZ8yQioiItJNFRzmOsYPZJOBF4G2gKWj+d6Lj3E8A/YH3gYvcvSYI+t8C5wB7gK+5+zHHsc2sVcMsIiJdgbvbkdqbDe7jQcEtInK4owW3PjkpIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSJJL8WMXhkZiS4jaSi4RSSp5WRmckZpKV8aMoSCrKxEl5MUFNwiR5FixtCSEopzchJdSpc2obiY8UVF5GVlMbZPn0SXkxTSEl2ASLI6sbSUE4qKaGxqomndOrbu2tX8QdLh/rFxIz3T09nX1MSa7dsTXU5SUHCLHIW7Y2bg3rq7hEiHqo9EeGbtWhyIJMGX4iUDBbfIUWz8+GOyMzLYtH0723S1nVCNCuyD6GtdRUSSlL7WVUSkk1BwixxDWXY25bm5dEtNTXQpIgcouEWOoiAjg28PHsw3Bg1ialERR/x/VpEEaMnNgvuZ2RIzW21mq8zs2qD9ZjOrNrPlweO8mGNuNLNKM1tjZmfHswMi8dIrPZ28jAyM6JV3eoqucyQ5tORmwSVAibu/aWY9gTeALwIXAbXuftsh+w8H/ghMAEqBhcCJ7h45xjn05qQknVQzTujenbOLi3lo/Xp2R476EhaJiza/Oenum9z9zWD5E+AdoOwYh0wD/uTue919PVBJNMRFQiXizrrdu3lQoS1JplX/72dmA4GxwGtB09VmtsLMHjKz3KCtDKiKOWwjxw56EQB+/vNvccstMHIkDB8OpaWJrig6f3jPcQztyZMn8/DDwzjvPBgxAj7zGdD7onKoFn8Ax8x6ALOB69x9l5nNAH4CePDzduDrrXi+6cD01pUrndmoUYMpKYGzzoqub9oEq1dHl//6V6isBHfYvBk66wVwYWEhEybUMmJEdL2xEV5+Gfbtg40b4c9/jrbv3AmffJK4OiWxWhTcZpZONLQfc/enAdx9S8z2B4DngtVqoF/M4X2DtoO4+0xgZnC8xrjlAAtG9UpLP73qnjIlGtqRCMyfD3V10WD//e8TV2c87f8dpKfDmWdGl93h0kujyytXwpo10eVHHoEtWw5/Dum8WjKrxIAHgXfc/dcx7SUxu10ArAyW5wIXm1mmmQ0ChgLLOq5k6YqamqKh3dgIe/bA7t3R8O5K9v/DFYlAfX30d7B7d/R3I11LS664JwJfBd42s+VB278Dl5jZGKJDJRuAbwG4+yozewJYDTQCVx1rRolILPfoA6JDA8uDV9z8+bBuXXRbTU3nD6v9v4fGRli8GBoaoLoa5s6Nbq+t7Xr/cMmnmg1ud18KR/zswV+OcczPgJ+1oy7pgmpr4fnno8MfTU3RMdxt2xJd1fG3fDk88AC8/3709/DBBx3/D1VaWiZ5ef0PaqupeZ/GxoaOPZHEhb4dUJLGBx/AzTcnuorE+/WvoaKibceaGb16FWP26Shoeno2J5wwMfoVtYGM9O4U5484sN7YVM+atYtYufL5Ntctx4+CW6STGDjwVAoLT6Bv8VhSUj6dQ5iWkklRj5OxmLe0UlMy6Jnx6XzLHfXvs7nXalJS0mhqajyudUvrKbhFOoGsrF4MGTyJE0vOJT97KCl26ORvO+iK+1A5WQMo7XMyubl9+fjjDXGtVdpPwS3SCWRkdKNnj2IKug07Qmi3jFnKQVfqkrz0rTkiAkBxjzGccMKkRJchLaDgFhHMjMzUXvTs3ofMzJ6JLkeaoeAWEQC6ZxRQ0Hso+fkDEl2KNEPBLSIHpKVkkZHRLdFlSDMU3CJyQHGPkxk06LSD5oFL8tF/HRGJkUKKpR6YYVJQMFgzTZKQpgOKhJBZKqWlI0hJSWXLljX07l1K94xCrJ13xsxOz6U4dyT9+4+jV69ihg6YzLqNL/HWW7M7qHLpCApukRDq1auI8jGX0COriPXVL/F+1es0RGpxvF3RnWKppFgGpaUj6dennMF5/5P01B5s27aWjRuXN/8EclwouEVCqKRkBPk9hpKTNYCN6W9QX7+LfZGO+7rAzZvfZevW98gak8Pg/ClQDku5V+GdJBTcIiGTkdGNkuLh5GUPZnfDR+yq3cy+ffUd8twNkVqavAH3JiorXyEtLYOJozMZUjA1Jrz/SfTbnCVRFNwiIVNaOpK+BePITO3Ne7sWsHbt0b55+VORpgaa/OAvj9pev566fTUHtTWmbcOy68jLy2HDBqiqWs4HZcvIyx7CkIKp2Cmw1GZQVbUchXfiJEVwp6SkkJWVRV1dHe5d58WQlZVFSkoKF1xwAUVFRfzud79j165diS5Lklxubj+6ZxRRU/ceH1S/zq5dW+nZsw/uTTR5I7sbtvJJw4cHHbOzrorahk/vb5aSksKOnev55JNtnHJKOWcG90c744wpFBb2Zs+er3Pvvfdwzz338NrrD5NCCqP7X8LgvLP4ZNQmPvpoPXV1O45rv+VTSRHcI0eOZN68ecyYMYPNmzfzt7/9jXXr1tHUyW5zYhb9hrZx48YxduxYvvOd71BQUEBubi7p6elcc8013H777Tz66KPs2KE/Cjlcjx6F9CsdT27WIN7Z9mcqK5cCTm3tNqo+fANwdu76kI+3r485yvj443XU1HzACSecwJlnnklJSQnf/OatNDU10r17d3r16nXQebKz8/iP//gPpk+fzt13383s2X/BUlIZWfZlemWVkZ8/UOPdCZQUwZ2enk5paSk/+clPAKiurmbz5s3ceuut1NXV8fe//z3UV6LFxcWccsopXH755Zx00kkUFhZSWFh42H4DBgzgzjvv5JprruGOO+7gqaeeYovuAisxhgyZSGnvcTQ21fHJ7i3s3VsLgHsTK1fOY82axTQ01NHQsJuCggJOP/10cnNz+MEP/kBqaiq9e/emrKysRecyM4qKivjpT3/Kueeey2233cs/qx5j3YZXqK5eEc9uSjOaDW4zywL+AWQG+z/l7j8ObgT8JyAfeAP4qrs3mFkm8AgwHvgY+F/uvqE1RZWVlVFWVsbjjz+Ou7N8+XJ2797NH/7wByoqKnjzzTeJRJL3NpbDhg0jLy+PqVOnMnXqVPLz8znppJNadKyZMWTIEH77299y5ZVXcv/99zN79myqq6vjXLUku27dcikqOInM1J5srl3B+g2v0NCwO9jWjdGjR5Kdnc2PfvQjunXrRk5ODiNHjuyQc0+cOJFx48ZRUbGCH//4Daqq0mho0G3OEsbdj/kg+q5Hj2A5HXgNOA14Arg4aL8P+D/B8pXAfcHyxcDjzZ1j/Pjx3hJNTU2+d+9ef+655/zpp5/2yZMn+7Bhw7xnz55O9J2ShDzy8/N92LBhfsEFF/icOXP8ww8/9KamJm9qampRv5rr86pVq/yHP/yhl5aWJrSf8X7ccsstCa8h0Y8LL7zQy8vLj7q9uPgknzLlGr/oop/46NHj/MEHH/Q5c+b44sWLO+w115zGxkafN2+eT5482bt165bw31lnfvjRcvloG464M3QD3gROBT4C0oL204H5wfJ84PRgOS3Yz471vC0N7lhNTU3e2NjojY2NvmDBAr/vvvv80ksv9aKiIk9LS4vrLzMzM9OLiop8xIgRPmPGDF+2bJk3NjZ6JBJp459C8yKRiL/33ns+fvx4z8jISPgLKh4PBffhwW1mXlhY6EVFRX7TTTf5fffd7w8++N9eWVntjY2NxyWoj2T/398zzzzjZ511lgI8Tg8/Smaat2AWh5mlEh0OGQLcA/wKeNXdhwTb+wHz3H2kma0EznH3jcG2tcCp7v7R0Z6/vLzcK9p6d9QYDQ0NNDQ0MGvWLLZt28bcuXNZs2YN+/btY9++fW1+XjMjOzub9PR0vvnNbzJ69Gi+9KUvHWg/1i2hOpK7U19fz5NPPsm9997Lm2++2a5+JZPU1FQ+//nPs3Dhwi41s+hQI0aMYMeOHZx++ukMGjSIzMxMrrjiCrKzs8nKyiI1Nfm+N6S+vp4FCxYwY8YMFi9eHOohlG7dunHiiScybdo0AGpra3nggQdobGxMyKw3dz9iuLQouA/sbJYDzAH+H/Bwe4LbzKYD0wH69+8//v333299r5qxY8cO6uvrefHFF3nhhReYM2cONTU1LZqtkpKSQv/+/fnc5z5Hfn4+V199NampqRQUFCTFH8/OnTt59tlnuf/++3nppZdCGXZmRlpaGhdeeCFjxozhq1/9aqJLShq5ublkZmYmuoxWqa2tZenSpdx1113Mnz8/FLPCUlKi37N3xhlnMGrUKK677jry8/PJyckBIBKJ8NFHHxGJRLj77rupqalh/vz5VFVVxY5ExE2HBDeAmd0E1AE/AordvdHMTgdudvezzWx+sPyKmaUBm4FCP8aJOuqK+1jcnQ0bNrB3715+//vfs3LlSv71r3/xzjvvHNine/fufPaznyUjI4Prr7+e0tJS+vbtG9e62mv79u0sXLiQO++8k5deeinR5bRITk4OZ555Jueffz6TJk1i8ODBpKenJ7os6SC1tbW88sor3HrrrSxatCgpLyoGDhzI6NGjufLKK+nfvz/FxcUHwro5VVVV7N69m7/+9a8sWbKEhQsXsmfPnrjUebTgbsm4diGQEyxnAy8Cnwee5OA3J68Mlq/i4Dcnn2juHG0Z4+4IVVVVvnTpUv/yl7/sjz76qL/xxhsJGzNsrx07dvhTTz3lp5xyiptZwsfmDn2UlZX5pEmT/KmnnvLly5eH9vcsLbdnzx5fsGCBT548OSnelznppJP83HPP9SVLlvjatWs7pI+RSMQrKir8pZde8q9//es+ceJELysrS/wYt5mdDMwCUol+f/cT7v5fZjaY6HTAPOAt4FJ33xtMH3wUGAvUEA33dcc6x/G44u4q6urqeP755/nFL37B22+/nbAx8O7du9O/f39yc3P5wQ9+wLBhwxg+fHhCapHEikQiLFy4kF/+8pcsW7YsblenhyooKKBPnz5873vfo7CwkAkTJlBcXBz3865evZo1a9Zw++23U1NTwwcffMDu3bvb9FzeUUMl8aDg7ljuTiQSYfbs2fzqV79i5cqV7N27N+7n7dmzJ7179+baa69l+PDhnH322UB0HPF4vYEryWn/leLzzz/PHXfcwauvvhqXAM/Pz6e0tJSrr76a8ePHM2bMmIS8/tz9wBj/ggUL2LhxIwsWLGDp0qXU19e3+JPRCu4uyINZKPfeey833XRTXP5QsrOzKSkp4bLLLmPKlCmMHz+e7OzsA2/6iByqvr6e+fPnc99997FkyZJ2XVRkZGTQrVs3rrjiCrp3785XvvIV+vbtS1ZWVtJdLDQ0NLBv3z7effddnnvuOZ588kk2bNhwoP1IFNxd2L59+1i/fj233XYbjz32WLsDPCUlhZNPPplTTz2Vb3/72/Tr14+8vLyk+0OR5FZbW8uLL77I3Xff3apZKCkpKRQVFfGFL3yB888/n/Ly8qSZ7dUa27dvZ+/evSxYsICXX36ZtWvXsmTJkoOu1hXcQiQSobKykltvvZWnn366VV9kVVhYyMSJE8nNzeX73/8+hYWF9OnTJ47VSlfRklkomZmZTJ06lfT0dL773e/Sv39/BgwYkIBq42fXrl1s2rSJDz/8kLvuuot58+ZRX1+v4JYod+ftt9/mzjvv5Nlnn2Xbtm1H3G/gwIEMGDCA66+/nkGDBunNRYmruro6li5dys9//nNefvllevTowfDhw7nssssYO3YsY8eODd1VdXucfPLJrFixQsEth3vrrbeYOXMmc+bMYcuWLQwaNIjBgwdzzTXXMHz4cIYMGZLoEqWLaWxsZPHixeTk5DBhwoREl5Mw5eXlVFRUKLjlyJqamli1ahXLli3j/PPPp6Cg4MB3h4tIYhwruJPi+7glsVJSUhg1ahSjRo1KdCki0gKasyUiEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhEyzwW1mWWa2zMz+aWarzOw/g/aHzWy9mS0PHmOCdjOzu8ys0sxWmNm4eHdCRKQracmXTO0FznL3WjNLB5aa2bxg2/Xu/tQh+58LDA0epwIzgp8iItIBmr3iDu5CXxuspgePY30X7DTgkeC4V4EcMytpf6kiIgItHOM2s1QzWw5sBV5w99eCTT8LhkN+Y2aZQVsZUBVz+MagTUREOkCLgtvdI+4+BugLTDCzkcCNwGeAU4A84EetObGZTTezCjOrONqts0RE5HCtmlXi7juAJcA57r4pGA7ZC/w3sP8eQ9VAv5jD+gZthz7XTHcvd/fywsLCtlUvItIFtWRWSaGZ5QTL2cBU4N3949YWvb/VF4GVwSFzgcuC2SWnATvdfVNcqhcR6YJaMqukBJhlZqlEg/4Jd3/OzBabWSFgwHLg28H+fwHOAyqBPcDXOr5sEZGuq9ngdvcVwNgjtJ91lP0duKr9pYmIyJHok5MiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkDF3T3QNmNknwJpE1xEnBcBHiS4iDjprv6Dz9k39CpcB7l54pA1px7uSo1jj7uWJLiIezKyiM/ats/YLOm/f1K/OQ0MlIiIho+AWEQmZZAnumYkuII46a986a7+g8/ZN/eokkuLNSRERablkueIWEZEWSnhwm9k5ZrbGzCrN7IZE19NaZvaQmW01s5UxbXlm9oKZvRf8zA3azczuCvq6wszGJa7yYzOzfma2xMxWm9kqM7s2aA9138wsy8yWmdk/g379Z9A+yMxeC+p/3MwygvbMYL0y2D4wkfU3x8xSzewtM3suWO8s/dpgZm+b2XIzqwjaQv1abI+EBreZpQL3AOcCw4FLzGx4Imtqg4eBcw5puwFY5O5DgUXBOkT7OTR4TAdmHKca26IR+L67DwdOA64K/tuEvW97gbPcfTQwBjjHzE4DbgF+4+5DgO3AN4L9vwFsD9p/E+yXzK4F3olZ7yz9Apji7mNipv6F/bXYdu6esAdwOjA/Zv1G4MZE1tTGfgwEVsasrwFKguUSovPUAe4HLjnSfsn+AJ4BpnamvgHdgDeBU4l+gCMtaD/wugTmA6cHy2nBfpbo2o/Sn75EA+ws4DnAOkO/gho3AAWHtHWa12JrH4keKikDqmLWNwZtYVfk7puC5c1AUbAcyv4G/xs9FniNTtC3YDhhObAVeAFYC+xw98Zgl9jaD/Qr2L4TyD++FbfYHcAPgaZgPZ/O0S8ABxaY2RtmNj1oC/1rsa2S5ZOTnZa7u5mFduqOmfUAZgPXufsuMzuwLax9c/cIMMbMcoA5wGcSXFK7mdnnga3u/oaZTU50PXEwyd2rzawP8IKZvRu7MayvxbZK9BV3NdAvZr1v0BZ2W8ysBCD4uTVoD1V/zSydaGg/5u5PB82dom8A7r4DWEJ0CCHHzPZfyMTWfqBfwfbewMfHudSWmAh8wcw2AH8iOlxyJ+HvFwDuXh383Er0H9sJdKLXYmslOrhfB4YG73xnABcDcxNcU0eYC1weLF9OdHx4f/tlwbvepwE7Y/5XL6lY9NL6QeAdd/91zKZQ983MCoMrbcwsm+i4/TtEA/zfgt0O7df+/v4bsNiDgdNk4u43untfdx9I9O9osbv/b0LeLwAz625mPfcvA58DVhLy12K7JHqQHTgP+BfRccb/m+h62lD/H4FNwD6iY2nfIDpWuAh4D1gI5AX7GtFZNGuBt4HyRNd/jH5NIjquuAJYHjzOC3vfgJOBt4J+rQRuCtoHA8uASuBJIDNozwrWK4PtgxPdhxb0cTLwXGfpV9CHfwaPVftzIuyvxfY89MlJEZGQSfRQiYiItJKCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQ+f+jN84hlY/jywAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F5paWqo7tWL2",
        "colab_type": "text"
      },
      "source": [
        "## Policy Gradient\n",
        "\n",
        "現在來搭建一個簡單的 policy network。\n",
        "我們預設模型的輸入是 8-dim 的 observation，輸出則是離散的四個動作之一："
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J8tdmeD-tZew",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class PolicyGradientNetwork(nn.Module):\n",
        "\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.fc1 = nn.Linear(8, 16)\n",
        "        self.fc2 = nn.Linear(16, 16)\n",
        "        self.fc3 = nn.Linear(16, 4)\n",
        "\n",
        "    def forward(self, state):\n",
        "        hid = torch.tanh(self.fc1(state))\n",
        "        hid = torch.tanh(self.fc2(hid))\n",
        "        return F.softmax(self.fc3(hid), dim=-1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ynbqJrhIFTC3",
        "colab_type": "text"
      },
      "source": [
        "再來，搭建一個簡單的 agent，並搭配上方的 policy network 來採取行動。\n",
        "這個 agent 能做到以下幾件事：\n",
        "- `learn()`：從記下來的 log probabilities 及 rewards 來更新 policy network。\n",
        "- `sample()`：從 environment 得到 observation 之後，利用 policy network 得出應該採取的行動。\n",
        "而此函式除了回傳抽樣出來的 action，也會回傳此次抽樣的 log probabilities。"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zZo-IxJx286z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class PolicyGradientAgent():\n",
        "\n",
        "    def __init__(self, network):\n",
        "        self.network = network\n",
        "        self.optimizer = optim.SGD(self.network.parameters(), lr=0.001)\n",
        "\n",
        "    def learn(self, log_probs, rewards):\n",
        "        loss = (-log_probs * rewards).sum()\n",
        "\n",
        "        self.optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        self.optimizer.step()\n",
        "\n",
        "    def sample(self, state):\n",
        "        action_prob = self.network(torch.FloatTensor(state))\n",
        "        action_dist = Categorical(action_prob)\n",
        "        action = action_dist.sample()\n",
        "        log_prob = action_dist.log_prob(action)\n",
        "        return action.item(), log_prob"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ehPlnTKyRZf9",
        "colab_type": "text"
      },
      "source": [
        "最後，建立一個 network 和 agent，就可以開始進行訓練了。"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GfJIvML-RYjL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "network = PolicyGradientNetwork()\n",
        "agent = PolicyGradientAgent(network)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ouv23glgf5Qt",
        "colab_type": "text"
      },
      "source": [
        "## 訓練 Agent\n",
        "\n",
        "現在我們開始訓練 agent。\n",
        "透過讓 agent 和 environment 互動，我們記住每一組對應的 log probabilities 及 reward，並在成功登陸或者不幸墜毀後，回放這些「記憶」來訓練 policy network。"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vg5rxBBaf38_",
        "colab_type": "code",
        "outputId": "c829577a-a343-4d59-a322-6dffe8d54c12",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 120,
          "referenced_widgets": [
            "768c24e7c9734fe6a649874afdc9b581",
            "139598a4e42e412587aac7b2476e27ac",
            "2e0caf29163745148a8f19db071c93ce",
            "8b6960000f7d4f5aa56a777ec424d23f",
            "83437c368b3f4412b3342aa66db98fac",
            "d0d7cf2908c0443f86f74a4505d68fa2",
            "75979e278c40405da4e546df1137c3d1",
            "bb8b16d9fc324bc2a4bb60d1f09c70c5"
          ]
        }
      },
      "source": [
        "agent.network.train()  # 訓練前，先確保 network 處在 training 模式\n",
        "EPISODE_PER_BATCH = 5  # 每蒐集 5 個 episodes 更新一次 agent\n",
        "NUM_BATCH = 400        # 總共更新 400 次\n",
        "\n",
        "avg_total_rewards, avg_final_rewards = [], []\n",
        "\n",
        "prg_bar = tqdm_notebook(range(NUM_BATCH))\n",
        "for batch in prg_bar:\n",
        "\n",
        "    log_probs, rewards = [], []\n",
        "    total_rewards, final_rewards = [], []\n",
        "\n",
        "    # 蒐集訓練資料\n",
        "    for episode in range(EPISODE_PER_BATCH):\n",
        "        \n",
        "        state = env.reset()\n",
        "        total_reward, total_step = 0, 0\n",
        "\n",
        "        while True:\n",
        "\n",
        "            action, log_prob = agent.sample(state)\n",
        "            next_state, reward, done, _ = env.step(action)\n",
        "\n",
        "            log_probs.append(log_prob)\n",
        "            state = next_state\n",
        "            total_reward += reward\n",
        "            total_step += 1\n",
        "\n",
        "            if done:\n",
        "                final_rewards.append(reward)\n",
        "                total_rewards.append(total_reward)\n",
        "                rewards.append(np.full(total_step, total_reward))  # 設定同一個 episode 每個 action 的 reward 都是 total reward\n",
        "                break\n",
        "\n",
        "    # 紀錄訓練過程\n",
        "    avg_total_reward = sum(total_rewards) / len(total_rewards)\n",
        "    avg_final_reward = sum(final_rewards) / len(final_rewards)\n",
        "    avg_total_rewards.append(avg_total_reward)\n",
        "    avg_final_rewards.append(avg_final_reward)\n",
        "    prg_bar.set_description(f\"Total: {avg_total_reward: 4.1f}, Final: {avg_final_reward: 4.1f}\")\n",
        "\n",
        "    # 更新網路\n",
        "    rewards = np.concatenate(rewards, axis=0)\n",
        "    rewards = (rewards - np.mean(rewards)) / (np.std(rewards) + 1e-9)  # 將 reward 正規標準化\n",
        "    agent.learn(torch.stack(log_probs), torch.from_numpy(rewards))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:7: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
            "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
            "  import sys\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "768c24e7c9734fe6a649874afdc9b581",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=400.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vNb_tuFYhKVK",
        "colab_type": "text"
      },
      "source": [
        "### 訓練結果\n",
        "\n",
        "訓練過程中，我們持續記下了 `avg_total_reward`，這個數值代表的是：每次更新 policy network 前，我們讓 agent 玩數個回合（episodes），而這些回合的平均 total rewards 為何。\n",
        "理論上，若是 agent 一直在進步，則所得到的 `avg_total_reward` 也會持續上升，直至 250 上下。\n",
        "若將其畫出來則結果如下："
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wZYOI8H10SHN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "plt.plot(avg_total_rewards)\n",
        "plt.title(\"Total Rewards\")\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mV5jj4dThz0Y",
        "colab_type": "text"
      },
      "source": [
        "另外，`avg_final_reward` 代表的是多個回合的平均 final rewards，而 final reward 即是 agent 在單一回合中拿到的最後一個 reward。\n",
        "如果同學們還記得環境給予登月小艇 reward 的方式，便會知道，不論**回合的最後**小艇是不幸墜毀、飛出畫面、或是靜止在地面上，都會受到額外地獎勵或處罰。\n",
        "也因此，final reward 可被用來觀察 agent 的「著地」是否順利等資訊。"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "txDZ5vlGWz5w",
        "colab_type": "code",
        "outputId": "4a3871a8-6b56-4c68-b04f-27f6d0410519",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281
        }
      },
      "source": [
        "plt.plot(avg_final_rewards)\n",
        "plt.title(\"Final Rewards\")\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAEICAYAAAC3Y/QeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOy9abjsRnktvF6pe+9zjufh2MYTNsYJH8QQiDEO85QEEhKThHCdL8klkODLkABJbhITvo8M9zExBBLmwWGOHQIhEJzYGDxgg8HH5njC07F9PJ/JPvO4d3dLqvtDKtVbpSpJ3S317r13refZT++W1FJJKr1atWrVWySEgIeHh4fH8kKw0AXw8PDw8Jg8fPD38PDwWIbwwd/Dw8NjGcIHfw8PD49lCB/8PTw8PJYhfPD38PDwWIbwwd9jUYKI9hHRUxrYz98Q0cVNlGmhQETXEtEfLnQ5PBYXfPD3mGoQ0cNENJcFe/l3vBDiYCHEgy0f+6VElGTH3EtE9xLRG9s8pofHpOCDv8diwK9mwV7+bZrgsTcJIQ4GcCiAPwHwz0T00xM8fg5K4Z9Zj0bgK5LHogQRCSJ6avb/F4noE0R0WcbQbySi09i2HyGix4hoDxHdTEQvGvZ4IsXlAHYAeGa234CIzieiB4hoOxF9jYiOzNZ9iYj+LPv/hKy8b8++n0ZEO7LfH0FE/01EW4loZ/b/iazs1xLRBUT0QwAHADyFiH6BiNYR0W4i+jgAYts/lYiuy9ZtI6KvjnB5PZYBfPD3WCo4F8DfAjgCwHoAF7B1PwbwswCOBPCvAP6diFYMs/MsUP8agKOz/QPAHwN4LYCXADgewE4An8jWXQfgpdn/LwHwIIAXs+8/EEIkSJ/BLwB4MoCTAcwB+Lhx+N8DcB6AQwDsBvANAP9fVpYHALyAbft/AHw3uw4nAvjYMOfpsXzgg7/HYsB/EtGu7O8/Hdt8UwhxkxAiAnAJ0mAPABBCXCyE2C6EiIQQHwIwC6CudHM8Ee1CGpS/CeBPhRC3ZuveAuA9QogNQogegL8B8Doi6iAN/i/MZJoXA/gAVJB+SbYeWbn+QwhxQAixF+lL6yVGGb4ohLgrO7dXA7hLCPF1IcQAwIcBbGHbDpC+SI4XQswLIa6veZ4eyww++HssBrxWCHF49vdaxzY8AB4AcLD8QkT/m4juyaSQXQAOQ8qa62CTEOJwpJr/RwG8nK17MoBvyhcTgHsAxACOFUI8AGA/0pfQiwD8N4BNWX9BHvyJaBURfYaIHiGiPQC+D+BwIgrZcR5j/x/Pv4s0MyNf/xdIZaCbiOguInpTzfP0WGbwwd9jSSPT9/8CwOsBHJEF8t1gOnkdZMz+LwGcQUTyBfQYgFezF9PhQogVQoiN2frrALwOwEy27DoAb0AqydyWbfNnSFshzxNCHAolDfHy8dS7mwGcxM6P+HchxBYhxJuFEMcD+F8APin7Rjw8OHzw91jqOARABGArgA4RvRcpix8aQog+gA8BeG+26NMALiCiJwMAEa0monPYT64D8EdI2TwAXJt9v14IEbPyzQHYlXUW/3VFMS4D8Awi+o1MXnoHgOPkSiL6LdZhvBPpiyMZ9lw9lj588PdY6vgOgCsA3AfgEQDz0GWSYfF5ACcT0a8C+AiASwF8l4j2AlgD4Hls2+uQBncZ/K8HsIp9B1LNfiWAbdnvryg7uBBiG4DfAnAhgO0ATgfwQ7bJcwHcSET7srK9s+3xEB6LE+Qnc/Hw8PBYfvDM38PDw2MZwgd/Dw8Pj2UIH/w9PDw8liF88Pfw8PBYhugsdAHq4OijjxannHLKQhfDw8PDY1Hh5ptv3iaEWG1btyiC/ymnnIK1a9cudDE8PDw8FhWI6BHXOi/7eHh4eCxD+ODv4eHhsQzhg7+Hh4fHMoQP/h4eHh7LED74e3h4eCxDtB78iehwIvp6Nu3cPUT080R0JBFdSUT3Z59HtF0ODw8PDw+FSTD/jwC4QgjxNADPQjrhxfkArhZCnA7g6uy7h4eHh8eE0GrwJ6LDkE5O8TkgzYcuhNgF4BwAX8o2+xLSeVCXBNY8uB3rn9g3kWPt3N/Hx66+H99b94S2/PI7NmPH/v5EyuDh4bE40TbzPxXpJBpfIKJbieizRHQQ0mnuNmfbbAFwrPlDIjqPiNYS0dqtW7e2XMzm8O5v3IFPXru+esMG8J27tuBDV96Hd3zl1nzZnvkB3nbJLfjmrRtLfunh4bHc0Xbw7wB4DoBPCSGejXROU03iyeYgLUwqIIS4SAhxphDizNWrraOTpxK9QYxBPJk5EvpxOkHT/n6UL5vvpxNE9SM/eZOHh4cbbQf/DQA2CCFuzL5/HenL4HEiehIAZJ9POH6/6BALgSSZTPCPspcMP1wvC/qJn6THw8OjBK0GfyHEFgCPEdFPZ4teAeBupNPLvSFb9gYA32qzHJNEnIiJBd7Y8pKRwT+aUOvDw8NjcWISid3+GMAlRDQD4EEAb0T60vkaEf0B0nlVXz+BckwEUSKsQbmtY0kIIUBE6EWp7BMnXvbx8PBwo/XgL4S4DcCZllWvaPvYC4E4nhzzj2IV4HtRghXdUDH/Cb2APDw8Fif8CN+GESUCk4q7PMDLoN8bpJ+Tan14eHgsTvjg3zDiCco+sRb8Y+3TM38PD48y+ODfMKIkmZzsw4N/xvhlC8Azfw8PjzL44N8gkkzymZzbR9f8+WfkO3w9PDxK4IN/g4izoD8p1s0Hk8lBXX3P/D08PGrAB/8GIQPupEh3qebvff4eHh4l8MG/QUgNfkE0f+/28fDwGAI++DeIOGPb8VRo/j74e3h4uOGDf4OQnawL4vMf6LKPZ/4eHh5l8MG/QSjNf7KJ3QDv9vHw8BgOPvg3CMnEJznIa7aT3kKv+Xt4eAwDH/wbRDzxDt8EB82m6Zmk3NOP/QhfDw+Pavjg3yAm7faJE4FVMyEANsLXM38PD48a8MG/QUj3zSRTOh80I5m/ofl7n7+Hh0cJfPBvEDLD8qQm0YoTgZWS+Ufe7ePh4VEfPvg3COmwmZTPfxAnmOkEmAmDAvOfVBlGhRACH7hiHTbtmlvoonh4LEu0HvyJKCSiW4nov7PvpxLRjUS0noi+ms3wtSQQL4DbpxMQZjtBQfOf9g7fuzfvwSevfQBv/9dbFrooHh7LEpNg/u8EcA/7/n4A/ySEeCqAnQD+YAJlmAhkwJ0U6Y4SgTAgzHYDi+wz3T7/MCAAwIFevMAl8fBoB0K45/a48cHt+Pe1j024RDpaDf5EdCKAXwHw2ew7AXg5gK9nm3wJwGvbLMMkYTL/T3xvPb5y06OtHq8TkCb79OPF0eHbDdOqN4in+yXl4TEqvrb2Mbzw/ddY1/2Pi9bgz7/+kwmXSEfbc/h+GMBfADgk+34UgF1CiCj7vgHACbYfEtF5AM4DgJNPPrnlYjYDGXCl1fMfvnMvAOC3z2qn/FEsEAYBwpDyF85isXp2gzT4933w91ii2LBzDpt3zyNJBIKspTtNaI35E9FrADwhhLh5lN8LIS4SQpwphDhz9erVDZeuHSzEIK9uSAiJBf9Fks+fsmfBM3+PpYpcCZhS80WbzP8FAH6NiH4ZwAoAhwL4CIDDiaiTsf8TAWxssQwTRbQAPv8wIAQB5S+cxTaH72DK5SkPj1EhH8FJkcFh0RrzF0K8WwhxohDiFADnArhGCPE7AL4H4HXZZm8A8K22yjBpKOY/ueN1AkJAPPgvDuYvn4dB5Jm/x9KEfCbLvBdiAV8MC+Hz/0sAf0pE65H2AXxuAcrQCvL0DhPM6hkGAUKivIIpq+d0B1WB9Bp5zd9jqaKO7LOQLd+2O3wBAEKIawFcm/3/IICzJnHcSWPSGp9k/kTqmItlhK+8RItFnvLwGBY58y+JB70oxkxnYcba+hG+DWLy0zgm6ISEMCAkiUAUJ7nkNO1BVZZu2l9SHh6jIqmhBPQXUPb0wb9ByIFVk1Jcooz5h1mHb49VpHjKO1KntRPMw6MpqA5f9zY9H/yXBqJJz+Gbaf5EhFioijTTCaaf+U938Tw8xoaMA2WtWx/8lwgm7/MXqexDadNS6v0HzYSLQE6Z9vJ5eIyHpEY88LLPEgHP7TMJC1csff6Z1VM6fVbNdKbf7eNjv8cSR90O34WCD/4NgrPtSRDvKEnQzQZ5xYnS/A+aDZGIyVlOR8H0lszDoxlIF3NZK9wz/yUCrrO3LbskiUAikPv8hVAsYmU2u9e0DisHPPP3WPooG+TVyXL9eM1/iYCnUW5b95cvmk5ICII00OfMP5vda5p1f+G5v8cSR5ns0wll8Peyz5IAH6zadvCXgV1q/nEi8ibkKsn8pzn4T2/RPDwaQdmgT5nS3Ms+SwSc+bcdeGWHrvT5C8HcPrNhts30Rlgf/D2WOmQdt5k/ZrLg72WfJQIebNs228iXi0zsFhtuH77NNMLLPh5LHWpyp+K6XPYZ+OC/JKC7fdoNbjIhVBgGmeyDguY/zXZPz/w9ljrKBnlJ2ae3gIkNffBvEJrbZ0Kaf8r8ock+q2YXAfOf3qJ5eDQCUdLhmwf/ge/wXRLQmP+ENP8w0/y5z3+VZP5TnN+Hyz7TPB7Bw2NUlI3472ayz0KmNF+Wwf/Ojbtxx4bdje+XB9u245nG/AN9hO+isHqyovmc/h5LEWWJ3RTz98F/orjw2+vwvsvvaXy/mtundc1fMf80vYMKorLDd6rdPux/H/w9pgXv+rdbccr5lzWyr6RE85fhYcm6fYjoJCL6HhHdTUR3EdE7s+VHEtGVRHR/9nlEm+Uw0YviViYOjyYo+9y1aQ8A4JSjDkJIaQXrDWIQASu6i4H5q7ItpNfZw4PjP2/b1Ni+ymQfuWwp+/wjAH8mhHg6gLMBvJ2Ing7gfABXCyFOB3B19r01mD7bKBGtuHEm5fZJEoFLb9uEI1Z18TMnHKZknyjBbCfIbWRT7fZh//vg77EUodI7lDH/JdrhK4TYLIS4Jft/L4B7AJwA4BwAX8o2+xKA17ZVhivu3IxT3305Ht1+IF8WZ3lxmsakcvv89aV34ep1T+BFp69Wsk/W4TsTprl+ZBlueXQnTjn/Mvz44R219n3GX38Hv/LRH7RWdgn+bmyjFeaxOLBuyx7873+/fcFbqaecfxl+97M3NrpPyb1sErB8MSxZ2YeDiE4B8GwANwI4VgixOVu1BcCxlu3PI6K1RLR269atIx/3iz96GADwyI79+bIoFq2kXJ5UVs9Nu+YAAO/+5acBQDqBu0iD6EwnQJgzf4Ef3LcNAPD9++pdw729KJeU2oW6QD74L1/8aP12fP3mDdh1oL/QRcH167c1ur84Z/7FdWWtgklhIsGfiA4G8B8A3iWE0CKLSKNw4QoIIS4SQpwphDhz9erVIx978+55AMARq2byZZNg/m3KPlEi8KyTDseTDlsJAHlit/TwlGcMjBORWyqptdKMBn79fexfvpCEaXp7p0ZHWWI3uWghpzNtPfgTURdp4L9ECPGNbPHjRPSkbP2TADzR1vE370qDP7/GUZK0kl5gUrl94mzuXgkp+wghEFDqAAJkCyfbiKYr/Ast+C/FR9+jDvgESDYM4gSnnH8ZPn/9QxMsVTOQrL5M9lnIqt+224cAfA7APUKIf2SrLgXwhuz/NwD4VltlkDZC/oZNJzpp/li6z79N5p/kAR5APoF7IgQCInSC9LbyoDpdoV/vhPeTuS9fSMLkkmH39yIAwIevum9iZWoK8vGznVsyBcy/0/L+XwDg9wDcQUS3Zcv+CsCFAL5GRH8A4BEAr2+5HNpFjpIEiWj+vaeP8G1899px5CARAHlK50RAZ/5JMrXNaV4uH/yXL6Kasg9NuOWaJAJBMN4xyxK7yTq/kFW/1eAvhLgebtL5ijaPDQBzfWWj0jRmLoc0iEnl9okSgRVdQ/YRaUUi0jV/eaJTpvp42ccDANP8HVVgoYJjIgSCMdvLdQZ5LWnNfyGxvx/lLFhozL99n3/bmr8u+6SVSAiBIODMX/Vs0JQJP1puHx/7ly0U87dXgoXqsmqCvCl2X6b5++DfCo4+eBZfftNZAEx3yeIO/lFc7PCV5xQQ5YO8NM1/umK/1s73ss/yhewUrXpcJl19m6iSZTN5LfkO32mADHo8wMSiHdmHB9s2R9eazD8I0gncU83fsHpOaVzlxfKyz/KFcvs4mP8CVeAm6qTchW1fZZ3Bk8KSD/5BFv214B+3w/yjJMlTtbYq+wiRO3qAtJM3ztw+REDI3D7T6vMXnvl7oIbmn31OvMO3UdmnuE545t8+ZPDXff7tDPLiLpw2c+kXNP9M9hEG848Y85822Uez3vpBXssWUW71tK+X9WTS1beJOqncPtNp9VwGwT/91Jh/S5p/lAjMdIL8/7YQJYmu+TOZh1s94ymOqprs45n/soUa4euSfSZZGoUm4kOZo8dr/hMA5bKPWhYlSWua/0woJZcWNf/Y0PxJ+foLzD/bZtLN5ir4QV4egGohVzL/Rej2KU3pLNct9dw+CwmT+YssB04bHS0Rl31aZf4id/QAiukPYgEiYsx/cXT4+mkcly/KAmS6XP63+DR/NYF7cZ33+U8ASvPX9be2NP/ZTjG1QhvHsTF/KfvIzmDe7zBlxF+L/t7ts3xRNcJ3oYhBEw33sgncvc9/AsjdPtnNjCqYxjhI3T7pJR2wwNt0KyNKim6f9Jip7BMyx1EbCeyagB/k5QFUu33kczpmpoWh0QjzL53JS/9cCCz54G/6/Ftl/rHq8OWaf9PHKo7wNZm/0vxljJ26Eb7e6ukB7vYpl30mrvk38NDmscbq9ikf3zAJLPngHxgdvlWDSsaBy+3TdHAz3T6yM3eQmJp/smDD46vgc/t4ANX5/BeKGDRxWLkPm+tbeObfPqQ6UtT8m7/qqc+/OMir6WMVff7pZxQnqdWTGPPPMGWx32f19ABQnc9fPreTbrk24vYR1czfa/4tosj8E+17k0iZf5j+r2n+zR+n45R9CEFAIJJun+kMrPr8CtNZRo/2Udftsxhln9KZvPJtxj7MyFgGwT/9VHNm6t+bBPf5R5rm39yxksy+GbIO31z2yTp8gVT3n+YRvrrss3Dl8FhY1PX5TxpNkKY6E7gvS82fiF5FRPcS0XoiOr/F4wBQF7tqOPk4iJIEM52i5NLk213u1+bzjxKRB/kwyFI+ZNtMW4cv4Jm/R/UIXxlAJ11725R9BEssuexkHyIKAXwCwKsBPB3AbxPR09s4lpnbp23NPx/h29KUjrL8us8//YxiwZh/MNU+f83t4zt8ly1q5/aZ+ExeDezDkcJBr/vjH2dULBTzPwvAeiHEg0KIPoB/A3BOGwcyZZ92ff72Eb5izBu8Z36AOzfuzvab7szM5y/XSTUoZf7tpLFoArxYdVnWb3zyh/jY1fePfMxT330Z3nbJzSP/vm3s2N/HFXduWehiTBTTPJPXOODsftu+HvqRXQZedswfwAkAHmPfN2TLchDReUS0lojWbt26deQDmR2+VZVtVEgtvttpXvP/lxsewW9+6kdIEmFl/maHL8A0f7RzvuNC9/mXb/uTDbvwh1/6MW55dBc+dOV9+PR1D4yklQoBXH7H9AbXb966EW+5+GYc6EcLXZSJoWomr7JnZ+3DO7Bl93wr5Rr3meV1+ss3PII/+tdbrOuEAO7atBvfvHVDvuz9V6zD8953Fb637omxylCFqe3wFUJcJIQ4Uwhx5urVq0fej2uQV9PBULLXGQvzH7ci7ZkboBcl6MeJ0vwtzF/m9gGY5j8F2qIN2gjfiuj/rq/ehqvuUQ/Chd9eh5sf2dla2RYKg6znu828UJPG3vmBNpe2iapBl2WJ3V736Rvwqo98f+wylpWrqd9/9+7H8/9N5v+Vmx7F3/3X3fmyHz2wHY/v6eG2x3aNVYYqLFTw3wjgJPb9xGxZ43Dn9mneew9AjfDVNP/x9t3Lmoy9QcKYP0vvIDt8M58/kL4ceAWctnAyzCAvm9q7lAKkhLwmU/aeHgtn/M138YL3X+NcXzXosuo27zowGLlsZRi3epXFF3N0exQLvT4bsaotLFTw/zGA04noVCKaAXAugEvbOJBrhG/zo27dzH9cO1ce/OPYyvz5oC55vmFIms9/+pi/QlXZAgvtsy1b7JgG+18b2LG/71xXNcJXOJh/kyaBsgnWR4X5exkXzHVCpM+tNv+30T/ZFjqt7t0BIURERH8E4DsAQgCfF0Lc1caxzA7fuKVBXpLpz1g1//H23YvSZnNvoDpwrW6fLLcPkLl9mNVz2uLJMPn87cG/8SJNDabtXrWJardP+mlalfsNDg6xPZ/jvlxM1t5l1mxT9omYnJseW+6jXSvQggR/ABBCXA7g8raPY07mEhnZNpuykMlK3M0nc2lO88+Zf6RkHXMCd3lMp+Y/ZTLJMIO8bLdoyk6nESQttUqnGXE+yGu4Dt/eoLnAaDv2uD5/M25LIwig193EwvxNZ2JbmNoO36YgY6Sp+QPNBhBT848a9PnLit6LYofPv9gKSN0+7WUWHRdiiEFeNubPrXNLBdOQ5nfSqMzn75B9ZGu4CdiOPe77t0z2MVu9cRb8zRi1VDX/iUHl888uaEse28gI/lpna1OyT2R3+7B6pTT/fITvdLLJYQZ5BZZaOliCOSFyW+7Udc+3h7o+f/P1P98g87c9G2O7fYx9dkM785eaPz+mZ/4NwdXhmy5r7uLmzD9U6ZWbOo7d7eNi/rrPX8bIaetE1GSfEZh/bwkz/ym7Va2iyoDhlH2aZP42zb/hDt/Zjr3DVzJ/gF+LdF1sywXdIJZ88KfsDPMO35aybRaZf5MdvkXZh+f24cGxkNvHMcR8ocGLU1U2W7/MUmT+skIup+BfPZNX+mnWgbZf/mMHf1Pzd7h94kQUmL/5MmgLSz74m7l92mP+Mu1CAKKmNf+i7MN9/rZWgMzt02Yuo3Gg6Z4j+PyXtuY/XfeqTeRun4oRvmYdkMw/bMD2ZZ1mcczqVZB9Oqqc5uh2GTvMVlDbbp9lEPzTT3OEL9AO8+8EhICMAVZjHqjP3D4xO46ErcNXMv94ETD/atmnuGwpMn+l+S8fVGv+Dtkn0/ybCP62Q4zv9inT/EW2jCCEyOf7Nqd99Mx/TLgmc0mXNdjhGystPiRqNKWz0vzjvPw2n3/6f8b8w9Ttk+QP15SFFI39DK/5N+nznhbkzH/a3tQtomomr/xxNarAfMb8O00Ef+txm9X8bR2+YUCG5q+PQfJunzFh5vbRO1uaOw7X4omafcn0Kpg/fxGYPv9plRKSIWSf5WP1XF6av0yGCAyf2K1J5m+Vfca8B2bg1jp882c4yH3+/DeTGuG79IM/9Nw+5iCvpsC1+DAgXfMfM07ZrJ62gA+YPv/hZJ9Jtg402WeEQV5LkfmjIhAuNcQ1iJga4atDEqJGmH8bso/x867m808/FfPPNP9Yl3088x8TSvNPP9se5NXJZJ92RvjGuVupU9HhK/sdhhk1yjdp+0Wgd3oNz/wH0dILkK7JP5Yq6vSLqdw+ptsnk33CBkKY5dDj1v86uX06AaU+f1Pz98y/GSjNv3hBmx3kpZqhqezTTMeyEEJ1+A7szD/kzD+7o6nmLyo71DjaksRsGGqEr6WW9uPmfN7TglwCWSa6j/aMOLZx1cMmmX8bg7wKmr/F51/U/A35x7t9xoOMi/JeNMnIOTTmX0itMPpxuJ9Z0/xDLvuA/S81/2BozZ9v0XYfgZ7bZwTm3/IAmIXAckvvENeQYF1Wz/lBc1ZPa4dvw5o/f0nJVZ0s+JtBP7el+0Fe44EoZeK23D5tWD0DGfwb8vnz4N+PEqvbx+b8kbl9bB3dLgwjxYwL/UVTvq1tkNfS7vBdHtGfEyT3IC8p++jLZYdvM5q/pcN3XOZfUj3l8YKAsg5f6fNPP2NLrGoDSz74AylztGv+zV3chDF/Kmj+o++XD2PXRvg6gz9z+2iDvGqcg2jnxWgF0z2r3T7FZcN2+C6mgLp4Sjoe6hAxxfzbG+Frq35Np3fQnq3sM9X8Rd4C8iN8W0BALs2/uWNwLd7s8B3nceapa11uH6vPPyDEQozMJifF/MOsnGVowuq5GGL/MK20pYA6/W+SQbuyeo7rygHs7qpx92v+3taqDnPmr8cmm0TdBpZF8CeN+fOmZhuaf2b1bOglwxmu7vNn0ziW5PbJA0qNWGkOO28Tpt2tDE2M8F0MAVV1+C5sOSaFOtOMuhO7ZRJJE7q4lfnX++mmXXPWlqsZW/RxLeln6vMXBca/6H3+RPQPRLSOiH5CRN8kosPZuncT0XoiupeIfqmtMkgETPNvyoVjgjPyNLcP6/Ad4yZqzH8QO5h/WVbPITT/IRw440Jw9lNxfZrQ/BdDJ+pSZv62e1znWXQtlx2+TQRI2x7qPLPb9/Xw4g98D9ese6KwzuQmeh+XqvtCKCJjWj7nBzG27J6vPoER0SbzvxLAzwghngngPgDvBgAiejrSOXufAeBVAD5JRGGL5cg0/2InSjuJ3ahR5q9r/gniuNjRZZOAwiBAHIucZdTT/NX/ouX+VHmsbhigirw1wfwXw8ApWcIlGPutQVq3MjpkH6fPP2P+DQR/+wjf6v3umY8QJcI6R7FZLt4SkP9Kxx53+/DtHtq2H2f//dWt5bFqLfgLIb4rhIiyr2sAnJj9fw6AfxNC9IQQDwFYD+CstsoBlHX4DrefT1/3AK64cws+fd0DAIAHtu7DH3/lVrz14pvx+esfBpAGYltityvu3ILv3rVl6LJf+O11+f+9iDF/ntLZltUz8/lzzf/j19yPP/3qbdi2r4coTvCeb96Rlf0h7DrQx99ffk++n6rK/9UfP4ofP7yjsvxCCFx6+ya8/ZJb8LZLbsY3btmAd3zlVnzphofz8vJjXXHnZlx59+PaPnjL5sQjVuKog2asHX73btmLf/7+g45yuMv47Ts24+p7Hndv0CBuemgHDvQj6zqR36t6+/rMdQ/g/sf3NlW0HHXv7TCwBek6BMkc4duLYrzv8nvygMv3sX1fD++/Yh3iRGD33ADvu/wefOu2jbhmXXpvv33HZrz9kltw+2O7tGPYrveeuQgXXHY3elGM+UGMCy67G/t76X17aNt+/O1/3YXH98wXyqD2afe8w0sAACAASURBVMg+rLpy5g+oVmyUJNbr1Jb2P6k5fN8E4KvZ/ycgfRlIbMiWaSCi8wCcBwAnn3zyWAcn1uE7KvMfxIkWiN/4glPwvXVP4L9u34TTVh+EMCA8/7SjcMyhswhIZ6aJAD53/YMIiPCLzziu9jF7UYwbH0ofwqcddwj6rtw+mtE/W1bI6inwwe/eBwD4pZ85DqcfczAuufFRAMCtj+7CsYeuyL/L7cvwT1fejxc89Wg895QjS7e79/G9eMdXbsVMJx13cMWdW7QH3XT7vOXiWwAAD1/4K/kyHvw/+tvPxoXfXmdlQ5f9ZBM+es16vPnFTymsKzudt15SPGYb2D03wLkX3YD3/foZOPesYp1WrbTqehknAn//7XWYG8R417GHNFrOf7ryfrzw9Op7OwxSG6PewI9qzK1hXos7N+7BRd9/MK///Hn+wf3b8KlrH8BvPudEfPFHD+HiNao+P3zhr+CSGx/F9eu34YQjVuJZJ+UqtLXN8eGr74MQwMlHrkKUCPzzDx5CJwzwl696Gr637gl84YcP48s3PJKVoVgXZV/dLz79WHz37setkqo8B/nyuPXRXXj2yUcU9jWVwZ+IrgJgi2bvEUJ8K9vmPQAiAJcMs28hxEUALgKAM888c6yzD4is+fyH6fA1bwBBsfv/+uMXYtWMupQy8ErIgRzd0KJflEAW7y9f9TSseXA7ds0N7CN8rfn89ayemqQj1PkcMtuBgNA81+b2NvBWRRn291LZ6jO/93M4/z9+gsf39LT15rWygb/bOgFhJgys7FnuRghRkAmmQUfvRwkSAcwN7KOTh0np3GYSuLr3dqh9WrQ9vcPXfjyV3kH/rhwyxcGUiRA40C9e43zugAIrtzF3uU+1TDJ/uajMkin7JN71yp/CozsOaPuR/5sD1D5y9f24a9NuS7mnMPgLIV5Ztp6Ifh/AawC8QqgrvhHASWyzE7NlrSFwMv/6+zBvAB+ZZ97EoJDSOe147QTD3URZ5oAy62Zid/t0rJq/biPTPfyq7N1OmlmwTKO0IU6SWpVSTW8ZaGWWkOUsAxkd2jOdALvn3DptIgD+nr3izs0469SjKsvaNqpmVZPL6wTeNgeExYw0NAVbXRlK9iH7drbnmU9iZNt2mFMLSOUPkq1N85rbjiVf8CtnQo188t/bnoebHirKbW0x/zbdPq8C8BcAfk0IcYCtuhTAuUQ0S0SnAjgdwE1tlQPQdeVR0y6YljLOns2bWNT80wo57D2U2wekRg3nI4lZcAstUzrKF4KssOZDIr/LCSWKL7fysqVOouqOKD4i2TYUXw5xLwP/WSckdEOyun1swfOH67fhLRffgg9csa6w/aTBWyY2DGP1tDHTppAy/2b3WaVlV6V3cH23jRXgUyPatjX3UXa9iShPyubqZLYG/3667cpuqGUYSI+fftqeB9t1N1vlTaFNzf/jAGYBXJkxtzVCiLcIIe4ioq8BuBupHPR2IUSrWbrI1eE7xDUtyiL2QAzA4vYRmue+LvjQ9k6YafhJktlJ1UE58+e5fQDVmaT3QTDmHwYYxHGhAleVNU5ErdwjvI/CNhS/juzD0QkIM53QqvkrZq32tz3rGNy4a672MdpClZVTdfjWZ/5tyFm8r6gp2AJYnaCmOnz1BI0SQqSyTZCNlpX7tbVcXEkOy66hbGkC6lkyd2170eTMvxsWTA2m5s9hq9dTqfmXQQjx1JJ1FwC4oK1jm+A+/1E7fG3BMU6SPJ2DeTx926xCiuEcrdJumTL/IGc0JmNw5fYBVIXtG7mGJGufCQPsE5FV1ioDH0NQtZ0so43p1BnkxQ8TBgG6IVndPjYdfJrSOlSlbBbGZ/m+9M8mESWi8etWxfyrXohmgkaOKBGYCXSCZ2X+Ml+++QIpKXdAaiIWFfyrmb/U/FfMBGn8YetMt49tXzNhkHcaLzrZZ5oQEOUsf9RBXjZZxBaIAd16mW4rCh7eOjA1/yhJ8/ObjME22je3kWUVaMCCpZShgJT5J4nIxw/wbcrgesAK27H5B0YP/mq97PC1MaRhNPOFgKgonxqNXZ/5t6P5i7EnIDJRpfnXdfuUpV/mKVzKNX+z9VDO/OVELK5g7Ar+AWV5/Env15KH61gMIPKazLAU0D74jwFXh+8wA39sHaJxXAzEgGG9hOpgHVX2kZlCXcxfz+2TfsqK5ZJ9cs2/k00oMQTzF9nvh2X+tsreCYPK/fCHk18LEzZmXaWjt50/haPKoTMMm5etwqZfdPm9bWC//L5Zg3Etq2f6ScZ3Dtvct/aXjV22KbveRIpQyWeoDvOf68eZ3k+a8sB/b+vwleDOwLbcPssi+HPNf9SRt0Mx/4LNcDQ2lVd8oty6GVuYP5edyMX8jXEHXPPnLwPz2DYoi1v1CfH5B0Kb24eq7wM/TCegrJ/Cpvln21seNBcmmRo6D+6OE841/xqkpEpCGhVKFx9/x7xstv6hWondRP4QOLczy+wyI7jPrZz5y2Pmso9x0V2a/8qZVOYl2PNmlaWj9sy/IQQB0/xHzLNvVqbcvmmZRs6McbKDdViWluf9JhjM333bctmHqjR/HvyHY/7m5BNl4G4fW2XvBEGlzMHLIvdTxvx5aoqqS87TZ7SNSs2/opVi31ezgWGYe1sFHmRtRKFOYjfz9GwvpdzFk5c9KX3ZmEUpu4RBoO6XfJbMVpHtRTM3iLGimwb/gMg6yKtsIho+529bk7osj+DPc/uwGzcMu3H5/Idh/sM+p/KQQc783VKTPCT3+fN9cKbMrZ0zDuZfdm2GyTfO3T4uzb9KYuCH6QSEMCQMSmWf+i/4JvPCV8HmRtLWy0FetYK/3GcjRcsxihfeBY35l8gwAJzRXwb0XPax3C6zzNU+//qtXB47eg63jy31zvwglX3yfWiERLWGXfDMvyE0kdvHfPsKgdztY8IMckIIRHEyNEvjHb5BkE7OUvXC4bl9OEzZR7KV1Odvl7VcGI75K5bjulZVuzE1/25g7ydQmnk1o5TgWVPbhgrYDoljCB2/6kUyKlxe+FHA9zGq20dJn+7tzDJX+/z15WUyG5GykEoJ1WypWpl/X8k+IIOQZJuXMX8+4XvTtluJZRH8eW4fTWccIvrbrJ71mf+4Pn/G/JPEyhhkMVQ+f/3WcreP6fPn1k/z2DbkzH8on3+J22cI2Ue2IGzuKesIWsuu+fEmKfuoPgn7+mGkHNuLrgkMkwJ8GFRp/q6j1WHpUsqVm1a5fYr1xlVqaRZJ/3d1+Lo0/xUdyfztVs+yDl+d+S+yrJ7TBD68ml/IcTt8bZ2v6fH070kivdP1jweoSlnl8wdUR685wleCyySy7IAK/oXzK6lv+VyjQzJ/1wjfqv3w1WFAuRPCnMRdltnmrODgTGqSsk/lIC/js96+GigYwzD3tgrDMH/Xs2FmOrVr/npQttVnQM2xMcwIX1uHrynz2NM7JFiRd/iSUSfTz7rM32v+Y4BbPfVMgvUvaoGBZOzCxWbN347D/PPcPrKTueSFY2r+EpGh+fP0DomwpK8oCUHyt3Wao3z+AWu5h/b5B3mrxjUqmS9WAdX+QpiuDl87My3bV1uDsZp4qfCi2Uf41pd9hPGdw675uzuYzX3UHWQ41CCvfoyV3bSepoYTtU7l9vGaf+swJ3ORb9XxNX9hbbrZZJ9R8qXwDl8pdQxiu9sn1/wDB/Pnbp9El32AYsum7HkwZxwqA59/wPqiZPemKucNkL7ccubvyESqaf6WXfKfTVLzL2Ov+vr6+2p8MJYcBTsJ5s8JScU+yvo4htf8hyN9ueY/RPCfYx2+BDO9Q/pZ2+3jg//o0HL7CJVaedz0DnU1/1E70ZTmz9I1xPZOZnlM0+cv0S90+Gbswxi9aB7bhpF8/gEVXpTELKx8WxPcGif7PwBbsr3idc6vIaiwDJgu2WcYKactq2eTmj8/j6oRvq43nuokl9/VdrKO233+lpdNvp2+vOxUE6GO6RrhW+nzd2r+ZcFfrfPMfwyYuX26Hcn8619UW2K31OdfLfvIztbhNf/0BwFRnrmzN4gdmj+ybdNPM9C6ErvNSBYd2Vm0DVLuqTN5dpXmnybk0rc1YfqiQ5li19FJbWti28oPTFr20T9NyMX1ZJ/yfY2KeIgXUCV48K/I518phVlssF1jGsSc4LlmxHK8MKtGs8tqNnDkBnKld+A+f74JH7nvhg/+jYBf/ChWUs0wl9Q2Atbd+ap/d1WaKnDZR7KEXlTO/M3cPhJ6QFRN7q6Rq1wduw7zrz4f7vYxyy1E+rKK2YvZBrlY/r5rMD5zO5vVU9P82e848286h72Jauavf9bZV3uafxPMX+2jSvOvSulss8HKuhsbHb425p8ww0Whw7fiHIr9ffo28ljXrHsce+YHAHSfv5nSWRj12Q5+7XzwHxlmbh/JGMYZ5JV2mtbz+fcdLoMq8A5fqfP3oqQe8y8ZQKJZPTsy+BfPz4VRNH85StlEyOY+cFVyWZac+WefJpu0ySa2pj5fzzX/th4yiao5eofp8G3N5z8iUbGB76HS7ePYR1mHr+y7U2VW+zXrc1lal7LrnSTF+2UaHaJEYP0T+/CmL67FX3/rLkRxgkEs9BG+nHxB1md3+BWWOtw0lkXw13P7JDljGKazzMr8Y1HQ94FiYjflDx6i0Kx8pDH/2OHzL9f8zbJzqycvI9/GheGYv0p7bS13wG24NZl/RSf1MBZDLvu0NWmGhNKvXS+57HOIfTUu++T9L+PvS2f+FcHfcR5lHb6K+evbRIko1Gf9WPYWo+v4VTJRnCR4aNt+AMDOA/38xSMdO2T8RlazMubPj9BWvVwWwb+g+Y/Q4WtL7+DS/M38/q7p36qgM3+p+ScOt4/8tLt9zP0WNH8z+Jc8EcP6/E3GbpZ7274e9s4PnJW8oPnnzN+l+atyyQeR3xK+nss+T3/vd/Djh4vT6DUF23zK2voh2HxbHb62eW4f23EA/+MzN+SSRl00MsLXaLnxcnU7puavnnGzL0efwc88ivsaClFsSdoSu23enU4WdPzhK1U+K0bI9JanXp/txy2/dk2g9eBPRH9GRIKIjs6+ExF9lIjWE9FPiOg5bZfBzO3THcHqaRsB60qyZuZ6M5uldSHvfx3NXw3ykmUor1hF5l+fDQ3l9mG5iFxz+PaiBK//zJpK5h/mzN8xyMsq+1j83o7gDwD/csMjZaczFmx9EhwqwNXYVz6grYGCMdicVx++6n7c+NAOXHHnFgDA+if21dsZK1ul28cBJfsU763J/OW6QZwU7mvZi6bc7aMz/0GcwOy7ToTIZ4p70qErCukbuOzMj1cmzXIsSs2fiE4C8IsAHmWLX4103t7TAZwH4FNtlgHQEyvFscibY0Np/k6f/wQ0/4Ax/8ju9ikyf/etlZ3VvKym1bNU87cEiLJty5j/fY+ngeSezXucIxnNCa/dg7zkZ1FucGmovYHOEO97fG/5CY2B6gncp4H5F/crAy8BuGPDbrzyH6/D3Zv21Cij+t+cLAjQX8xVso/t3s4Y8p9cNz9ICvsr0/zLZZ/iiHHzmm/cOYdv35G+GLudQMtkC8gO3+I5lRI0XobFGPwB/BPSSdx56c8B8GWRYg2Aw4noSW0Wggi44cHtuOLOzYiSesz/G7dswJ997fb8+zA+f1P2ceUBr4Ly+Su9vMrtQzWYf8I6q+V27Wn+Ku21rdxb9/YAAD917MG1fP6AYkym1TMPrmyxjcnyB9FkiPeWBP+3XnwzPnnteuu6h7btxynnX1YaFCs1f+cvLdtWtCKGxWs+9gN8/vqH7KNgs////Os/wa9+/HoAwLdu24ir7n7cub/r7tuKs//+6vy7i/mr7LP8/ghc9P0H8Pie+cI1szN/3Uo9148KxyrT/Es7fIU+OPP1n7kBl/1ks7bNw9sP4NEdB/LjxEZ9NVM6y/Vl0ixft+iYPxGdA2CjEOJ2Y9UJAB5j3zdky8zfn0dEa4lo7datW8cqiwyMl9z4qMbWyx6cNQ9ux1X3qMptz+3jcPuYg7yMxFN1IQ8pc/sA6YukVlbPig5f+eCZsxSpbaqZvxDVL7Qq5n/R//w5AMDBsx23z9/oIOuOwPz5tq7/5TldcNnd1nLcsXE31m22vxyuvDtlft+4ZYN1PS+XW/YZhflXbloLD29LA5iN+dvK85nvP4g//PJa5/4uXqPLZ7bBdJrzji1/YOt+vO/ydXjrxTczn38KHqjzidWNUcnzllHbZakkyi6hMJj+nRvLWzxcUuXMn98nWU4+ivcj5/4sLvyNM/Lvv//8U/GaZ6aceCo1fyK6iojutPydA+CvALx31H0LIS4SQpwphDhz9erV4xQzn1ylHyWIEiX7lD1kvSjROo2GYf5NyT58MhcezMutntVuHz4FpdxsEBXPzwXeXK9iJfwFaXshPe24Q/Hypx2DfmwfmMPL8pwnHwEAJa2V4ktWdgrzxHZc87flJ/rxwzut5RDCnc/I5UCylc8t+2SfNcwdTfv840ROzVnszB/lCGaxTHkNSK+VfJFr9ywrw/5eXLinnGxIH33fyLNvSpiAPiDRvL5ldT0R9V6wxx+2It++GPz1xG5mahUAeOX/cyx+LqvfAHDwig4u+PUztO2bRmecHwshXmlbTkRnADgVwO2ZBHIigFuI6CwAGwGcxDY/MVvWGmRA7EUJEib7lD03vUHaaSSEABE5fP7lE6tI9Ee1emrMvzz4B0aHbx2fv0yXANgkFHe5uDZfxUp05m/nGjNhkL2Y7VFPCOBVzzgOH/ytZwFQHb7DMX89sR3fN8cZJxzmDAa2NNISrjQZ5nmY5dPX6yy3DHKbpuKC1LJtrdRR3i/mdbIy/1i55WxWSM6YbbKP9NFLkpanYbAcq2+McNcL6z6PRLjvOccRB81g0+75/CUKMLcP7H1OPPjPdgItmVvI05gsJqunEOIOIcQxQohThBCnIJV2niOE2ALgUgD/M3P9nA1gtxBic9n+xgWx4B/VlH16UQwhlKOk6PZJg6B9XlqH5j9ihy8ZzL8sq6fN57+iq5dRspNOGIwk+/CgW+X44S9I1wtpppMG/zLmzy+za5CXTTbJ+ye0F5b9XICUTbrKIbOq2tDNy+S+HraWib5e364MTQ/yksHfNsJ3JOZvfJ93MX/LNKi2DKzC+A4gz50jB+qZk65w6DPZlZeVI2X+1VdgRTfMR/KqvFlc81eQ62c67JkO9eAv814BU8r8R8TlAH4ZwHoABwC8se0DyjjYi2Itt08p888Cdi+KMdMJLPnuS9Irm7LPiLl9lM/fZP4lWT1lhy97Aa3shpoOypm/W/Zxl4tfi+GYvzv4yxezDUnW+pKoSknBd2PT/LWXg3FTZrsBsj7oAoRwZ2btGKNN7edRPL62/8I/bqh9VW9bB5Kx2q7XKNJSLebPBlzaWmN8ZKztxSlTJpvTK1qZf+Rm/uZ3OXGSLJftGvOEhEDaeg2INILAn0lba9R88fHvPBdWW26fiQT/jP3L/wWAt0/iuBK57DNIpQWV0rmM+Sf55yEoJjHLO01LRttKuGYAqgJ/CMIK5m9q/tzqmTaP1QAdIZQWPz7zLz+nmOVScnVCz1YwfyH0a2pmc8zLbJnMRd431wvLDFIruqHzYbMN8JGQUlSZ7FOl+Q/V4cuCUxOQ2rbN7TPKEcxzdDF/Jfuw42X/8xn4bJLZbMcu+5h1GSiXfcxLuLIbYm8vyre13Y+ZMMBcos5pthtk6clVXZP1nUhP6czzXWn7ZMw/IMoJ3KJz+0wTZNiYH8RIhLopZc9NnwV/YNjcPvZ9jZfbp6bmnx2bv5SkNprvN2N4AVH+0hjF5w8Mx/xtqTCAmrIP+6nK529ncDbmr4/wdJe/G7onlOc2PhPyQS5j/pUTuGeL69SSYSSiyn3JgM+Yv7bfEQ5h/sTl9rE9i/m4Ak3zz8rKtgsDwkwY5K1auc4W/PVpTMvLOsuel0TY4wQP1EBKYIiUIiDLJ8/Drvnrz8OMxvxTBSGgKXX7LBbICn2gn76p66R0lmxCuhRs9rA4qZfPP0/vMGS55T1Pc/uoW1Unnz/fZlZjFGk5kox1uZm/u1y8/6OqYvI5h5UGqm8z0wnQK3X76Ne0k1s97Z3UurMie3lzt4f2IOrHSgcEuoOz63RdDiTzPMzy6evrM3/bmIZRkWdVFcrtU2X1NMtRtbxK8+c6P2/xmq4mvl1AhNlOkD+r5qQrHL3svsj6X1bWlTPqeUmEvbVXDP5hOh+1sAR/UCH4hwEV5GEe/HkL3jP/MSAfyLmsAtaZyatXwfzz3D4VwV+mLwDSSj3K1JEF5m/NJ6QfW+/wVUyGz9mbVsB0uStVgg3DMn+zTKbWOZu5fVyBM9X81XcVaIdh/vVkn4Dc00rGjkDA91PL6ukI2MPo+I0yf3bd7Jq/+7cumcv8jdvnX3wW5csodcnIoF/cb0Cp3NIzWta2Y8kXQjcMLIO89G1XsufF9cKfMetwJ9P8E1gGeZmERB9jk58Pe2b5M+OZ/xgwg0qdxG7SQSArkjufvz1fDf+/X+I0KAOfzKXa7ePu8OVun5kw0FJTyN+5bJM2DKX5J6Lg8y88ONnDxtmh2QHIHxQzp4sqsyj8No6LwayswzcMKmSfktYJUO5+quzwVU0X5z7UvuS5Vm5ava9Efqrzs8kwNrhmQjN/42L++bPCWxqqyVvIYcRfvkFAmO2E+bNaKvtky2Yy8lNWVh78k8Sh+XeKer3s2K0a5BUnCUKiQguYg5sk/ATuY6BvXDybw8CEbEpKxmDLHe9m/ur/TkCFWbTqQlb8Wm6fQG2bfrcz/zAkxvyDQioKiTr5/IFqD7LN5981H5zsfkhZDtAfFlPzd2f1LP5Wvpy0e6C9CPTy8jxQJlIW6G4VAEXXlP77YsvE3H/Zeo4mc/vwfdnmaig7hGsOZPMautw+cqAhPwSfA8I8T35tiKDJPrnP39bhK5l/JyiUzTy/FQXN3xL8C8w/ROCQfQrpHRJoxMuGIODMfxH5/KcN5hSFdXL7cKsnYPH5JyVz+BosvV/S2VQGzecf6vssHNPI7cOxInNFdALKJ0xXzL/82DYMzfxDnfkXOro6tuCvM3Wd+dtlH7vPv9hy48zelHFMW552Lg7nB9+POVhO28bSMtHKj/L12rYNyj58JrVhff6uaTB5sOuGZGf+sRpoaOuEJ7B+kmwd3y4gym3CgLomZVbPbliU9cwqLMcPyONZZR9T88/dPpZBXqRP4xgnCcKQrM+qOrf0s1PSEh0XyyP4jyL7yOA/sGv+pp2Lg0sunUzPlhiK+WebmrJPWTI5G5uQlVk9aG7dMT92Cdng16KqScqlMZfmLx+kOWfw15PlOa2eloBo0/xtzgu+7zLXUVUKilKff6JvW9yH/lmGKtvoMEhYwLdm9axBkkzw36zohm63T0iFEbCyDDwtglLEePDX962snsUCyxjQCYLiIC9jgSRLcp3tfpkEJnX7ZJq/jA0hd/vodbKK+csY4jX/MVEM/uXMP2LOE5fmL/dZ5fMPiLTKOJLmH+hST5nUZKtQUvPPm9hCaE1uG8qZ/5BunwrNX37fz7IxapqzMK2e9Sdwt8kYZbndZdPdhBCi1O0j91nu9ikP2MNIOVXOoWHAXzrK7aPWl8qjDtmH/8IcZCghiUFxBKxy5pitJV6u3O2TO/LS5Va3T7ZspmPT/HXozN/R4Wt1++gjfHk/nEk4AlLB33weAN215zX/MWAygU6F5s9ZimzWmsxfMYly2Seg8tGFZZBb1mH+ZbKPHAwjmX6S6E1u67FLisnjW5XsI4/Dy11w+3SLzJ8fPxGm1TNj/o4JaGwSglP2EUJjcQHZH3abHKIfuzr4V0o1QzzjZtqDccCvkT2fvxsu2Yf/aOVMaN0u7zMzpDYZ7AhKLrHJPkSEWQvzl88bf2ZkDLDJPraBfuo0hLXDe6YTGt+D3ClmDuJKz0Ovk5x4mS8SgHX4hp75jwWzA0hOXeh6BvXg72D+Wcde2ZSK6XoqTypVApfVs4z523avcuvojgTu83cd24bhmH8xt0+3ox/T3uGrB2hbh685yMvGDm3uG3OKPG1wDdkftiqnjnL7lMg+lpaJbf1QPv+GO3z5tIl1JpSv4/ZZ2Q2tLYQ8uWD6gxz59a9k/rLD157bh5OMAVtWlH307yvrDPKy9FtJq2feYZ2bMGC0bESm+dv7wAAm+1iSSjaFZRH83bKP/aJypi6blOYN6Jcw/1Bj/vr64Tp8008ymX9JM9F2Ttw5oGv+QYns4y6XpvnXSOxWcPs4NH9n8E/03D7yetim1jR/awvkvDokQncfBYF9kJfcp4vY2xLIFfeh78uEXFonnuf7asAIorl9LE6ocubvGpuh/l/RDdGPk8J15SlG+Bru9lGtpawsRoev5vYxisIZNff5V8s+ww/y6mZjZrTEbpL5GwMHU80/KGX+JEfqe81/PAzr9uFNVMX89X3kmn/FIC+zYTCMRst9/nWZvy2wyOJ0jApa1ulU1+1TZ5CXqfm7g7/S/HXNufgStfmfbQHCxprMVoU5stJ27rns4zhfecx6mn+5dDRch+/4gYHLPrZ7W9rha3HxpL/RmT9QfFGotOLF1higyyW2lk7K/LnPXy8oD6q6z18vq/m7OoO8zLw8MkNuLIozednSOwSkCJtV9mEjfH3wHwNFzb/c7WOTfcxAU6r5c2fKWMxfMSBe2cpeOLZT4qMFZXDjD54NdXP71LF6mm4fm0ca0Jm/OW+qecrd0JJp1Sb7WJi47vMXeh51h7WuKtjW6fCt6/MvG1SlyqP/ZhyoQV72Ga/GZf556mVD95cEhIztuWxiloFvl2r+fISvXgZez+Q23Y5N89d/t6LGIC/zGQwD5IndJFHUff7meQf5dra01togLx/8R0dR86/o8B3w4C99/mbw19/uHPxemvk7Rh7kVdPnb5V9OPNnFbSc+bvLpbHDSqtn0e1js8kBptVTL4t5HVPmrqAFwwAAIABJREFUr99XW4eqTZbSZvJKLMzfEs/kMufo32xxHdnHnQ9H/yxDW5q/zQk1muavsDIfwW0w/5i5fTRmnG6ndfha7q0rt4+EJvuUaf5G2U23j+38iy1RntgtXZZn9YRZJ4WW9sXu9sn2EbY3yGsh8vkvOIaSfRw+/9xRUMPqyTFKhy9R/WkcbefEmb+A0nZLff6lzD9h/9dg/mG52yeXfQZc9tGDkFnMbki1snqWdd4CaVDXmb/93PN9V8k+NQZ5uS5Z1THs+2pA9pH9GUIY2U/Tz1LZx+n2UT8yZ9zKj8vcPry1o3z+xU5nXpZc9nFkzeVBdVCm+ZfIPq5BXuYj2AmIJXbLrKqS+QfFl1snVBbwMtln0TJ/IvpjIlpHRHcR0QfY8ncT0XoiupeIfqnNMthQldWzltsn1/zdE6sARdlnmGdVblvU/N3HtGv+Sj+ULKt6hK+7XKNr/hXpHXr2Dl+X5l+0ehaZqlXzL5F9pG5rIq4I3Er2qWb+zg7fXPaphpnzZhxwOcqq+ZeUyOapl/uSWDXjYP6JQBAUB3nlmj+VWz0l8+9n07Oa94a7yvqa5l9f9nEN8jIJmGxFxzWZf0CEZxx/KH7tWcfjH1//rML+c8LmcJ81gdaYPxG9DMA5AJ4lhOgR0THZ8qcDOBfAMwAcD+AqIvopIYSDQjQPOeVevQ5f6faxd/hWun3GkX14h69lhCuHsnq6ZZ8gIIhEzdXaRG6fSrePxedfTOxWdPvww9s0/04QFAd5GSNohRDWwUWm579ryD4ia+rzayNfGK4HkadIcKFK8x9ukFeDzD/X/IV2b21s20Qtq+dMMXFfetxMegzSkbyDOMG9W/Yyn3/xPHWfv6o7/TgpZf56egd3WdNtuNvH0Zo2g3+oS6qAnmad7yIRKSHqhgE++tvPLu6c7X+xMv+3ArhQCNEDACHEE9nycwD8mxCiJ4R4COl0jme1WI4C5PSFQgjsnhtgzYPbNb35jg178v9vfGgHBpZc8w9u3Q/APqLWHOTF8cj2A6Vle2LPPNY+vANRnOSVLiB9n2XpHWyxOK+EUG6WKub/kw27IYTA43vmsa8XYfPuuXydyQ537O9j7/xA+/1jOw5g5/4+9vai/AUpy22+MOVDuq/nln3M69y1DH6RwWrz7nnsOtDHb336BmzbV5yTMRGqzEmiBnnx8RSJAPbOD7Bjfz//bpZr8+45NVcDK0pVvn53bh/5e335hp0H8nPdtCs9ptLArbvSsHHXXOlLyeX22deLsObB7dhj3FsOq3+flQ9QfTxRIrBl9zx6UYy98wPsPDDIff4CwBV3bsFrPnY9/u6/7waAXEMH0lbDD9dvw7a9/Xy/KfNPXyyPbD9QuG4un38/SrD24R1Y8+B2HOhHhWeG90klwj6Bu/ncBJQ66Q70I2zePQ9A1XMZa9T1secEs6ETuueXGBdtBv+fAvAiIrqRiK4joudmy08A8BjbbkO2TAMRnUdEa4lo7datW8cqyPNOPVL7fujKbpaHQ+C937oT5160Bh+75n4AwI79ffzTVfcBAI46aAaPbD+A/7h5A6JE4OiDZ/N9XL3uiWxfxcbTISvSZQfPdgo3+dyL1uAnG3Y5y/q7n7sRr/v0DfjGrRtVoDHqyaErisd8/mlHAQBOPHJlYd3Bs+n2h2XnnQjV5DYZjMSlt2/Cui178eqP/ADnfPx6vPJD16kMp6yTNEoEzvvyWrzv8nvy3965cTde9IHv4ZxP/DArbxcAcPiqLsKAcPQhs9qxDprtYCYMjOAP7X+zhVJm9fzAFffi1z7+Q6x9ZKf13OJE4C3/cjMuuOweJEKN1nz1GU/KH+o4EXj+hdfgOf/nyvS7wTx7UYxXfOg6/MfNG/LtJary3bhln+LLYef+Pl72wWvx3bu2YK6fHvM/Wd3g+xrECTbs1MnFnvkBXvbBa3HZHZutx+T7MH3+H/zOvTj3ojW4c+Me10+xdV8a6HYfGODs912NN37hJjz1Pd/GXZvUb9QsZwnO/vur8Y6v3Irzv3EHAPUsJkIUXjKmBPc7n70RV9y1ha0HjjworVtvufjmwnVdzeqZcvsE2D03wOs+fQPOvWgNPnzV/QVR67CVXXZt7PcrJMLvPO9kPPWYgwEAJx25CiERfvTAdnzy2gfS8jGrp9kPZesrtCFscTKXsWQfIroKwHGWVe/J9n0kgLMBPBfA14joKXX3LYS4CMBFAHDmmWeOdfZfetNZ2DM/wL75CAf6MZ523CG5pW/3XFrhZMXbN58GoDe/6FT8v897Ml72wWuxZ36AOBE444RD8UcvPx2/+akfAQBe8NSj8PNPOapwvJecvhrffNvzsfqQWbz9X28trN++v19YJvFENnv4nrlB7oKRrPf7f/4y7DzQxzNPPKzwuz944an4pWcch5OOXJUvu/29vwhQ2oH1MycchqesPghv+PxNlbl9fvusk/GVmx7F7rmU+Ur2KyezjxORaq1Zi2j7/j6O3KfOSW7/6I4DWNEN8OYXp7f92ENX4Lo/fyk27JzDp7IHBEg11m+/60V4Yk8PNzywDR+9Zn1hjlqb7OPK5y+PDQBHHzxbYP+xENi2v4fDVnUztw/hxr96BY5YNYPPXv9gvq+98+xllLPj9Pv8IMGBfpzfy6qBZek2cl/W1VbNf9fcAIM4vcZzgxhzgxg79vdxSPZC5XHp7/7rbvzLmkdw6///CzjioBkAwP5ehH6UYIelBaTKpYI/L/vOA27GL/H1mzfgHS8/HbvmBtiyZx5b9swXtpF9KlJ3/85dj+PspxyJld0Qb3vpabhkzSOZ1Kb/juDuUwDS4PqrzzweH7tmPeb7cSFIn/fip+AXnn4s3vlvt1lTPqzshtgzNygw+yMPmsEN7345XvPR6+GawJ2IcMGv/wySRGDT7jmceMSqAkFRzJ9g9kOtdMitf/XLT8Nnrnsw//4Pr3tmI/06NowV/IUQr3StI6K3AvhGNmH7TUSUADgawEYAJ7FNT8yWtYYV3RAruiGOOUQt62QjOWPjoZZM4xnHH4YnHbYiXyf96iez4PpzJx9h1cyDgPDsk48AANhe8GXNON5xq2SfdNnJR63CyUetsv6OiLTADwCHrVIM5qys9aN0SXdun58+NmUzhakds69RkmC2G2BvT0kFWvBj/z/n5CO0DrQTj1iVN4s5Tlt9ME5bfTA27UrlJVPWsA7yMmWfwl6BNe9+Of70a7fj0ts3sfMQ6i+TlI49NL3XIbv+2rmb2nNi/wTcDqj6g7x4K0Llqs/rKuuE5Ntek7VG9/WiPPir31gPqW+T6G6fsjELAPB7Zz8Z/7LmEdy+YTdOOLzY4pTIJ7c3jBRnnpLWDamJm9eFqDz4ExE6YYAzn3wEfnD/tkKQXDUT5vd1EBetzatmQsRJsTt7thPiuMNW5Engyjp8g4Bw4hGrtGXmNmnfhVouNX8bznvxaTjvxafl32X520Cbss9/AngZABDRTwGYAbANwKUAziWiWSI6FcDpAG5qsRxWyOChKr4+kjcMlMMmThKrRm5z+tiOY6JOMy5iAbWmPFgLAalA4vL5y8R3pp4b5ddI5FqrDP7aNIks0pR1TlvLl11SM7gVmL/F/2zTZvl9VOch8j+zP8E1s5mZ2M1MFV02R4BZPheTU/0KahmfTCivq7HSoc2+EUDvHzLrd9lxhdDPuyr4S3nT7CswIWUfzUU3SPKWrbR0mjKeEG4JDVB1QkojhZYDy5zZj5OCtXk2a8Wa9Ya3uBNLiyQ9ZnW9ViN8KTufrM7E9hkAJ402ff6fB/B5IroTQB/AG7JWwF1E9DUAdwOIALx9kk4fiTQ3jCg8xDxPv2SBMlCERiK0OrqdjVnXsW7FsUDSce9jVGi5fUK77COZmvngxewayaZ8lLFF1wQvNoZTdj6mZVWNdahm/uZllS0b8wUng5X8s7mzzFgp921mCVUvBbWtm/nLz/ovB241ziejT4T1RWGrV7b5DIrlUi8wvl0Z6wagEYAy15e09pouOvl7afUsvHCFqAj+ykBgtj6BtBUn+3z7UVHmnOnYXxrSQUQEJ/O3VeFCHSUl+yA7RyI92eFCorXgL4ToA/hdx7oLAFzQ1rHrQLIF8yGO2MjdIKssMkiYTLlOj73p8wfKH0Q+EbhL7x4H0nlQxvylS8IclMMn95bsSLaK9KkdOfMvHwdhQiWoS7/L5878jXzgOQoPP3NbmOchX1pCGE4qsu8rMQIzD8SAzvbdmn+57JNr/lz2YYMMeV21OYfyIG65F2UjsV1uH3P6UxNyngjz5W9CWqv1kfNJHmSlvdZ8LuJEOHMHpb9LP+Vob9v9l/VmkDlsuIWyEwZW2UeaGWS5anC1dJ9GHeUdvkDmWgMhSpLabp82sfBtjwVCJxskVGi+S+afTz0Y5IEiDCjPtif3UQXbTS5rgtse8LJgOSzkQBQ1h29xm1z2cTH/WAV/GUh15q9+VzXHsWudKWvYmtQF5m98Ny2m/DzyP6GPIZAPrDnQy8X4bdNEuliw+UIzIYxPQJ9GlNfVMuavjcCO6zN/IfQxHH3X6N0MeR2IRen+ZX3iaVZ6UaKYfy5FFtN1lDF/KjB/c716dvpRgk6o6nsYpJlho6T40ujkwV8x/0I9tpyuS8nJ63T2PRH1iGPbWLbBX2n+2cNrPCQ8GVmciHSwEg3P/G3BtSz/S2x5wJsO/lzasjL/nKm5mb+UfeJYZC9R9ZBqzL8i/YVrnRncisy/6PYxgyrvlNPOIxaI4iR/aXHGlh/f0aqQLwWTNFTJL2n5ypm/Lb1DL7Ix/8S6L/PF5FpWOG6ittU1/3LKO9vV+31cyGVEzvwHsdL8kQZGW9p01zzZABsFG5LWUpbg/T29SNf8ZTqG2CL78P0nIr1vZhnsObTs5TTTrfN8VwuJZRv8ZYehyYxiFhjlp2Q2HUMjH535ux+UiD3gPLdPUyDS01GXyz4m85eBKDE0f5P5q//tmr+7fHLzouZvbGfV/B3Mv6D5J3nASnXY4gvdvEUqnYK9BWAO37ehdm4ftp4nFuR11faikP/a5lsoY+au3D7Vmr9eB1ywyYjzTPYhlnKEQw685Ll2OGRdcTF/PlWicvuk6xTzdyevyDX/pJiPyvbCcAd//TfxEIO82sSyDf4ut4+s/Po0akneMagz/xpunyE1f878XXr3OEjnFFaeZ1sd7Dg6fPkLshMEqlVkPPy65j8c8zdZkmCZTbUyBrbJXPR9yfvj1Pxj2eHLy5adgyn7GEHfRRrM/23lq87tw5i/Q/NX/QOsjKXMvzrhHJ+IBKh2+6gO36R0/zYy0TdkHyGKL5C5rOUp+xZMqISFQYGAyPXyEe2XMn/7/ZD+/MTK/IvbuwJ6NlcZ6y/ywX9B0WGBC6hg/lmw6BgaeR3mP4zbhz8AcaJmEGq0wzfgU1Daff7ODl8W8OTDM4rbp47sI4xAadP8zdjkYv6m7MM1f/PBrpJ9pAxgdqRqzN8hl5idxiZsAb3a7cMCvSFJ8f9LNX9W5zS3T2XwZ8y/RCJyucc0qyeK11wy/9mOnfnLapTP6Wz8PmSOPWn1JPbC6AQBorhC9knSe26mIbe1F1zPqdL8VV3xwX8BEeYduQajS1RglJ/cEji028dyhausgHIb+X3SzL/r8PlzZimbzYMsB5Hu81e/q5rj2LWu4PM3fmRj/i7N32x9DWLl9kmEsMo+PJAIoU/llzpTDLePRXs3Yb7QCuuhGLiEy+dvk4hUEC/2v5Rq/kJ9xolKn10p+3Sl46tK8w+s++N+ehvzlzO7VTN/Je3o61kHfv78IvsNKpm/lH1smv8wso/Zj8Uz3S4klm3wdzL/WDL/IP+UgcLsIK3j87dVCFcTWes0ZQ94s5q/mlA+DO1ZPWvJPmHK/PuMmZrbAeP4/KF91vP5G8zfmEdAQgaJ/KVu6fDlgSQRRoeuKMqF/JY63T6WF4Vefv0TcLt9rB2+kvnHxXtRR2pMMjIkrY51ZZ8qzV/WAbMlKUd+E9SLh0PJPi7NX7XOgWJLJTCfV/Y9IEInlG4fe7llh2/q9jE1fxvzL9f8uRV3GgZ5LXwJFghcsgCGYf76PqpgS5zmelDMAJqmFW52kFdAqGb+gV320Zl/gE5A1pTX1Zq/u3zmg+LO7TOEz9/4sSqzKNju5LZzzOlkylpcHrGN8K3W/K2r1SAvrvk73D5yH/yU8w5Fray6m81eLnUOCQv+JfEcANhYD/tAKAnXiHEl+1Dm9tHXy3Tc7uCffrqYf0hmKnQ1abqcI8Pm88/3H8h8/jbZpwin5m9Imanm7zjoBDEFRVgYSNnAqfmHilUozV/XyGu5fWzM3/EgRsZDm4hmJR9Ayj7qBWd1+3RczF8FolTzV/Onxha2CYyj+euyhm3aPFdWT/PYvAgzYaDp6EmizxIm7xdP8W0GN+5M4axZosrt4075XDwPXfNXdbVswJim+Q/h85dZPW0zS9kgZZ/UOuve/4zDPSatorLD1/XSdLl9yGD+pjWVd/jK7eRviNSzXdbhK33+dWQfV7WWiwV7yXrmv4CQKYG5tRKwuH0CwiBKR4KaN6zODbS9H5zM3wigiShOYjIuAlJT2qUPQ3GbPBdLDc3fNs1lsz7/eszfntcnG6thSHWag0aYsk/6qTP/YuK6os+/mvmXTeYihP1c5FgLbUxKYu/wtR2/jttHvcDS/20Titsgt4uTihG+OZnQW5IFn7/jBeLW/NPPUMpUkSn76HWNkx2N+TuKLlOhCFG0epYle3OVk7fMvOa/gOiEavAWoCqe6fYJmbRhavx1biAfTi7helDMAJqIZiUfwND8Xcw/tD+sKuAludvHNsF9Iz5/I6gWNP9Q1/xtl9Tm9umGgVbmRAh98p3sfz7rVByLQkA1R/byuFXVoW8LHDb5BjCYP2PxZS+Sod0+hs+/LvPvBgG6oWoZu2BL7AYYso8o5lnKtxtC8+e/D8mc/pSY7KP681zCT0BwM/+S8pjg9uWy85w0lm3wr+v26YSUV1pXytbS42Q3nmuGw2j+rTD/XPMPhhvkxV6QaW4U0jRpdR4Vbp+Skxoutw/vlHUzMc7su7zMcYIkMdihlH1KmD/PQJkzf8f5c9gGZtnKr6d0Ls/tI8vDMbTbJ1HHTZm/uh5y/l0bwlCxZ35Mk6m7rZ7pvgOSDiphPZ5zkFd2GK7587IHpLds6zB/vr3K7SPyfguJYWQfZfVU98EH/wWE6fYxP3ng6DGZhKNWh28e/NWlruX2yWWfhpk/1BymYWCXpZQ7w+724T5/2wT3/EVg6/Oo0+E7bG4fW/C3Mf9OYGj+whjklf0/1zfuBbsUer1JCsd3+/zluRXX8UU68y/P7cP3mx9/SOYvO6uFSHXzGearLwv+nYCYG06w3+j5IkNHfcpH+EKN8D1opphr0iX7yMFTISMM2nzMARUkP/lVaf6624f/niCZv60FayEbNZi/vNY++C8gTLeP+ZBwzb8/BvOXgYdPJl2X+SeihQ5fdsedE7iTLneZ5ZNWtQ67NqZNVcJmh20it09IpuZf3JfK6smYf4c0t08s7Pn8C24fjZmz+mLIheb/HGW5fRJj/xJ5/0Rsun3cLz6r26eU+at1/SjBjMb83Yl/Ze6cyJjj2mTqAcn+IYfmT6rVYXvZrKga5MXKK58z/vyq8qr6nnYGp8kduezDn1PZ4Tuuz18RmqKsvJBYtsG/E8g0sOl3FdiUJCI/bVPA2b7bIOuSzvyrg7+cZarh2F9wK9lOgaB7+PPySR95In3+6troPnfG/EfV/I1Aaf6mqPlbmH/u81fLuqzMMoWGbeAe1/yTpJg6Oe8kZdo7z3dkQ1luH03zZ8Gob4xJAFAYlWqeu55eW36Waf7q/1Q6UResmvkXNX+zz0BKLKYPn1s4BdLrZtP3VzrKYA7yAtRzJhe56rt8IcVCv5a87OlLCVbmb03s5uzwTZdLacss80Jh2QZ/szKaLQDOHCRTLLp9hpB9OsNq/kkhMDUBl/tB3waZh9+e2E1q/mGASp//sFbPQm6fXPYpvnglY93Xi7TJ39X5BYXfdg2rZ7qdhfn3TZ+/2q/N558I5Y+v9vkX1zs7fF35/EtaPcMyf96qMa2esyWdv5L5myN8zd9IiaV8hG9aVtNPD9Qf5AWo58z2YuD1nUiN4eEw+wxcuX3szN9azNzqmQhlMJmG4N/aZC5E9LMAPg1gBYAIwNuEEDdR+nR/BMAvAzgA4PeFELe0VQ4XuE0RKGr+eW6f0K35m6P+bLBq/rV8/u1ZPSVcKZ0pY2qFaRzzTk7p9lGBNBFpa8XMtjlsYre8c6xK9smn7hN42yW3WIMUn0A7XxYSenPFNABm2bjsEyfCkH0sbh8ZNHt1fP7udeb/WlZPp+YvnOMMuEPLBbPDmNdVV14dIK3/OfOPy4N/YCETtsRutrrhegHJ+8ZJWdeUfSqYf2S8SPU+AyCJYXXd2e5wVeppAfWiXeqyzwcA/K0Q4mcBvDf7DgCvRjpv7+kAzgPwqRbL4EQauHRHB//Umf8Ymj+No/k33+FrMn+y1ACCZP5FzT9J1KhYs3WgMl+qZcNP5qIz/zKff7oeeHz3PDbvnivsy6b7cqtnfkyDHQJFt48m+yRFzV9n/vZAq15oFuZv2Q7gbp8Eus9f7yPgrdhh3T7mOo35OzpbgYz5h0W3j/nCcGr+bLpEAbf/3RX8zUFegBqdzhk+L2+u+QdpDEg1fwWX5m8Wa7h8/tlvWAtpqQ/yEgAOzf4/DMCm7P9zAHxZpFgD4HAielKL5bCiELgcPn9eaQs+/xq5fWRd4tu63T4m82/D56/+74Qu2Udn9bx8nLmExgP9hR8+lOeHkbBV8qFy+yTFcqf7paxMCfpxkqcC4LD7/IsMVO/wTT/njRG+mpQiii6xRIhccrjtsd3Yvq9XKI/5Qrv9sV341LUPZMnxXMzfPsLX7IPQJUyBR7cf0O7Fjx/eiT/56m2FMvFzkJipYP6rZkLMdoLc7fPNWzfio9esV78xXhgEWOvTitzqqRK7mYRqtuMwJaCY3gHgsk9xnT23j675a24fUrl9Cs+JVfZxBf90+Sv+8Tp8/eYNeVkWGm0G/3cB+AciegzABwG8O1t+AoDH2HYbsmUaiOg8IlpLRGu3bt3aeOFCFgRms4mcgRaYf1CUfdzMn5Unbsvnz1luYN+/U/PXmYu5zfsuX4e7N++pofm7y5ezpOyJlGMSzBGWPI1vP0q0Dlp1frLVZdf88+148Hcwf60zXvP5q+Asg+ZXbnoU//yDhwrlUZp/el7nfOKHeP8V63DLIzuNDl8FfYSvalnxKiQSPWPm5t3zeNmHrsX37n1CK/cVd27RyrNh5wGc8dffwf2P79OWc+Z/zKGzhfN45ytOx9ff8vzUSmm5mccftlL7njN/VsYTDl+ZH4egErt1QsLJR67Kt0uDf+EQ6X4ZQZPIO3wt63h9l9Km2Yp660tPY+WWGV2LdfZlTzumWB5XObPl/SjBP3znXq18C4mxgj8RXUVEd1r+zgHwVgB/IoQ4CcCfAPjcMPsWQlwkhDhTCHHm6tWrxymmFTw9wGwn0B4s3jyUPmb5G3MfVbD7/B2af6yXxxyA1ATqaf72zjDu6uD5/DnmB/GYmn+mj2YPpHpB6wxUMX+BXlTO/Hkwk5N2a8fkAcKq+Se6uyaxMX9o/vg984NCeXiQOcBaFvNRojF5UYP5my0FHvy37u0hTgS27u1pWrwpdz22Yw57exEe3KYHf97pufrgWdz0V6/Iv3/xjc/Fm154Ks448TAA9mfgyUevwlV/+pL8u6xPEqetPghf/oOzwDcQkAnPAnz/L16G33/+KQDS0b3uVMnpp83tI+8jEeWykc78lc9fXqGHL/wVvPbZiofy3D689XHbe38Bv/qs4wvlkeU4bfVB+Mqbz9bO38Q0MP+xOnyFEK90rSOiLwN4Z/b13wF8Nvt/I4CT2KYnZssmCl5hZrth7hYxm56mT9i1D+dxRtD8Z7shZK75dpm/PuiFj6a1SVpxkmhuBVsF7kXJeD7/7DLJSySDmikl5Mw/FuhHsd19kW3DXxxd68uoWDbN7RPrHb7W3D6J0FoYZmc5oAd13lLpDWK9A5f9VI1GNkf4su2N4L8/q8u9SJ9hKxHpqOY8y2b2MuAvIsB4WQaEYw5dkX9ffcisRmRs97cTEJ56zMH5dzLq09f+18/jqINVi0IxbKX5h/m9C+CqLTlBs/j8+Qt9tpO29kI2yEvaT6VRwVYlA5I2X72OuMY+yPL8wtOPw8+fdhTbj/rx2U85Emse3OEcuDZJtFmCTQDk6//lAO7P/r8UwP+kFGcD2C2E2NxiOazgTf2ZMMgrQRzrnU6u/9PvNdw+uezDNX8H8zdaIu1o/nYNlJ9bqtEWj8sHxUmfv4leVM38y33+UvOXzF/O5mS8eEPlqe9FicbU+fmZv7WVR0/pnH4eKNH80wRoutsnEXpCNJNly99J8JdLz2T+KLJ1LbdPbGr+eoevLLt5L+SxzP/Na8fPw0zMV5Q+i3XAXBaQ6bop9glID7zZSc9lH/PW8WkcVdmpsK1skdmYPwAMksT6giHG/G39QsXzlufnruCX/OHZ+MLvPxcv/emibDRptGb1BPBmAB8hog6AeaTOHgC4HKnNcz1Sq+cbWyyDE7xS5zMSWTqdXP/bvtsgN6mn+avgLwNC06YAXmTuew4DleqZyF6BeYIzJ/Mf6Gxz+JTO6ae8RC7ZR+5XdvjaR/gGhd9WtUTkPZWzSKXHKHawmsw/FvpLxuxXkL+T4AG3FyUGk88+E5HfkzK3jzCZf1b29F4Ug/9BGemWrY+5CubPYaYwsN1fc5nU1/N9hMX1AqljSP5WHma2E+aEZaYTaPKe3KVN8+fllPclDFSKZyJFXqJYWElWkLWGzeDvzNsPsq43f2vrL1gItBb8hRDXA/g5y3IB4O1tHbcueIVRE1GLgt2MB4uRRvhaNf9yt8/rPm2gAAAYAklEQVRsJ8SBfty61bPDOsDM2axsjK6g+VsCaS9KDOZvSexWckpq4osK5p9t14/sgV+WEdAlI3t5qPC/mc+/mNXT0PwTfXCUbQpEHoc12SeKNbYvz0e3b5b5/PX9HehJ5p9YmL/+0gEssk/ovl62GdVMFINf0XVjrpfMP8+CS+reya27oRn8i8G2KweOabKuW/MHUinMVieV1VNvrbpa4/IelgX/acLCC08LBF6p+YxE5kQLpcy/zjSOQ8g+udunG7SX24cz/5BPaF39YrO5fUz0olhjpaNO5pLLPtnDbqYMUAy9KK+Y2/BgZiuPltgtO/7+vu72MTV2lWpBJXbj99gm+wgH8zdfYMI4d0C3l0opgpeH7y9n/lFcqGt9m+xjXMODZhUnrEpmaL+exeDHSZS5Pk02qBMveR+41dM2eMwsQ675a8w/zI6r5/aR5ejHImftZrltso8L8pYUzm86Y3+rss9UQ2f+Sj8uav5uvbhxt09B828/t09aRmgpa013Bi+fssLat+lFieYwGTm3TxajJPs1H3wZTMqCv53528pTDEw68y/68G3ZYPl+qmSf+YGp+avtRL5cn1MgYi8cM7cPZ8RK89fLbZZL2kjNnDs8+NtYPEcd5p8uY/fArACS+cdKcs0TIjLJbsaw+9pz+xSXzWhuH/VbWYeiOIFN9JcmiESUt1YlHI9146aNprCMmT9vFirZp2nNP0/sxoJXteYf5g942yN85TIzv42d+Svd2cX8+5GuM9sncxme+ZsJv2xB2oRsmWmavy0wWTrzzEFTus+/OCI8EfqMYDa3D7/tPGV0b2Af5CUD9UwYFHP7aJq/3pLI3T42zX9QZP4mDilh/rYcSyZsv+FEw/S4B1Qc4Rsyti/PtduxB/+O1uGrJ3YDgFnmADJz+wCZ3FQi+9Rm/tlru7htsQU6DZiu0kwQduafBjeXzm+6FIbJ7TMzis9/Arl9ZBnru32Y5m91+5ia/3AnIJ8bGdtcmr8sL++YNTGS28fSUWhaK62av9D3Y3X7ODt8YyP46/tYNRtmxEQleTNlKLMlIX9vzi1g0/xNzJRcr3pun6LsEVoCdb4efISvHrxnO0Gee6fI/LPjGYP40nXsuWXzcWvMP5d9Eofsk94LUdN1J2+huak8pi1p3UJi2Qb/0BL8h2X+deKakn3Uxq7JPnSffzHdcBOw5bExZR7Tl83LV+320XXmOikwtPLJDl8o9ss751TZ03tms3ia28yWuFeAYv4XCSl/RLGhsRuBWHrU+X5sgZUrMKbbx6b5SynnoJlOJfOXwV9/ARWZv67526+d1jkbmnW+GNhNmAGetyxdMqAQ0Nw+fIyGPAWz38eW22eGOXvM89Fy+5CqH1GcOHz+qQup7kh7lYHWPL8sBtScHnNSmK7STBC2CiMH0vDme9mDUIcNyN8Po/nPhFzzbzb42wKdyfwDsr90eACSI3xNmMx/2JeXmdunF6VzyprXQZZ3f6+O5q9kH9uwehfzP2hWd4FJJALF70IUAq8JTfPvG24fLfjr+1g1E+o+/0TPRMmZ/8FMsulb3T663GRDx3E9ABSsx2ZGUMBOkjoWLZ4fQ8o+BZ9/N8gH2JkpPjiLl1A+/2JroBMofs/rfBTbZR0iQpJIzb+6HstLbbYiFPOfrnA7XaWZIEJ2I7jVcxjmXwe2G+9Kryv19JkOd/sMfciK8hTPJ7Axf5vmH/MO35IRvprPf7gqlnf45pp/bE0uZvPju7apdPtwzZ8VV04paKZTiIWewliOxtY1f5vbR/0/x5h6b5DoVs+81SNln04mSfJj6i8f2YfAg7+N+deRfXRbpt1iK2EjMmU+fzfz1yc2ryf7FJm/yu1T3E6v44oIDBJhHeQVZOVKao63ccs+xXo4DZiu0kwQTrdPIko0/xGCvwxAnWrmzwd5pd+TVq2enVxfpeJDbqnt3G7YCQIH868e4VsGcwL3fpyU5uov6/CV23Q1m6FFc+YvRPa/nMUqFgbTNlsCSfo3jNtHBv9DV3SKg7yk00ky/26Y25AlBqxDWmRWz5kw0OqZzerZqyH7lDnczPpoMy/YLNA8S64NshO9aPUMWYevvSw2zf//tneuQXJU1x3/nZnZnV2h1XOFHiskrR4gZIGF3iCCgwxGAioKRFSUOAl5uKgQ5CTlciWiSCXkQz44FYjz8KOSoODEjoHYcSCkXAQbApUHKJKRZEmAkCypZAlphZ4rrbS7M3PzoW933+6+PTv7mIe19181Nd23b3efOd1z+vT/nHtuvI6//x145xHP357t4wV87fX8bVDWKv8Ex47TVvVGY0lTQ0SzfeKcv32Qy1A8/3CQV7hvWrZPkOqp5ekvVre2j39sW2pnWp5/wajtYzX+sQyTwXP+3reZ626rKR94/mU5f6+PLb3V1s87v0n7eF50sRgfgRstWx14/oaY8ZIN3m8Kl/2H1rjWJp3nb/P8Neefz0aC7RCNG5U059/SFH0gWwd5VZDtk81I4KUmMndi65V4/v4xzW8TUo72yWUC3cdpE1uev1+vKhLb8imnbAaCjBwj26eY4vlnSK3nb4NZGytyHIsNaASMWuNvG+EbZPukePuDpTBgcHn+Zqon6CyEEef8Qy8onNgiGeC1eW8Rzj+bTvsMlOdfDgHnXwoNoI328eWtxPM3YeecjWUz4GvQPpHCbiX7iN/4nz6eP2/L829ryelsH7Of9x1y/jmUgn7DWJvH9jn/lqZs5DfHS20A9BYr4PyzEubLJwK+0b5Wz7+M3tNKfPvZPgnPvykTPBgTtI/lgZKRZLVam+dvcv79pZI1FlS2nr8FvpyWYQyA4/wbBqnZPsUR5vwtAd8BPf+c7/nbh50PB0GQLObtVub5l4J4hef521I9Y9k+QzX+RrqjjSu1lWGII2vZz2p8Uq5x4PlbAqxm3CaYdS127LhnHad9mrMZWpuyycJuRrzDkyObOF6/xfi3NmcjD3FrYbdYiqkN2YwEmSkDjfAtWT1/25uapmMsToWA8UYZLc+Qz2WD+ymZ5588n38v25I2sjpv3+wHni6tnr/4nH9lAV9fE2mev6N9GgTmDZPg/E1vv0xtn0rg259K8vzjnL9n/EfW+pu1TcK2gf/kEC0rnLO8LcAI5PlrNZkDnWy0j3/9ytE+Vs/fIrNtkBeERjdZPz8+Sbqe2jJ2reKetXnZL/cVyecy5HPa+Bv9ktk+Ob0e/tYo5+89TFqbspEHsj3gW1m2T3wuXB8Vcf4WvYfetz3mYlaLheggL/+tK58S8DVPJzp+ZZ7GP5ZCGZx/KEt6YTeznn9icwJpU476+zrPv0EwpBG+Q+DsAtrHCFYVivY/XZDq6Rv/QjUKu0XlgmTVRSjD+RvZPumc/3CyffQf1TCAtoBvwPn3DpztY2KgWkNWzr+kMC+ZqQcInYb4oeOedby2T74poyu4xgd5xTj/ZovnX4inepbIx2mfgVI9y3j+IedvN7g+RoLzzwiRWJLfBp7x91WTzPP3v6MP72yc9vHLOJSUkYsf9fxtPoof8K10vE2gihQduWyfBoE126do8fxTlivF4Dh/L97g/+H6UwafDAdmYatQRktKX0pVTzPbJ62w23A8f797xPMvw/lXUtvHhLWqp+klGvuMaUrz/GOcf9GjhQamfcLlKzqFNd+U8VI9jW0q2D9M9YSopx6lfXzPPxbw7a+ssFscuUzGWiPHtj54z992Pwj9Pp1oxKHAM/gB7ZPi+cfb4mNQ/GVzcFxGwnvIK5ttoaMkLKI3uIBvtD2cinSE/8zDxKg1/pF6/okRviOY7TNIzt/Mn++rIu1jHrZyzr8Cz7+C2j6VyBdw/v3F8p5/Bdk+Jgau7RMu+wFUW2G3gfL8IVnWOc7553MZmvWcwtbyDv3hIC+Ieuq2gG9rRZ5/ZXn+zSmcf1x9Vs7fNmdCGc9frJ6/T/uEnH/c87cb/+hIXu83GGU6SuG+YbZPmRG+Qwn4xh4kafNQ1xuNJU0NUbtsH++7Is5fVxSNvI6O8BWy0T62Qm722j4hpZPL2LN94qNKB0uV+WL5xrCvUEoUdfPPD3C5bG2fgevOQDz4Hbb7qZPxwm7FUpLzL1YU8A2XL/cVaQ44//gI3/CtpzmbCX5HWsBXGdk+5u8rlFRisJmtqmccJuefLOcQ9/zT5062taVl+/iT1oQjgb1tkWyflIBvXL5cRjBvO5vnb+b591soO//4JVX5jHppnn9f0S5/vTEsaUTkIRHZKyIlEVke2/a4iBwQkfdF5B6jfZ1uOyAiW4Zz/uGgbJ5/Su3xITj+VtonPjOU2Z41Aqn9KcPOhwN7kGyoef62bJ+R8fzjBjCOcITv8D3/ePzDRz6nPf9ispBaMtsnaSjjxjU6h6/3UMs3ZSz1/L3vPh3v8GWOGv/om8Jli+cPydpHleb5+/drf8podPPcyf3Tr5fV80eCh1nS808v75AWpM1m7bSPXy/L60fEySpXz7/SPP/wwRJt998ArzbOfw/wIPCm2Sgii4BNwMeAdcCXRSQrIlngS8B6YBHwC7pvzVFxtk/GbhgqRdoAD9ufxhtdHPK2/YVq5PlH5fKXRyrP/0qMZx5ynr8Z8C2T7TPSef4m8obnX26Er58Kmgz4Dkz7+FN2mobDnL84b/D45sPEzPkvlcKAb9zwxmsfmXRRKuefDQO+thnJTAze87dl+4RJELYRvr7IcfovzVvPZaK1oOycfxhb87J9kscK8/wHl+oZ/882Ku0zrMlclFLvgtUobgCeU0r1AodE5ACwUm87oJT6kd7vOd1333DkGAqief4erfDkS3s51d3LslkTrf2GdB6/nGzMmN7zxTcTvsbJC1dozoXeW3dvwZp/PBwEnn/sjSatTns2I0Emy77jF3jq1f1euyVOAHCmpy+yX2LijgHl876f/Z/D/Nuu45y51FvW8z/T0xdUhUzrE/ldtlTPlGucz2XJZTN8551jlJTnFBRKiqde3c/5y/2aFoBHv7EjkSUGsOXbuyMToxw53RMc42xPX5Dq2dNXZPM3f+DJlxE+6LrI3U+/wYkLV2jL5wKZu3sLwTm7ewvBsT73wi5OX+ylpSkZhD99qTey/vp7Xdz99BsAXOwtWHWXFaGtxZM7PhlMHC2WYLy9eF6y5k7QXySYOS1eryefC4PPSeNvOmZ+6WUSef5+4F7EGD9gPFQv9hYY15I0hRkJH36V3Ma+cU8L7I7JJ3VVT1RrJq8O4C1j/ce6DeBorH2V7QAi8gh60vdZs2aNuIA3Tm/j55dfx4QxTdzUMZ5fXDWLc9pwbVw2M+i3bPZEfm7pTDrbxwRtW391eWqOdBwr5kzikTvmsrhjPH9w341MG9/CK3tPJkZeAiyYOpZlsyexfM4kHlzawZX+IhuWdFiOOnTcufBadh49x63zJgdtn127gBkTWnnkjrnsOnoOgPWLp3PkdA933nAt+092s2jGOF7ZewKAqeNaaB+b57b5k3nglg4+NmMcPX1F5k65hu/uOYEA6xZP4/i5y5EZwkz82UMf57qJrYl2EeG3187nwKmLAFw/rY0NS2Yk+rWPbebX13Ry4sJlFneMp/tKgdMXe1kzv52MCHuOnWdF56Sg/3OPrObY2cvcOredB27pYNH0cXR1X6H7SoHrp7ZFjv35T13Pjz66xIo5E3n0E/N45+hZwLuWR0730NV9BYC1C6fyvwdPc7m/wMdnTmDd4mksmz2Rsz19bDt0hvOX+yPHXTB1LJ9cOJX/PvhRcG1nTmzl8OlLlJRi5ZzJ3DpvMq+9dzLov3ruZFbP9fTcWyhGznnnDdey7dAZLvUVWDJrAhuWdHC2p49sRrhr0bW88f4p+ool1i2ezr7jF5gzeQxvfnAqkOeGaW3cdeNUvv9eF7fNm8zuH5+npSlD+9g8X9h4M1v/6xCrOr375OXP3s6OI2cT1+GZh1fw4s5jTLymmfaxefYdP8+M8S0A/Otja9hz7DwA9900naNne7jvpumJYzy0fCYKRXM2w5r57QDcNq+d3/zEPG6Y1kZn+zX0Fkr80urZAMycOIaDpy5GHqx/dP8idh49x5r57eRzWaa05YNtn/mpuXT3Fvi12zrJZoSDXRd5bO18WpuyfHrVLM729LFyziTiuPem6Rw50xPcz8tmTyxbRXbL+oWMa81x/83R+/XBpR0c/ugSm9fOT923HhAb9xzpIPI9YJpl0xNKqRd1n/8EPq+U2q7X/xp4Syn1db3+DPBdvd86pdRndPsvA6uUUpvLybB8+XK1ffv2in+Ug4ODgwOIyA6l1HLbtgE9f6XUXUM45zHgOmN9pm6jTLuDg4ODQ41QrQjES8AmEcmLSCewANgG/B+wQEQ6RaQZLyj8UpVkcHBwcHBIwbA4fxF5APgrYArw7yKyUyl1j1Jqr4i8gBfILQCPKaWKep/NwCtAFtiqlNo7rF/g4ODg4DBoDMj5NwIc5+/g4OAweJTj/Bsr8dTBwcHBoSZwxt/BwcFhFMIZfwcHB4dRCGf8HRwcHEYhfiICviJyCjgyjEO0Ax+NkDgjCSfX4ODkGhwaVS5oXNmuNrlmK6Wm2Db8RBj/4UJEtqdFvOsJJ9fg4OQaHBpVLmhc2UaTXI72cXBwcBiFcMbfwcHBYRRitBj/v6m3AClwcg0OTq7BoVHlgsaVbdTINSo4fwcHBweHKEaL5+/g4ODgYMAZfwcHB4dRiKva+DfKZPFalsMi8kMR2Ski/qQ3k0TkVRH5QH9PHOg4IyTLVhHpEpE9RptVFvHwl1qHu0VkaY3lelJEjmm97RSRe41tj2u53heRe6oo13Ui8rqI7BORvSLyO7q9rjorI1dddSYiLSKyTUR2abn+WLd3isjb+vzP67Lu6NLvz+v2t0VkTo3lelZEDhn6WqLba3bv6/NlReQdEXlZr1dXX0rPTn+1ffBKRh8E5gLNwC5gUR3lOQy0x9r+FNiil7cAX6iRLHcAS4E9A8kC3Is3C5sAq4G3ayzXk3izxMX7LtLXNA906mudrZJc04GlerkN2K/PX1edlZGrrjrTv3usXm4C3tZ6eAHYpNu/Cjyql38L+Kpe3gQ8XyV9pcn1LLDR0r9m974+3+eAfwJe1utV1dfV7PmvRE8Wr5TqA/zJ4hsJG4Cv6eWvAT9bi5Mqpd4EzlQoywbgH5SHt4AJIpKciLV6cqVhA/CcUqpXKXUIOIB3zash14dKqR/o5W7gXbw5qeuqszJypaEmOtO/+6JebdIfBawFvqXb4/ry9fgt4JMilUyZPmJypaFm976IzATuA/5OrwtV1tfVbPw7SE4WP7KzoQ8OCvgPEdkh3uT0AFOVUh/q5RPA1PqIVlaWRtDjZv3avdWgxuoil37FvgXPa2wYncXkgjrrTFMYO4Eu4FW8t4xzSqmC5dyBXHr7eWByLeRSSvn6+hOtrz8XEX/291pexy8CvweU9Ppkqqyvq9n4NxpuV0otBdYDj4nIHeZG5b3DNUTebSPJAnwFmAcsAT4EnqqXICIyFvg28LtKqQvmtnrqzCJX3XWmlCoqpZbgzdO9ElhYaxlsiMslIouBx/HkWwFMAn6/ljKJyP1Al1JqRy3PezUb/3KTyNccSqlj+rsL+A7eH+Kk/xqpv7vqJV8ZWeqqR6XUSf2HLQF/S0hT1FQuEWnCM7DfUEr9i26uu85scjWKzrQs54DXgVvxaBN/6ljz3IFcevt44HSN5Fqn6TOllOoF/p7a62sN8DMichiPnl4L/AVV1tfVbPwbZrJ4EblGRNr8ZeBTwB4tz8O628PAi/WQTyNNlpeAX9GZD6uB8wbVUXXEONYH8PTmy7VJZz50AguAbVWSQYBngHeVUk8bm+qqszS56q0zEZkiIhP0citwN1484nVgo+4W15evx43Aa/pNqhZyvWc8wAWPVzf1VfXrqJR6XCk1Uyk1B89OvaaU+jTV1tdIRqsb7YMXrd+Pxzc+UUc55uJlWewC9vqy4PF03wc+AL4HTKqRPN/EowP68bjE30iTBS/T4Utahz8EltdYrn/U592tb/rpRv8ntFzvA+urKNfteJTObmCn/txbb52VkauuOgNuBt7R598D/KHxP9iGF2j+ZyCv21v0+gG9fW6N5XpN62sP8HXCjKCa3fuGjD9NmO1TVX258g4ODg4OoxBXM+3j4ODg4JACZ/wdHBwcRiGc8XdwcHAYhXDG38HBwWEUwhl/BwcHh1EIZ/wdHBwcRiGc8XdwcHAYhfh/vl4er3HWuIUAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "u2HaGRVEYGQS",
        "colab_type": "text"
      },
      "source": [
        "## 測試"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5yFuUKKRYH73",
        "colab_type": "code",
        "outputId": "a3f9cbcb-c9e6-4e00-c47c-dc959230f79d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 770
        }
      },
      "source": [
        "agent.network.eval()  # 測試前先將 network 切換為 evaluation 模式\n",
        "\n",
        "state = env.reset()\n",
        "\n",
        "img = plt.imshow(env.render(mode='rgb_array'))\n",
        "\n",
        "total_reward = 0\n",
        "\n",
        "done = False\n",
        "while not done:\n",
        "    action, _ = agent.sample(state)\n",
        "    state, reward, done, _ = env.step(action)\n",
        "\n",
        "    total_reward += reward\n",
        "\n",
        "    img.set_data(env.render(mode='rgb_array'))\n",
        "    display.display(plt.gcf())\n",
        "    display.clear_output(wait=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/numpy/core/fromnumeric.py\u001b[0m in \u001b[0;36m_wrapreduction\u001b[0;34m(obj, ufunc, method, axis, dtype, out, **kwargs)\u001b[0m\n\u001b[1;32m     78\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 79\u001b[0;31m             \u001b[0mreduction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     80\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: 'bool' object has no attribute 'any'",
            "\nDuring handling of the above exception, another exception occurred:\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-43-fcf2ad804c22>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m     \u001b[0mimg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrender\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'rgb_array'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m     \u001b[0mdisplay\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdisplay\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgcf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m     \u001b[0mdisplay\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclear_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/IPython/core/display.py\u001b[0m in \u001b[0;36mdisplay\u001b[0;34m(*objs, **kwargs)\u001b[0m\n\u001b[1;32m    304\u001b[0m             \u001b[0mpublish_display_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetadata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmetadata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    305\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 306\u001b[0;31m             \u001b[0mformat_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmd_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minclude\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minclude\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexclude\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mexclude\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    307\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mformat_dict\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    308\u001b[0m                 \u001b[0;31m# nothing to display (e.g. _ipython_display_ took over)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/IPython/core/formatters.py\u001b[0m in \u001b[0;36mformat\u001b[0;34m(self, obj, include, exclude)\u001b[0m\n\u001b[1;32m    171\u001b[0m             \u001b[0mmd\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    172\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 173\u001b[0;31m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mformatter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    174\u001b[0m             \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    175\u001b[0m                 \u001b[0;31m# FIXME: log the exception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<decorator-gen-9>\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, obj)\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/IPython/core/formatters.py\u001b[0m in \u001b[0;36mcatch_format_error\u001b[0;34m(method, self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    215\u001b[0m     \u001b[0;34m\"\"\"show traceback on failed format call\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    216\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 217\u001b[0;31m         \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    218\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mNotImplementedError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    219\u001b[0m         \u001b[0;31m# don't warn on NotImplementedErrors\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/IPython/core/formatters.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    332\u001b[0m                 \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    333\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 334\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mprinter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    335\u001b[0m             \u001b[0;31m# Finally look for special method names\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    336\u001b[0m             \u001b[0mmethod\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_real_method\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprint_method\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/IPython/core/pylabtools.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(fig)\u001b[0m\n\u001b[1;32m    239\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    240\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;34m'png'\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mformats\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 241\u001b[0;31m         \u001b[0mpng_formatter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfor_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mFigure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mfig\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mprint_figure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'png'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    242\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;34m'retina'\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mformats\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m'png2x'\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mformats\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    243\u001b[0m         \u001b[0mpng_formatter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfor_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mFigure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mfig\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mretina_figure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/IPython/core/pylabtools.py\u001b[0m in \u001b[0;36mprint_figure\u001b[0;34m(fig, fmt, bbox_inches, **kwargs)\u001b[0m\n\u001b[1;32m    123\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    124\u001b[0m     \u001b[0mbytes_io\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBytesIO\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 125\u001b[0;31m     \u001b[0mfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcanvas\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprint_figure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbytes_io\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    126\u001b[0m     \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbytes_io\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetvalue\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    127\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfmt\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'svg'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/backend_bases.py\u001b[0m in \u001b[0;36mprint_figure\u001b[0;34m(self, filename, dpi, facecolor, edgecolor, orientation, format, bbox_inches, **kwargs)\u001b[0m\n\u001b[1;32m   2077\u001b[0m                             print_method, dpi=dpi, orientation=orientation),\n\u001b[1;32m   2078\u001b[0m                         draw_disabled=True)\n\u001b[0;32m-> 2079\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfigure\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdraw\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrenderer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2080\u001b[0m                     \u001b[0mbbox_artists\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"bbox_extra_artists\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2081\u001b[0m                     bbox_inches = self.figure.get_tightbbox(renderer,\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/artist.py\u001b[0m in \u001b[0;36mdraw_wrapper\u001b[0;34m(artist, renderer, *args, **kwargs)\u001b[0m\n\u001b[1;32m     36\u001b[0m                 \u001b[0mrenderer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart_filter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 38\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mdraw\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0martist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrenderer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     39\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0martist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_agg_filter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/figure.py\u001b[0m in \u001b[0;36mdraw\u001b[0;34m(self, renderer)\u001b[0m\n\u001b[1;32m   1734\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdraw\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrenderer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1735\u001b[0m             mimage._draw_list_compositing_images(\n\u001b[0;32m-> 1736\u001b[0;31m                 renderer, self, artists, self.suppressComposite)\n\u001b[0m\u001b[1;32m   1737\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1738\u001b[0m             \u001b[0mrenderer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose_group\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'figure'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/image.py\u001b[0m in \u001b[0;36m_draw_list_compositing_images\u001b[0;34m(renderer, parent, artists, suppress_composite)\u001b[0m\n\u001b[1;32m    135\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mnot_composite\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mhas_images\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    136\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0ma\u001b[0m \u001b[0;32min\u001b[0m \u001b[0martists\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 137\u001b[0;31m             \u001b[0ma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdraw\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrenderer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    138\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    139\u001b[0m         \u001b[0;31m# Composite any adjacent images together\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/artist.py\u001b[0m in \u001b[0;36mdraw_wrapper\u001b[0;34m(artist, renderer, *args, **kwargs)\u001b[0m\n\u001b[1;32m     36\u001b[0m                 \u001b[0mrenderer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart_filter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 38\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mdraw\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0martist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrenderer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     39\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0martist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_agg_filter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/axes/_base.py\u001b[0m in \u001b[0;36mdraw\u001b[0;34m(self, renderer, inframe)\u001b[0m\n\u001b[1;32m   2628\u001b[0m             \u001b[0mrenderer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstop_rasterizing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2629\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2630\u001b[0;31m         \u001b[0mmimage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_draw_list_compositing_images\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrenderer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0martists\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2631\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2632\u001b[0m         \u001b[0mrenderer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose_group\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'axes'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/image.py\u001b[0m in \u001b[0;36m_draw_list_compositing_images\u001b[0;34m(renderer, parent, artists, suppress_composite)\u001b[0m\n\u001b[1;32m    135\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mnot_composite\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mhas_images\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    136\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0ma\u001b[0m \u001b[0;32min\u001b[0m \u001b[0martists\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 137\u001b[0;31m             \u001b[0ma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdraw\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrenderer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    138\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    139\u001b[0m         \u001b[0;31m# Composite any adjacent images together\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/artist.py\u001b[0m in \u001b[0;36mdraw_wrapper\u001b[0;34m(artist, renderer, *args, **kwargs)\u001b[0m\n\u001b[1;32m     36\u001b[0m                 \u001b[0mrenderer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart_filter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 38\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mdraw\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0martist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrenderer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     39\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0martist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_agg_filter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/axis.py\u001b[0m in \u001b[0;36mdraw\u001b[0;34m(self, renderer, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1225\u001b[0m         \u001b[0mrenderer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen_group\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgid\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_gid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1226\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1227\u001b[0;31m         \u001b[0mticks_to_draw\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_update_ticks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1228\u001b[0m         ticklabelBoxes, ticklabelBoxes2 = self._get_tick_bboxes(ticks_to_draw,\n\u001b[1;32m   1229\u001b[0m                                                                 renderer)\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/axis.py\u001b[0m in \u001b[0;36m_update_ticks\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1109\u001b[0m             \u001b[0mtick\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_label1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1110\u001b[0m             \u001b[0mtick\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_label2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1111\u001b[0;31m         \u001b[0mminor_locs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_minorticklocs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1112\u001b[0m         \u001b[0mminor_labels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mminor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformatter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat_ticks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mminor_locs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1113\u001b[0m         \u001b[0mminor_ticks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_minor_ticks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mminor_locs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/axis.py\u001b[0m in \u001b[0;36mget_minorticklocs\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1351\u001b[0m         \u001b[0;34m\"\"\"Get the array of minor tick locations in data coordinates.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1352\u001b[0m         \u001b[0;31m# Remove minor ticks duplicating major ticks.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1353\u001b[0;31m         \u001b[0mmajor_locs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmajor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlocator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1354\u001b[0m         \u001b[0mminor_locs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mminor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlocator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1355\u001b[0m         \u001b[0mtransform\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_scale\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/ticker.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   2232\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2233\u001b[0m         \u001b[0mvmin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvmax\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_view_interval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2234\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtick_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvmin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvmax\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2235\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2236\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mtick_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvmin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvmax\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/ticker.py\u001b[0m in \u001b[0;36mtick_values\u001b[0;34m(self, vmin, vmax)\u001b[0m\n\u001b[1;32m   2240\u001b[0m         vmin, vmax = mtransforms.nonsingular(\n\u001b[1;32m   2241\u001b[0m             vmin, vmax, expander=1e-13, tiny=1e-14)\n\u001b[0;32m-> 2242\u001b[0;31m         \u001b[0mlocs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_raw_ticks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvmin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvmax\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2243\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2244\u001b[0m         \u001b[0mprune\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_prune\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/ticker.py\u001b[0m in \u001b[0;36m_raw_ticks\u001b[0;34m(self, vmin, vmax)\u001b[0m\n\u001b[1;32m   2179\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_nbins\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'auto'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2180\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maxis\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2181\u001b[0;31m                 nbins = np.clip(self.axis.get_tick_space(),\n\u001b[0m\u001b[1;32m   2182\u001b[0m                                 max(1, self._min_n_ticks - 1), 9)\n\u001b[1;32m   2183\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/axis.py\u001b[0m in \u001b[0;36mget_tick_space\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   2188\u001b[0m         \u001b[0mends\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maxes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransAxes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2189\u001b[0m         \u001b[0mlength\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mends\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mends\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maxes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfigure\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdpi\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;36m72\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2190\u001b[0;31m         \u001b[0mtick\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tick\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2191\u001b[0m         \u001b[0;31m# There is a heuristic here that the aspect ratio of tick text\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2192\u001b[0m         \u001b[0;31m# is no more than 3:1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/axis.py\u001b[0m in \u001b[0;36m_get_tick\u001b[0;34m(self, major)\u001b[0m\n\u001b[1;32m   1941\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1942\u001b[0m             \u001b[0mtick_kw\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_minor_tick_kw\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1943\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mXTick\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maxes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m''\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmajor\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmajor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mtick_kw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1944\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1945\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_get_label\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/axis.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, axes, loc, label, size, width, color, tickdir, pad, labelsize, labelcolor, zorder, gridOn, tick1On, tick2On, label1On, label2On, major, labelrotation, grid_color, grid_linestyle, grid_linewidth, grid_alpha, **kw)\u001b[0m\n\u001b[1;32m    155\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_tickdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtickdir\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    156\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 157\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtick1line\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tick1line\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    158\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtick2line\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tick2line\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    159\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgridline\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_gridline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/axis.py\u001b[0m in \u001b[0;36m_get_tick1line\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    463\u001b[0m                           \u001b[0mlinestyle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'None'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmarker\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tickmarkers\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    464\u001b[0m                           \u001b[0mmarkersize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 465\u001b[0;31m                           markeredgewidth=self._width, zorder=self._zorder)\n\u001b[0m\u001b[1;32m    466\u001b[0m         \u001b[0ml\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maxes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_xaxis_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwhich\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'tick1'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    467\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_set_artist_props\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/lines.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, xdata, ydata, linewidth, linestyle, color, marker, markersize, markeredgewidth, markeredgecolor, markerfacecolor, markerfacecoloralt, fillstyle, antialiased, dash_capstyle, solid_capstyle, dash_joinstyle, solid_joinstyle, pickradius, drawstyle, markevery, **kwargs)\u001b[0m\n\u001b[1;32m    391\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_markerfacecoloralt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    392\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 393\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_markerfacecolor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmarkerfacecolor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    394\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_markerfacecoloralt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmarkerfacecoloralt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    395\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_markeredgecolor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmarkeredgecolor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/lines.py\u001b[0m in \u001b[0;36mset_markerfacecolor\u001b[0;34m(self, fc)\u001b[0m\n\u001b[1;32m   1251\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mfc\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1252\u001b[0m             \u001b[0mfc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'auto'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1253\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0many\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_markerfacecolor\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mfc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1254\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstale\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1255\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_markerfacecolor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<__array_function__ internals>\u001b[0m in \u001b[0;36many\u001b[0;34m(*args, **kwargs)\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/numpy/core/fromnumeric.py\u001b[0m in \u001b[0;36many\u001b[0;34m(a, axis, out, keepdims)\u001b[0m\n\u001b[1;32m   2315\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2316\u001b[0m     \"\"\"\n\u001b[0;32m-> 2317\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_wrapreduction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogical_or\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'any'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeepdims\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkeepdims\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2318\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2319\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/numpy/core/fromnumeric.py\u001b[0m in \u001b[0;36m_wrapreduction\u001b[0;34m(obj, ufunc, method, axis, dtype, out, **kwargs)\u001b[0m\n\u001b[1;32m     78\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     79\u001b[0m             \u001b[0mreduction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 80\u001b[0;31m         \u001b[0;32mexcept\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     81\u001b[0m             \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     82\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAe10lEQVR4nO3deXxV9Z3/8dfnZiWAhDWERUCMtIAIiIh1Q9BqaRWr1uLjJ1Krk06r1T7aTqsdZ6ozjy7Yjv6ouJRaFJ22SqdYkTpFQNrqjIph3zHsBGLCEpYAWT/zxz2hV7ZsN7k5yfv5eJxHzvme7fMN4Z2T7z3nXnN3REQkPCKJLkBEROpHwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiHTZMFtZjeY2UYzyzezh5rqPCIibY01xX3cZpYEbAKuA3YBHwJ3uPu6uJ9MRKSNaaor7tFAvrtvcfdy4BVgYhOdS0SkTUluouP2BnbGLO8CLj3TxmamxzdFRE7i7na69qYK7lqZWS6Qm6jzi4iEVVMFdwHQN2a5T9B2grvPAGaArrhFROqjqca4PwRyzGyAmaUCk4C5TXQuEZE2pUmuuN290szuB+YDScBMd1/bFOcSEWlrmuR2wHoXoaESEZFTnOnFST05KSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBp1GdOmtk24DBQBVS6+ygz6wK8CvQHtgG3u/uBxpUpIiI14nHFfY27D3f3UcHyQ8Aid88BFgXLIiISJ00xVDIRmBXMzwJuboJziIi0WY0NbgfeMrOlZpYbtGW5+55gvhDIauQ5REQkRqPGuIEr3L3AzHoAC8xsQ+xKd3cz89PtGAR97unWiYjImZn7aXO1/gcyexQ4AvwDMNbd95hZNvAXdx9Uy77xKUJEpBVxdztde4OHSsysvZl1rJkHPgusAeYCU4LNpgCvN/QcIiJyqgZfcZvZecBrwWIy8Ft3/5GZdQVmA+cC24neDri/lmPpiltE5CRnuuKO21BJYyi4RUROFfehEhERSQwFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREKm1uA2s5lmVmRma2LaupjZAjP7KPjaOWg3M/uFmeWb2SozG9mUxYuItEV1ueJ+EbjhpLaHgEXungMsCpYBPgfkBFMu8Gx8yhQRkRq1Bre7/w3Yf1LzRGBWMD8LuDmm/SWPeh/INLPseBUrIiINH+POcvc9wXwhkBXM9wZ2xmy3K2g7hZnlmlmemeU1sAYRkTYpubEHcHc3M2/AfjOAGQAN2V9EpK1q6BX3xzVDIMHXoqC9AOgbs12foE1EROKkocE9F5gSzE8BXo9pvyu4u2QMcDBmSEVEROLA3M8+SmFmvwPGAt2Aj4EfAn8EZgPnAtuB2919v5kZMJ3oXShHgbvdvdYxbA2ViIicyt3tdO21BndzUHCLiJzqTMGtJydFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMrUGt5nNNLMiM1sT0/aomRWY2YpgmhCz7mEzyzezjWZ2fVMVLiLSVtXlw4KvAo4AL7n70KDtUeCIu//8pG0HA78DRgO9gIXABe5eVcs59JmTIiInafBnTrr734D9dTzPROAVdy9z961APtEQFxGROGnMGPf9ZrYqGErpHLT1BnbGbLMraDuFmeWaWZ6Z5TWiBhGRNqehwf0sMBAYDuwB/qO+B3D3Ge4+yt1HNbAGEZE2qUHB7e4fu3uVu1cDv+LvwyEFQN+YTfsEbSIiEicNCm4zy45Z/CJQc8fJXGCSmaWZ2QAgB1jSuBJFRCRWcm0bmNnvgLFANzPbBfwQGGtmwwEHtgFfA3D3tWY2G1gHVAL31XZHiYiI1E+ttwM2SxG6HVBE5BQNvh1QRERaFgW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMjUGtxm1tfMFpvZOjNba2YPBu1dzGyBmX0UfO0ctJuZ/cLM8s1slZmNbOpOiIi0JXW54q4EvuPug4ExwH1mNhh4CFjk7jnAomAZ4HNEP909B8gFno171SIibVitwe3ue9x9WTB/GFgP9AYmArOCzWYBNwfzE4GXPOp9INPMsuNeuYhIG1WvMW4z6w+MAD4Astx9T7CqEMgK5nsDO2N22xW0nXysXDPLM7O8etYsItKm1Tm4zawD8AfgW+5+KHaduzvg9Tmxu89w91HuPqo++4mItHV1Cm4zSyEa2r9x9zlB88c1QyDB16KgvQDoG7N7n6BNRETioC53lRjwa2C9uz8Rs2ouMCWYnwK8HtN+V3B3yRjgYMyQioiINJJFRznOsoHZFcA7wGqgOmj+AdFx7tnAucB24HZ33x8E/XTgBuAocLe7n3Uc28zqNcwiItIWuLudrr3W4G4OCm4RkVOdKbj15KSISMgkJ7qA1qZrWhoXd+vGgoKC+t1mcwaRSDI9e34Ks4b9ji0q+oiKimNxqEREWgoFdxxd1qMHPxs9mt4ZGfx89Wqe3bCB6kYOReXkXMWQT32e9ORO9d63ouoo+dv/wvLlc2rfWERCQ8EdR+5OVno6DlyRlcXzmzZRVlXV4OO1a9eJ/ueO4YKunyM9ObPe+x88vouPO2wgOTmNysqyBtdxOl3T0vjmoEGUVFQwY9MmjjainyJSPwruOHq/uJh/WbaMzNRUfrlhQ6OHSpKSUmnXrhOpSR0atP85ab3p2W0InTv3pbg4v5HVfNL4nj25Kiv6sOzxqiqe27QprscXkTNTcMfZq1u2APV8jPQMkpJSiFjL/CdadeAAe8vKKCkv5487diS6HJE2pWWmQojF877GgQMvp2eHYRhJDT6GYQ1+YfNsNhw6xKMrV3KgvJzC48fjfnwROTPdDtiCJSUlE7Ekos80NUzPDhcxcODlcazq79YdPMieY7pjRaS5KbhbMTMjKZJGSkp6oksRkThScLdQHTv2IKvbpzknrW/tG4tIm6LgbqFSUtJJT+tESiSjUcdpl9yZruecR5cu/eJUmYgkmoK7BUlJaUdSUkp8j5nUjnZpnWnXLnofeCSSTFJSalzPISLNS3eVtBDZ2YO58MIbOXy4iC1b3qOi4mhcj28GXbsOoH//0bRLP4e8pa9y/Pih2ncUkRZHwd1CdO9+Pn07X0py1zTKy0vZuvX9uB4/OTmNCy4Yy6C+nyUjpRvHjh9k6dLZcT2HiDQPDZW0ABkZnenTazg92g+movoYpaX74n6OiooyDhzYScRSaJfchY4dskhNbdz4uYgkhoK7BcjI6EKH9B5UeSW7Dyxj27YPm+Q8BQWr2HtkI2nJ59CjyyA6dcpukvOISNPSUEkLMGDAaLq3H8yBY5vZtXslx48fIiMjk/KqI2w5sBBo+AM4AKVlRVRVlXH0aAlHjhZRUVVK13Y59O49jOLizfHphIg0GwV3gmVkdKZLp/60S8mk6Mhaioo+AqC09ACrVr9ey951U15+jMLCjYCzecv/0KvLcHp0GEpmp95EIslUV1fG5Twi0jxqDW4z6wu8BGQRfSuOGe4+zcweBf4BKA42/YG7vxns8zBwD1AFPODu85ug9lahd+8L6dVlBCmR9hwpK6SkZBcAZWWH2bZtSdzPt2vXSvacv5rsjiPp2fVCevb8FLt3r4n7eUSk6dTlirsS+I67LzOzjsBSM1sQrHvS3X8eu7GZDQYmAUOAXsBCM7vA3fWGzafRo0cOndL7UFS6hm3bl1Be3rTv/VFVVUlJyS6O9txLh9QsMjN7s3v3WuL79lgi0pRqfXHS3fe4+7Jg/jCwHuh9ll0mAq+4e5m7bwXygdHxKLa12n14OUfKijh0qJCmD1Bn9+61FJWu5VjFPjp27NGoN7ESkeZXrzFuM+sPjAA+AC4H7jezu4A8olflB4iGeuxNyLs4e9C3aatX/4kt7d8nLa0DBQXNM2Sxb982tuz8H4qL8yks3IB7dbOctzY//vHXSEr6JS+/DNXVUFICu3cnuqrmNXbsWL7ylT3Mnr2R7duhqgo++ij6VaRGnYPbzDoAfwC+5e6HzOxZ4N+JXiL+O/AfwFfrcbxcILd+5bY+hw4VBlfazae8vJSlS19t1nPWxYUXnkd2NowbF13eswfWrYvO//nPkJ8P7lBY2HqDrHv37owefYQhQ6LLlZXwv/8LFRWwaxf88Y/R9oMH4fDhxNUpiVWn4DazFKKh/Rt3nwPg7h/HrP8VMC9YLABi39KuT9D2Ce4+A5gR7K8BVjmhZuSmV6/oBHDNNdHQrqqC+fPh2LFosP/nfyauzqZU8z1ISYGrr47Ou8Odd0bn16yBjRuj8y+9BB9/fOoxpPWqdYzbogOgvwbWu/sTMe2xT298Eaj5O38uMMnM0sxsAJADxP/2CGlTqqujoV1ZCUePQmlpNLzbkppfXFVVcPx49HtQWhr93kjbUpcr7suBycBqM1sRtP0AuMPMhhMdKtkGfA3A3dea2WxgHdE7Uu7THSVSV+7RCaJDAyuCn7j582HLlui6/ftbf1jVfB8qK+Htt6G8HAoKYO7c6PojR9reLy75u1qD293f5fSP7r15ln1+BPyoEXVJG3TkCPzpT9Hhj+rq6BhucXHt+7U2K1bAr34F27dHvw87drT+X1RSP3pyUlqMHTvg0UcTXUXiPfEE5OUlugppyfQmUyGTO2QI7ZP1+1akLVNwh0QE+OEll/DY6NH87vrrGZOVleiSRCRBFNwhYWbcnpNDz/bt+ey555KTmZnokkQkQRTcIVHlzv1//Ssr9+7lhrlz+a/NejtWaV3M7JRJTk+DpSGyuKCA619/nY91H5iE2MUXX0zPnj1PaZ88eTJDhw49sbx+/XpefPFF8vLy+FhPGH2CgjtkFNrSkrRv354LL7zwlPa+ffty//33n3afwYMH061bt1qPPWTIEG677TbWrVvH3r17WbhwIW+99RarVq3iWBv/f6DgFpFPSE5OZsCAAZ8Yqujdu/dpg7hz586MHTv2tMeJ11DH4MGDAbjyyit57LHHWLhwIYcPH+bll19m3bp17Nu3j3374v85rS2ZgltEAOjSpQv33HMPw4YN4/bbbycS+ftLYGZGJBJJ6Lhzzbmvu+46ACZOnIi7s2zZMvLy8nj66afZu3cvJSUllJeXJ6zO5qDgFmnDUlJSSEtL46tf/SoPPPAA/fr1IzkkzwkkJSUBMHr0aC655BKmTJlCdXU1c+bMYcWKFcycOZOKigqOHz+Oe+t6H7tw/AuJSFyZGWPHjuUb3/gGl19+Od27dw9NYJ+OmdG+fXsApkyZwp133sn3vvc9qqqqeOqpp1i3bh1vvvkm7t4qQjy8/1IiUm+9e/dmxIgR3HvvvYwdO5ZOnToluqQmkZSURHZ29A1Mp06dytGjR9m2bRuHDh3i8ccfJz8/n9WrVye4yoZTcIu0Af369WPSpEncfffdDBo0KNHlNLuMjIwTL3LOmTOHPXv2kJ+fz9atW3nuuefIz8+nOETvaKbgFmmlMjIyGD16NA888ABDhw7l/PPP10MtgezsbLKzs7nyyiuZPHkyy5cvZ/v27bz33nu8/vrr7Nixg+PHjye6zDNScIu0ImZGt27dGDZsGA8++CATJkxI+N0gLZ2ZMXLkSEaOHMnEiRP58Y9/zNy5c1mzZg3Tp09vkVfieuRdpBUwM3r27Mljjz3GunXrmDdvHjfeeCNJSUkK7XqIRCIkJydzyy238Mgjj7Bq1SoeeeQRunbt2qK+j7ribqUMuKBzZ8qrqth66FCiy5EmEolEuOmmmxg6dCj3338/PXr0aFEBE2aRSOTEL8Ovf/3rTJs2jbVr1564OyWRFNytkAGf6dWLS7Ky2F9WRuGmTRyrrEx0WRJHycnJXH/99Tz88MOMHDmSdu3aJbqkVisSidCrVy+mTp1KaWkpH3zwAT/96U9ZtGgR1Qn6aKK6fFhwupktMbOVZrbWzB4L2geY2Qdmlm9mr5pZatCeFiznB+v7N20X5GTJkQhZGRk4cE5qKplpaYkuSeLk05/+NNOmTeOdd97htdde4/LLL1doN6P27dszbtw43njjDRYsWMD48eNJS8D/r7qMcZcB49z9ImA4cIOZjQGmAk+6+/nAAeCeYPt7gANB+5PBdtKMKqqreS0/n11HjrBg+3b2lJYmuiRphM6dOzNo0CCmT5/O4sWL+eY3v8mYMWNISUlJdGltVlpaGuPGjeOtt97ijTfeYPz48fTv37/5Cqh5kqguE5ABLAMuBfYCyUH7ZcD8YH4+cFkwnxxsZ2c7biQScaKfFq8pjpO1gBrqM02dOjXhNSR6+tKXvuSjRo1ywM855xy/8847/d133/XKykqvrq52aXmqq6u9srLSi4qK/PHHH/devXq5mcXl58HPkJnmdRhkN7MkYClwPvA08DPgfY9eVWNmfYH/dvehZrYGuMHddwXrNgOXuvveMx1/0KBBfscddzBr1iwKCwtb9P2T0nS+8IUvsHjx4oS/8JNIQ4YMoWPHjlx55ZVMmDCB4cOHk5qamuiypI6qq6spKSnhueee4+mnn6awsLBR4+DuftpXmusU3Cc2NssEXgP+BXixMcFtZrlALsC555578bZt29i3bx87d+7kueee4/3332fNmjUJG/yX5hGJRMjJyeHqq6/mK1/5CgMGDEh0SQmXmppKly5dEl2GNIK7s3fvXp555hlmzpzJrl27GpRlZwrueg2VBCH/r8A/EcehkosvvviUPz8KCwt99erVPmXKFL/ssssS/iespvhOGRkZPmnSJF++fLkXFBTE669WkRZnx44d/t577/lNN93kKSkp9fp/4g0dKjGz7kCFu5eYWTvgLaIvOE4B/uDur5jZc8Aqd3/GzO4DLnT3fzSzScAt7n772c4xatQoz8vLO+P6AwcOsH79en7729+Sl5dHXl4eVVVVZ61bWp7u3btzwQUXkJuby5AhQxgxYsQn3vNZpDWrrKzkww8/5Cc/+Qlvv/02pXW4acAbOlRiZsOAWUAS0btQZrv7v5nZecArQBdgOXCnu5eZWTrwMjAC2A9McvctZztHbcEdq7y8nIULF7JkyRJmzpzJzp0767SfJEZKSgoDBw7k6quvJjc3l5EjRya6JJGEcnf+9re/8cQTT/CXv/yFQ2d5QK7Bwd0c6hPcNdydnTt3Mn/+fGbPns3q1aspKipqMy9stUtKIsWMw5WVtMQed+vWjX79+vHd736XW2+9laSkJF1di8Sorq7mr3/9K8888wzvvvsuxcXFp4wktLrgjlVWVsbhw4d5/vnnWbZsGW+++SbHjh0LxQub6enpJz7Jw905evRorftkpqTwtfPOo2d6Oi9v386ykpKmLrNOUlNT+cxnPsM111zD5MmT6dmzJ+np6XoEW+QsysvLOX78ODNnzuSpp55i+/btJwK8VQd3rGPHjnHw4EFefPFFVq1axe9//3uqqqoSciVe8zl9NTIzM7n11ls/EWT33nsvffr0AaL/gNOmTaO0tJTCwkLmzZsHRH8z19SflpLCl847jyuCT/tYXFzMqwkcLopEInTr1o0vfvGLXH/99Vx77bV07NgxYfWIhJW7s3//fmbOnMnzzz/Ppk2b2k5wxyovL2fr1q0sXryY+fPns2jRIg4fPhz389QYP378J0Lr2muv5dprrz2xnJqaSv/+/et0BXr06NET4/fz589n8eLFACz74AP6dOjAZR07knbsGDM2b2ZvAj4YNRKJcN111/H973+fAQMG0K9fP11Zi8TJ7t27ufTSS9m5c2fbC+5Y7s7KlSvZuHEj06dPZ/PmzezZs6fO+6ekpDBq1KgTwxoAd911F0OGDAGiV9cXXXQRGRkZca+9hruzZs0ajpaWsuKdd/jTn//MgfJy9u7dy4YNG5rsvDXat2/PiBEj+PznP8/YsWMZMWJEQt6nQaQtGDVqFHl5eW07uE+2evVqPvroI372s59RUFBAJBL5ROhecskl3HbbbSeWU1NTGT9+fIv8QNXdu3ezdOlSAGbPnn1i/tChQxQUFDT6+Oeffz6DBw/m29/+NldddZWurEWagYL7DGpuZt+5cycpKSlkZWWdWHfy+HRYxI6Hb9u27cQQy5w5c1ixYgUARUVFtd4Hn5mZyc0338xVV13FLbfcQseOHUP5/RAJq7MFd8u7fGxGZoaZ0a9fv0SXEjex4Tpw4EAGDhwIwOTJk6mqqqK6upoXXniBkpISiouLeeGFFwCoqKggKSmJAQMG8OUvf5kJEyYwbNgwvQOdSAvUpq+427qKigr2798PwJIlS8jJySErK4vOnTsnuDIR0RW3nFbs8NCNN96Y4GpEpK40aCkiEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIytQa3maWb2RIzW2lma83ssaD9RTPbamYrgml40G5m9gszyzezVWamDxkUEYmjujzyXgaMc/cjZpYCvGtm/x2s+yd3/6+Ttv8ckBNMlwLPBl9FRCQOar3i9qgjwWJKMJ3tnakmAi8F+70PZJpZduNLFRERqOMYt5klmdkKoAhY4O4fBKt+FAyHPGlmNR+F0huI/RDEXUGbiIjEQZ2C292r3H040AcYbWZDgYeBTwGXAF2A79fnxGaWa2Z5ZpZXXFxcz7JFRNquet1V4u4lwGLgBnffEwyHlAEvAKODzQqAvjG79QnaTj7WDHcf5e6junfv3rDqRUTaoLrcVdLdzDKD+XbAdcCGmnFri34A4c3AmmCXucBdwd0lY4CD7l73T+UVEZGzqstdJdnALDNLIhr0s919npm9bWbdAQNWAP8YbP8mMAHIB44Cd8e/bBGRtqvW4Hb3VcCI07SPO8P2DtzX+NJEROR09OSkiEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGTM3RNdA2Z2GNiY6DqaSDdgb6KLaAKttV/QevumfoVLP3fvfroVyc1dyRlsdPdRiS6iKZhZXmvsW2vtF7TevqlfrYeGSkREQkbBLSISMi0luGckuoAm1Fr71lr7Ba23b+pXK9EiXpwUEZG6aylX3CIiUkcJD24zu8HMNppZvpk9lOh66svMZppZkZmtiWnrYmYLzOyj4GvnoN3M7BdBX1eZ2cjEVX52ZtbXzBab2TozW2tmDwbtoe6bmaWb2RIzWxn067GgfYCZfRDU/6qZpQbtacFyfrC+fyLrr42ZJZnZcjObFyy3ln5tM7PVZrbCzPKCtlD/LDZGQoPbzJKAp4HPAYOBO8xscCJraoAXgRtOansIWOTuOcCiYBmi/cwJplzg2WaqsSEqge+4+2BgDHBf8G8T9r6VAePc/SJgOHCDmY0BpgJPuvv5wAHgnmD7e4ADQfuTwXYt2YPA+pjl1tIvgGvcfXjMrX9h/1lsOHdP2ARcBsyPWX4YeDiRNTWwH/2BNTHLG4HsYD6b6H3qAL8E7jjddi19Al4HrmtNfQMygGXApUQf4EgO2k/8XALzgcuC+eRgO0t07WfoTx+iATYOmAdYa+hXUOM2oNtJba3mZ7G+U6KHSnoDO2OWdwVtYZfl7nuC+UIgK5gPZX+DP6NHAB/QCvoWDCesAIqABcBmoMTdK4NNYms/0a9g/UGga/NWXGf/H/geUB0sd6V19AvAgbfMbKmZ5QZtof9ZbKiW8uRkq+XubmahvXXHzDoAfwC+5e6HzOzEurD2zd2rgOFmlgm8BnwqwSU1mpl9AShy96VmNjbR9TSBK9y9wMx6AAvMbEPsyrD+LDZUoq+4C4C+Mct9graw+9jMsgGCr0VBe6j6a2YpREP7N+4+J2huFX0DcPcSYDHRIYRMM6u5kImt/US/gvWdgH3NXGpdXA7cZGbbgFeIDpdMI/z9AsDdC4KvRUR/2Y6mFf0s1leig/tDICd45TsVmATMTXBN8TAXmBLMTyE6PlzTflfwqvcY4GDMn3otikUvrX8NrHf3J2JWhbpvZtY9uNLGzNoRHbdfTzTAbws2O7lfNf29DXjbg4HTlsTdH3b3Pu7en+j/o7fd/f8R8n4BmFl7M+tYMw98FlhDyH8WGyXRg+zABGAT0XHGf050PQ2o/3fAHqCC6FjaPUTHChcBHwELgS7Btkb0LprNwGpgVKLrP0u/riA6rrgKWBFME8LeN2AYsDzo1xrgX4P284AlQD7weyAtaE8PlvOD9eclug916ONYYF5r6VfQh5XBtLYmJ8L+s9iYSU9OioiETKKHSkREpJ4U3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEzP8BunXo1ZMaMzMAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Aex7mcKr0J01",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(total_reward)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wUBtYXG2eaqf",
        "colab_type": "text"
      },
      "source": [
        "## 參考資料\n",
        "\n",
        "以下是一些有用的參考資料。\n",
        "建議同學們實做前，可以先參考第一則連結的上課影片。\n",
        "在影片的最後有提到兩個有用的 Tips，這對於本次作業的實做非常有幫助。\n",
        "\n",
        "- [DRL Lecture 1: Policy Gradient (Review)](https://youtu.be/z95ZYgPgXOY)\n",
        "- [ML Lecture 23-3: Reinforcement Learning (including Q-learning) start at 30:00](https://youtu.be/2-JNBzCq77c?t=1800)\n",
        "- [Lecture 7: Policy Gradient, David Silver](http://www0.cs.ucl.ac.uk/staff/d.silver/web/Teaching_files/pg.pdf)\n"
      ]
    }
  ]
}