TD7_ResNet.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TD7_ResNet.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyO8NnIEQ4CzLsWhUwu5jGYW",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/thinhngo-x/deep-learning/blob/master/TD7_ResNet.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lY9FwLodbb5q",
        "colab_type": "text"
      },
      "source": [
        "# 0. Setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fVjZRSBX7VkU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch.nn as nn\n",
        "import torch.utils.model_zoo as model_zoo\n",
        "from tqdm.notebook import tqdm"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NT4dEo7ZKDsc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data.sampler import SubsetRandomSampler\n",
        "from torchsummary import summary"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x7Qj_9nmOCMa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def init(m):\n",
        "    if isinstance(m, nn.Conv2d):\n",
        "        nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n",
        "        if m.bias is not None:\n",
        "            nn.init.constant_(m.bias, 0)\n",
        "    elif isinstance(m, nn.BatchNorm2d):\n",
        "        nn.init.constant_(m.weight, 1)\n",
        "        nn.init.constant_(m.bias, 0)\n",
        "    elif isinstance(m, nn.Linear):\n",
        "        nn.init.normal_(m.weight, 0, 0.01)\n",
        "        nn.init.constant_(m.bias, 0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xGPupe5hbdSp",
        "colab_type": "text"
      },
      "source": [
        "# 1. ResNet"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fXIz0oLkUMIt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def trainer(train_loader, test_loader, model, use_cuda, loss_fn=nn.CrossEntropyLoss(), epoch=10, rate=1e-3):\n",
        "    loss_eval = []\n",
        "    err_eval = []\n",
        "    #   data = torch.utils.data.DataLoader(dataset, batch_size)\n",
        "    optimizer = torch.optim.Adam(model.parameters(),lr=rate)\n",
        "    j = 0\n",
        "    for i in tqdm(range(epoch)):\n",
        "        for imgs, labels in train_loader:\n",
        "            j += 1\n",
        "            if use_cuda and torch.cuda.is_available():\n",
        "                imgs = imgs.type(torch.cuda.FloatTensor)\n",
        "                labels = labels.type(torch.cuda.LongTensor)\n",
        "\n",
        "            optimizer.zero_grad()\n",
        "\n",
        "            output = model(imgs)\n",
        "            loss = loss_fn(output, labels)\n",
        "            if j % 500 == 1 :\n",
        "                loss_eval.append(loss.item())\n",
        "                err_eval.append(1 - accuracy(test_loader, model, use_cuda))\n",
        "            loss.backward()\n",
        "\n",
        "            optimizer.step()\n",
        "        \n",
        "    return loss_eval, err_eval\n",
        "\n",
        "def accuracy(dataloader, model, use_cuda):\n",
        "    model.eval()\n",
        "    #   test_loader = torch.utils.data.DataLoader(test_data, batch_size=len(test_data))\n",
        "    corr = 0\n",
        "    total = 0\n",
        "    for imgs, labels in dataloader:\n",
        "        if use_cuda and torch.cuda.is_available():\n",
        "            imgs = imgs.type(torch.cuda.FloatTensor)\n",
        "            labels = labels.type(torch.cuda.LongTensor)\n",
        "        with torch.no_grad():\n",
        "            output = model(imgs)\n",
        "            output = [torch.argmax(i) for i in output]\n",
        "            correct = [i for i,j in zip(output, labels) if i == j]\n",
        "            corr += len(correct)\n",
        "            total += len(labels)\n",
        "    model.train()\n",
        "    # print(len(correct))\n",
        "    return(corr / total)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7BqTSz6Bm1Aw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "\n",
        "\n",
        "class SimpleBlock(nn.Module):\n",
        "    def __init__(self, in_planes, planes, stride=1):\n",
        "        super(SimpleBlock, self).__init__()\n",
        "\n",
        "        assert((stride == 1) or (stride == 2 and planes == in_planes * 2))\n",
        "\n",
        "        padding = 0\n",
        "        self.conv1 = nn.Conv2d(in_planes, planes, 3, stride=stride, padding=1, bias=False)\n",
        "        self.bn1 = nn.BatchNorm2d(planes)\n",
        "        self.conv2 = nn.Conv2d(planes, planes, 3, stride=1, padding=1, bias=False)\n",
        "        self.bn2 = nn.BatchNorm2d(planes)\n",
        "\n",
        "    def forward(self, x):\n",
        "        out = F.relu(self.bn1(self.conv1(x)))\n",
        "        out = self.conv2(out)\n",
        "        out = F.relu(self.bn2(out))\n",
        "\n",
        "        return out\n",
        "\n",
        "      \n",
        "class ResNetBlock(nn.Module):\n",
        "\n",
        "    def __init__(self, in_planes, planes, stride=1):\n",
        "        super(ResNetBlock, self).__init__()\n",
        "\n",
        "        assert((stride == 1) or (stride == 2 and planes == in_planes * 2))\n",
        "\n",
        "        padding = 0\n",
        "        self.conv1 = nn.Conv2d(in_planes, planes, 3, stride=stride, padding=1, bias=False)\n",
        "        self.bn1 = nn.BatchNorm2d(planes)\n",
        "        self.conv2 = nn.Conv2d(planes, planes, 3, stride=1, padding=1, bias=False)\n",
        "        if stride == 2: \n",
        "            self.shortcut = nn.Conv2d(in_planes, planes, 1, stride=2, padding=0, bias=False)\n",
        "        else: \n",
        "            self.shortcut = nn.Identity()\n",
        "        self.bn2 = nn.BatchNorm2d(planes)\n",
        "\n",
        "    def forward(self, x):\n",
        "        out = F.relu(self.bn1(self.conv1(x)))\n",
        "        out = self.conv2(out) + self.shortcut(x)\n",
        "        out = F.relu(self.bn2(out))\n",
        "\n",
        "        return out\n",
        "\n",
        "class ResNet(nn.Module):\n",
        "    def __init__(self, block, num_blocks, num_classes=10, num_filters=16, input_dim=3):\n",
        "        super(ResNet, self).__init__()\n",
        "        self.in_planes = num_filters\n",
        "\n",
        "        self.conv1 = nn.Conv2d(input_dim, num_filters, kernel_size=3, stride=1, padding=1, bias=False)\n",
        "        self.bn1 = nn.BatchNorm2d(num_filters)\n",
        "\t\t\n",
        "        layers_resnet = []\n",
        "        for i in range(len(num_blocks)) :\n",
        "            if i == 0 :\n",
        "                planes = self.in_planes\n",
        "                print(planes)\n",
        "                layers_resnet.append(self._make_layer(block, planes, num_blocks[i], stride=1))\n",
        "            else :\n",
        "                planes = self.in_planes * 2\n",
        "                print(planes)\n",
        "                layers_resnet.append(self._make_layer(block, planes, num_blocks[i], stride=2))\n",
        "\n",
        "        self.resnet = nn.Sequential(*layers_resnet)\n",
        "\t\t\n",
        "        self.fc = nn.Sequential(\n",
        "            nn.Linear(self.in_planes, num_classes),\n",
        "            nn.LogSoftmax(1)\n",
        "        )\n",
        "\n",
        "    def _make_layer(self, block, planes, num_blocks, stride):\n",
        "        layers = []\n",
        "\n",
        "        if stride != 1 :\n",
        "            downsample = block(self.in_planes, planes, stride)\n",
        "            layers.append(downsample)\n",
        "            num_blocks -= 1\n",
        "            self.in_planes = planes\n",
        "\n",
        "        for i in range(num_blocks) :\n",
        "            layers.append(block(self.in_planes, planes, 1))\n",
        "            self.in_planes = planes\n",
        "        \n",
        "        return nn.Sequential(*layers)\n",
        "\n",
        "    def forward(self, x):\n",
        "        out = F.relu(self.bn1(self.conv1(x)))\n",
        "        out = self.resnet(out)\n",
        "        out = F.avg_pool2d(out, 4)\n",
        "        out = out.view(out.size(0), -1)\n",
        "        out = self.fc(out)\n",
        "        return out\n",
        "\n",
        "\n",
        "# (1 + 2*(1 + 1) + 2*(1 + 1) + 2*(1 + 1) + 2*(1 + 1)) + 1 = 18\n",
        "def ResNet18(input_dim=3):\n",
        "    return ResNet(ResNetBlock, [2, 2, 2, 2], input_dim=input_dim)\n",
        "\n",
        "def PlainNet18(input_dim=3):\n",
        "    return ResNet(SimpleBlock, [2, 2, 2, 2], input_dim=input_dim)\n",
        "\n",
        "def ResNet36(input_dim=3):\n",
        "  return ResNet(ResNetBlock, [4, 4, 4, 4], input_dim=input_dim)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2z0j-1B2H84O",
        "colab_type": "code",
        "outputId": "d8ea3bbe-0007-4e33-dade-49d88d5910b6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "summary(ResNet18(3).cuda(), input_size=(3,32,32), batch_size=32)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "16\n",
            "32\n",
            "64\n",
            "128\n",
            "----------------------------------------------------------------\n",
            "        Layer (type)               Output Shape         Param #\n",
            "================================================================\n",
            "            Conv2d-1           [32, 16, 32, 32]             432\n",
            "       BatchNorm2d-2           [32, 16, 32, 32]              32\n",
            "            Conv2d-3           [32, 16, 32, 32]           2,304\n",
            "       BatchNorm2d-4           [32, 16, 32, 32]              32\n",
            "            Conv2d-5           [32, 16, 32, 32]           2,304\n",
            "          Identity-6           [32, 16, 32, 32]               0\n",
            "       BatchNorm2d-7           [32, 16, 32, 32]              32\n",
            "       ResNetBlock-8           [32, 16, 32, 32]               0\n",
            "            Conv2d-9           [32, 16, 32, 32]           2,304\n",
            "      BatchNorm2d-10           [32, 16, 32, 32]              32\n",
            "           Conv2d-11           [32, 16, 32, 32]           2,304\n",
            "         Identity-12           [32, 16, 32, 32]               0\n",
            "      BatchNorm2d-13           [32, 16, 32, 32]              32\n",
            "      ResNetBlock-14           [32, 16, 32, 32]               0\n",
            "           Conv2d-15           [32, 32, 16, 16]           4,608\n",
            "      BatchNorm2d-16           [32, 32, 16, 16]              64\n",
            "           Conv2d-17           [32, 32, 16, 16]           9,216\n",
            "           Conv2d-18           [32, 32, 16, 16]             512\n",
            "      BatchNorm2d-19           [32, 32, 16, 16]              64\n",
            "      ResNetBlock-20           [32, 32, 16, 16]               0\n",
            "           Conv2d-21           [32, 32, 16, 16]           9,216\n",
            "      BatchNorm2d-22           [32, 32, 16, 16]              64\n",
            "           Conv2d-23           [32, 32, 16, 16]           9,216\n",
            "         Identity-24           [32, 32, 16, 16]               0\n",
            "      BatchNorm2d-25           [32, 32, 16, 16]              64\n",
            "      ResNetBlock-26           [32, 32, 16, 16]               0\n",
            "           Conv2d-27             [32, 64, 8, 8]          18,432\n",
            "      BatchNorm2d-28             [32, 64, 8, 8]             128\n",
            "           Conv2d-29             [32, 64, 8, 8]          36,864\n",
            "           Conv2d-30             [32, 64, 8, 8]           2,048\n",
            "      BatchNorm2d-31             [32, 64, 8, 8]             128\n",
            "      ResNetBlock-32             [32, 64, 8, 8]               0\n",
            "           Conv2d-33             [32, 64, 8, 8]          36,864\n",
            "      BatchNorm2d-34             [32, 64, 8, 8]             128\n",
            "           Conv2d-35             [32, 64, 8, 8]          36,864\n",
            "         Identity-36             [32, 64, 8, 8]               0\n",
            "      BatchNorm2d-37             [32, 64, 8, 8]             128\n",
            "      ResNetBlock-38             [32, 64, 8, 8]               0\n",
            "           Conv2d-39            [32, 128, 4, 4]          73,728\n",
            "      BatchNorm2d-40            [32, 128, 4, 4]             256\n",
            "           Conv2d-41            [32, 128, 4, 4]         147,456\n",
            "           Conv2d-42            [32, 128, 4, 4]           8,192\n",
            "      BatchNorm2d-43            [32, 128, 4, 4]             256\n",
            "      ResNetBlock-44            [32, 128, 4, 4]               0\n",
            "           Conv2d-45            [32, 128, 4, 4]         147,456\n",
            "      BatchNorm2d-46            [32, 128, 4, 4]             256\n",
            "           Conv2d-47            [32, 128, 4, 4]         147,456\n",
            "         Identity-48            [32, 128, 4, 4]               0\n",
            "      BatchNorm2d-49            [32, 128, 4, 4]             256\n",
            "      ResNetBlock-50            [32, 128, 4, 4]               0\n",
            "           Linear-51                   [32, 10]           1,290\n",
            "       LogSoftmax-52                   [32, 10]               0\n",
            "================================================================\n",
            "Total params: 701,018\n",
            "Trainable params: 701,018\n",
            "Non-trainable params: 0\n",
            "----------------------------------------------------------------\n",
            "Input size (MB): 0.38\n",
            "Forward/backward pass size (MB): 98.00\n",
            "Params size (MB): 2.67\n",
            "Estimated Total Size (MB): 101.05\n",
            "----------------------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QznQ6WoDH9W7",
        "colab_type": "code",
        "outputId": "9a84e593-c03f-4a7d-c184-8200c8144d77",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "summary(PlainNet18(3).cuda(), input_size=(3,32,32), batch_size=32)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "16\n",
            "32\n",
            "64\n",
            "128\n",
            "----------------------------------------------------------------\n",
            "        Layer (type)               Output Shape         Param #\n",
            "================================================================\n",
            "            Conv2d-1           [32, 16, 32, 32]             432\n",
            "       BatchNorm2d-2           [32, 16, 32, 32]              32\n",
            "            Conv2d-3           [32, 16, 32, 32]           2,304\n",
            "       BatchNorm2d-4           [32, 16, 32, 32]              32\n",
            "            Conv2d-5           [32, 16, 32, 32]           2,304\n",
            "       BatchNorm2d-6           [32, 16, 32, 32]              32\n",
            "       SimpleBlock-7           [32, 16, 32, 32]               0\n",
            "            Conv2d-8           [32, 16, 32, 32]           2,304\n",
            "       BatchNorm2d-9           [32, 16, 32, 32]              32\n",
            "           Conv2d-10           [32, 16, 32, 32]           2,304\n",
            "      BatchNorm2d-11           [32, 16, 32, 32]              32\n",
            "      SimpleBlock-12           [32, 16, 32, 32]               0\n",
            "           Conv2d-13           [32, 32, 16, 16]           4,608\n",
            "      BatchNorm2d-14           [32, 32, 16, 16]              64\n",
            "           Conv2d-15           [32, 32, 16, 16]           9,216\n",
            "      BatchNorm2d-16           [32, 32, 16, 16]              64\n",
            "      SimpleBlock-17           [32, 32, 16, 16]               0\n",
            "           Conv2d-18           [32, 32, 16, 16]           9,216\n",
            "      BatchNorm2d-19           [32, 32, 16, 16]              64\n",
            "           Conv2d-20           [32, 32, 16, 16]           9,216\n",
            "      BatchNorm2d-21           [32, 32, 16, 16]              64\n",
            "      SimpleBlock-22           [32, 32, 16, 16]               0\n",
            "           Conv2d-23             [32, 64, 8, 8]          18,432\n",
            "      BatchNorm2d-24             [32, 64, 8, 8]             128\n",
            "           Conv2d-25             [32, 64, 8, 8]          36,864\n",
            "      BatchNorm2d-26             [32, 64, 8, 8]             128\n",
            "      SimpleBlock-27             [32, 64, 8, 8]               0\n",
            "           Conv2d-28             [32, 64, 8, 8]          36,864\n",
            "      BatchNorm2d-29             [32, 64, 8, 8]             128\n",
            "           Conv2d-30             [32, 64, 8, 8]          36,864\n",
            "      BatchNorm2d-31             [32, 64, 8, 8]             128\n",
            "      SimpleBlock-32             [32, 64, 8, 8]               0\n",
            "           Conv2d-33            [32, 128, 4, 4]          73,728\n",
            "      BatchNorm2d-34            [32, 128, 4, 4]             256\n",
            "           Conv2d-35            [32, 128, 4, 4]         147,456\n",
            "      BatchNorm2d-36            [32, 128, 4, 4]             256\n",
            "      SimpleBlock-37            [32, 128, 4, 4]               0\n",
            "           Conv2d-38            [32, 128, 4, 4]         147,456\n",
            "      BatchNorm2d-39            [32, 128, 4, 4]             256\n",
            "           Conv2d-40            [32, 128, 4, 4]         147,456\n",
            "      BatchNorm2d-41            [32, 128, 4, 4]             256\n",
            "      SimpleBlock-42            [32, 128, 4, 4]               0\n",
            "           Linear-43                   [32, 10]           1,290\n",
            "       LogSoftmax-44                   [32, 10]               0\n",
            "================================================================\n",
            "Total params: 690,266\n",
            "Trainable params: 690,266\n",
            "Non-trainable params: 0\n",
            "----------------------------------------------------------------\n",
            "Input size (MB): 0.38\n",
            "Forward/backward pass size (MB): 83.00\n",
            "Params size (MB): 2.63\n",
            "Estimated Total Size (MB): 86.01\n",
            "----------------------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6En1zWhUMbcI",
        "colab_type": "text"
      },
      "source": [
        "The more blocks there are, the deeper the network is."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KoFHrQrLgTER",
        "colab_type": "text"
      },
      "source": [
        "## On CIFAR-10"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fv0YOntO4EQl",
        "colab_type": "code",
        "outputId": "bfabb4f1-2f59-4b68-9d9d-c40a830b646e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "batch_size=64\n",
        "transform = transforms.Compose(\n",
        "    [\n",
        "     transforms.ToTensor(),\n",
        "     transforms.Normalize((0.5,), (0.5,)),\n",
        "])\n",
        "trainset = torchvision.datasets.CIFAR10(root=\"./data\", train=True, download=True, transform=transform)\n",
        "trainloader = torch.utils.data.DataLoader(trainset,batch_size=batch_size,shuffle=False)\n",
        "testset = torchvision.datasets.CIFAR10(root=\"./data\", train=False, download=True, transform=transform)\n",
        "testloader = torch.utils.data.DataLoader(testset,batch_size=10000,shuffle=False)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qQGOG7dZitlZ",
        "colab_type": "code",
        "outputId": "7d558b74-489c-4644-aa7b-ad0e80d4072a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168,
          "referenced_widgets": [
            "29e6a2afadba4ae2b972febcf479fb8d",
            "a18b978fddd4499a9ef55f94551e540d",
            "94966049d0014a2fbfa38084a3da4928",
            "9759d8c39ca8425f8ce7a68d212ce64d",
            "497c965df9c8421ea06ba3382fe2a8c2",
            "4d24bcbf8a5f46e1a236c1f76c46601d",
            "baa1a1601d85439b861551f05f01d6ff",
            "28e624943aa5493ca83bcac584232537"
          ]
        }
      },
      "source": [
        "model_resnet18 = ResNet18()\n",
        "model_resnet18.apply(init)\n",
        "\n",
        "#net = AlexNet(10)\n",
        "use_cuda = True\n",
        "if use_cuda and torch.cuda.is_available():\n",
        "    print(\"using cuda\")\n",
        "    model_resnet18.cuda()\n",
        "losses, val_errs = trainer(trainloader, testloader, model_resnet18, use_cuda, epoch=3)\n",
        "print(accuracy(testloader, model_resnet18, use_cuda))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "16\n",
            "32\n",
            "64\n",
            "128\n",
            "using cuda\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "29e6a2afadba4ae2b972febcf479fb8d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "0.7327\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sCnlwU266xJf",
        "colab_type": "code",
        "outputId": "a7d8db0c-3e13-46bc-d533-39b688a5ac6b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168,
          "referenced_widgets": [
            "4421293507d1491ab116c97d1b54662a",
            "6448c0931628482690517d3c5257654e",
            "a7ce475310974d42bd0b014434fd5b9e",
            "96d5fe0a71f94d05b69e177af1e0a2c0",
            "b7e85178f8b244ae946cda9cefec3dc3",
            "a6827e08328d43b78a288aa11f52e497",
            "2f02d222f49e402e9c577785501875bf",
            "b006256bc6454fedb22fecbd6540d9e0"
          ]
        }
      },
      "source": [
        "model_plainnet18 = PlainNet18()\n",
        "model_plainnet18.apply(init)\n",
        "#net = AlexNet(10)\n",
        "use_cuda = True\n",
        "if use_cuda and torch.cuda.is_available():\n",
        "    print(\"using cuda\")\n",
        "    model_plainnet18.cuda()\n",
        "losses_, val_errs_ = trainer(trainloader, testloader, model_plainnet18, use_cuda, epoch=3)\n",
        "print(accuracy(testloader, model_plainnet18, use_cuda))\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "16\n",
            "32\n",
            "64\n",
            "128\n",
            "using cuda\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4421293507d1491ab116c97d1b54662a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "0.68\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CNaWaWJxBXCB",
        "colab_type": "code",
        "outputId": "71de87f0-2f9b-49e0-e48c-bb2ba6fcc707",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        }
      },
      "source": [
        "plt.plot(losses, label='ResNet training loss')\n",
        "plt.plot(val_errs, label='ResNet validation error')\n",
        "plt.plot(losses_, label='Plainnet training loss')\n",
        "plt.plot(val_errs_, label='Plainnet validation error')\n",
        "plt.legend()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7f8ab067acf8>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdd3gUxR/H8fekXgrpgUBCkxJKekJoIr0pgoUiHQtVEPkpoiLNigoqIIKAShEQFREpKkWqKJBQQwcJEAiQ3nvm98clRwJpwCWXMq/nyWNud3Z3DuFzezO73xVSShRFUZTKy8jQHVAURVFKlwp6RVGUSk4FvaIoSiWngl5RFKWSU0GvKIpSyZkYugMFcXJykvXq1TN0NxRFUSqM4ODgSCmlc0HrymXQ16tXj6CgIEN3Q1EUpcIQQlwpbJ0aulEURankVNAriqJUciroFUVRKrlyOUavKBVNRkYGYWFhpKamGrorSiWn0Whwc3PD1NS0xNuooFcUPQgLC6NatWrUq1cPIYShu6NUUlJKoqKiCAsLo379+iXeTg3dKIoepKam4ujoqEJeKVVCCBwdHe/7m6MKekXRExXySll4kL9nlSbos7Mlb2xdy/5Llw3dFUVRlHKl0gR9WFwkv9+ay8u7xnI1JtLQ3VGUMmdsbIyPjw8eHh48+eSTxMbG3vc+du/ejRCCTZs26Zb16tWL3bt3F7nd8uXLuXHjxn2vK8rixYtZuXJlkW2CgoJ45ZVX7nvfBZk5cyZz5szRy77Km0oT9HXsnZni9xFZxjfpt/FF4tMSDd0lRSlTFhYWHDt2jJCQEBwcHFi4cOED7cfNzY0PPvjgvrZ50KDPysoqdJ9jxoxh2LBhRR43ICCA+fPnl7yjVVSlCXqAId5d6VfnLZII5dlfXiI1U13qplRNrVu35vr16wBcunSJHj164O/vT7t27Th79iwAP/30Ex4eHnh7e/PYY4/ptvX29sbW1pbt27ffs9/g4GDat2+Pv78/3bt3Jzw8nJ9//pmgoCAGDx6Mj48PKSkpuvYFratXrx5TpkzBz8+Pn376iaVLl9KiRQu8vb159tlnSU5OBvKfYXfo0IEpU6YQGBhI48aN2bdvH6D9BtKrVy9d+xdeeIEOHTrwyCOP5PsAeO+993B3d+fRRx9l4MCBxZ65Hzt2jFatWuHl5cXTTz9NTEwMAPPnz6dZs2Z4eXnx3HPPAbBnzx58fHzw8fHB19eXhISE+/g/VTaKvbxSCFEbWAnUACSwREo57642g4EpgAASgLFSyuM560JzlmUBmVLKAH2+gbtN79SfC+uiOZa6iOGbx/P9k4swNS759aaK8rBmbTrF6Rvxet1ns1o2zHiyeYnaZmVlsXPnTl588UUARo0axeLFi2nUqBEHDx5k3Lhx/PXXX7z77rv8+eefuLq63jPMM3XqVKZNm0bXrl11yzIyMpgwYQIbN27E2dmZdevWMXXqVL799lu+/PJL5syZQ0BA/n/effv2LXCdo6MjR44cASAqKoqRI0cC8M477/DNN98wYcKEe95XZmYmhw4dYuvWrcyaNYsdO3bc0+bs2bPs2rWLhIQE3N3dGTt2LMeOHWP9+vUcP36cjIwM/Pz88Pf3L/LPcNiwYSxYsID27dszffp0Zs2axRdffMHs2bO5fPky5ubmuj+zOXPmsHDhQtq2bUtiYiIajabIfRtCSa6jzwRek1IeEUJUA4KFENullKfztLkMtJdSxgghegJLgJZ51neUUpbJwLkQgmXPjqLHN7GcjlvDKzsn82WXuRgbGZfF4RXFYFJSUvDx8eH69es0bdqUrl27kpiYyIEDB+jXr5+uXVpaGgBt27ZlxIgR9O/fn2eeeSbfvnLP8Pfv369bdu7cOUJCQnThn5WVRc2aNR+orwMGDND9HhISwjvvvENsbCyJiYl07969wG1y++jv709oaGiBbZ544gnMzc0xNzenevXq3Lp1i7///ps+ffqg0WjQaDQ8+eSTRfYtLi6O2NhY2rdvD8Dw4cN1f35eXl4MHjyYp556iqeeegrQ/jn+73//Y/DgwTzzzDO4ubnd159FWSg26KWU4UB4zu8JQogzgCtwOk+bA3k2+Rcw6DvVmBqzqv9EnliRyH5+4539M/ig3bsYiUo1UqWUUyU989a33DH65ORkunfvzsKFCxkxYgR2dnYcO3bsnvaLFy/m4MGDbNmyBX9/f4KDg/Otnzp1Ku+//z4mJtqYkFLSvHlz/vnnn4fuq5WVle73ESNG8Ouvv+Lt7c3y5csLnfg1NzcHtJPOmZmZRbYprt2D2rJlC3v37mXTpk188MEHnDx5kjfffJMnnniCrVu30rZtW/7880+aNGmi1+M+rPtKPiFEPcAXOFhEsxeB3/O8lsA2IUSwEGJUEfseJYQIEkIERURE3E+3ClTbwZIvn3iV9MjObL68kY8PfYyU8qH3qyjlnaWlJfPnz2fu3LlYWlpSv359fvrpJ0Ab1sePHwe0Y/ctW7bk3XffxdnZmWvXruXbT7du3YiJieHEiRMAuLu7ExERoQv6jIwMTp06BUC1atUKHZsuah1AQkICNWvWJCMjg9WrVz/cmy9A27Zt2bRpE6mpqSQmJrJ58+Yi29va2mJvb6+bB1i1ahXt27cnOzuba9eu0bFjRz7++GPi4uJITEzk0qVLeHp6MmXKFFq0aKGbAylPShz0QghrYD3wqpSywAFIIURHtEE/Jc/iR6WUfkBP4GUhxGMFbSulXCKlDJBSBjg7F1g7/7491tiZ8T7jSI96lDVn17Dg6AK97FdRyjtfX1+8vLxYu3Ytq1ev5ptvvsHb25vmzZuzceNGACZPnoynpyceHh60adMGb2/ve/YzdepU3QeAmZkZP//8M1OmTMHb2xsfHx8OHNB+mR8xYgRjxoy5ZzK2uHWgnSht2bIlbdu2LZUz4RYtWtC7d2+8vLzo2bMnnp6e2NraFrnNihUrmDx5Ml5eXhw7dozp06eTlZXFkCFD8PT0xNfXl1deeQU7Ozu++OILPDw88PLywtTUlJ49e+r9PTwsUZKzXCGEKbAZ+FNK+VkhbbyADUBPKeX5QtrMBBKllEVOeQcEBEh9PXgkO1syclUQB2IXY2J3iFf9XuVFzxf1sm9FyXXmzBmaNm1q6G4ohUhMTMTa2prk5GQee+wxlixZgp+fn6G79cAK+vsmhAgu7GKXYs/ohfZ+22+AM0WEfB3gF2Bo3pAXQljlTOAihLACugEhJXwvemFkJPisvw/V0wdhlOzHF0e+YO3ZtWXZBUVRDGzUqFH4+Pjg5+fHs88+W6FD/kGU5KqbtsBQ4KQQIndG522gDoCUcjEwHXAEvsqpw5B7GWUNYEPOMhNgjZTyD72+gxKwtTDl66EteHphKnb1s/jw4IdYmljSp2Gfsu6KoigGsGbNGkN3waBKctXNfrTXxxfV5iXgpQKW/wfcO/BnAE1cbJj9rA8T12XQwCOb6QemY2lqSde6XYvfWFEUpQKrUtcb9vFx5fk2jbgU0g83iya8sfcN9oXtM3S3FEVRSlWVCnqAtx9vSou6Nfgv5DlqWzdg0u5JHL552NDdUhRFKTVVLuhNjY1YOMgPa7NqxF4aRk2rWozfOZ6TEScN3TVFUZRSUeWCHqC6jYavBvtxI9oE+/gJOGgcGLNjDOdjCrwqVFEqhPJapvh+jRgxgp9//hmAl156idOnT9/TZvny5YwfP77I/ezevVt3nT+UrOxxZVUlgx6gRT0Hpj7RlL1n0nnMZhoaEw2jto0iNC7U0F1TlAdSXssUP4xly5bRrFmzB9r27qAvSdljfZFSkp2dXejrwui7ZEOuKhv0ACPa1KOPTy2W/BXD6MafIJGM3D6SG4n6/wurKGWpvJQpPnv2LIGBgbrXoaGheHp6AvDuu+/SokULPDw8GDVqVIElSjp06EDuzZPfffcdjRs3JjAwkL///lvXZtOmTbRs2RJfX1+6dOnCrVu3CA0NZfHixXz++ef4+Piwb9++fGWPCytDXFg55Lt9+umntGjRAi8vL2bMmKF7b+7u7gwbNgwPDw/27duX7/W1a9eYPHkyHh4eeHp6sm7dOkD7gdSuXTt69+79wB9qxSnJdfSVlhCCj57x5Gx4Ah9ujOKLofN4+99xjNw2kuU9luNsqZ9SDEoV8/ubcFPPcz4untBzdomalqcyxU2aNCE9PZ3Lly9Tv3591q1bp6tcOX78eKZPnw7A0KFD2bx5c6GVJcPDw5kxYwbBwcHY2trSsWNHfH19AXj00Uf5999/tZVrly3jk08+Ye7cuYwZMwZra2tef/11AHbu3KnbX2FliKH4csjbtm3jwoULHDp0CCklvXv3Zu/evdSpU4cLFy6wYsUKWrVqRWhoaL7X69ev59ixYxw/fpzIyEhatGih+4A9cuQIISEh1K9fv0T/j+9XlT6jB7A0M2HxUH+ysiRzNycyr8NCIlIiGLV9FLGp9z/GqSiGklum2MXFhVu3bt1TptjHx4fRo0cTHh4O3ClTvHTp0nue9FRcmWIfHx/ef/99wsLCiu1X//79dWeveYN+165dtGzZEk9PT/766y9dgbSCHDx4kA4dOuDs7IyZmVm+MsdhYWF0794dT09PPv300yL3AwWXId67d69ufXHlkLdt28a2bdvw9fXFz8+Ps2fPcuHCBQDq1q1Lq1atdG3zvt6/fz8DBw7E2NiYGjVq0L59ew4f1l7xFxgYWGohD1X8jD5XfScrPhvgw8iVQfz8tw0LOi1g3I5xjNkxhmXdlmFtZm3oLioVSQnPvPWtvJYpHjBgAP369eOZZ55BCEGjRo1ITU1l3LhxBAUFUbt2bWbOnElq6oM9EW7ChAn873//o3fv3uzevZuZM2c+0H5yFVcOWUrJW2+9xejRo/MtDw0NzVd+GbjndWFK2u5BVfkz+lxdm9VgfMeGrAu6xn/XavJZh884F32Ol3e+TErmvRX3FKW8Km9lihs0aICxsTHvvfee7kw8N9SdnJxITEzUXWVTmJYtW7Jnzx6ioqLIyMjQvR/QnqG7uroC2qqTuQrrU2FliEuqe/fufPvttyQmap9Lff36dW7fvl3sdu3atWPdunVkZWURERHB3r17881flCYV9HlM6tqYdo2cmLHxFHZ481G7jzgWcYxJuyaRnpVu6O4pSomVpzLFoD2r//777+nfvz8AdnZ2jBw5Eg8PD7p3706LFi2KfD81a9Zk5syZtG7dmrZt2+ar3Dhz5kz69euHv78/Tk5OuuVPPvkkGzZs0E3G5lVQGeKS6tatG4MGDaJ169Z4enrSt2/fEj0n9umnn8bLywtvb286derEJ598gouLS4mP+zBKVKa4rOmzTPH9iklKp9eC/Ugp2TThUfaGb2X6gel0qdOFT9t/iomRGu1S7qXKFCtlSe9liqsaeyszFg/xJzIpnVd+OErvBk/xZuCb7Li6g+l/TydbFn8trKIoSnmigr4Anm62vN/Hg78vRjF32zkGNx3MBN8JbPpvEx8e/FA9klBRlApFjUMUon+L2hy9FstXuy/hXduOkZ4jScxI5LuQ77A0tWSS3yRy6uwriqKUayV5wlRtIcQuIcRpIcQpIcTEAtoIIcR8IcRFIcQJIYRfnnXDhRAXcn6G6/sNlKaZvZvh7WbLaz8e53JkEpP8JjHAfQDfhXzH0pNLDd09RVGUEinJ0E0m8JqUshnQCu0Dvu++T7cn0CjnZxSwCEAI4QDMAFoCgcAMIYS9nvpe6sxNjPlqiD9mJkaMXhVMcnoWb7d8m94NerPg6AK+P/29obuoKIpSrGKDXkoZLqU8kvN7AnAGcL2rWR9gpdT6F7ATQtQEugPbpZTRUsoYYDvQQ6/voJS52lmwYKAvlyISmbL+BALBrDaz6FKnCx8f/pgNFzYYuouKoihFuq/JWCFEPcAXOHjXKlcg790WYTnLClte0L5HCSGChBBBERER99OtUte2oROTuzdh84lwvtl/GRMjEz5+7GPaurZlxoEZ/HG5zB+Dqyj3yFumuF+/fiQnJwNgbV38nd1t2rTRe3+OHTvG1q1b73tdUW7cuEHfvn2Lbff4448/UJnmu4WGhuLh4fHQ+zG0Ege9EMIaWA+8KqWM13dHpJRLpJQBUsoAZ+fyV0xsTPtH6N68Bh/9fpaD/0VhZmzG5x0+x6+GH2/te4s91/YYuotKFZe3TLGZmRmLFy8u8bZ5y/nqy4MGfVGlemvVqlXsXbQAW7duxc7OrmQdrQJKFPRCCFO0Ib9aSvlLAU2uA7XzvHbLWVbY8gpHCMGcft7UdbDk5TVHuRWfioWJBV92+pImDk343+7/cTD87i86imIY7dq14+LFi/mWJSYm0rlzZ/z8/PD09NTdIQt3zvp3795Nhw4d6Nu3L02aNGHw4MG6y4nr1avHjBkzdNvnljtOSkrihRdeIDAwEF9fXzZu3Eh6ejrTp09n3bp1+Pj46IqaAQWumzlzJkOHDqVt27YMHTqU0NBQ2rVrh5+fH35+froPorxn2MuXL+eZZ56hR48eNGrUiDfeeEN3jHr16hEZGUloaChNmzZl5MiRNG/enG7duunu3D18+DBeXl74+PjoygcXJTU1leeffx5PT098fX3ZtWsXAKdOnSIwMBAfHx+8vLy4cOECSUlJPPHEE3h7e+Ph4ZHv/RtCsZdXCu01hN8AZ6SUnxXS7DdgvBDiB7QTr3FSynAhxJ/Ah3kmYLsBb+mh3wZRTWPK10P96bPwb8atPsLaka2wNrNmUZdFPP/n80z4awJLui7Bp7qPobuqGNDHhz7mbPRZve6ziUMTpgROKVHbzMxMfv/9d3r0yD8dptFo2LBhAzY2NkRGRtKqVSt69+59z2XCR48e5dSpU9SqVYu2bdvy999/8+ijjwLa2jRHjhzhq6++Ys6cOSxbtowPPviATp068e233xIbG0tgYCBdunTh3XffJSgoiC+//DLf/s3MzO5ZN3PmTE6fPs3+/fuxsLAgOTmZ7du3o9FouHDhAgMHDqSgu+WPHTvG0aNHMTc3x93dnQkTJlC7du18bS5cuMDatWtZunQp/fv3Z/369QwZMoTnn3+epUuX0rp1a958881i/1wXLlyIEIKTJ09y9uxZunXrxvnz51m8eDETJ05k8ODBpKenk5WVxdatW6lVqxZbtmwBtPV4DKkkZ/RtgaFAJyHEsZyfx4UQY4QQY3LabAX+Ay4CS4FxAFLKaOA94HDOz7s5yyqsRjWq8UlfL4KvxPDBFu0jzuw0diztthRnC2fG7Rin93/kilISuWWKAwICqFOnjq4efS4pJW+//TZeXl506dKF69evc+vWrXv2ExgYiJubG0ZGRvj4+OQr1VtQCd9t27Yxe/ZsfHx86NChA6mpqVy9evW++9+7d28sLCwAbcG0kSNH4unpSb9+/Qp8nCBA586dsbW1RaPR0KxZM65cuXJPm/r16+Pj45Ov37GxsSQkJNC6dWsABg0aVGz/9u/fz5AhQwBtnf26dety/vx5WrduzYcffsjHH3/MlStXsLCwwNPTk+3btzNlyhT27duHra3tff956FOxZ/RSyv1AkXcGSe13u5cLWfct8O0D9a6c6uVVi2NXY1m2/zI+dex42tcNJwsnlnZbyvA/hjN6+2i+6/Edj9g+YuiuKgZQ0jNvfcsdoy/M6tWriYiIIDg4GFNTU+rVq1dgaeDcMr1wb6negkr4SilZv3497u7u+fZz8OD9DWXmLdX7+eefU6NGDY4fP052djYajabAbYrqa2FtCiq69jAGDRpEy5Yt2bJlC48//jhff/01nTp14siRI2zdupV33nmHzp0731fhNH1TJRAe0Js9m9CyvgNv/XKS0ze0c9O1rGuxrNsyBIKR20YSllD8QxkUpazExcVRvXp1TE1N2bVrV4Fnvw+ie/fuLFiwQDeWf/ToUaDo0sVFrcvta82aNTEyMmLVqlX3PBjlYdnZ2VGtWjXdh9EPP/xQ7Dbt2rVj9erVAJw/f56rV6/i7u7Of//9xyOPPMIrr7xCnz59OHHiBDdu3MDS0pIhQ4YwefJkjhw5otf+3y8V9A/IxNiILwf5YWthypjvg4lLzgCgrk1dlnRbQmpmKiO3jeR2cvF1qhWlLAwePJigoCA8PT1ZuXIlTZo00ct+p02bRkZGBl5eXjRv3pxp06YB0LFjR06fPn3PZGxx6wDGjRvHihUr8Pb25uzZs6XyYI5vvvmGkSNH4uPjQ1JSUrHDK+PGjSM7OxtPT08GDBjA8uXLMTc358cff8TDwwMfHx9CQkIYNmwYJ0+e1E3Qzpo1i3feeUfv/b8fqkzxQwq+EsNzS/7hsUbOLB0WgJGRdpTrZMRJXtr2Ei5WLnzX4zscNA4G7qlSmlSZ4oonMTFRd7XR7NmzCQ8PZ968eQbuVcmoMsVlzL+uPdN6NWPn2dt8uevO5Wyezp582flLrideZ8z2McSn6/3WA0VRHsKWLVt0N5jt27fP4GfdpUkFvR4MbVWXZ3xd+XzHeXaduzNU08KlBV90/IILsRd4ecfLJGckG7CXiqLkNWDAAN0NZlu2bKE83qipLyro9UAIwQdPe9LExYZXfzjGteg7gf6o66N88tgnnIg8wcRdE0nLSjNgTxVFqYpU0OuJhZkxi4f4IaVk9KpgUjPuXCXQtW5X3m3zLv+G/8vre14nIzvDgD1VFKWqUUGvR3UdrfjiOR9Oh8czdUNIvidR9WnYh6ktp7L72m6m7p9KVrZ+LxdTFEUpjAp6PevUpAYTOzdi/ZEwVh/Mf3fgc02e41W/V/n98u+89+976pGEiqKUCRX0pWBi50Z0cHdm1qZTHL0ak2/di54vMtJzJOsvrGdO0BwV9oreVKQyxfdr9+7d9OrVC4DffvuN2bNnF9iuuPcaGxvLV199pXtd0rLHFZ0K+lJgZCT4YoAPLrYaxn5/hMjE/BOwE3wnMLjpYFaeXsmi44sM1EulsqlIZYofRu/evUtUhKwgdwd9Scse68vdJRqKKsmc18PeGayCvpTYWZqxaLA/McnpTFhzlMysbN06IQRvtHiDpxo+xaLji1hxaoUBe6pURuW5TDFAq1atOHXqlO51hw4dCAoK4tChQ7Ru3RpfX1/atGnDuXPn7nlvy5cvZ/z48QBcvnyZ1q1b4+npme86+MLe65tvvsmlS5d0pYnzlj0urAxxUeWQ8woODqZ9+/b4+/vTvXt3wsPDde/t1VdfJSAggHnz5t3zeufOnfj6+uLp6ckLL7xAWlqa7s97ypQp+Pn58dNPPxX+P7sEii1qpjw4D1dbPnjak9d/Os6nf57jrcfv3MlmJIyY2XomyRnJzAmag6WpJf0a9zNgbxV9ufnhh6Sd0W8FU/OmTXB5++0StS3vZYpBew37jz/+yKxZswgPDyc8PJyAgADi4+PZt28fJiYm7Nixg7fffpv169cX+l4nTpzI2LFjGTZsGAsXLiz2vc6ePZuQkBBd8be8lTkLK0MMxZdDzsjIYMKECWzcuBFnZ2fWrVvH1KlT+fZbbT3H9PR0XZnlTZs26V6npqbSqFEjdu7cSePGjRk2bBiLFi3i1VdfBcDR0VEvdXLUGX0p6+vvxpBWdfh673/8fjI83zpjI2Nmt5vNY26P8d4/77H5v80G6qVSGVSkMsX9+/fXDZn8+OOPunHyuLg4+vXrh4eHB5MmTcp31l+Qv//+m4EDBwIwdOjQ+36veRVWhhiKL4d87tw5QkJC6Nq1Kz4+Prz//vuEhd0pajhgwIB87XNfnzt3jvr169O4cWMAhg8fzt69ewvd7kGpM/oyMK1XM0Kux/P6T8dpVMOahtWr6daZGpsyt/1cXt75Mu/sfwcLEws61+lswN4qD6ukZ976VpHKFLu6uuLo6MiJEydYt26dbj5h2rRpdOzYkQ0bNhAaGkqHDh2Kfd93fyO5n/daUsWVQ5ZS0rx5c/75558Ct7+7KFtJi7Tpq5hbsWf0QohvhRC3hRAhhayfnOeBJCFCiCwhhEPOulAhxMmcdRWjSlkpMDcxZtEQPzSmxoxeFUxiWv6/JBoTDfM7zae5Y3Mm75nMgRv6nxhTlPJUphi0Z6uffPIJcXFxeHl56fro6uoKaMfGi9O2bVtdieHcEsK5+ynovRbVp8LKEJeEu7s7ERERuqDPyMgo9ttI7nahoaG6+ZRVq1bRvn37Eh3zfpRk6GY50KOwlVLKT6WUPlJKH7SPCdxz11OkOuasL7CqWlVR09aCBYN8uRyZxOSfjt9zWaWVqRVfdfmK+rb1mfjXRI7cMmz9aqXyKU9ligH69u3LDz/8QP/+/XXL3njjDd566y18fX1LdEXKvHnzWLhwIZ6enly/fudx1IW9V0dHR9q2bYuHhweTJ0/Ot6/CyhCXhJmZGT///DNTpkzB29sbHx+fEl3JpNFo+O677+jXrx+enp4YGRkxZsyYYre7XyUqUyyEqAdsllIW+fRcIcQaYJeUcmnO61AgQEoZeT+dqkhliu/Xkr2X+HDrWd5+vAmjHmtwz/rIlEie/+N5IlMiWdZ9Gc0dmxugl8r9UmWKlbJksDLFQghLtGf+eafIJbBNCBEshBhVzPajhBBBQoigiIgIfXWr3BnZ7hEe93Rh9u9nOXDp3s+/3EcS2pjZMGb7GC7GXCxgL4qiKCWnz6tungT+vmvY5lEppR/QE3hZCPFYYRtLKZdIKQOklAGVuVyoEIJP+npT38mKCWuOEh537/MrXaxcWNZtGaZGpozaPopr8dcM0FNFUSoLfQb9c8DavAuklNdz/nsb2AAE6vF4FZa1uQlfDw0gNSOLsd8fIS3z3rveatvUZknXJWRkZ/DStpe4mXTTAD1V7ocqZ6GUhQf5e6aXoBdC2ALtgY15llkJIarl/g50Awq8cqcqaljdmjn9vDl2LZb3Np8uuI19QxZ3XUx8ejwjt40kKiWqjHuplJRGoyEqKkqFvVKqpJRERUWh0Wjua7tir6MXQqwFOgBOQogwYAZgmnPQ3GIaTwPbpJRJeTatAWzIucbVBFgjpfzjvnpXyfX0rMnoxx7h673/4VPbnr7+bve0ae7YnIWdFzJ6+2hGbx/NN92/wda86IcYK2XPzc2NsLAwKvP8klI+aDQa3NzuzYqiqHlKZl8AACAASURBVIeDG1hmVjZDvznEkasxrB/bBg/XgkP8wI0DjN85nqYOTVnSbQlWpvq5kUJRlMpBPRy8HDMxNmLBIF8crMwYuzqY2OT0Atu1qdWGT9t/yqmoU7zy1yukZj74XX6KolQtKujLASdrc74a7MfNuFQm/nCM7OyCv2V1rtOZ9x99n8M3D/PantfIyFKPJFQUpXgq6MsJ3zr2zHiyOXvOR/DFzguFtuv1SC+mtZ7G3rC9vLX/LfVIQkVRiqWKmpUjg1vW4di1WObvvIC3my2dm9YosF2/xv105Y0tTCyY1WYWRkJ9ZiuKUjCVDuWIEIL3n/KgeS0bJq07xpWopELbDm8+nLHeY/n14q98cvgTdVmfoiiFUkFfzmhMjVk8xB8hBKNXBZOSXvjQzFjvsQxtNpTVZ1az4OiCMuyloigViQr6cqi2gyXznvPh3K0E3t5wstCzdSEEkwMm82yjZ1l6cinfnPymjHuqKEpFoIK+nOrgXp1JXRqz4eh1Vv1beN1wIQTTWk3j8fqP88WRL1h7dm2hbRVFqZrUZGw5Nr5jQ45fi+XdTadpXssG/7oOBbYzNjLm/UffJzkzmQ8PfoiliSV9GvYp494qilJeqTP6cszISPDZAB9c7S0Yt/oItxMKv0nK1MiUOe3n0LJmS6YfmM72K9vLsKeKopRnKujLOVsLUxYP8ScuJYPxa46SkZVdaFtzY3Pmd5yPl5MXb+x9g31h+8qwp4qilFcq6CuApjVt+OgZTw5djubj388W2dbS1JKFXRbSyK4Rk3ZP4vDNw2XUS0VRyisV9BXE075uDG9dl2X7L7Pp+I0i29qY2bC462JcrV0Zv3M8JyNOllEvFUUpj1TQVyBTn2iGf117pqw/wflbBT/JPpeDxoElXZdgr7FnzI4xnI85X0a9VBSlvFFBX4GYmRjx1WA/LM1MGLMqmPjUooua1bCqwbJuy9CYaBi1bRShcaFl01FFUcoVFfQVTA0bDQsH+XIlOpnXfzxebOkDt2puLO22lGyZzcjtI7mRWPSwj6IolU+xQS+E+FYIcVsIUeBjAIUQHYQQcUKIYzk/0/Os6yGEOCeEuCiEeFOfHa/KWj7iyFs9m7Dt9C0W7blUbPtHbB/h665fk5SexMhtI4lMiSyDXiqKUl6U5Ix+OdCjmDb7pJQ+OT/vAgghjIGFQE+gGTBQCNHsYTqr3PHio/Xp5VWTOX+eY/+F4oO7qWNTvuryFREpEYzcNpLY1Ngy6KWiKOVBsUEvpdwLRD/AvgOBi1LK/6SU6cAPgLpdU0+EEHz8rBcNq1vzyg9HuR6bUuw2PtV9WNBpAVfjrzJmxxgS0xPLoKeKohiavsboWwshjgshfhdCNM9Z5gpcy9MmLGdZgYQQo4QQQUKIIPWA5ZKxMjdh8RB/0jOzGft9MKkZxT+EpGXNlsztMJdz0ed4eefLpGQW/wGhKErFpo+gPwLUlVJ6AwuAXx9kJ1LKJVLKACllgLOzsx66VTU84mzN3P7enAiLY9amUyXapkPtDnzY7kOO3j7KoC2DOHDjQCn3UlEUQ3rooJdSxkspE3N+3wqYCiGcgOtA7TxN3XKWKXrWvbkL4zo0YO2ha6w7fLVE2/Ss35P5neaTkpnC6O2jGbtjLBdjLpZyTxVFMYSHDnohhIsQQuT8HpizzyjgMNBICFFfCGEGPAf89rDHUwr2Wjd3Hm3oxLSNpzgRVrKJ1g61O/DbU7/xmv9rHL99nGc3Pct7/7xHVEpUKfdWUZSyVJLLK9cC/wDuQogwIcSLQogxQogxOU36AiFCiOPAfOA5qZUJjAf+BM4AP0opSza2oNw3YyPB/IG+OFubM/b7I0QnpZdoOzNjM0Z4jGDLM1t4zv05frnwC09seIJlJ5eRmll4tUxFUSoOUR6fNRoQECCDgoIM3Y0K6fi1WPot/oeWjziw/PlAjI3EfW1/Oe4ynwV/xu5ru6lpVZNX/F7h8fqPq4ePK0o5J4QIllIGFLRO/eutZLxr2/Fun+bsuxDJ59vvv75Nfdv6LOi0gG+7f4uduR1v7XuLwVsGE3wruBR6qyhKWVBBXwk9F1iHAQG1+XLXRbaduvlA+2jh0oIfev3AB49+wO2U24z4YwSTdk3ianzJJnsVRSk/VNBXUrP6NMfT1ZbXfjzO5cikB9qHkTCid4PebH56My/7vMzfN/6mz8Y+fHzoY+LS4vTcY0VRSosK+kpKY2rMoiF+mBgLxqwKJjk984H3ZWFiwRjvMWx5egt9GvRhzdk1PP7L46w6vYqMrKIraCqKYngq6CsxN3tL5g/05fztBN5cf7LYSpfFcbZ0ZmabmfzY60eaOzbnk8Of8NTGp9hxZcdD71tRlNKjgr6Sa9fImde7ufPb8Rt893eoXvbp7uDO112/5qvOX2FqZMqk3ZMY8ccIQiILLHCqKIqBqaCvAsa2b0CXpjX4cOsZDl1+kPp09xJC0M6tHT/3/plpraYRGh/KwC0DeXPfm4QnhuvlGIqi6IcK+irAyEjw2QBv3OwteHnNEW7H6+9GKBMjE/q792fL01t4yfMldlzZwZO/Psm8I/NUdUxFKSdU0FcRNhpTvh4aQGJqJuNWHyEjK1uv+7c2s2ai30Q2PbWJLnW7sOzkMp7Y8AQ/nvuRzOwHnwhWFOXhqaCvQtxdqjH7WU+CrsTwwZYzpXKMmtY1md1uNmufWEs9m3q89+979P2tL/vC9qkJW0UxEBX0VUwfH1eeb1uP5QdC2Xis9IqJejh5sLzHcj7v8DkZ2RmM2zmO0dtHcy76XKkdU1GUgqmgr4LefrwpLerZ8+b6k5wMK70bn4QQdKnbhV/7/MobLd7gVNQp+m/uz8wDM9VzaxWlDKmiZlXU7fhUnvxyPxEJaTzl68qETo2o72RVqseMS4vj6xNfs/bsWkyNTHnB4wWGNx+OhYlFqR5XUaqCooqaqaCvwiIS0liy9xKr/r1CemZ2mQX+1firfB78OTuu7qC6ZXVe8X2FJxs8qSpkKspDUEGvFMlQgR98K5g5h+cQEhVCU4emvB7wOoE1A0v1mIpSWT1U0AshvgV6AbellB4FrB8MTAEEkACMlVIez1kXmrMsC8gsrBN3U0FvGIYI/GyZze+Xf2fekXmEJ4XToXYH/uf/P+rb1i+1YypKZfSwQf8YkAisLCTo2wBnpJQxQoiewEwpZcucdaFAgJTyvmbeVNAb1u2EVJbs+Y/vD5Zd4KdmpvL9me9ZdnIZaZlp9HPvx1jvsdhr7EvtmIpSmTz00I0Qoh6wuaCgv6udPRAipXTNeR2KCvoKK2/gZ2RJnvJxZUKnhtQrxcCPSoniq2Nf8fOFn7EysWKU1ygGNR2EmbFZqR1TUSqDsgz614EmUsqXcl5fBmIACXwtpVxSxLajgFEAderU8b9y5Uqx/VLKhiEC/1LsJeYGzWXf9X24Wrvyqv+rdK/bnZzn0CuKcpcyCXohREfgK+BRKWVUzjJXKeV1IUR1YDswQUq5t7jjqTP68skQgX/gxgHmBM3hQswFvJ29eT3gdXyq+5Ta8RSloir1oBdCeAEbgJ5SygIfVCqEmAkkSinnFHc8FfTlW27gr/r3CpnZpR/4WdlZbLy0kQVHFxCZEkmPej2Y6DcRt2pupXI8RamISjXohRB1gL+AYVLKA3mWWwFGUsqEnN+3A+9KKf8o7ngq6CuG2wmpfL3nP77PCfynfV0Z37H0Aj85I5nvTn3H8pDlZMkshjQdwkteL2FjZlMqx1OUiuRhr7pZC3QAnIBbwAzAFEBKuVgIsQx4FsgdVM+UUgYIIR5Be5YPYAKskVJ+UJIOq6CvWMo68G8m3WTB0QVsurQJW3NbxnqPpZ97P0yNTEvleIpSEagbppQyUdaBfybqDHOC5nDo5iHq2dTjtYDXaO/WXk3YKlWSCnqlTJVl4Esp2RO2h7lBcwmNDyXQJZDXA16nqWNTvR9LUcozFfSKQdyOT2Xxnv9YffBO4E/o1JC6jvoP/IzsDH469xOLji8iLi2OJxs8ySu+r1DDqobej6Uo5ZEKesWg7g78Z3xdGV9KgR+fHs+yE8v4/sz3GAtjRniM4Pnmz2Npaqn3YylKeaKCXikXyjLwwxLCmHdkHn+E/oGzhTPjfcfTp0EfjI2M9X4sRSkPVNAr5UpZBv6x28f4NOhTTkScoLF9Y14LeI02tdro/TiKYmgq6JVyqawCX0rJn1f+5IvgL7ieeJ12ru14LeA1Gtg10OtxFMWQVNAr5drt+FQW7bnEmoNXycyWPOvnyviOjajjqN9x9bSsNNacWcPSE0tJzkzm2UbPMs5nHI4Wjno9jqIYggp6pUIoq8CPSY1h0fFF/HjuRzQmGl7yfImhzYZibmyu1+MoSllSQa9UKLmBv/rgVbJKMfAvx13ms+DP2H1tNzWtajLRbyI96/dUjzRUKiQV9EqFdCs+lcVlEPiHwg8xJ2gOZ6LP4OHoweQWk/Gr4afXYyhKaVNBr1RoZRH42TKbTZc2Mf/ofG4n36Zr3a5M8ptEbZvaejuGopQmFfRKpXArPpVFuy+x5tBVsrMlz/q58XLHhnoN/JTMFFacWsG3Id+SkZ3BwCYDGe01GltzW70dQ1FKgwp6pVIpKPDHd2pIbQf9BX5EcgRfHvuSDRc2UM2sGmO8x/Cc+3OYGqsKmUr5pIJeqZTKIvDPRZ9jbtBc/gn/hzrV6vBMo2eoblkdJwsnnC2ccbJwwtbcVlXMVAxOBb1SqZV24Esp2X99P58Ff8bF2Iv3rDcxMsHJwgknjZP2v5ZOd17n/p7zoy7hVEqLPp4w9S3QC7hdyFOmBDAPeBxIBkZIKY/krBsOvJPT9H0p5YrijqeCXnkQdwd+X3/tGL4+Az8pI4mIlAgiUyKJSonS/X7365jUGCT3/tuqZlYtX/Dn/jhbOONo4aj73dbcVl3mqdwXfQT9Y0AisLKQoH8cmIA26FsC86SULYUQDkAQEABIIBjwl1LGFHU8FfTKw7gZp71Kp7QCvyQyszOJTo3WfQgU9ZOSmXLP9ibCBAcLB93wUFE/GhNNmb0vpfzSy9BNMc+N/RrYLaVcm/P6HNrHD3YAOkgpRxfUrjAq6BV9KA+BXxwpJcmZyUSmRBKRHEFkas43g+Scbwp5XkenRhf8LcG0Wr5vA7rfLZ1x0jjhaOGIs6UzduZ26ltCJVZU0Jvo6RiuwLU8r8NylhW2vKBOjgJGAdSpU0dP3VKqMhdbDTN7N2dM+wa6wP85OKxcBb4QAitTK6xMrahrU7fItpnZmcSmxd75ECjg51TUKSJTIknOTL5ne2NhjKPGURf8ThZOOGru/J73x8LEorTesmIA+gr6hyalXAIsAe0ZvYG7o1QiFSHwS0I36WvhVGzb5IxkXfgXNKcQkRzBmagzRKVGkS2z79neytTqnnmDvL87WWi/Kdib26sa/xWAvoL+OpD3FkK3nGXX0Q7f5F2+W0/HVJT7Uljg9wtwY1yHihP4JWFpakkd0zrUsSn623FWdhYxaTH3TCzn/TkbfZb9KftJyki6Z3tjYYyDxqHQ+YO6NnVpZN9IDRkZmL7G6J8AxnNnMna+lDIwZzI2GMgtHHIE7WRsdFHHUmP0Slm4GZfKot0XWXvoGtlSVsrA16fkjGSiUqKITI28M6eQEklU6p05haiUKKJSo8iSWbrt7MztaOHSghYuLWjp0pL6tvXVfQelQB9X3axFe2buBNwCZgCmAFLKxTmXV34J9EB7eeXzUsqgnG1fAN7O2dUHUsrvijueCnqlLKnA169smU1MagyRKZGciznHofBDHLp5iPCkcACcLJx0oR/oEohbNTcV/HqgbphSlBIIj0th8e5LKvBLgZSSsMQwXegfunmIyJRIAGpa1dQGf01t8LtYuRi4txVT1Qn6Ez9C3bZgW+CFPYpSIvcGfm1e7tgAN3sV+PoipeRy/GVd8B++eZjYtFgA6lSrowv+Fi4tSjT5rFSVoE+Jhc+bg8yGtq9Cmwlgpv5hKg8uPC6FRbsv8UOewB/csg7uLtUwNVaTi/qULbO5EHNBe7YffoigW0EkZiQC0MC2AYE1Awl0CSSgRgB2GjsD97Z8qhpBDxATCtunw+mNYFsbus6C5s+AGv9THkLewE/PysbMxIimLtVo7mqLp6stHrVsaexijbmJusxQXzKzMzkbfVYX/EduHyElMwWBwN3BnUAXbfD71/DH2sza0N0tF6pE0Espufr8C1h4eWH/WENMgz6GmyehTmvo8RHU8i2l3ipVxe34VP75L4pTN+I5GRZHyI04ElIzATA1FjSuUQ1PV1vdB0ATl2poTFX460NGVgYhUSG6oZ5jt4+Rnp2OsTCmmWMzXfD7VPfB0rRqfpOvEkGflZhE+FtvkrBjJxgbY9O9Ow4tnbC4sgySIsFnMHSeDtVqlFKvlapGSsnV6GRCrsdz8nocp27EcfJ6HLHJGQAYGwkaVbfGI/fM39WGZjVtsTBT4f+w0rLSOH77OAdvHuTwzcOcjDhJpszExMgELycv3VCPt7M3ZsZmhu5umagSQZ8r/do1Yr7/ntif15OdlISFtycOfpZUS9mMMDOHdq9Bq3FgqgpBKfonpeR6bAoh1+N0HwAh1+OISkoHwEhAA2frfGf+zWrZYG1ebm5Sr5CSM5I5evuoNvjDD3M6+jTZMhtzY3N8qvvozvibOzXH1KhyPjymSgV9rqzEROJ++YXolavICAvD1KU69p4m2FULxrh6Hej+ATTppcbvlVInpeRmfOqdM//r2jP/2wlpgPavYH0nKzxq2eZ8ANjg4WqLjaZyBlJZiE+PJ/hmsO5SzvMx5wGwNLHEr4afNvhrBtLEvkmlKeFQJYM+l8zKInHXLqJXrCT58GGExhy7xhKH2tcw82wDPWaDyz03+ypKqbsdn0rIjbh8HwA34lJ16+s5Wuab8PVwtcHOsmoMQ+hbTGoMh28e1gX/5bjLgPb5AAE1AnSXcja0a1hhyzVU6aDPK/X0aaJXriJu82bIysTaLQuHhnFY9nwO0XkaWKnrdRXDikxM49SN+JyhH+2Zf1jMnXr1bvYWOeP9OT+1bHC0Vk+tul8RyRG66/cPhh8kLDEMAAeNQ77gr2dTr8LctauC/i6ZERHErP2BmLVryIqJxdwuA4dm2dgMfwWjtuPARJ01KeVHbHI6IdfjCcmZ7D11PY7QqDtliGvZau6c+ecM+1Svpuag7seNxBu6SzkP3TzEreRbAFS3qE6LmjnlGmoG4mpdfm/GVEFfiOy0NOI3byb6m6Wk/XcFY/Ms7D3NsR8/DZOW/dT4vVJuxaVkcDr3zD/nA+ByZBK5/5yrVzPPN+Hr4WqDi42mwpydGpKUkqsJV/MFf3Sqtg6jq7UrgS6BtHBpQaBLIDWsys9VfCroiyGlJPngQaIXziHx8CmEkcTGww6HiTPQtO1ZZv1QlIeRmJZ5J/xzPgAu3k4kO+efuJO1Gc1r2eYZ+rHB1c5ChX8xpJRcir2kG98/fPMw8enxANSzqacN/potaFGjBY4Wjgbrpwr6+5B26QIxn71D7J7jyEyBZQNHHF6ejHX3XgjjyjE7r1QdyemZnAlPyDfmf+F2Ilk56W9vaZpnvF/7IVDbQYV/UbJlNueiz+mCP/hWsK5Wf0O7hrrx/YAaAdia25ZZv1TQP4Cs8MvEzp1E9F+nyUw2xtTZFocXx2Dbtx/G1lYG7ZuiPIzUjCzO3kzId+Z/7mYCGVnaLLDRmNwJ/5wJ33qOVhgZqfAvSGZ2JqejTuuGeo7ePkpqVioCQROHJrqqnH41/LAyLb3sUEH/EOT1EyR8OZHofVdJiTTDyFKDXb8B2A8dgpmbm6G7pyh6kZaZxYVbibobvEKux3HmZgLpmdrHDFqbm9Cslo1uvN/T1Zb6TtYYq/C/R3pWOicjT+rG949HHCcjOwNjYYyHk4fuGn4fZx80JvqbNFdB/7CkhLNbSFn1NtHBccRfswSMqNa5Mw7Dh2Hh76++6iqVTkZWNhduJeZc668d9jkTHk9qhjb8Lc2MaVbTJs/Zvw0Nna0xUZU980nJTOF4xHFd8IdEhpAlszA1MsXb2VsX/F5OXpgaP/hNcvp4wlQPYB5gDCyTUs6+a/3nQMecl5ZAdSmlXc66LOBkzrqrUsrexR2v3AV9rsw0+HcRGb/PJeaMEbGhdmQlp6Np1gyH4cOw6dkTYaYuzVQqr8ysbC5FJOmC/9SNOE7diCc5XfvoQHMTI5rWvHPm37B6NapXM8e5mrkq8JYjKSOJI7eOcOjmIQ6GH+Rs9FkkEo2xBn8XfxZ2WvhAd+s+VNALIYyB80BXIAw4DAyUUp4upP0EwFdK+ULO60Qp5X3VES23QZ8r4Rb89R7Zh78n7oYz0aHVSb8RjbGzEw6DBmE3YAAmDg6G7qWilImsbMnlyCRtUbecqp6nrseTkJaZr101cxOcq5njZK0Nft1Pzuvc5Y7WZlWq3n9cWhxBt4I4fPMw0anRfPLYJw+0n4cN+tbATCll95zXbwFIKT8qpP0BYIaUcnvO68oX9LluHIM/3kJeOUBSmjvR1+qQFHwKYW6Obe8nsR86FE3jxobupaKUuexsyZXoZEKjkohMSCMiMY2IhDw/iWlEJqQRn5pZ4PYOVmZ5PgDM7vpg0OBUTbve3tJMTRLneNig7wv0kFK+lPN6KNBSSjm+gLZ1gX8BNym1j4EXQmQCx4BMYLaU8tdCjjMKGAVQp04d/ytXrpTw7RmYlHBqg/aBJ3HXSHPsQvT1usT9sQuZmopVm9Y4DB+OVbt2CKOqc5aiKCWRmpFFZOLdHwDpRCSm5lsWkZCmmxvIy9hI3PkgsC7i20I1c6qZm1TqubSyDPopaEN+Qp5lrlLK60KIR4C/gM5SyktFHbPCnNHnlZECB76E/Z9BdiaZniOJvVGTmB9+JvP2bczq1cN+2FDsnnoKI8uq+WAERXlQUkqS0rPyfytISM3zwXBneWRiGpnZ9+aauYlRocNFdy+viPMJZTZ0I4Q4CrwspTxQyL6WA5ullD8XdcwKGfS54m/Ajllw4gewroFs/w7xN+2JXrmK1JMnMbKxwb5/P+wHD8a0Zk1D91ZRKp3sbElsSkb+bwp5hovyfihEJ6dTUATq5hPu+gBwvuuDwcGq/MwnPGzQm6CdjO0MXEc7GTtISnnqrnZNgD+A+jJnp0IIeyBZSpkmhHAC/gH6FDaRm6tCB32usCD4400IOwy1fJHdPyIlSkP0ypUkbNsGQmDTvRsOw4Zh4eNj6N4qSpWUkZVNdFJ6viGiwj4YEgqYTxAC7C3N7vkAcLY2z5lH0OiW2VmYlup8gj4ur3wc+ALt5ZXfSik/EEK8CwRJKX/LaTMT0Egp38yzXRvgayAbMAK+kFJ+U9zxKkXQA2RnQ8jPsH0GJNwAj2ehyywykoyIXr2G2J9+IjshAY23Fw7DhmHTrRvCVD1sQlHKo9SMrEK/GeiWJ6ZxOz6NtMx75xNMjAROug+Au4eLNDhZm1HdRkN9pwe7e1bdMGVo6Umw/ws4MB8Q0HYitJ1IdoYk9tdfiVm5ivQrVzBxccF+8CDs+/XD2M7O0L1WFOUBSClJTMsscsgod27h7vkEByszjkzr+kDHVUFfXsRe1Z7dn/oFbFyhyyzw7Kv9i7FnD9ErV5L8z78IjQbbp/rgMGwY5o88YuheK4pSSnLnE3I/AFIzsujS7MFKH6ugL2+u/AN/TIHw4+AWCD1ng6s/AKnnzhO9cgXxmzYj09OxeqwdDsOGY9W2TaW+NExRlIejgr48ys6GY6th57uQdBu8B0LnGWCjvRInMyqKmHXriFmzlqzISMwaNsBh2DBse/fGSKOeHqQoSn4q6Muz1HjYNxf+/QqMTKHdJGg9HkwtAMhOTyd+61aiV6wk7cwZjO3ssHtuAPYDB2Fao7qBO68oSnmhgr4iiP4Ptk2Ds5vBtg50ew+a9dE9zlBKSfLhw0SvXEnizr/A2Bibnj1xGD4cC4/mBu68oiiGpoK+Irm8F/54C26FQN220GM21PTK1yT92jWiV60i7uf1ZCcnY+Hvj8OwYVTr3AlhYmKgjiuKYkgq6Cua7Cw4sgL+eh+So8FvKHSaBtb5h2qyEhKI++UXold9T0ZYGKa1amE/dCh2fZ/FuFo1A3VeURRDUEFfUaXEwt5P4eBiMLGA9m9AyzFgkr/mvczKIuGvv4hZsZLkoCCMLC2xfeYZHIYOwaxuXQN1XlGUsqSCvqKLvAB/ToULf4LDI9DtA3DvqRu/zyvl1CliVq4kbuvvkJmJdceOOAwbhmXLQHV5pqJUYiroK4uLO+CPtyHyHDzSAbp/BDWaFdg04/ZtYn/4gZi1P5AVE4N5kybaMgtPPI6RuXmZdltRlNKngr4yycqAoG9h14eQFg8BL0CHt8HKscDm2ampxG/erL0888IFjB0dsX/uOewHPoeJk1MZd15RlNKigr4ySo6G3R/B4W/A3Bo6vAUtXoJCHi4spST5n3+IXrGSxD17EKam2PTqhU2P7pi7u2NSo4Ya2lGUCkwFfWV2+4z2csz/doFTY+1wTqMuRW6SdvkyMau+J3bDBmRKCgBGNjZoGjfGvHFjzN3dMW/cCPNGjTG2frBKeoqilC0V9JWdlHD+D+2EbfQlaNQNun8ITo2K3CwrMYm0M6dJPX+etHPnSTuv/clOTta1MXVz0wW/xt0d88aNMatTR12vryjljAr6qiIzHQ59DXs+gYxkCBylvSTTwr7Eu5DZ2WTcuEHauXOknT9Pas4HQHpoqLY+DyDMzTFv0CDnA6AxGnftNwE15q8ohqOPB4/0AOahffDIMinl7LvWjwA+RfsEKoAvpZTLctYNB97JWf6+lHJFccdTQf+QEiNg1/sQvEIb8p2mgt8IMH7ws/Ds1FTSLl3Kc+Z/jtTzF8iKjNS1MXZ01J75N3a/MwTUsIEqwqYoZeBhHyVojPZRgl2BMLSPEhyY93GAOUEfZarWowAAD3VJREFUcPcDw4UQDkAQEABIIBjwl1LGFHVMFfR6cvOkdvw+dB9UbwY9PtJelqlHmVFROWf+50g7f0H7TeDiRWRamraBkRFmdevmBH9j7TyAuzumrq4Io/LxrE1FqQyKCvqSnOIFAhellP/l7OwHoA9Q5HNfc3QHtkspo3O23Q70ANaWpOPKQ3LxhOGb4MxvsO0dWNkHmvSCru+CYwO9HMLE0RGT1q2xat1at0xmZZF+9aru7D/1/DlSz5wh4c8/dW2MLC0xb9Qo3+SvpnFj9WQtRSkFJQl6V+BantdhQMsC2j0rhHgM7dn/JCnltUK2dS3oIEKIUcAogDp16pSgW0qJCKGtgtmoO/y7EPbOhQutoNVYaPc6aGz0f0hjY8zr18e8fn3o0V23PDspibSLF/Od/Sds20bsTz/p2pjUqJHvzN+8cWPM69dHmJkVdChFUUpAX5dObALWSinThBCjgRVAp/vZgZRyCbAEtEM3euqXkstUA+1eA5/B2oed/D0Pjq0Fj2fA1g1saoGNG9i6grXLQ43nF8bIygoLb28svL11y6SUZN6+rR33P3dOdwVQ1D//QkaGtpGJifaD4+7JXxcXde2/opRASf41Xwdq53ntxp1JV/7f3rnGxnFdB/g7++BzX3qZpEg5quMk6ANprAqGiyCt0TSA0RZyigSoUTitmwYFUgRpkR99Ag3q/glaoOgbieEYSJs+0iZBoQoOghR2kh+NXcuvyrIqWbJrSRQpUVKWyyUlch+nP+4sd3Z2dnc2XO6Sq/MBA96Ze6/m8Ipzzpx7ztwLoKo3fKdPAn/q6/tgoO+3uxXS6CHpafjw37mPq/7zs/Dyl2Gj2NhGYk7ZZ2fd3raZ2eZyagpi8S2LIyIkp6ZITk2R+sAHNq9rqcT6W2/V5/3PnWPtxRcpnDix2SaWyQSCv++23H/DCCFKMDaBm475IE5xvwD8sqqe9rWZUdUFr/yLwO+q6gNeMPZF4IjX9CVcMPZmu3taMLaPqMLtZShcgcI8LF8OlOdheR7Ktxr7SRzSMz4DcNDzDHzGYPIu6HHAtVIoePP+gdz/1dXNNsm5uabgr+X+G8POloKxqloWkU8B38SlVz6lqqdF5HHgpKoeBz4tIseAMnATeMzre1NE/gRnHAAe76TkjT4jAuM5d7RYIA1VuPV9p/QLV+oGoFZeeAXOPg3l2439YglIH6wbgzAPYWJ/V8YgnskwcfQoE0frf8+qSmn+CuvnzjZkABW/8x2oVNyvOTLC6L33NgZ/3/Mey/037gjsgymjN6i69XcKl0OMwXz9emWjsV98xIsPtDMG+0KXZO5EdX2djQsXGt7+b587S2XJl/u/d2/9zb82BXTvO4mNj291RAyjr9iXscbOoFqFtRtO6S97RiBYLixAtdTYLzHW2RiM74lsDMo3b25O+WxmAL3xBnrb80hiMUbuvrtp6Yfk3Jzl/hs7FlP0xu6hWoXVpRbGoHZ+BbTS2C850dkYjGVbGgOtVChdurS55IP78vccpYuXnLcCyMQEI7MHiWWzxLM54jnvZzbrjly2Xs5miWVzxCYnLDPI6Aum6I3holqB4tXweEGtvLIAWm3sl5xsn0mUmW36rqC6ttaQ+19eXKSyvNxw1FYADSWR8BmCXIMhiOeyntHwGQzPWMRSKfMejK7Y6pexhrGziMW9t/eDMBf6dw2VMhQXfZ7AfGP5/BlnLAi86Ixm6p5BdpZYZpbxzCzjR2bhwfe6utHGjder6+tU8stUlvNU/UYgvxwwCnlKVxdZP3uWSj7fsEpo8+8Ya/QOcgGDEOpBZIlnMkh862mvxnBhit4YTuIJl+6ZnWvdplJyb/6tjMHiKVi91twvOQnpKZdempoilp4mlp4mmZqGfVNw+BCkjradKgLQjQ0qhUKIYchTWV52RqN27foNNi686a6vrLT+nUSIZTKNnkPAMDR4ETnP08hkLP10iLH/WePOJZ6E3N3uaEV5vdEYrCzAyqI7ilddaum5q1Babe6bGHMfqKWmGwwD6WlITyOpaRLpaRL77ukqq0jLZSorK84raOVB5POb5Y1LF6nml6kUCpvxhjBiqVT7aaWasfBNQcWyWWK2PMWOxxS9YbQjMQp7DrujHesrjQagZhCKV93Pq6/DhWfdPr9B4qOeAZjyGYbp5vL4XojFkESCxJ49JPZE32cA3F4D1ZWVcA8i3+xFlBbq8Yja9whhyMREZw9i0zjk6tdt+eq+YYreMHrBaNodHXb1YmM1YAyuulhCzUhcfwPe+q77WjlILFH3DhqMgect1K5P7g9dnkJ88/7doKpUV1cb4xA+jyEYi1h/88Lm9c31ikKQ0dHmOITfILSIRciEZTJ1iyl6w+gnI5NuiehOy0SXbtW9gU1PYdEZhpUF+P5bcPF7cCvkQ3OJQ+quZgMQNAyTByItXicixFMp4qkUzIUuPhuKqqJray4OUTMMDUah0YsoXbzE7VOvUcnn6/sZhJFMts1kaghO+z2IVOqONRCm6A1jJ5IcjzZlVF73DIJnAPzGobjoUk4vvwBr15v7Sswpe1/coDGeUPMeplw8o0tEBJmcJDY5SXJmpqu+1du3fYbBZxBCYhGlK1e4feaMS3Vtl8kUj4dOL4WmuPrjEOn0rk91NUVvGLuZxGjngDK4DKPiNZ9nEOIlLLzqPlYLfn8Abk2iUGMw1egpJEZ78mvFxsaIjY2RnJrqql91Y6MxQO03DPl8g9EoLy2xfv68Oy8WW/+jIsQzGS/FNRdiLHLNnkQuRzyd3jGZTDtDCsMwtpd40n0Ulu0w9VIpO2XvNwBBL+HqaWc0gl8ng1uKwm8AJva52MXIJIyk3DGaan2+xaWvYyMjxA4cIHHgQFf9tFTyMplCgtMhXsTG22+7+k6ZTOl089RSLtfyQ7l4Lkdi374tjUEYpugNw6gTT0Bmxh3tqFbcukVhAeVaoPn6Gy6GUGoznRIkORHBKHjXGgxI8NxrG9HDkGSSxN69JPbujS4rbumMSqHQ2ovwxyLyy5Tm5921QsEt9xEgvmcP7/7ef3UlQxRM0RuG0T2xWsD3Lug0/V6tuGyjjaL7ub7S+jysbu0m5C9650XYWAmfXgqVMxnBKNTOa+VU3ViEtfUFdCUed2muP0iqa7HYlOYapvx7gSl6wzC2l1jcrSHUq/2JVd3eB+tFzyAUfUbAO9rVbazC6nVnMGp1lTZZPg1IwEAEjUKKZm8k3VQnI5PER1PED87AoUOdb7tFIil6EXkI+EvcxiNPqurnAvWfAT6B23hkCfi4qr7t1VWAU17Ti6p6rEeyG4ZxJyLispKS40B3c/EtqZQCRmHVGQJ/OVjnNyDFRV9dsXl7znbER+tGITMHH/9Gb34nHx0VvYjEgb8FPgRcBl4QkeOq+rqv2cvAUVVdE5FP4vaM/SWv7paqvq/HchuGYfSOeNIFkse7m4JpSbXqYhOhU1LFZqNQMxg9yloKEuWN/n7gvKq+CSAi/wI8DGwqelV91tf+OeDRXgppGIaxq4jF3Fv6aAroLkV0W8SJ0GYWuOQ7v+xda8WvA37fY0xETorIcyLy4VadROQ3vHYnl5aWIohlGIZhRKGnwVgReRQ4Cvy07/I7VHVeRO4BnhGRU6p6IdhXVZ8AngC38Ugv5TIMw7iTifJGPw/4w8Jz3rUGRORngT8EjqnqZghbVee9n28C3wbu24K8hmEYRpdEUfQvAO8SkR8SkRHgEeC4v4GI3Ad8Aafkr/mu7xGRUa+8H3g/vrl9wzAMY/vpOHWjqmUR+RTwTVx65VOqelpEHgdOqupx4M+AFPBv3upwtTTKHwa+ICJVnFH5XCBbxzAMw9hmbHNwwzCMIaDd5uC7e+1NwzAMoyOm6A3DMIacHTl1IyJLwNs/YPf9QMguCwPH5OoOk6s7TK7uGEa53qGqoWtC7EhFvxVE5GSreapBYnJ1h8nVHSZXd9xpctnUjWEYxpBjit4wDGPIGUZF/8SgBWiBydUdJld3mFzdcUfJNXRz9IZhGEYjw/hGbxiGYfgwRW8YhjHk7FpFLyIPichZETkvIr8XUj8qIl/x6p8XkcM7RK7HRGRJRF7xjk/0QaanROSaiLzWol5E5K88mf9HRI5st0wR5XpQRJZ9Y/VHfZLrkIg8KyKvi8hpEfmtkDZ9H7OIcvV9zERkTET+W0Re9eT645A2fX8eI8rV9+fRd++4iLwsIidC6no7Xqq66w7c4moXgHuAEeBV4EcCbX4T+LxXfgT4yg6R6zHgb/o8Xj8FHAFea1H/c7jNYgR4AHh+h8j1IHBiAH9fM8ARr5wGzoX8P/Z9zCLK1fcx88Yg5ZWTwPPAA4E2g3geo8jV9+fRd+/PAP8U9v/V6/HarW/0m9sbquoGUNve0M/DwJe88leBD4q3tOaA5eo7qvpd4GabJg8Df6+O54CciMzsALkGgqouqOpLXnkFOEPzrmp9H7OIcvUdbwxqu2EnvSOY5dH35zGiXANBROaAnweebNGkp+O1WxV9lO0NN9uoahlYBvbtALkAPuK5+18VkUMh9f2m2+0i+8lPeq73N0TkR/t9c89lvg/3NuhnoGPWRi4YwJh50xCvANeAb6lqy/Hq4/MYRS4YzPP4F8DvANUW9T0dr92q6Hcz/wEcVtX3At+ibrWNZl7Crd/x48BfA//ez5uLSAr4GvDbqlro573b0UGugYyZqlZU9X24HejuF5Ef68d9OxFBrr4/jyLyC8A1VX1xu+9VY7cq+ijbG262EZEEkAVuDFouVb2h9a0WnwR+YptlikKk7SL7jaoWaq63qj4NJMXtVLbtiEgSp0z/UVW/HtJkIGPWSa5Bjpl3zzzwLPBQoGoQz2NHuQb0PL4fOCYi/4eb3v0ZEflyoE1Px2u3KvqO2xt657/qlT8KPKNeZGOQcgXmcY/h5lkHzXHgV7xMkgeAZVVdGLRQIjJdm5cUkftxf6/brhy8e34ROKOqf96iWd/HLIpcgxgzETkgIjmvPA58CPjfQLO+P49R5BrE86iqv6+qc6p6GKcjnlHVRwPNejpeHbcS3IlotO0Nvwj8g4icxwX8Htkhcn1aRI4BZU+ux7ZbLhH5Z1w2xn4RuQx8FheYQlU/DzyNyyI5D6wBv7bdMkWU66PAJ0WkDNwCHumDsQb3xvUx4JQ3vwvwB8DdPtkGMWZR5BrEmM0AXxKROM6w/Kuqnhj08xhRrr4/j63YzvGyJRAMwzCGnN06dWMYhmFExBS9YRjGkGOK3jAMY8gxRW8YhjHkmKI3DMMYckzRG4ZhDDmm6A3DMIac/we+aNyyNIn1ZQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B-zMmdXTMlfe",
        "colab_type": "text"
      },
      "source": [
        "ResNet proved to be more efficient than a plain network though they have the number of parameters nearly equivalent (700k vs 690k) and the same depth."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cJlun734gnMJ",
        "colab_type": "text"
      },
      "source": [
        "## On FashionMNIST"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "3EpULhPSUApK",
        "outputId": "cf60a676-ec4a-413d-9dd9-e3d331d2fa5e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 369,
          "referenced_widgets": [
            "a6314a0e6f234c8980d5672c91dd18f9",
            "29d588a3e2664bd5bb43181fdac9d5b7",
            "598ddc70f05348e19381fb66336d9a93",
            "1723ff12a86a462d8500329dc8fde61d",
            "aec3c5be3e784fe1a9ccc16fb60d66a3",
            "570e62dddb93451995285b3299ad20c2",
            "97d4592eb9cc409cbf78eb621c816ab1",
            "e3d0a717046b45cc97c6c441d8fc11ee",
            "cb6b9f12a65f41d79a121ce8096f3d2d",
            "73883e27b91a4ad4a8e49a620b42862e",
            "9845fa9cd4ec4cd4a5daded3648e47e7",
            "94cc4cf62f83496f941f30b3df95cfe5",
            "c1bec21213c143b6933685169c67b1be",
            "2680930248d54c4aa37d1b9e220342c0",
            "5636f594ab7a42948a6a9f0c912ce080",
            "aedf6e2e1e1a4dcbafb1e3eab8183246",
            "873610c22abb46348865d6b1f02536fc",
            "5b2ffc1d06324509b058e20a475f4d8f",
            "5b9a739ad63c4efa8fbc0b764f4755f8",
            "0dcf2f68d94a4ee6bc3c13a55e49aa7e",
            "da426cd559994c0c8ec8765322024974",
            "766fbd9fccff44f18776ef803e1bfb35",
            "c7eedba45f0a48d4968fdb784d574241",
            "5147bd4eb2044344965994366f9b20df",
            "04d3d52c83024c7fa39afea634ed7277",
            "2cea5e627f844c4cace45c01128f3480",
            "11b9d473b923494d80d9660370a4656a",
            "c16780aac50b4b34b8618e4f87be6214",
            "5652aec3cb9e44ea8cedf35256c00b92",
            "12a188f185b148c38295d23017e6d51b",
            "61e4d3e86196481a9363e82381f71c96",
            "46f394961fc54e1b9454333834b94fff"
          ]
        }
      },
      "source": [
        "batch_size=32\n",
        "transform = transforms.Compose(\n",
        "    [\n",
        "     transforms.ToTensor()\n",
        "])\n",
        "trainset = torchvision.datasets.FashionMNIST(root=\"./data\", train=True, download=True, transform=transform)\n",
        "trainloader = torch.utils.data.DataLoader(trainset,batch_size=batch_size,shuffle=False)\n",
        "testset = torchvision.datasets.FashionMNIST(root=\"./data\", train=False, download=True, transform=transform)\n",
        "testloader = torch.utils.data.DataLoader(testset,batch_size=batch_size,shuffle=False)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz to ./data/FashionMNIST/raw/train-images-idx3-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a6314a0e6f234c8980d5672c91dd18f9",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ./data/FashionMNIST/raw/train-images-idx3-ubyte.gz to ./data/FashionMNIST/raw\n",
            "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz to ./data/FashionMNIST/raw/train-labels-idx1-ubyte.gz\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "cb6b9f12a65f41d79a121ce8096f3d2d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ./data/FashionMNIST/raw/train-labels-idx1-ubyte.gz to ./data/FashionMNIST/raw\n",
            "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz to ./data/FashionMNIST/raw/t10k-images-idx3-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "873610c22abb46348865d6b1f02536fc",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ./data/FashionMNIST/raw/t10k-images-idx3-ubyte.gz to ./data/FashionMNIST/raw\n",
            "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz to ./data/FashionMNIST/raw/t10k-labels-idx1-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "04d3d52c83024c7fa39afea634ed7277",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ./data/FashionMNIST/raw/t10k-labels-idx1-ubyte.gz to ./data/FashionMNIST/raw\n",
            "Processing...\n",
            "Done!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/pytorch/torch/csrc/utils/tensor_numpy.cpp:141: UserWarning: The given NumPy array is not writeable, and PyTorch does not support non-writeable tensors. This means you can write to the underlying (supposedly non-writeable) NumPy array using the tensor. You may want to copy the array to protect its data or make it writeable before converting it to a tensor. This type of warning will be suppressed for the rest of this program.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "19c3ebb9-01a5-4141-b7ec-64a2bc467dbc",
        "id": "4k9Cnh1AUTK9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 202,
          "referenced_widgets": [
            "b617361961aa4a9c9b928e3d6a221ffc",
            "c02c2743c0ef40c2898ad68be913a90f",
            "23534f0043c8417a986b5c83492681c2",
            "a3eed46616e74a4daa3706e5a108dca9",
            "7fd845879bc948f68693808e939faef0",
            "c93e23f889914378a92d864716a1e926",
            "cee606008fd64c34b72ba93d6a068d69",
            "d073bdc1ef674276a889d5560c16ad7d"
          ]
        }
      },
      "source": [
        "model_resnet18 = ResNet18(1)\n",
        "model_resnet18.apply(init)\n",
        "\n",
        "#net = AlexNet(10)\n",
        "use_cuda = True\n",
        "if use_cuda and torch.cuda.is_available():\n",
        "    print(\"using cuda\")\n",
        "    model_resnet18.cuda()\n",
        "losses, val_errs = trainer(trainloader, testloader, model_resnet18, use_cuda, epoch=3)\n",
        "print(accuracy(testloader, model_resnet18, use_cuda))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "16\n",
            "\n",
            "\n",
            "32\n",
            "64\n",
            "128\n",
            "using cuda\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b617361961aa4a9c9b928e3d6a221ffc",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "0.9079\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "9828a73c-2385-495b-f861-242ff5fcd9ba",
        "id": "XnhaSaA3UW2e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168,
          "referenced_widgets": [
            "8c6100525c234a20b66dfae3e971209b",
            "e38d0d6e8551409790fbfd31fe9fc663",
            "798dc56309544e9685fd19ddac9971d8",
            "1f48eee9b1cd4173a513d8cff2f92fa8",
            "c73261fe8b714d59901ef6306cb3b774",
            "1f72284a8f39472e92c58c8542f6847f",
            "219ac7df0b0b4192a4cfb268ce952819",
            "27f33cb27c5e440bb4d2d76acf7dfa5c"
          ]
        }
      },
      "source": [
        "model_plainnet18 = PlainNet18(1)\n",
        "model_plainnet18.apply(init)\n",
        "#net = AlexNet(10)\n",
        "use_cuda = True\n",
        "if use_cuda and torch.cuda.is_available():\n",
        "    print(\"using cuda\")\n",
        "    model_plainnet18.cuda()\n",
        "losses_, val_errs_ = trainer(trainloader, testloader, model_plainnet18, use_cuda, epoch=3)\n",
        "print(accuracy(testloader, model_plainnet18, use_cuda))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "16\n",
            "32\n",
            "64\n",
            "128\n",
            "using cuda\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8c6100525c234a20b66dfae3e971209b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "0.9029\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DRA1WR_KBzjW",
        "colab_type": "code",
        "outputId": "4cfb8555-45cd-41e7-b3b7-0cb701b3c612",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        }
      },
      "source": [
        "plt.plot(losses, label='ResNet training loss')\n",
        "plt.plot(val_errs, label='ResNet validation error')\n",
        "plt.plot(losses_, label='Plainnet training loss')\n",
        "plt.plot(val_errs_, label='Plainnet validation error')\n",
        "plt.legend()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7f8ab05809e8>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdd3wU1d7H8c/spmx6IQmkQUInPaGDSLGADRugqCD4PHotoF69XFAs6L0+YsEKXhRFbPeKgIogKiJdvPSA9AQIpAHpfbPtPH9ssgRIzyab3Zz367Wv7M7MzpwN5Dtnz8z8RhFCIEmSJNk/la0bIEmSJFmHDHRJkiQHIQNdkiTJQchAlyRJchAy0CVJkhyEk602HBAQICIiImy1eUmSJLu0d+/eXCFEYG3zbBboERER7Nmzx1ablyRJskuKopypa54ccpEkSXIQMtAlSZIchAx0SZIkB2GzMXRJskd6vZ6MjAy0Wq2tmyI5OI1GQ1hYGM7Ozo1+jwx0SWqCjIwMvLy8iIiIQFEUWzdHclBCCPLy8sjIyCAyMrLR75NDLpLUBFqtlk6dOskwl1qVoih06tSpyd8EZaBLUhPJMJfaQnP+n9ldoB8/V8Kr645SWmmwdVMkSZLaFbsL9IyCcj7ceorj54pt3RRJsgm1Wk1CQgIxMTHccsstFBYWNnkdmzdvRlEU1qxZY5l28803s3nz5nrft2zZMrKyspo8rz6LFy/m888/r3eZPXv28Pjjjzd53bWZN28eb775plXW1d7YXaD3DfZEccnhcGbT/xNLkiNwc3MjOTmZQ4cO4e/vz6JFi5q1nrCwMF555ZUmvae5gW40Gutc58MPP8zUqVPr3e6AAQN47733Gt/QDsruAn137q949ljArswTtm6KJNnc0KFDyczMBODkyZOMGzeO/v37M2LECI4dOwbAihUriImJIT4+nquvvtry3vj4eHx8fPj111+vWO/evXsZOXIk/fv3Z+zYsWRnZ7Ny5Ur27NnDvffeS0JCAhUVFZbla5sXERHB7NmzSUpKYsWKFSxZsoSBAwcSHx/PnXfeSXl5OXBpj3nUqFHMnj2bQYMG0bt3b7Zt2waYv1HcfPPNluUfeOABRo0aRffu3S8J+n/84x/06dOHq666ismTJzfYE09OTmbIkCHExcVx++23U1BQAMB7771HVFQUcXFx3H333QBs2bKFhIQEEhISSExMpKSkpAn/Um3D7k5bjO4UDcCRvMPAONs2RurQXlpzmCNZ1h36iwrx5sVbohu1rNFo5LfffuN//ud/AHjooYdYvHgxvXr1YufOnTz66KNs3LiRl19+mV9++YXQ0NArhmfmzp3L888/z3XXXWeZptfrmTlzJqtXryYwMJDly5czd+5cli5dysKFC3nzzTcZMGDAJeuZMGFCrfM6derEvn37AMjLy+PBBx8E4LnnnuOTTz5h5syZV3wug8HArl27WLduHS+99BIbNmy4Ypljx46xadMmSkpK6NOnD4888gjJycmsWrWKAwcOoNfrSUpKon///vX+DqdOncr777/PyJEjeeGFF3jppZd45513mD9/PqdPn8bV1dXyO3vzzTdZtGgRw4cPp7S0FI1GU++6bcHuAr27T3fUuJKtTcFgNOGktrsvGZLUIhUVFSQkJJCZmUm/fv247rrrKC0tZceOHUycONGyXGVlJQDDhw9n2rRpTJo0iTvuuOOSdVX32Ldv326Zdvz4cQ4dOmQJeaPRSHBwcLPaetddd1meHzp0iOeee47CwkJKS0sZO3Zsre+pbmP//v1JS0urdZmbbroJV1dXXF1dCQoK4vz58/z+++/ceuutaDQaNBoNt9xyS71tKyoqorCwkJEjRwJw//33W35/cXFx3Hvvvdx2223cdtttgPn3+NRTT3Hvvfdyxx13EBYW1qTfRVuwu0BXq9SEuffkVHk6p3LL6N3Zy9ZNkjqoxvakra16DL28vJyxY8eyaNEipk2bhq+vL8nJyVcsv3jxYnbu3MmPP/5I//792bt37yXz586dyz//+U+cnMxxIIQgOjqaP/74o8Vt9fDwsDyfNm0a33//PfHx8SxbtqzOA7Curq6A+eCvwVD72WzVyzS0XHP9+OOPbN26lTVr1vDKK6/w559/MmfOHG666SbWrVvH8OHD+eWXX+jbt69Vt9tSdtm9jQ2MQaXJ4mBmnq2bIkk24+7uznvvvceCBQtwd3cnMjKSFStWAOZQPnDgAGAeWx88eDAvv/wygYGBpKenX7Ke66+/noKCAg4ePAhAnz59yMnJsQS6Xq/n8OHDAHh5edU5dlzfPICSkhKCg4PR6/V89dVXLfvwtRg+fDhr1qxBq9VSWlrK2rVr613ex8cHPz8/yzj9F198wciRIzGZTKSnpzN69Ghee+01ioqKKC0t5eTJk8TGxjJ79mwGDhxoOUbRnthloA8LS0RRGfhv+hFbN0WSbCoxMZG4uDj+85//8NVXX/HJJ58QHx9PdHQ0q1evBmDWrFnExsYSExPDsGHDiI+Pv2I9c+fOtQS9i4sLK1euZPbs2cTHx5OQkMCOHTsAcy/74YcfvuKgaEPzwHzAcvDgwQwfPrxVerYDBw5k/PjxxMXFccMNNxAbG4uPj0+97/nss8+YNWsWcXFxJCcn88ILL2A0GrnvvvuIjY0lMTGRxx9/HF9fX9555x1iYmKIi4vD2dmZG264weqfoaUUIYRNNjxgwADR3BtcnC0+y03f3USoYQo//8/frdwySarb0aNH6devn62bIdWhtLQUT09PysvLufrqq/noo49ISkqydbOarbb/b4qi7BVCDKhtebsbQwcI9wrHGQ+yKlIQQshLsSVJAsxn+hw5cgStVsv9999v12HeHHYZ6IqiEOrei5Pas2QXaQnxdbN1kyRJagf+/e9/27oJNmWXY+gAMQHRqFzPcSAj19ZNkSRJahfsNtCv6pqEopjYfvaArZsiSZLULthtoPfvEgfAn7mHbNwSSZKk9sFuA72ze2dc8CGzXNZ0kSRJAjsOdEVRCHbrTYUqjWKt3tbNkaQ2017L5zbVtGnTWLlyJQD/+7//y5EjV15XsmzZMmbMmFHvejZv3mw5Tx4aV47XUdltoIO5UJfKJZd9Z7Nt3RRJajPttXxuS3z88cdERUU1672XB3pjyvFaixACk8lU5+u6WLtUQTW7DvQRXRNRFMHWM1fWr5CkjqC9lM89duwYgwYNsrxOS0sjNjYWgJdffpmBAwcSExPDQw89RG0XM44aNYrqCw0//fRTevfuzaBBg/j9998ty6xZs4bBgweTmJjItddey/nz50lLS2Px4sW8/fbbJCQksG3btkvK8dZVHreuMr2Xe+ONNxg4cCBxcXG8+OKLls/Wp08fpk6dSkxMDNu2bbvkdXp6OrNmzSImJobY2FiWL18OmHc8I0aMYPz48c3eeTXELs9DrzY8PBGAgzl/AuNt2xip4/lpDpz707rr7BILN8xv1KLtqXxu37590el0nD59msjISJYvX26ptDhjxgxeeOEFAKZMmcLatWvrrISYnZ3Niy++yN69e/Hx8WH06NEkJpr/zq+66ir++9//oigKH3/8Ma+//joLFizg4YcfxtPTk7/97W8A/Pbbb5b11VUeFxou07t+/XpSUlLYtWsXQgjGjx/P1q1b6dq1KykpKXz22WcMGTKEtLS0S16vWrWK5ORkDhw4QG5uLgMHDrTsSPft28ehQ4eIjIxs1L9xU9l1oPtp/HARAZwtkwdGpY6jvZbPnTRpEsuXL2fOnDksX77c0jPdtGkTr7/+OuXl5eTn5xMdHV1noO/cuZNRo0YRGBgImMvvnjhh/vvOyMjgrrvuIjs7G51O12Ao1lceFxou07t+/XrWr19v2aGUlpaSkpJC165d6datG0OGDLEsW/P19u3bmTx5Mmq1ms6dOzNy5Eh2796Nt7c3gwYNarUwBzsPdIAuml6kGY6hM5hwcbLrESTJ3jSyJ21t7bV87l133cXEiRO54447UBSFXr16odVqefTRR9mzZw/h4eHMmzcPrVbbrM89c+ZMnnrqKcaPH8/mzZuZN29es9ZTraEyvUIInnnmGf7yl79cMj0tLe2SssDAFa/r0tjlmsvuEzCqUzQq5wL2ZaQ3vLAkOZD2Vj63R48eqNVq/vGPf1iGW6rDOyAggNLSUstZLXUZPHgwW7ZsIS8vD71eb/k8YO5xh4aGAuYqidXqalNd5XEba+zYsSxdupTS0lIAMjMzuXDhQoPvGzFiBMuXL8doNJKTk8PWrVsvOb7Qmuw+0KvH0Tel7bNxSySp7bWn8rlg7qV/+eWXTJo0CQBfX18efPBBYmJiGDt2LAMHDqz38wQHBzNv3jyGDh3K8OHDL6k0OG/ePCZOnEj//v0JCAiwTL/lllv47rvvLAdFa6qtPG5jXX/99dxzzz0MHTqU2NhYJkyY0Kj7iN5+++3ExcURHx/PmDFjeP311+nSpUujt9sSdlk+t6YibQnDvx5OtPsElk9q/D+WJDWHLJ8rtaWmls+1+x66j8YLF9GZs6XHbd0USZIkm7L7QAfo7NqTUnG6USf0S5IkOaoGA11RlHBFUTYpinJEUZTDiqI8UcsyiqIo7ymKkqooykFFUdq0qnw/v2hwKuFA9pm23KwkSVK70pgeugF4WggRBQwBHlMU5fLLnG4AelU9HgL+ZdVWNmBY1YHR307vbWBJSZIkx9VgoAshsoUQ+6qelwBHgdDLFrsV+FyY/RfwVRSl4SsRrOTaHokIoWL/eStftSdJkmRHmjSGrihKBJAI7LxsVihQ8+TWDK4MfRRFeUhRlD2KouzJyclpWkvr4evmjrMxhLTSY1ZbpyRJkr1pdKAriuIJrAKeFEIUN2djQoiPhBADhBADqi/ttZYgl14Um07XWvhHkhxJzfK5EydOpLy8HABPT88G3zts2DCrtyc5OZl169Y1eV59srKymDBhQoPL3Xjjjc0qH3y5tLQ0YmJiWrweW2tUoCuK4ow5zL8SQnxbyyKZQHiN12FV09pMH79+oKrgcM6pttysJLW5muVzXVxcWLx4caPfW7PMrLU0N9DrKyEbEhLS4FWlAOvWrcPX17dxDe0AGnOWiwJ8AhwVQrxVx2I/AFOrznYZAhQJIdq0SPmQsAQANpxs+cVKkmQvRowYQWpq6iXTSktLueaaa0hKSiI2NtZyxShc7MVv3ryZUaNGMWHCBPr27cu9995r+XYbERHBiy++aHl/dRnesrIyHnjgAQYNGkRiYiKrV69Gp9PxwgsvsHz5chISEiwFuYBa582bN48pU6YwfPhwpkyZQlpaGiNGjCApKYmkpCTLDqdmj3nZsmXccccdjBs3jl69evH3v//dso2IiAhyc3NJS0ujX79+PPjgg0RHR3P99ddbrmTdvXs3cXFxJCQkWMra1ker1TJ9+nRiY2NJTExk06ZNABw+fJhBgwaRkJBAXFwcKSkplJWVcdNNNxEfH09MTMwln98WGlOcazgwBfhTUZTqyj/PAl0BhBCLgXXAjUAqUA5Mt35T63dN9zj+b78T+84fBO5q681LHdBru17jWL51j9v09e/L7EGzG7WswWDgp59+Yty4cZdM12g0fPfdd3h7e5Obm8uQIUMYP3485r7ZRfv37+fw4cOEhIQwfPhwfv/9d6666irAXHtl3759fPDBB7z55pt8/PHHvPLKK4wZM4alS5dSWFjIoEGDuPbaa3n55ZfZs2cPCxcuvGT9Li4uV8ybN28eR44cYfv27bi5uVFeXs6vv/6KRqMhJSWFyZMnU9sV5MnJyezfvx9XV1f69OnDzJkzCQ8Pv2SZlJQU/vOf/7BkyRImTZrEqlWruO+++5g+fTpLlixh6NChzJkzp8Hf66JFi1AUhT///JNjx45x/fXXc+LECRYvXswTTzzBvffei06nw2g0sm7dOkJCQvjxxx8Bc70ZW2ow0IUQ2wGlgWUE8Ji1GtUcnb09UOvDOF0sD4xKjq26fC6Ye+jV9dCrCSF49tln2bp1KyqViszMTM6fP39FPZFBgwYRFhYGQEJCAmlpaZZAr1la9ttvzaOs69ev54cffrDcPEKr1XL27Nkmt3/8+PG4ubkB5sJfM2bMIDk5GbVabSmVe7lrrrkGHx8fAKKiojhz5swVgR4ZGWn5vVSXxC0sLKSkpIShQ4cCcM8997B27dp627d9+3ZmzpwJmOu8d+vWjRMnTjB06FBeeeUVMjIyuOOOO+jVqxexsbE8/fTTzJ49m5tvvpkRI0Y0+fdhTXZfPremAJee5Ji2YjAZcFI51EeT2qHG9qStrXoMvS5fffUVOTk57N27F2dnZyIiImotWVtdPhauLCFbW2lZIQSrVq2iT58+l6xn587LT3qrX80Ssm+//TadO3fmwIEDmEwmNBpNre+pr611LVNb8bCWuOeeexg8eDA//vgjN954Ix9++CFjxoxh3759rFu3jueee45rrrmmSQXArM0hLv2v1ts3CqHoOJaX2vDCkuSgioqKCAoKwtnZmU2bNnHmjHWuoB47dizvv/++Zax9//79QP0ldeubV93W4OBgVCoVX3zxBUaj0Sptrebr64uXl5dlp/P11183+J4RI0bw1VdfAXDixAnOnj1Lnz59OHXqFN27d+fxxx/n1ltv5eDBg2RlZeHu7s59993HrFmz2LfPtlVfHSrQB4fGAbBJXjEqdWD33nsve/bsITY2ls8//5y+fftaZb3PP/88er2euLg4oqOjef755wEYPXo0R44cueKgaEPzAB599FE+++wz4uPjOXbsWKvcAOKTTz7hwQcfJCEhgbKyMsvQTV0effRRTCYTsbGx3HXXXSxbtgxXV1e++eYbYmJiSEhI4NChQ0ydOpU///zTcqD0pZde4rnnnrN6+5vC7svn1nQqp4Txa0YzIOAalo1/zarrliSQ5XPtUWlpqeXsnvnz55Odnc27775r41Y1TlPL5zrUQHNEJ0/QhXFSHhiVJKnKjz/+yKuvvorBYKBbt24sW7bM1k1qNQ4V6CqVQienHuQbfkNn1OGidrF1kyRJsrG77rrLcks8R+dQY+gAvXyiQDFyLE/e8EKSpI7F4QJ9YIj5fonbzsp7jEqS1LE4XKAP69YTk8GD3dkHbd0USZKkNuVwgd6rsxdCG8bJ4qO2bookSVKbcrhA1zir8VV3p1CfQbm+3NbNkSSrs6fyuU21efNmbr75ZgB++OEH5s+fX+tyDX3WwsJCPvjgA8vrxpbjtXcOF+gAPbz7gSI4mi976ZLjsafyuS0xfvz4RhXTqs3lgd7YcrzWcnlpgvpKBdfU0itlHTLQBwSbD4zuzKy73oUkOYL2XD4XYMiQIRw+fNjyetSoUezZs4ddu3YxdOhQEhMTGTZsGMePX3lW2rJly5gxYwYAp0+fZujQocTGxl5yNWZdn3XOnDmcPHnSUjK3Zjneusrj1lemt6a9e/cycuRI+vfvz9ixY8nOzrZ8tieffJIBAwbw7rvvXvH6t99+IzExkdjYWB544AEqKystv+/Zs2eTlJTEihUr6v7HbgSHOg+92sCuXfnopA+7sw/YuimSAzv3f/9H5VHrXsTm2q8vXZ59tlHLtvfyuWA+B/ybb77hpZdeIjs7m+zsbAYMGEBxcTHbtm3DycmJDRs28Oyzz7Jq1ao6P+sTTzzBI488wtSpU1m0aFGDn3X+/PkcOnTIUsQsLS3N8p66yuNCw2V69Xo9M2fOZPXq1QQGBrJ8+XLmzp3L0qVLAXMN+Oor4NesWWN5rdVq6dWrF7/99hu9e/dm6tSp/Otf/+LJJ58EoFOnTlapA+OQPfSoYG+MFWGkFMohF8nxVJfPHTBgAF27dq2zfG5cXBzXXnutpXzu5arL56pUKkv53Go1y+dWT1+/fj3z588nISGBUaNGNap87qRJkyxDHd98841lHLuoqIiJEycSExPDX//610t68bX5/fffmTx5MgBTpkxp8metafv27dx3333ApeVx4WKZXo1GYynTW9Px48c5dOgQ1113HQkJCfzzn/8kIyPDMv/yC5iqXx8/fpzIyEh69+4NwP3338/WrVvrfF9zOWQP3dfdBU8iKTIcpqiyCB/X+ovxSFJzNLYnbW32VD43NDSUTp06cfDgQZYvX24Z73/++ecZPXo03333HWlpaYwaNarBz335N4ymfNbGaqhMrxCC6Oho/vjjj1rff3lxscYWG7NWUTKH7KEDRHqbK8wdzq1/zy9JjqY9lc8Fc+/z9ddfp6ioiLi4OEsbQ0NDARpVW2X48OGW0rfVpW2r11PbZ62vTXWVx22MPn36kJOTYwl0vV7f4LeL6velpaVZjnd88cUXjBw5slHbbAqHDfT+Xcz/cZIv/GnjlkhS22pP5XMBJkyYwNdff82kSZMs0/7+97/zzDPPkJiY2KgzQN59910WLVpEbGwsmZkX7z9f12ft1KkTw4cPJyYmhlmzZl2yrrrK4zaGi4sLK1euZPbs2cTHx5OQkNCoM4c0Gg2ffvopEydOJDY2FpVKxcMPP9yobTaFQ5XPremXw+d46vfJDAztx7KbPmj4DZLUCLJ8rtSWmlo+12F76FHB3hi1YZwoPGLrpkiSJLUJhw30MD83nA1dKTHkkVOeY+vmSJIktTqHDXRFUYj0Mo+nHco9ZOPWSI7EVsOUUsfSnP9nDhvoAAmdoxFC4U8Z6JKVaDQa8vLyZKhLrUoIQV5eHhqNpknvc8jz0KvFhQaxMrsze2QpXclKwsLCyMjIICdHDuNJrUuj0RAWFtak9zh0oFcfGD1ecAQhRK0XJkhSUzg7OxMZGWnrZkhSrRx6yKVnkCdKZTjlxmIySzMbfoMkSZIdc+hAd3FSEe5uvgLsUJ4cR5ckybE5dKADxAX1BaGWJQAkSXJ4Dh/oMaH+GLUh7D8vD4xKkuTYHD7QzaV0QzmWfxSjqWV3A5EkSWrPHD7Q+4WYz3SpNFVwptg6VeckSZLaI4cPdG+NM51degHywKgkSY7N4QMdICawFwgXWQJAkiSH1iECPTrUF2NFKAdzZKBLkuS4Okagh5jvMXo8/zh6o97WzZEkSWoVDQa6oihLFUW5oChKrd1bRVFGKYpSpChKctXjBes3s2Wiqg6MGoSOlMIUWzdHkiSpVTSmh74MGNfAMtuEEAlVj5db3izr6uKtwZMIQJbSlSTJcTUY6EKIrUB+G7Sl1SiKQlRQJCrhwZE8eQcjSZIck7XG0IcqinJAUZSfFEWJrmshRVEeUhRlj6Ioe9q6/GhMiC+G8lBZG12SJIdljUDfB3QTQsQD7wPf17WgEOIjIcQAIcSAwMBAK2y68aKCvdFXhJJamEqFoaJNty1JktQWWhzoQohiIURp1fN1gLOiKAEtbpmVRYV4Y6oIwySMHM8/buvmSJIkWV2LA11RlC5K1Z0jFEUZVLXOvJau19q6B3ig0ncF5IFRSZIcU4N3LFIU5T/AKCBAUZQM4EXAGUAIsRiYADyiKIoBqADuFu3whotOahV9A8NIFz6yBIAkSQ6pwUAXQkxuYP5CYKHVWtSKokO8STsfJnvokiQ5pA5xpWi1qGBvtKUhnCk+Q4muxNbNkSRJsqqOFegh3hi14QDyfHRJkhxOhwr0vl28MWlDAXlgVJIkx9OhAt3D1YlIvyA0BHI4T95jVJIkx9KhAh0u3sFI9tAlSXI0HS7Qo4K9KSkKJrssm7yKdne6vCRJUrN1vEAP8cakDQOQwy6SJDmUDhfo0SHeGLWhgMLhXBnokiQ5jgYvLHI0QV4aAty9cVZC5BWjkiQ5lA7XQwfzsIuoDOdQ7iHaYZUCSZKkZumYgR7sTWFBZ/K1+ZwrO2fr5kiSJFlFxwz0EG905VUXGMlhF0mSHESHDPToEG9MlcGoUMvz0SVJchgdMtAjOnng5qTBW91VnukiSZLD6JCBrlYp9A32QqkM50jeEUzCZOsmSZIktViHDHQwHxgtKOhMib6Es8Vnbd0cSZKkFuuwgR4d4kNZSQggD4xKkuQYOmygR4V4Y6oMwlnlKsfRJUlyCB020Pt09kKlqPFTR8gzXSRJcggdNtDdXNR0D/REpe/KsfxjGEwGWzdJkiSpRTpsoIP5fPSigi5ojVpOFp60dXMkSZJapEMHelSwNzn5QYAspStJkv3r2IEe4o3QdcJN7SHH0SVJsnsdOtD7BXsDKgJceshAlyTJ7nXoQA/wdKWztytqXVdSClKoNFbaukmSJEnN1qEDHcwXGBUXdcEgDBzPP27r5kiSJDVbhw/0qGBvss4HAMhhF0mS7JoM9BBv9DofvJ395JkukiTZNRnowd6AQpBrT1kCQJIku9bhA72rvzuerk44G7pyqugUZfoyWzdJkiSpWTp8oKtUCv2CvSgpCkYgOJJ3xNZNkiRJapYOH+hgHnZJz+4EIIddJEmyWzLQMR8YLa1wI9Cti6yNLkmS3ZKBjvlcdIDOrj3lqYuSJNktGehAzyBPnFQKLoZuZJZmUqAtsHWTJEmSmqzBQFcUZamiKBcURam166qYvacoSqqiKAcVRUmyfjNbl8ZZTc8gT8st6eSBUUmS7FFjeujLgHH1zL8B6FX1eAj4V8ub1faigr05m+0HyCtGJUmyTw0GuhBiK5BfzyK3Ap8Ls/8CvoqiBFurgW0lKsSbC0Uqwj27ygOjkiTZJWuMoYcC6TVeZ1RNu4KiKA8pirJHUZQ9OTk5Vti09USFeAPQRdNbnrooSZJdatODokKIj4QQA4QQAwIDA9ty0w0ylwAAjakbORU5nC87b+MWSZIkNY01Aj0TCK/xOqxqml3xdXch1NeNsmLzgVE57CJJkr2xRqD/AEytOttlCFAkhMi2wnrbXL9gbzLO+6FW1HLYRZIku+PU0AKKovwHGAUEKIqSAbwIOAMIIRYD64AbgVSgHJjeWo1tbdEh3mw8dp6EHj1kKV1JkuxOg4EuhJjcwHwBPGa1FtlQVIg3JgEhbr1IztuOEAJFUWzdLEmSpEaRV4rWUH1g1M0USVFlERklGTZukSRJUuPJQK8hzM8Nb40T5aXm0+jlgVFJkuyJ3QV65cmTnHv5HwidzurrVhSFqBBvMi/44qJykVeMSpJkV+wu0PWHdlDw739T/CZnZRMAACAASURBVOPaVll/VLAPx7PL6ePfRwa6JEl2xe4C3SMqGBdvPfkfLcR8PNa6okK8qdAbCffow9H8oxhNRqtvQ5IkqTXYXaArPa/BP8kT7elsKnbvsfr6qw+MuosIKgwVnC46bfVtSJIktQa7C3QUBZ/7HkbtYiRv8TtWX33PIE+c1QraUnnFqCRJ9sX+Ah1QDZqCb18jpX/sQ3f2rFXX7eKkondnLzJyvHB3cpfj6JIk2Q27DHRcPPCbeBsogvyPrV9+PSrYm6NZpUR1ipIlACRJshv2GeiA83VP4NNVS9HqtRhLSqy67qgQb3JLK+nu1ZfjBcfRG/VWXb8kSVJrsNtAx7cr/uOSMFUaKFz+H6uu2nJgVIlEb9JzouCEVdcvSZLUGuw30AHNbU/hHlhJ/rJPEAaD1dbbr+pmF7oy83065Di6JEn2wK4DnW7D8R8cgCG3mJJff7Xaar01znT1dyf9ggZfV19ZeVGSJLtg34GuKHje/RjOngbyP3zfqquOCvbmWHYJsQGxbDi7gTUn17TKhUySJEnWYt+BDijxd+EfI6g4dpqK5GSrrTcqxJvTeWXMiH+KCO8Int3+LA/88gCpBalW24YkSZI12X2g46zBd+JkVM4m8pZY7xTG6BBvhICKcn++vPFLXhj6AicKTjBxzUQW7FlAub7catuSJEmyBvsPdEA14hF8e1ZQsmkb+qwsq6wzqurA6JGsYlSKiom9J7L29rWM7zmeZYeXMf778axPWy+HYSRJajccItDxDsH/phEgTOR/9qlVVtnFW4OfuzNHsost0/w0frw07CW+uOELfF19eXrL0zy84WHOFJ+xyjYlSZJawjECHXAe+wTe4VoKV6zAVFbW4vVV10Y/nFV8xbyEoAS+vvlr5gyaw8Gcg9y++nYW7l+I1qBt8XYlSZKay2ECnbCB+A8PxVReSeGqb62yyqhgb46dK8FgNF0xz0nlxL397uWH237g+ojr+fDgh9y2+ja2pG+xyrYlSZKaynECXVFwu+1x3DrpyP/0I4Sx5XXMo0N80BlMnMqtu8cf6B7I/BHz+eT6T3BVuzJj4wxmbpxJZmlmi7cvSZLUFI4T6ADRt+Mfr0afnUvp5s0tXl3NA6MNGRQ8iJW3rOSv/f/Kzuyd3Pb9bSw5uASd0fq3ypMkSaqNYwW6kwted07D2d1A/pLFLV5d9wAPXJxUlxwYrY+z2pkHYh5g9a2rGRE2gvf2v8edP9zJH1l/tLgtkiRJDXGsQAeUwf+LX99KypMPoT1ypEXrclKr6NvFi99Tc8kva3xPO9gzmLdGvcW/rv0XRmHkoV8fYtaWWZwvO9+i9kiSJNXH4QIdzyB8b7kelZMgf+nHLV7dhP5hHMku5qrXNvLaz8eaFOxXhV7Fd7d+x6MJj7Lx7EbGfz+ezw5/ht4ky/FKkmR9iq0ujBkwYIDYs8f69wQFIGs/5x4eT8FJb3pu3IRz56AWrS71Qgnv/ZbKmoNZuDmruX9YBA+O6I6/h0uj15FenM6ru15lW+Y2evn1Yu7gufTv3L9F7ZIkqeNRFGWvEGJAbfMcr4cOEJKI/9W9wGii4KsvW7y6nkFevDc5kV//ejXX9uvM4i0nm9xjD/cOZ9E1i3hn9DuU6kqZ9vM05m6fS15FXovbJ0mSBI7aQwc49C3pTzxFRXEnem7djsrNzWqrbmmPvVxfzpI/l7Ds8DLcnNx4PPFxJvaeiFqltlobJUlyTPX10B030I16yufEcWYNdJk3D7+777L6Ji4P9qlDI3hwRCSdPF0b9f5TRaf4v//+HzvP7SSqUxTPD3memIAYq7dTkiTH0TEDHRBbF5A2+wNMXpF0/3k9iqp1RphSL5Tw/sZUfjjQ9GAXQvBz2s+8sfsNcitymdB7Ak8kPYGPq0+rtFWSJPvWYQOd8nyKZsaT9bsn4R99iOfVV7fq5loS7KW6Uj448AH/PvpvPJw9eCzhMSb1mYSTyqlV2yxJkn3puIEOiG8fI/UfG3CNH0zXZZ+3+vagZcF+ouAEr+9+nZ3ZO+np25NZA2cxLGRYG7RakiR70KEDnfOHyZ15PTkHvYn8YTWa3r1bf5tVUi+U8v7GFEuwTxnajYdGdG8w2IUQbErfxBu73yCjNINR4aP424C/0c27Wxu1vHUU64oxmAz4a/xt3RRJslsdO9AB479uIGVRGt633kHIK6+0yTZrSr1QysKqYHd1UjN1WOOCXWfU8eXRL/nwwIfoTDqm9JvCQ3EP4eni2UYtt45zZef47PBnrEpZhZuTGytuWUGQe8uuDZCkjqrDBzpH15I961GKzvjQc/NmnDp1apvtXqa5wZ5bkct7+97j+9Tv8dP48UTSE9za49Z2f5pjakEqnx7+lHWn1iEQXNftOrZkbCG6UzRLrl8ijw9IUjO0ONAVRRkHvAuogY+FEPMvmz8NeAOorhm7UAhR73X3bRroJiOVLydw6msDATNnEPjYY22z3To0N9gP5x3mtV2vsf/Cfvr592P2oNnt8mrTfef3sfTQUrZkbMHNyY07e93JlKgphHiG8MPJH5i7fS5/ifsLMxJn2LqpkmR3WhToiqKogRPAdUAGsBuYLIQ4UmOZacAAIUSj/0LbNNABdizk7NwFaLWd6bl5CyrXxp0r3pqaE+zVpzku2LOA8+XnGRsxlqf6P0WIZ0gbtvxKJmFic/pmPj30Kck5yfi5+jG532Qm95mMr8b3kmWf//15VqeuZvF1i+UBX0lqopYG+lBgnhBibNXrZwCEEK/WWGYa7T3QKwopmxXL2Q2eBL/yCr533tF2227A5cF+35CuPHh1d4K8NHW+p8JQwbJDy1h6aCkCwbToaTwQ8wDuzu5t2HLQG/WsPbWWZYeXcaroFKGeoUyNmsrtvW7Hzan2q3MrDBXc8+M95Gvz5Xi6JDVRSwN9AjBOCPG/Va+nAINrhndVoL8K5GDuzf9VCJFey7oeAh4C6Nq1a/8zZ9r25spi7VOc/ucaCOhN5Jq1KIrSpttvyMmcUhZuTGV1ciZOahWTB4bzl5E9CPGtu2zBubJzvLX3LX46/RNB7kH8tf9fuSnyplb/bGX6MlaeWMnnRz7nQvkF+vj1YXrMdMZGjG3U2PjJwpNM/nGyHE+XpCZqi0DvBJQKISoVRfkLcJcQYkx9623zHjpAzgkKn76a7F1+dP10KR5Dh7bt9hspLbeMxVtOsmpfBgB3JoXxyKgedOvkUed79l/Yz/xd8zmSd4S4wDjmDJxDbGCs1duWW5HLv4/+m6+Pf02JroRBXQYxPWY6w0OGN3knIsfTJanpWn3I5bLl1UC+EKLea9dtEuiA6dPbSH33GG6DRxL+4Ydtvv2myCys4KMtJ/nP7nSMJsGt8SE8OroHPYO8al3eJEysTl3Ne/vfI7cil/E9xvNE0hNWGdI4W3yWZYeXsTp1NXqTnmu7Xcv06Okt3mnI8XRJapqWBroT5mGUazCfxbIbuEcIcbjGMsFCiOyq57cDs4UQQ+pbr60CnRPryXl2OrmHvOm+bh2u3SPbvg1NdKFYy5Jtp/jyv2fRGozcGBPMY6N7Wu55erkyfRlLDi7h8yOf46Ry4sHYB5kaPRVXddMPBB/OO8zSP5ey4ewG1Iqa8T3GMy16GhE+ES38VGZyPF26XIWhgh1ZO7gq9Kpm/Z91dNY4bfFG4B3Mpy0uFUK8oijKy8AeIcQPiqK8CowHDEA+8IgQ4lh967RZoJtMGN4YQOrnFfhMupvgF19s+zY0U36ZjqXbT/PZjjRKKg1c2y+IGWN6kRDuW+vy6cXpLNi7gN/O/kaoZyhPD3iaa7te2+DQiBCCP7L/YOmhpezM3omnsyeT+kzivn73EegeaPXPJcfTpWoZJRk8uelJjhccJ9QzlFkDZzEmfEy7O95lS/LCosvt/Iisef+kONuPXpu3oPatPRDbq6IKPZ/vSOOT309TWK5nRK8AZozuyeDutV8wtTN7J6/tfo2UghQGdhnI7IGz6ePf54rlDCYDG85sYOmhpRzNP0qgWyD3Rd3HxN4T8XKpfZjHWqrH0x+Ke4iZiTNbdVtS+7Qjawd/3/p3TMLEw3EP813qd6QWpjIkeAhzBs2hh28PWzexXZCBfrnKErTPx3D6B3cCn3qKgIcetE07Wqis0sBXO8/w0dbT5JZWMijCn5nX9OSqngFX9GgMJgOrTqxiYfJCinXF3NnrTmYkzsBf44/WoGV16mqWHV5GRmkGEd4RTI+Zzs3db8ZF3fjb7LWUZTz92sUMC5Xj6R2FEIJPD3/Ku/vepYdvD94d9S7h3uEYTAaWH1/OouRFlOvLubvv3TwS/0iHLy0tA702Pz/D2fnfUGkKoedvG1GcnW3XlhbS6o18vessi7ec4lyxlvhwX2aO7sk1/YKuCPaiyiIWH1jM18e+xs3JjXGR4/jt7G/ka/OJC4jjgZgHGN11NCql7e9OKMfTO55yfTkv7HiBX9J+YWzEWF4e9vIV11IUaAtYuH8hK1NW4uPiw4zEGdzZ6852X/qitchAr03+KUrnDCV9qz8hb7yBzy03264tVlJpMPLtvkw+2JxKen4F/YK9mTG6J+NiuqBWXRrspwpP8fqe1/k983euCr2KB2IeYEDnATYfqzxVeIq7f7xbjqd3AGeLz/LEpic4VXSKJ5OeZFr0tHr//x3LP8b8XfPZe34vff37MmfQnHZZ+qK1yUCvg/jqLk69dwBVWDQRK1faPMysRW808UNyFos2p3Iqp4wegR48Nron4+NDcFJf2vOuMFTUeUWnrcjxdMe3LWMbs7fNRqWoeP3q1xt9yqoQgl/O/MKCPQs4V3aOcRHjeHrA03Tx6NLKLW4/6gv0tv9e3Y4oQx7Gv1cR2sNHqNi719bNsRpntYo7+4fx619HsvCeRJzVKp765gBjFmzh611n0RlMlmXbW5gDjO8xntt73s6Sg0vYkbnD1s2RrEgIwZKDS3jst8cI9Qzl65u+btL1B4qiMC5iHD/c9gMPxz/MpvRN3PLdLSw+sBitQduKLbcPHbqHjhCY3h1C6rIS3K+6hrCF79u2Pa3EZBL8duwC729M4WBGEcE+Gh4e2YO7BoajcW6f45ByPN3xlOnLeG77c2w4u4EbI29k3rB5Le5QZJVmsWDPAtafWU+IRwhPD3ia67pd5zDftmsjh1zqs+dTLvzf8+Qd9abH+l9wCQ+3dYtajRCCbSm5vL8xhd1pBQR4ujJ1aDeCfTS4OKlwdVLh6qTGxUllee3ipMJFXf1afXG6WoVK1bp/NHI83XGcLjrNk5ue5EzxGZ7q/xRToqZYNXR3n9vNq7teJaUghUFdBjF70Gx6+7Xd3cnakgz0+ujK0f8zitRV7vjdex9dnn3W1i1qE/89lcfCjalsT81t9jqc1QouahWuzuoaoa+y7BBqznOtmtcv2Jv7h0Xg4tS40T45nm7/Nqdv5pltz+CscubNkW8yKHhQq2zHYDKw8sRKFiYvpERXwqTek5iROMPhTnOUgd6QX18gc8HnlF7wp+eWLai9WvcimvYkr7SScp2RSoMJncFEpcGIzmBCZzRRqTf/1NWYV1nPvMvfd3GdJnQGI1q9icxC89k3b06MIzqkcX9oL/z+At+nfi/PT7czJmHiwwMf8sGBD+jn3493R79LsGdwq2+3qLKIhfsX8s2Jb/By8WJGwgwm9J7gMN/wZKA3pPAsFS/2J+2XAIJmz6bT9Gm2bpHDWn/4HHO/P0RBmY7HRvfksdE9G+yty/F0+1OiK+HZ7c+yOX0z43uM5/khz6NxuljfX6s3UqzVU6I1UFxh/lmiNVRN06MzmOgR6ElUiDfhfu7NGt47UXCC13a9xq5zu+jt15s5g+YwsMtAa35Mm5CB3hjLp3Dmg53o1WH0WL8exckx9ubtUWG5jpfWHOG7/ZmN7q2fKjrF3WvvJqpTFB9f/7HD9LbskcFouiSALw/msyVprM99lVLTBSKVyXjorqZUa7xkGZ3R1PCGqni4qOkX7E2/YG+iQsw/+3T2ws2l4QP6Qgg2nN3Am7vfJKssi+u6XcffBvzN5nf4agkZ6I1xZgcl/7idjO3+hL7zDt7jxtq6RQ7v1yPnefa7Pyko0/Ho6J7MaKC3vubkGp7d/iwPxj7I40mPt2FLOy6t3siaA1ks351OekE5JVoD5Tpjncs7eR5GE7IchAtuBdPxVfXBS+OEt5szXhpn83PLz+rpTnhpnC3TvTROOKlUpFwo4UhWMUezizmSXczR7BJKKw0AqBSIDPAgKsSHfsFeRFWFfV13+dIatCw7vIxP/vwEgWB6zHQeiHmgXZ622xAZ6I0hBOJfV3FyWSFOPeKJ+M+/bd2iDqFmb71vFy/enBhPTGjdvfUXd7zIdynf8a9r/8Xw0OFt2NKOJaOgnC//e5blu89SUK6nV5AnSV39Loavm9MlAe3hqrDm7DK+TvmUmE4xvD36batf7GMyCTIKKjiSXcSR7Ithn1lYYVkmwNPF3JOv0aPvHuBhuaDuXNk53trzFj+l/UQXjy48PeBpxnYba1enOcpAb6z9X5H/xizO7/Mh4pvluMXF2bpFHcYlvfVRPZgxpletvfXq8fS8ijxW3LKCzh6dbdBaxySE4I+TeSzbkcaGo+cBuC6qM/cPi2Bo9051hl6xrpg5W+ewLXMbt/e8nblD5rZpHfOicj1HzxVf0ptPOV9qGdZxcVLRp7NXVch7ERXig9YphfeT3+RY/jH6d+7PzMSZxAXG4axq/zWdZKA3ll6L8fVoUr/R4HnNOELfWmDrFnUoheU6Xl57hG/31d9bl+Pp1lVWaeDb/Zl8viONlAul+Lk7c/egrtw3pBuh9dzPFiClIIUnNz1JVlkWzwx6hom9J7aL3q7eaOJkTukVQzb5ZTrLMmH+rnTqnEyW6ju0pmI0ag2xgbEkBCaQ1DmJ+MD4Vi8b3Rwy0Jti4z85/+6H5Kd60/PXX3EOsd+DJ/ZqQ1VvPa+qtz6zlt66HE9vuVM5pXzx3zOs3JNBSaWB2FAf7h8Wwc1xwY26gnh92nqe+/05PJw9eGvUWyQGJbZBq5tPCMH54kpLwB/JLuZoVjGnC/JQu5/A2zcDb98MCoxnMAkjCgq9/HqRGJRIQlACSUFJBHsE23yHJQO9KYqz0b8SR+oPAfg/8ACdZ82ydYs6pKJyPS+tPWzprb8xIZ7YsEt76+1hPF1v1LMjawfrTq/jYM5BkjonMSZ8DENDhl5RBrY9MJkEm09c4LMdZ9hyIgdntcKNscHcPyyCxHDfRoWV0WTk/f3v88mhT4gPjOetUW/Z9amkZZUGNh/PYcXedLaeyMFEJVGRBUSG5aBVneRQ3kHKDeUABLkHkRSUZAn4Xn692vwbogz0plr5P2R8vIWyPD96bd6MysPD1i3qsH47ep5nvr3YW58xpieuTubeo63G040mI7vP7+bn0z/z65lfKdYV4+PqQ0JgAvsv7KdYV4yr2pUhwUMY03UMI8NG0smt9rtJtZWicj0r9qbz+R9nOJtfTpCXK/cO7sbkweF1nhlS63oqi/j71r+zI2sHE3tPZM6gOW16E5TWdr5Yy7f7MlmxN51TOWW4OasZGxPIkL46jM6nOZCTzL4L+zhfbj7G4O7kTnxgPIlBiSR2TiQuIK7Vd+Qy0JsqYw8V88eRtiGQzs89h/9999q6RR1aUbmel9ceYdW+DPp0No+tV/fW22o8XQjBgZwD/Jz2M7+k/UJuRS7uTu6M6TqGGyJvYGjwUJzVzhhMBvad38em9E1sSt9EZmkmCgrxgfGM7jqaMeFjrHaD7cY4dq6Yz3ac4fv9mVTojQyM8OP+YRGMje6Cs7ppxVaP5x/niU1PcKH8AnMHz+XO3ne2UqttTwjB/vRCVuzJYO2BLEoqDYT7u3FnUhh3JoXh5FLEvgv72H9hP/sv7CelIAWBQK2o6e3Xm6TO5l58YmCi1TsaMtCbY8kY0r48j8E1jB4//4Si6tCVhtuFjcfMvfXcUh2PjOzBzGvMvfXWGk8XQnCi4ATrTq/j59M/k1WWhYvKhavDruaGyBsYETai3vOYq9+/MX0jm85u4mj+UQAifSIZHT6aMV3HEBsQa/W7Q+mNJn49cp7PdqSx83Q+rk4qbksIZeqwbo0ut1DNJExklmbyR9YfvLH7DbxdvHl79NvEBXacM8AqdEbWHznHij0Z/H4yFyFgaPdOTOgfxg2xXXB3caJEV8KBnAPsv7Cf5AvJHMw5iNZoLucb6hlq7sFXPXr49mjRv7kM9OY4uILid2aQucOfsA8W4TVmjK1bJGHurf/jxyOs3Gvurb8xMY64MF/m7ZjHtynfWmU8/UzxGUuInyo6hVpRMyRkCDdG3sjo8NHNPvMhuzSbTemb2Ji+kb3n9mIQBgLcAhgZNpIxXccwOHhwi073yy2t5OtdZ/nyv2c5V6wlzM+NKUO6cdfAcHzdGx4WKdWVklqYyvH845woOMGJghOkFKZQpi8DICkoiQWjFhDgFtDsNtq7zMIKvt2bwcp9GZzJK8fT1YmbYoOZMCCMAd38LMcg9CY9x/OPs+/8PpJzktl/YT+5FeZCeF4uXjwY+yDTY6Y3qw0y0JvDoEO8FUvqSleE2hNNbAwuYWE4h4XjHBaKS3g4zmFhHaqQV3uy6dgF5nx7kNxSHQ+P7M5DI8OZvn4KueW5zRpPP1d2jp9P/8y60+s4mn8UBYWkzkncGHkj13W7Dj+Nn1XbX1RZxPbM7Ww8u5HtmdspN5Tj5uTGVaFXMTp8NFeHXd3oKoHJ6YV8viONtQez0RlNjOgVwNShEYzpG3TFrQfB3OtOL0m3hHZ1gGeWZlqW8XLxordf70se0Z2iO+x9PC8nhGDX6XxW7M1g3Z/ZlOuMRAZ4MKF/GLcnhhJy2emeQggySjLYn7Offef3MSRkCOMixjVr2zLQm2vL65Qvf4N84y3ocgrRp2dgKim5ZBG1jw/OVeHuEn5Z4AcH2/XNp9u7ogo9/1hr7q337uzJUzf68sKeB+nn349Pxn7S4Hh6vjaf9Wnr+en0T+y7sA+A6E7R3BB5A2MjxjZ4paMQguIKA5mFFWQVVpBbWgmASlFQqRRUCqhVCoqioFbMr83TFdQqLNNN6DlRnMzBvO0k5++gSJeHSlHT1zeegYEjGBh0NZ09uqBWqtZVte69Zwr47I8zHEgvxMNFzYT+YUwZGkHPIE9LG4t1xaQUpFhCO6UghZTCFCoMFVVtVdHNuxu9/XrTx6+PJby7eHSx+el59qKs0sC6P7NZsTeDXafzURS4qmcAEweEc31UZ6vfREYGenOVXoC3oyFyJMROBO9gjMILXbER/bk89Bnp6DIy0KdnoM/IQJeVBXr9xferVDh36VIV+NW9+nBcwkJxDg9H7e8v/2isYNOxCzzz7Z/klFZy3cBMfi9+r87x9GJdMRvPbuSn0z+xM3snRmGkh08Pboi8gRsib6Crd1fLslq9kXNFWrIKK8gsrCC76nlW9c/CinrrmjSPCZUmEyevwzh5HUHtegEAozYYQ0kUhpIoTJUhgPn/TfdAD+4fGsGtCV3I12eZe935Jyy97+yybMuafVx9Lgnt3v696eHT45IqiFLLnMkrY9XeDFbtyySzsAIvjRPj40OYOCCc+DAfq/y9y0BviQ3zYPvbV0539QavLuAVbH54ByM8umDQuaErVaEvqESXU4I+Kxt9ejq6zAyMOZfeTEJxd8clNPRi4IdVDeP4+aLy8EDt4YGq6iF7+vUrqtDzz7VHWLE3g6DIH6jQ7GDxtYsZHjqcCkMFWzK28NOpn9iWuQ29SU+oZyhXh1xHjM9IVPoQzhVrLT3t6uDOLdVdsZ0AT1dCfDWE+LgR4utmfu7rRrCPhiBvDSoFjCaBEOafJlH9qPHaBCYhMAqBEAJj1WuTqWq56veYBOcrMjiYv4PDhTs4U3YYgcDHOYg+3oOJ8O6KXm0O8dTCVCqN5m8IakVNpE/kFUMmQe5BsgPRRkwmwR+n8lixJ52fDp2j0mCiV5CneUgmKbRJp4peTgZ6S1WWQsk5KMmC4mwoqXoUZ1VNr3ptMlz2RgU8g6qCPwSTawB6nRe6Mmf0RSb0hVp0F4rQZ51Hl5mJKC+vswmKq6sl3C8+3M3B7+mJyv3yeTUenpftHNzcHPYPe9PxC8xZtZfSTm/h5lZGjP8gDub/jl5occEHD8MA9EVx5OZ1Rn9Z59rdRV0V0m6E+moIrhnaPm508dHY9B6seRV5bM3Yysb0jfyR9QeVxkr8Nf6WwO7jb+59d/fp7lDnhtu7Yq2eHw9ms2JPOvvOFqJWKTw+phdPXNurWeuTgd4WTCYozzOHfsm5qrCvDv7sizuE8rwr3+ukQXh2xqjujF7vg9HohsnkhMnohEmvwmhQMOkFJp0Jk9aISWvApNVh0lZiKtdiKiszP+rZIVxCpULl7l4V9p7mwPf0quW5J2ov80+VR43nXl6oPDxRubu1y9M5iyr0PLNmA1tLnwdU6ItjMZUkEOjcj1Bfd0toh/hU967dCPV1w9vNyW52dBWGCsr15Ta/YElqmtQLpazal8GgCH9G923e1bUy0NsTQ+Wlvfri7Bo7gapp2kLzt4Kqr9ANcvYAV0+EiycmxRMT7uaHcMVkcsFkdMZkUGM0qjHpFUx6zDuHSpN5x1BeibG8AlNZedWjDFN5RcPbVZSLwe/pgarWnYIH6uodgKcnKjcNKAqggKKgqMw/65xW9VCUy6epqhaveq1SVb2/appKxYGscygmZ4K93fF3c0aFAJMJYRIgTFXPTSCE+XHFvFqWq35eNa/mcwBFo0Hl5o7K3Q2VmxuKm5t55+lW9VoOnUktVF+gyzJ1bc3JFfy6mR8NMehAVwqVxeaAryy58nWNaUplKerKEtS6Uqg8f3F+ZQmYahysdap61HOFsjCByaCYvyHoFUwGp4s7BoMaU9XDqC/FpC/DpFcw5ioYs0CvExh1ApNOIPS26TAAEG018QAAB65JREFUVJ9QWlL1aBfUKlSuTqhcnMw/XZ1Rqn6qXF2qXrug0lQ9XF1Qubmah9w0LqjcNKg0rigaV1SuGoSTBpw0oHY1/99ycgW1+bVQuwCKeUcEgLi486oav6+efPk8MJ/Fg6DG+6namXLlDrZ6Z6pSXbGDrmv5K3bcl+2QFVdXVK6uKBoNiqur3Xx7siUZ6O2Zkws4+YO7f8vXZais2gkU19gJVP00Gszj/ya9+afRgGLSozYZUBsNOF827/JlLz7Xg8l4yTxh0GOq0GGs0GGqNCB0RoQwmoeoTKaqPYe4OK2qR4zJhBAXe8WYTICwZA8AwvwHfum0ix9ZCAVFqZpwMS9AETWeV82ummZ+Xr18deioqTrn8GJgVT9XqUBRW8LJpDdZhseE3mT+JmSgappAGMBkEFW/IgVhUDBpFYwGBX31a4OCyahCGGWA1aS4uJi/AVWFvErjiuLienGaqyuKi3PVc2dULi4ors7maS7VP51QnJ1QXJxQWX6qUNRV/8ZK1Te96n9n816q6t+3xjyUy5ZXUBT1pdMt3yQv7uQUlXkZlW8n1AHWrz0kA72jqO69ebTtmKsCqKseLSaEeYchjFXBX/XcdNnrms9RQKWu8YdV9Vylrvojq/m6xnxLr7EVCVF3u02mqp1hOaLcfHzEVFGOqbwMUV6BqaICoa2o2nnqwKgzD9EZ9WDQoZgqa0yrMd+gQzFqza8N1fMrzTt8gxaFqiPFl390hUt2llg698oV0y/ZwcKlO17lst+zUCEUddVrxTJPCBXCaELojZj0AmEwIQxaTIZyhB5MRoGoFJjKMO8kjWAyKlX/HRRE1cNkVCzbb086XduXoIXfWX29MtAl+6EooK4eL3IA1TuUOq6+VAB100qvtJxR///t3VuIVVUcx/HvzxnTNLBCkVJBISkkCEPCEiKyB6PIHioqigihly4WQVgPBT0VRJeHCEQtKbFiEpKQLmjQS0gzFuSlSCxvaY5UFl1wzsyvh72PHo8zznHcZ9bs3f/zsvdeZznzXzPjf++9zl7/A31/Q98/2fZ4w/5A/Q6sdvrW9eNafkdVa2rvb/g3tfqc3hm+Vn9+ku2AcZ3ZVfG4zpM/r3Gd+Ym4s+G4o6lf9pqd3Tm5Bq4N5PtmoJbfRfX1Q/9APgXl7D0R8pPtKdNQJ19zY3u9H5y44zylzynTWVmfCfOvbcuvryL/M0IIhegYn51FJo72maR96neJ/wdj75mzEEIIIxIJPYQQKqKlhC5piaTvJe2WtGKQ1ydIei9/fauk2UUHGkII4cyGTejKnsV5HbgZmAfcI2leU7dlwG+2LwNeAV4sOtAQQghn1soV+jXAbtt7bB8H3gWWNvVZCqzN97uAxYpVACGEMKpaSegzgP0NxwfytkH72K4Bx4DTHniW9JCkbkndvb29I4s4hBDCoEb1TVHbK20vsL1g2rRpo/mtQwih8lpJ6AeBWQ3HM/O2QftI6gSmAIOUFQwhhNAurSws+gqYK2kOWeK+G7i3qc9G4AHgS+AOYIuHKePY09NzVNLesw8ZgKnA0WF7lVeVxxdjK68qj69MYxuyst+wCd12TdIjwCdkC67W2N4h6Xmg2/ZGYDXwtqTdwK9kSX+4rzviORdJ3UOVj6yCKo8vxlZeVR5fVcbW0tJ/25uATU1tzzbs/wvcWWxoIYQQzkasFA0hhIooa0JfmTqANqvy+GJs5VXl8VVibMk+gi6EEEKxynqFHkIIoUkk9BBCqIjSJfThKj+WlaRZkj6XtFPSDknLU8dUNEkdkr6W9FHqWIom6UJJXZK+k7RLUns+kiYBSU/kf5PbJa2XNDF1TOdC0hpJRyRtb2i7WNJnkn7ItxeljHGkSpXQW6z8WFY14Enb84CFwMMVGlvdcmBX6iDa5DXgY9tXAFdRkXFKmgE8BiywfSXZWpRh15mMcW8BS5raVgCbbc8FNufHpVOqhE5rlR9LyfYh29vy/T/JEkJzEbTSkjQTuAVYlTqWokmaAlxPtsAO28dt/542qkJ1AufnZT0mAT8njuec2P6CbAFko8aKsWuB20c1qIKULaG3Uvmx9PIPCJkPbE0bSaFeBZ4CBlIH0gZzgF7gzXxKaZWkyamDKoLtg8BLwD7gEHDM9qdpo2qL6bYP5fuHgekpgxmpsiX0ypN0AfAB8LjtP1LHUwRJtwJHbPekjqVNOoGrgTdszwf+oqS37M3yueSlZCetS4HJku5LG1V75XWoSvk8d9kSeiuVH0tL0niyZL7O9obU8RRoEXCbpJ/IpslulPRO2pAKdQA4YLt+R9VFluCr4CbgR9u9tvuADcB1iWNqh18kXQKQb48kjmdEypbQT1R+lHQe2ZszGxPHVIj8E55WA7tsv5w6niLZftr2TNuzyX5nW2xX5irP9mFgv6TL86bFwM6EIRVpH7BQ0qT8b3QxFXnDt0m9Yiz59sOEsYxYS8W5xoqhKj8mDqsoi4D7gW8lfZO3PZMXRgtj36PAuvxCYw/wYOJ4CmF7q6QuYBvZk1hfU/Jl8pLWAzcAUyUdAJ4DXgDel7QM2AvclS7CkYul/yGEUBFlm3IJIYQwhEjoIYRQEZHQQwihIiKhhxBCRURCDyGEioiEHkIIFREJPYQQKuI/sr//Do9by60AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uv-_ZwOlOEYp",
        "colab_type": "text"
      },
      "source": [
        "On the dataset FashionMNIST, the difference is less visible due to its simplicity and the already efficiency of the plain network on it"
      ]
    }
  ]
}