BackwardFunctions.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BackwardFunctions.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/wiatrak2/BScThesis/blob/master/BackwardFunctions.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "uezPcmhMQamF",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Load data"
      ]
    },
    {
      "metadata": {
        "id": "v2r9aCdkugp1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "outputId": "c2570c55-57c0-45c7-c87d-7364593c2c7f"
      },
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "import sys\n",
        "sys.path.append('BScThesis/')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "7E7W98N6uoXl",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 372
        },
        "outputId": "a9ffeec0-5f91-417a-dbb5-539cb06251cb"
      },
      "cell_type": "code",
      "source": [
        "!pip3 install torch torchvision"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting torch\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7e/60/66415660aa46b23b5e1b72bc762e816736ce8d7260213e22365af51e8f9c/torch-1.0.0-cp36-cp36m-manylinux1_x86_64.whl (591.8MB)\n",
            "\u001b[K    100% |████████████████████████████████| 591.8MB 26kB/s \n",
            "tcmalloc: large alloc 1073750016 bytes == 0x62826000 @  0x7fda7e9502a4 0x591a07 0x5b5d56 0x502e9a 0x506859 0x502209 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x507641 0x502209 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x507641 0x504c28 0x502540 0x502f3d 0x507641\n",
            "\u001b[?25hCollecting torchvision\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ca/0d/f00b2885711e08bd71242ebe7b96561e6f6d01fdb4b9dcf4d37e2e13c5e1/torchvision-0.2.1-py2.py3-none-any.whl (54kB)\n",
            "\u001b[K    100% |████████████████████████████████| 61kB 21.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from torchvision) (1.14.6)\n",
            "Collecting pillow>=4.1.1 (from torchvision)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/85/5e/e91792f198bbc5a0d7d3055ad552bc4062942d27eaf75c3e2783cf64eae5/Pillow-5.4.1-cp36-cp36m-manylinux1_x86_64.whl (2.0MB)\n",
            "\u001b[K    100% |████████████████████████████████| 2.0MB 6.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from torchvision) (1.11.0)\n",
            "Installing collected packages: torch, pillow, torchvision\n",
            "  Found existing installation: Pillow 4.0.0\n",
            "    Uninstalling Pillow-4.0.0:\n",
            "      Successfully uninstalled Pillow-4.0.0\n",
            "Successfully installed pillow-5.4.1 torch-1.0.0 torchvision-0.2.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "V1UkM_eUuuzU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 167
        },
        "outputId": "8dc908b0-728b-4a45-be7d-d1fccb64194f"
      },
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/wiatrak2/BScThesis"
      ],
      "execution_count": 111,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'BScThesis'...\n",
            "remote: Enumerating objects: 14, done.\u001b[K\n",
            "remote: Counting objects: 100% (14/14), done.\u001b[K\n",
            "remote: Compressing objects: 100% (11/11), done.\u001b[K\n",
            "remote: Total 364 (delta 5), reused 9 (delta 3), pack-reused 350\u001b[K\n",
            "Receiving objects: 100% (364/364), 56.72 MiB | 10.07 MiB/s, done.\n",
            "Resolving deltas: 100% (209/209), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "5iUo1KQ7uwSS",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from importlib import reload "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "4Cq-GyfLuxEy",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "9mt-K4JYux_S",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import torchvision\n",
        "import torchvision.transforms as transforms"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "WXfJlKLouy3i",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import pickle\n",
        "import sympy\n",
        "from collections import defaultdict, namedtuple\n",
        "import copy"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Ghc6PxSduzji",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from mnist_m import Mnist_M\n",
        "from random_dataset import RandomDataset\n",
        "import datasets_merge\n",
        "import models\n",
        "import svhn_models\n",
        "import trainer\n",
        "import domain_trainer\n",
        "from test_model import test_model\n",
        "import plots\n",
        "import test\n",
        "import plotly_plots"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "7FOZ0XH8P1qO",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import backward_functions"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "hh2uAkdPu0fy",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import plotly\n",
        "from plotly import tools\n",
        "from plotly.offline import init_notebook_mode, iplot\n",
        "import plotly.graph_objs as go"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Grv2u1Rhu1Rq",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def convertMnist(img):\n",
        "  img = img.squeeze().numpy()\n",
        "  return torch.from_numpy(np.tile(img, (3,1,1)))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "s_WzLvOku2NS",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "batch_size=128\n",
        "num_workers=1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "XbHI-HSFu29S",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        },
        "outputId": "73e6d71a-a868-4a0c-a768-4d704f021761"
      },
      "cell_type": "code",
      "source": [
        "mnist_transform=transforms.Compose([\n",
        "                           transforms.ToTensor(),\n",
        "                           transforms.Normalize((0.1307, 0.1307, 0.1307), (0.3081, 0.3081, 0.3081)),\n",
        "                           transforms.Lambda(convertMnist)\n",
        "                       ])\n",
        "\n",
        "mnist_train_set = torchvision.datasets.MNIST(root='./data', train=True, download=True,\n",
        "                                     transform=mnist_transform)\n",
        "mnist_train_loader = torch.utils.data.DataLoader(mnist_train_set, batch_size=batch_size,\n",
        "                                         shuffle=True, num_workers=num_workers)\n",
        "\n",
        "mnist_test_set = torchvision.datasets.MNIST(root='./data', train=False, download=True,\n",
        "                                     transform=mnist_transform)\n",
        "mnist_test_loader = torch.utils.data.DataLoader(mnist_test_set, batch_size=batch_size,\n",
        "                                         shuffle=False, num_workers=num_workers)\n",
        "\n"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz\n",
            "Processing...\n",
            "Done!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "Vmzds_3Qu3xz",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "mnist_m_transform = transforms.Compose([\n",
        "                           transforms.Normalize((0.1307, 0.1307, 0.1307), (0.3081, 0.3081, 0.3081)),\n",
        "                       ])\n",
        "mnist_m_train_set = Mnist_M('drive/My Drive/BScThesis/keras_mnistm.pkl',\n",
        "                        transform=mnist_m_transform)\n",
        "mnist_m_test_set = Mnist_M('drive/My Drive/BScThesis/keras_mnistm.pkl',train=False,\n",
        "                     transform=mnist_m_transform)\n",
        "mnist_m_train_loader = torch.utils.data.DataLoader(mnist_m_train_set, \n",
        "                                                   batch_size=batch_size,\n",
        "                                         shuffle=True, num_workers=num_workers)\n",
        "mnist_m_test_loader = torch.utils.data.DataLoader(mnist_m_test_set,\n",
        "                                                  batch_size=batch_size,\n",
        "                                         shuffle=False, num_workers=num_workers)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "T8RfsNsvu5By",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "merge_train_set = datasets_merge.Merge_Datasets([mnist_train_set, mnist_m_train_set], get_labels=False)\n",
        "merge_test_set = datasets_merge.Merge_Datasets([mnist_test_set, mnist_m_test_set], get_labels=False)\n",
        "merge_train_loader = torch.utils.data.DataLoader(merge_train_set, \n",
        "                                                   batch_size=batch_size,\n",
        "                                         shuffle=True, num_workers=num_workers)\n",
        "merge_test_loader = torch.utils.data.DataLoader(merge_test_set, \n",
        "                                                   batch_size=batch_size,\n",
        "                                         shuffle=True, num_workers=num_workers)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "1MHy8I8Hu58R",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "log_interval = 100"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ifEUlREtu93p",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "use_cuda = True\n",
        "use_cuda = use_cuda and torch.cuda.is_available()\n",
        "device = torch.device(\"cuda\" if use_cuda else \"cpu\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "l0OQ5mAcu-io",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "lr = 0.01\n",
        "momentum=0.9"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "HPDb6hjIu_S4",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "Models = namedtuple('Models', ['model_f', 'model_c', 'model_d'])\n",
        "Optims = namedtuple('Optims', ['optim_f', 'optim_c', 'optim_d'])\n",
        "Criterions = namedtuple('Criterions', ['criterion', 'criterion_domain'])\n",
        "Loaders = namedtuple('Loaders', ['train_loader', 'train_loader_domain',\n",
        "                                'source_test_loader', 'target_test_loader',\n",
        "                                'merged_test_loader'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Z-gYDKa5vAFz",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "loaders = Loaders(mnist_train_loader, mnist_m_train_loader,\n",
        "                 mnist_test_loader, mnist_m_test_loader, merge_test_loader)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "lHCsY8HovAt5",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "criterion = F.nll_loss\n",
        "criterion_domain = F.nll_loss"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "9wjjXEROvBWI",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "criterions = Criterions(criterion, criterion_domain)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ylK_3Q4DvB6p",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "epochs=10"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "4Mzw0Hb-QWrV",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Testing functions"
      ]
    },
    {
      "metadata": {
        "id": "nBztAH3_vDZ5",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def test_grad_function(grad_function, epochs=4):\n",
        "  model_f = models.MnistFeatureExtractor().to(device)\n",
        "  model_c = models.MnistClassPredictor().to(device)\n",
        "  model_d = models.MnistDomain(grad_func = grad_function).to(device)\n",
        "  \n",
        "  optim_f = optim.SGD(model_f.parameters(), lr=lr, momentum=momentum, nesterov=True)\n",
        "  optim_c = optim.SGD(model_c.parameters(), lr=lr, momentum=momentum, nesterov=True)\n",
        "  optim_d = optim.SGD(model_d.parameters(), lr=lr, momentum=momentum, nesterov=True)\n",
        "  \n",
        "  train_models = Models(model_f, model_c, model_d)\n",
        "  optims = Optims(optim_f, optim_c, optim_d)\n",
        "  \n",
        "  model_trainer = trainer.Trainer(train_models, optims, criterions, device,\n",
        "                                train_domain=True, tune_lr=True)\n",
        "  \n",
        "  test_history = defaultdict(lambda:[])\n",
        "  model_trainer.train(4, loaders, test_history=test_history)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "nz1WdAq0AyW7",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "grad_log = []\n",
        "def grlog(x, lambd):\n",
        "  return backward_functions.GRLog(lambd, grad_log)(x)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "aYdkzJIm0356",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "grad_log = []\n",
        "def grinv(x, lambd):\n",
        "  return backward_functions.GRInv(-1, lambd, grad_log)(x)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "8m3UXoes5-e0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 578
        },
        "outputId": "3e6ad416-f755-47c6-a6a2-d2dd4e84e015"
      },
      "cell_type": "code",
      "source": [
        "test_grad_function(grlog)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train Epoch: \t\t\t\t\t1 [0/60000 (0%)]\tLoss: 2.302571, lr: 0.01000 lambd: 0.00000\n",
            "Train Epoch: \t\t\t\t\t1 [12800/60000 (21%)]\tLoss: 0.632978, lr: 0.00726 lambd: 0.26052\n",
            "Train Epoch: \t\t\t\t\t1 [25600/60000 (43%)]\tLoss: 0.591048, lr: 0.00580 lambd: 0.48792\n",
            "Train Epoch: \t\t\t\t\t1 [38400/60000 (64%)]\tLoss: 0.404582, lr: 0.00488 lambd: 0.66404\n",
            "Train Epoch: \t\t\t\t\t1 [51200/60000 (85%)]\tLoss: 0.264745, lr: 0.00425 lambd: 0.78820\n",
            "\n",
            "Target Domain Test set: Average loss: 1.2978, Accuracy: 6271/10000 (63%)\n",
            "Source Domain Test set: Average loss: 0.1211, Accuracy: 9614/10000 (96%)\n",
            "Domains predictor:  Accuracy: 14577/20000 (73%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t2 [0/60000 (0%)]\tLoss: 0.295873, lr: 0.00391 lambd: 0.84828\n",
            "Train Epoch: \t\t\t\t\t2 [12800/60000 (21%)]\tLoss: 0.283074, lr: 0.00351 lambd: 0.90812\n",
            "Train Epoch: \t\t\t\t\t2 [25600/60000 (43%)]\tLoss: 0.267564, lr: 0.00320 lambd: 0.94505\n",
            "Train Epoch: \t\t\t\t\t2 [38400/60000 (64%)]\tLoss: 0.297445, lr: 0.00295 lambd: 0.96740\n",
            "Train Epoch: \t\t\t\t\t2 [51200/60000 (85%)]\tLoss: 0.162265, lr: 0.00273 lambd: 0.98074\n",
            "\n",
            "Target Domain Test set: Average loss: 1.1145, Accuracy: 7147/10000 (71%)\n",
            "Source Domain Test set: Average loss: 0.0864, Accuracy: 9736/10000 (97%)\n",
            "Domains predictor:  Accuracy: 13037/20000 (65%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t3 [0/60000 (0%)]\tLoss: 0.189036, lr: 0.00261 lambd: 0.98661\n",
            "Train Epoch: \t\t\t\t\t3 [12800/60000 (21%)]\tLoss: 0.161942, lr: 0.00245 lambd: 0.99213\n",
            "Train Epoch: \t\t\t\t\t3 [25600/60000 (43%)]\tLoss: 0.078210, lr: 0.00231 lambd: 0.99537\n",
            "Train Epoch: \t\t\t\t\t3 [38400/60000 (64%)]\tLoss: 0.164317, lr: 0.00218 lambd: 0.99728\n",
            "Train Epoch: \t\t\t\t\t3 [51200/60000 (85%)]\tLoss: 0.218790, lr: 0.00208 lambd: 0.99841\n",
            "\n",
            "Target Domain Test set: Average loss: 0.9194, Accuracy: 7466/10000 (75%)\n",
            "Source Domain Test set: Average loss: 0.0728, Accuracy: 9761/10000 (98%)\n",
            "Domains predictor:  Accuracy: 13804/20000 (69%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t4 [0/60000 (0%)]\tLoss: 0.184848, lr: 0.00201 lambd: 0.99889\n",
            "Train Epoch: \t\t\t\t\t4 [12800/60000 (21%)]\tLoss: 0.241879, lr: 0.00192 lambd: 0.99935\n",
            "Train Epoch: \t\t\t\t\t4 [25600/60000 (43%)]\tLoss: 0.165163, lr: 0.00184 lambd: 0.99962\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "njSqcQ6a57q2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 697
        },
        "outputId": "92abfd5d-41e9-4426-f193-54a7ea1c8e31"
      },
      "cell_type": "code",
      "source": [
        "test_grad_function(grinv)"
      ],
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train Epoch: \t\t\t\t\t1 [0/60000 (0%)]\tLoss: 2.318720, lr: 0.01000 lambd: 0.00000\n",
            "Train Epoch: \t\t\t\t\t1 [12800/60000 (21%)]\tLoss: 0.810656, lr: 0.00726 lambd: 0.26052\n",
            "Train Epoch: \t\t\t\t\t1 [25600/60000 (43%)]\tLoss: 0.549511, lr: 0.00580 lambd: 0.48792\n",
            "Train Epoch: \t\t\t\t\t1 [38400/60000 (64%)]\tLoss: 0.457427, lr: 0.00488 lambd: 0.66404\n",
            "Train Epoch: \t\t\t\t\t1 [51200/60000 (85%)]\tLoss: 0.222553, lr: 0.00425 lambd: 0.78820\n",
            "\n",
            "Target Domain Test set: Average loss: 1.7708, Accuracy: 6613/10000 (66%)\n",
            "Source Domain Test set: Average loss: 0.1464, Accuracy: 9558/10000 (96%)\n",
            "Domains predictor:  Accuracy: 10350/20000 (52%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t2 [0/60000 (0%)]\tLoss: 0.371020, lr: 0.00391 lambd: 0.84828\n",
            "Train Epoch: \t\t\t\t\t2 [12800/60000 (21%)]\tLoss: 0.264524, lr: 0.00351 lambd: 0.90812\n",
            "Train Epoch: \t\t\t\t\t2 [25600/60000 (43%)]\tLoss: 0.335705, lr: 0.00320 lambd: 0.94505\n",
            "Train Epoch: \t\t\t\t\t2 [38400/60000 (64%)]\tLoss: 0.398240, lr: 0.00295 lambd: 0.96740\n",
            "Train Epoch: \t\t\t\t\t2 [51200/60000 (85%)]\tLoss: 0.152408, lr: 0.00273 lambd: 0.98074\n",
            "\n",
            "Target Domain Test set: Average loss: 1.3885, Accuracy: 6716/10000 (67%)\n",
            "Source Domain Test set: Average loss: 0.1081, Accuracy: 9662/10000 (97%)\n",
            "Domains predictor:  Accuracy: 13639/20000 (68%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t3 [0/60000 (0%)]\tLoss: 0.273087, lr: 0.00261 lambd: 0.98661\n",
            "Train Epoch: \t\t\t\t\t3 [12800/60000 (21%)]\tLoss: 0.390106, lr: 0.00245 lambd: 0.99213\n",
            "Train Epoch: \t\t\t\t\t3 [25600/60000 (43%)]\tLoss: 0.130011, lr: 0.00231 lambd: 0.99537\n",
            "Train Epoch: \t\t\t\t\t3 [38400/60000 (64%)]\tLoss: 0.231334, lr: 0.00218 lambd: 0.99728\n",
            "Train Epoch: \t\t\t\t\t3 [51200/60000 (85%)]\tLoss: 0.475967, lr: 0.00208 lambd: 0.99841\n",
            "\n",
            "Target Domain Test set: Average loss: 1.0248, Accuracy: 7316/10000 (73%)\n",
            "Source Domain Test set: Average loss: 0.0907, Accuracy: 9712/10000 (97%)\n",
            "Domains predictor:  Accuracy: 12666/20000 (63%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t4 [0/60000 (0%)]\tLoss: 0.357871, lr: 0.00201 lambd: 0.99889\n",
            "Train Epoch: \t\t\t\t\t4 [12800/60000 (21%)]\tLoss: 0.313296, lr: 0.00192 lambd: 0.99935\n",
            "Train Epoch: \t\t\t\t\t4 [25600/60000 (43%)]\tLoss: 0.105079, lr: 0.00184 lambd: 0.99962\n",
            "Train Epoch: \t\t\t\t\t4 [38400/60000 (64%)]\tLoss: 0.172043, lr: 0.00177 lambd: 0.99978\n",
            "Train Epoch: \t\t\t\t\t4 [51200/60000 (85%)]\tLoss: 0.175342, lr: 0.00170 lambd: 0.99987\n",
            "\n",
            "Target Domain Test set: Average loss: 0.9377, Accuracy: 7331/10000 (73%)\n",
            "Source Domain Test set: Average loss: 0.0753, Accuracy: 9746/10000 (97%)\n",
            "Domains predictor:  Accuracy: 14138/20000 (71%)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "4wTnJM3l6R6h",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "grad_log = []\n",
        "def grinv2(x, lambd):\n",
        "  return backward_functions.GRInv(1., lambd, grad_log)(x)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "P81rlicdI53Q",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 697
        },
        "outputId": "69705f59-c0fb-4cd4-d19d-5524f179e606"
      },
      "cell_type": "code",
      "source": [
        "test_grad_function(grinv2)"
      ],
      "execution_count": 95,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train Epoch: \t\t\t\t\t1 [0/60000 (0%)]\tLoss: 2.327919, lr: 0.01000 lambd: 0.00000\n",
            "Train Epoch: \t\t\t\t\t1 [12800/60000 (21%)]\tLoss: 0.745220, lr: 0.00726 lambd: 0.26052\n",
            "Train Epoch: \t\t\t\t\t1 [25600/60000 (43%)]\tLoss: 0.533236, lr: 0.00580 lambd: 0.48792\n",
            "Train Epoch: \t\t\t\t\t1 [38400/60000 (64%)]\tLoss: 0.456913, lr: 0.00488 lambd: 0.66404\n",
            "Train Epoch: \t\t\t\t\t1 [51200/60000 (85%)]\tLoss: 0.288192, lr: 0.00425 lambd: 0.78820\n",
            "\n",
            "Target Domain Test set: Average loss: 2.3159, Accuracy: 3049/10000 (30%)\n",
            "Source Domain Test set: Average loss: 0.0995, Accuracy: 9698/10000 (97%)\n",
            "Domains predictor:  Accuracy: 19797/20000 (99%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t2 [0/60000 (0%)]\tLoss: 0.233389, lr: 0.00391 lambd: 0.84828\n",
            "Train Epoch: \t\t\t\t\t2 [12800/60000 (21%)]\tLoss: 0.372094, lr: 0.00351 lambd: 0.90812\n",
            "Train Epoch: \t\t\t\t\t2 [25600/60000 (43%)]\tLoss: 0.303700, lr: 0.00320 lambd: 0.94505\n",
            "Train Epoch: \t\t\t\t\t2 [38400/60000 (64%)]\tLoss: 0.326173, lr: 0.00295 lambd: 0.96740\n",
            "Train Epoch: \t\t\t\t\t2 [51200/60000 (85%)]\tLoss: 0.291125, lr: 0.00273 lambd: 0.98074\n",
            "\n",
            "Target Domain Test set: Average loss: 2.3855, Accuracy: 2562/10000 (26%)\n",
            "Source Domain Test set: Average loss: 0.0701, Accuracy: 9779/10000 (98%)\n",
            "Domains predictor:  Accuracy: 19952/20000 (100%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t3 [0/60000 (0%)]\tLoss: 0.210821, lr: 0.00261 lambd: 0.98661\n",
            "Train Epoch: \t\t\t\t\t3 [12800/60000 (21%)]\tLoss: 0.246365, lr: 0.00245 lambd: 0.99213\n",
            "Train Epoch: \t\t\t\t\t3 [25600/60000 (43%)]\tLoss: 0.287860, lr: 0.00231 lambd: 0.99537\n",
            "Train Epoch: \t\t\t\t\t3 [38400/60000 (64%)]\tLoss: 0.165999, lr: 0.00218 lambd: 0.99728\n",
            "Train Epoch: \t\t\t\t\t3 [51200/60000 (85%)]\tLoss: 0.073122, lr: 0.00208 lambd: 0.99841\n",
            "\n",
            "Target Domain Test set: Average loss: 2.6827, Accuracy: 2394/10000 (24%)\n",
            "Source Domain Test set: Average loss: 0.0575, Accuracy: 9818/10000 (98%)\n",
            "Domains predictor:  Accuracy: 19988/20000 (100%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t4 [0/60000 (0%)]\tLoss: 0.164869, lr: 0.00201 lambd: 0.99889\n",
            "Train Epoch: \t\t\t\t\t4 [12800/60000 (21%)]\tLoss: 0.075003, lr: 0.00192 lambd: 0.99935\n",
            "Train Epoch: \t\t\t\t\t4 [25600/60000 (43%)]\tLoss: 0.145272, lr: 0.00184 lambd: 0.99962\n",
            "Train Epoch: \t\t\t\t\t4 [38400/60000 (64%)]\tLoss: 0.138908, lr: 0.00177 lambd: 0.99978\n",
            "Train Epoch: \t\t\t\t\t4 [51200/60000 (85%)]\tLoss: 0.106489, lr: 0.00170 lambd: 0.99987\n",
            "\n",
            "Target Domain Test set: Average loss: 2.9522, Accuracy: 2196/10000 (22%)\n",
            "Source Domain Test set: Average loss: 0.0562, Accuracy: 9821/10000 (98%)\n",
            "Domains predictor:  Accuracy: 19993/20000 (100%)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "tH-9P--GI9-P",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "grad_log = []\n",
        "def grandmult(x, lambd):\n",
        "  return backward_functions.GRandMult(1, lambd, grad_log)(x)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "arT7KyRHJC6G",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 697
        },
        "outputId": "7794e424-ce67-43f8-c59c-b8617a1941fc"
      },
      "cell_type": "code",
      "source": [
        "test_grad_function(grandmult)"
      ],
      "execution_count": 97,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train Epoch: \t\t\t\t\t1 [0/60000 (0%)]\tLoss: 2.356206, lr: 0.01000 lambd: 0.00000\n",
            "Train Epoch: \t\t\t\t\t1 [12800/60000 (21%)]\tLoss: 0.634399, lr: 0.00726 lambd: 0.26052\n",
            "Train Epoch: \t\t\t\t\t1 [25600/60000 (43%)]\tLoss: 0.509887, lr: 0.00580 lambd: 0.48792\n",
            "Train Epoch: \t\t\t\t\t1 [38400/60000 (64%)]\tLoss: 0.173120, lr: 0.00488 lambd: 0.66404\n",
            "Train Epoch: \t\t\t\t\t1 [51200/60000 (85%)]\tLoss: 0.351211, lr: 0.00425 lambd: 0.78820\n",
            "\n",
            "Target Domain Test set: Average loss: 1.9298, Accuracy: 3438/10000 (34%)\n",
            "Source Domain Test set: Average loss: 0.1113, Accuracy: 9651/10000 (97%)\n",
            "Domains predictor:  Accuracy: 19679/20000 (98%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t2 [0/60000 (0%)]\tLoss: 0.223685, lr: 0.00391 lambd: 0.84828\n",
            "Train Epoch: \t\t\t\t\t2 [12800/60000 (21%)]\tLoss: 0.170962, lr: 0.00351 lambd: 0.90812\n",
            "Train Epoch: \t\t\t\t\t2 [25600/60000 (43%)]\tLoss: 0.182587, lr: 0.00320 lambd: 0.94505\n",
            "Train Epoch: \t\t\t\t\t2 [38400/60000 (64%)]\tLoss: 0.096928, lr: 0.00295 lambd: 0.96740\n",
            "Train Epoch: \t\t\t\t\t2 [51200/60000 (85%)]\tLoss: 0.161787, lr: 0.00273 lambd: 0.98074\n",
            "\n",
            "Target Domain Test set: Average loss: 2.1723, Accuracy: 2546/10000 (25%)\n",
            "Source Domain Test set: Average loss: 0.0742, Accuracy: 9770/10000 (98%)\n",
            "Domains predictor:  Accuracy: 19886/20000 (99%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t3 [0/60000 (0%)]\tLoss: 0.254556, lr: 0.00261 lambd: 0.98661\n",
            "Train Epoch: \t\t\t\t\t3 [12800/60000 (21%)]\tLoss: 0.233278, lr: 0.00245 lambd: 0.99213\n",
            "Train Epoch: \t\t\t\t\t3 [25600/60000 (43%)]\tLoss: 0.178423, lr: 0.00231 lambd: 0.99537\n",
            "Train Epoch: \t\t\t\t\t3 [38400/60000 (64%)]\tLoss: 0.196384, lr: 0.00218 lambd: 0.99728\n",
            "Train Epoch: \t\t\t\t\t3 [51200/60000 (85%)]\tLoss: 0.183332, lr: 0.00208 lambd: 0.99841\n",
            "\n",
            "Target Domain Test set: Average loss: 2.0319, Accuracy: 2830/10000 (28%)\n",
            "Source Domain Test set: Average loss: 0.0538, Accuracy: 9836/10000 (98%)\n",
            "Domains predictor:  Accuracy: 19911/20000 (100%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t4 [0/60000 (0%)]\tLoss: 0.116799, lr: 0.00201 lambd: 0.99889\n",
            "Train Epoch: \t\t\t\t\t4 [12800/60000 (21%)]\tLoss: 0.159713, lr: 0.00192 lambd: 0.99935\n",
            "Train Epoch: \t\t\t\t\t4 [25600/60000 (43%)]\tLoss: 0.168556, lr: 0.00184 lambd: 0.99962\n",
            "Train Epoch: \t\t\t\t\t4 [38400/60000 (64%)]\tLoss: 0.187594, lr: 0.00177 lambd: 0.99978\n",
            "Train Epoch: \t\t\t\t\t4 [51200/60000 (85%)]\tLoss: 0.108288, lr: 0.00170 lambd: 0.99987\n",
            "\n",
            "Target Domain Test set: Average loss: 2.2315, Accuracy: 2308/10000 (23%)\n",
            "Source Domain Test set: Average loss: 0.0509, Accuracy: 9838/10000 (98%)\n",
            "Domains predictor:  Accuracy: 19960/20000 (100%)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "iaE4TggRCima",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "grad_log = []\n",
        "def grandmult(x, lambd):\n",
        "  return backward_functions.GRandMult(1e-1, lambd, grad_log)(x)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "53M7Cbo4C2Vn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 697
        },
        "outputId": "ede4435e-9b48-4be1-b639-e6ccea41fe75"
      },
      "cell_type": "code",
      "source": [
        "test_grad_function(grandmult)"
      ],
      "execution_count": 92,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train Epoch: \t\t\t\t\t1 [0/60000 (0%)]\tLoss: 2.312164, lr: 0.01000 lambd: 0.00000\n",
            "Train Epoch: \t\t\t\t\t1 [12800/60000 (21%)]\tLoss: 0.551612, lr: 0.00726 lambd: 0.26052\n",
            "Train Epoch: \t\t\t\t\t1 [25600/60000 (43%)]\tLoss: 0.499983, lr: 0.00580 lambd: 0.48792\n",
            "Train Epoch: \t\t\t\t\t1 [38400/60000 (64%)]\tLoss: 0.353211, lr: 0.00488 lambd: 0.66404\n",
            "Train Epoch: \t\t\t\t\t1 [51200/60000 (85%)]\tLoss: 0.277616, lr: 0.00425 lambd: 0.78820\n",
            "\n",
            "Target Domain Test set: Average loss: 1.5431, Accuracy: 4493/10000 (45%)\n",
            "Source Domain Test set: Average loss: 0.1045, Accuracy: 9670/10000 (97%)\n",
            "Domains predictor:  Accuracy: 19372/20000 (97%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t2 [0/60000 (0%)]\tLoss: 0.343850, lr: 0.00391 lambd: 0.84828\n",
            "Train Epoch: \t\t\t\t\t2 [12800/60000 (21%)]\tLoss: 0.236943, lr: 0.00351 lambd: 0.90812\n",
            "Train Epoch: \t\t\t\t\t2 [25600/60000 (43%)]\tLoss: 0.216584, lr: 0.00320 lambd: 0.94505\n",
            "Train Epoch: \t\t\t\t\t2 [38400/60000 (64%)]\tLoss: 0.133211, lr: 0.00295 lambd: 0.96740\n",
            "Train Epoch: \t\t\t\t\t2 [51200/60000 (85%)]\tLoss: 0.202780, lr: 0.00273 lambd: 0.98074\n",
            "\n",
            "Target Domain Test set: Average loss: 1.4694, Accuracy: 4692/10000 (47%)\n",
            "Source Domain Test set: Average loss: 0.0734, Accuracy: 9756/10000 (98%)\n",
            "Domains predictor:  Accuracy: 19642/20000 (98%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t3 [0/60000 (0%)]\tLoss: 0.237773, lr: 0.00261 lambd: 0.98661\n",
            "Train Epoch: \t\t\t\t\t3 [12800/60000 (21%)]\tLoss: 0.107190, lr: 0.00245 lambd: 0.99213\n",
            "Train Epoch: \t\t\t\t\t3 [25600/60000 (43%)]\tLoss: 0.184812, lr: 0.00231 lambd: 0.99537\n",
            "Train Epoch: \t\t\t\t\t3 [38400/60000 (64%)]\tLoss: 0.199738, lr: 0.00218 lambd: 0.99728\n",
            "Train Epoch: \t\t\t\t\t3 [51200/60000 (85%)]\tLoss: 0.153193, lr: 0.00208 lambd: 0.99841\n",
            "\n",
            "Target Domain Test set: Average loss: 1.5249, Accuracy: 4696/10000 (47%)\n",
            "Source Domain Test set: Average loss: 0.0575, Accuracy: 9820/10000 (98%)\n",
            "Domains predictor:  Accuracy: 19755/20000 (99%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t4 [0/60000 (0%)]\tLoss: 0.079153, lr: 0.00201 lambd: 0.99889\n",
            "Train Epoch: \t\t\t\t\t4 [12800/60000 (21%)]\tLoss: 0.149514, lr: 0.00192 lambd: 0.99935\n",
            "Train Epoch: \t\t\t\t\t4 [25600/60000 (43%)]\tLoss: 0.103344, lr: 0.00184 lambd: 0.99962\n",
            "Train Epoch: \t\t\t\t\t4 [38400/60000 (64%)]\tLoss: 0.132674, lr: 0.00177 lambd: 0.99978\n",
            "Train Epoch: \t\t\t\t\t4 [51200/60000 (85%)]\tLoss: 0.108807, lr: 0.00170 lambd: 0.99987\n",
            "\n",
            "Target Domain Test set: Average loss: 1.5512, Accuracy: 4521/10000 (45%)\n",
            "Source Domain Test set: Average loss: 0.0498, Accuracy: 9843/10000 (98%)\n",
            "Domains predictor:  Accuracy: 19818/20000 (99%)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "tnrIWkrCGf2-",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "grad_log = []\n",
        "def grand(x, lambd):\n",
        "  return backward_functions.GRand(1e-3, lambd, grad_log)(x)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "WrG0qreeGvyO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1615
        },
        "outputId": "fa287a82-e098-45c1-c0b1-de7d5a508f41"
      },
      "cell_type": "code",
      "source": [
        "test_grad_function(grand)"
      ],
      "execution_count": 89,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7f1853807358>>\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py\", line 717, in __del__\n",
            "    self._shutdown_workers()\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py\", line 713, in _shutdown_workers\n",
            "    w.join()\n",
            "  File \"/usr/lib/python3.6/multiprocessing/process.py\", line 122, in join\n",
            "    assert self._parent_pid == os.getpid(), 'can only join a child process'\n",
            "AssertionError: can only join a child process\n",
            "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7f185046d3c8>>\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py\", line 717, in __del__\n",
            "    self._shutdown_workers()\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py\", line 713, in _shutdown_workers\n",
            "    w.join()\n",
            "  File \"/usr/lib/python3.6/multiprocessing/process.py\", line 122, in join\n",
            "    assert self._parent_pid == os.getpid(), 'can only join a child process'\n",
            "AssertionError: can only join a child process\n",
            "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7f1853738c50>>\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py\", line 717, in __del__\n",
            "    self._shutdown_workers()\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py\", line 713, in _shutdown_workers\n",
            "    w.join()\n",
            "  File \"/usr/lib/python3.6/multiprocessing/process.py\", line 122, in join\n",
            "    assert self._parent_pid == os.getpid(), 'can only join a child process'\n",
            "AssertionError: can only join a child process\n",
            "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7f1853807358>>\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py\", line 717, in __del__\n",
            "    self._shutdown_workers()\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py\", line 713, in _shutdown_workers\n",
            "    w.join()\n",
            "  File \"/usr/lib/python3.6/multiprocessing/process.py\", line 122, in join\n",
            "    assert self._parent_pid == os.getpid(), 'can only join a child process'\n",
            "AssertionError: can only join a child process\n",
            "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7f185046d3c8>>\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Train Epoch: \t\t\t\t\t1 [0/60000 (0%)]\tLoss: 2.317810, lr: 0.01000 lambd: 0.00000\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py\", line 717, in __del__\n",
            "    self._shutdown_workers()\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py\", line 713, in _shutdown_workers\n",
            "    w.join()\n",
            "  File \"/usr/lib/python3.6/multiprocessing/process.py\", line 122, in join\n",
            "    assert self._parent_pid == os.getpid(), 'can only join a child process'\n",
            "AssertionError: can only join a child process\n",
            "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7f1853738c50>>\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py\", line 717, in __del__\n",
            "    self._shutdown_workers()\n",
            "  File \"/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py\", line 713, in _shutdown_workers\n",
            "    w.join()\n",
            "  File \"/usr/lib/python3.6/multiprocessing/process.py\", line 122, in join\n",
            "    assert self._parent_pid == os.getpid(), 'can only join a child process'\n",
            "AssertionError: can only join a child process\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Train Epoch: \t\t\t\t\t1 [12800/60000 (21%)]\tLoss: 0.712805, lr: 0.00726 lambd: 0.26052\n",
            "Train Epoch: \t\t\t\t\t1 [25600/60000 (43%)]\tLoss: 0.296397, lr: 0.00580 lambd: 0.48792\n",
            "Train Epoch: \t\t\t\t\t1 [38400/60000 (64%)]\tLoss: 0.255301, lr: 0.00488 lambd: 0.66404\n",
            "Train Epoch: \t\t\t\t\t1 [51200/60000 (85%)]\tLoss: 0.488634, lr: 0.00425 lambd: 0.78820\n",
            "\n",
            "Target Domain Test set: Average loss: 1.4624, Accuracy: 5065/10000 (51%)\n",
            "Source Domain Test set: Average loss: 0.1266, Accuracy: 9597/10000 (96%)\n",
            "Domains predictor:  Accuracy: 19248/20000 (96%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t2 [0/60000 (0%)]\tLoss: 0.316768, lr: 0.00391 lambd: 0.84828\n",
            "Train Epoch: \t\t\t\t\t2 [12800/60000 (21%)]\tLoss: 0.188924, lr: 0.00351 lambd: 0.90812\n",
            "Train Epoch: \t\t\t\t\t2 [25600/60000 (43%)]\tLoss: 0.195166, lr: 0.00320 lambd: 0.94505\n",
            "Train Epoch: \t\t\t\t\t2 [38400/60000 (64%)]\tLoss: 0.286140, lr: 0.00295 lambd: 0.96740\n",
            "Train Epoch: \t\t\t\t\t2 [51200/60000 (85%)]\tLoss: 0.112674, lr: 0.00273 lambd: 0.98074\n",
            "\n",
            "Target Domain Test set: Average loss: 1.5594, Accuracy: 4544/10000 (45%)\n",
            "Source Domain Test set: Average loss: 0.1062, Accuracy: 9645/10000 (96%)\n",
            "Domains predictor:  Accuracy: 18857/20000 (94%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t3 [0/60000 (0%)]\tLoss: 0.336886, lr: 0.00261 lambd: 0.98661\n",
            "Train Epoch: \t\t\t\t\t3 [12800/60000 (21%)]\tLoss: 0.251837, lr: 0.00245 lambd: 0.99213\n",
            "Train Epoch: \t\t\t\t\t3 [25600/60000 (43%)]\tLoss: 0.221007, lr: 0.00231 lambd: 0.99537\n",
            "Train Epoch: \t\t\t\t\t3 [38400/60000 (64%)]\tLoss: 0.228762, lr: 0.00218 lambd: 0.99728\n",
            "Train Epoch: \t\t\t\t\t3 [51200/60000 (85%)]\tLoss: 0.464130, lr: 0.00208 lambd: 0.99841\n",
            "\n",
            "Target Domain Test set: Average loss: 1.6057, Accuracy: 4844/10000 (48%)\n",
            "Source Domain Test set: Average loss: 0.0963, Accuracy: 9720/10000 (97%)\n",
            "Domains predictor:  Accuracy: 18671/20000 (93%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t4 [0/60000 (0%)]\tLoss: 0.212811, lr: 0.00201 lambd: 0.99889\n",
            "Train Epoch: \t\t\t\t\t4 [12800/60000 (21%)]\tLoss: 0.214758, lr: 0.00192 lambd: 0.99935\n",
            "Train Epoch: \t\t\t\t\t4 [25600/60000 (43%)]\tLoss: 0.165383, lr: 0.00184 lambd: 0.99962\n",
            "Train Epoch: \t\t\t\t\t4 [38400/60000 (64%)]\tLoss: 0.199856, lr: 0.00177 lambd: 0.99978\n",
            "Train Epoch: \t\t\t\t\t4 [51200/60000 (85%)]\tLoss: 0.125342, lr: 0.00170 lambd: 0.99987\n",
            "\n",
            "Target Domain Test set: Average loss: 1.5942, Accuracy: 4788/10000 (48%)\n",
            "Source Domain Test set: Average loss: 0.0893, Accuracy: 9712/10000 (97%)\n",
            "Domains predictor:  Accuracy: 18776/20000 (94%)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "8T3DaKH3Qe4E",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Gradient Inverse Layer test"
      ]
    },
    {
      "metadata": {
        "id": "79NDJ0UcNJQv",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def grinv(x, lambd):\n",
        "  return backward_functions.GRInv(-1, lambd)(x)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "eCN1QUTWGxWz",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "model_f = models.MnistFeatureExtractor().to(device)\n",
        "model_c = models.MnistClassPredictor().to(device)\n",
        "model_d = models.MnistDomain(grad_func=grinv).to(device)\n",
        "optim_f = optim.SGD(model_f.parameters(), lr=lr, momentum=momentum, nesterov=True)\n",
        "optim_c = optim.SGD(model_c.parameters(), lr=lr, momentum=momentum, nesterov=True)\n",
        "optim_d = optim.SGD(model_d.parameters(), lr=lr, momentum=momentum, nesterov=True)\n",
        "\n",
        "train_models = Models(model_f, model_c, model_d)\n",
        "optims = Optims(optim_f, optim_c, optim_d)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "YKmpjGiXNT72",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "model_trainer = trainer.Trainer(train_models, optims, criterions, device,\n",
        "                                tune_lr=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "tas3a2UjNV0n",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1717
        },
        "outputId": "44c76376-892a-44f5-c69c-cbde9dd54fd4"
      },
      "cell_type": "code",
      "source": [
        "test_history = defaultdict(lambda:[])\n",
        "model_trainer.train(epochs, loaders, test_history=test_history)"
      ],
      "execution_count": 104,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train Epoch: \t\t\t\t\t1 [0/60000 (0%)]\tLoss: 2.312509, lr: 0.01000 lambd: 0.00000\n",
            "Train Epoch: \t\t\t\t\t1 [12800/60000 (21%)]\tLoss: 0.470715, lr: 0.00865 lambd: 0.10626\n",
            "Train Epoch: \t\t\t\t\t1 [25600/60000 (43%)]\tLoss: 0.401973, lr: 0.00766 lambd: 0.21015\n",
            "Train Epoch: \t\t\t\t\t1 [38400/60000 (64%)]\tLoss: 0.208745, lr: 0.00690 lambd: 0.30951\n",
            "Train Epoch: \t\t\t\t\t1 [51200/60000 (85%)]\tLoss: 0.331025, lr: 0.00630 lambd: 0.40253\n",
            "\n",
            "Target Domain Test set: Average loss: 1.2261, Accuracy: 6299/10000 (63%)\n",
            "Source Domain Test set: Average loss: 0.1081, Accuracy: 9680/10000 (97%)\n",
            "Domains predictor:  Accuracy: 14677/20000 (73%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t2 [0/60000 (0%)]\tLoss: 0.371136, lr: 0.00595 lambd: 0.46212\n",
            "Train Epoch: \t\t\t\t\t2 [12800/60000 (21%)]\tLoss: 0.307323, lr: 0.00551 lambd: 0.54178\n",
            "Train Epoch: \t\t\t\t\t2 [25600/60000 (43%)]\tLoss: 0.395909, lr: 0.00514 lambd: 0.61276\n",
            "Train Epoch: \t\t\t\t\t2 [38400/60000 (64%)]\tLoss: 0.271887, lr: 0.00483 lambd: 0.67507\n",
            "Train Epoch: \t\t\t\t\t2 [51200/60000 (85%)]\tLoss: 0.285515, lr: 0.00455 lambd: 0.72904\n",
            "\n",
            "Target Domain Test set: Average loss: 1.2675, Accuracy: 7115/10000 (71%)\n",
            "Source Domain Test set: Average loss: 0.0991, Accuracy: 9695/10000 (97%)\n",
            "Domains predictor:  Accuracy: 13576/20000 (68%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t3 [0/60000 (0%)]\tLoss: 0.158411, lr: 0.00439 lambd: 0.76159\n",
            "Train Epoch: \t\t\t\t\t3 [12800/60000 (21%)]\tLoss: 0.225826, lr: 0.00417 lambd: 0.80288\n",
            "Train Epoch: \t\t\t\t\t3 [25600/60000 (43%)]\tLoss: 0.284502, lr: 0.00397 lambd: 0.83768\n",
            "Train Epoch: \t\t\t\t\t3 [38400/60000 (64%)]\tLoss: 0.182873, lr: 0.00379 lambd: 0.86678\n",
            "Train Epoch: \t\t\t\t\t3 [51200/60000 (85%)]\tLoss: 0.272523, lr: 0.00364 lambd: 0.89098\n",
            "\n",
            "Target Domain Test set: Average loss: 0.9923, Accuracy: 7328/10000 (73%)\n",
            "Source Domain Test set: Average loss: 0.0817, Accuracy: 9736/10000 (97%)\n",
            "Domains predictor:  Accuracy: 12820/20000 (64%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t4 [0/60000 (0%)]\tLoss: 0.145459, lr: 0.00354 lambd: 0.90515\n",
            "Train Epoch: \t\t\t\t\t4 [12800/60000 (21%)]\tLoss: 0.103794, lr: 0.00340 lambd: 0.92267\n",
            "Train Epoch: \t\t\t\t\t4 [25600/60000 (43%)]\tLoss: 0.321543, lr: 0.00328 lambd: 0.93706\n",
            "Train Epoch: \t\t\t\t\t4 [38400/60000 (64%)]\tLoss: 0.179386, lr: 0.00316 lambd: 0.94884\n",
            "Train Epoch: \t\t\t\t\t4 [51200/60000 (85%)]\tLoss: 0.218501, lr: 0.00306 lambd: 0.95846\n",
            "\n",
            "Target Domain Test set: Average loss: 1.0416, Accuracy: 7215/10000 (72%)\n",
            "Source Domain Test set: Average loss: 0.0614, Accuracy: 9807/10000 (98%)\n",
            "Domains predictor:  Accuracy: 13779/20000 (69%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t5 [0/60000 (0%)]\tLoss: 0.169267, lr: 0.00299 lambd: 0.96403\n",
            "Train Epoch: \t\t\t\t\t5 [12800/60000 (21%)]\tLoss: 0.177790, lr: 0.00290 lambd: 0.97084\n",
            "Train Epoch: \t\t\t\t\t5 [25600/60000 (43%)]\tLoss: 0.220318, lr: 0.00281 lambd: 0.97637\n",
            "Train Epoch: \t\t\t\t\t5 [38400/60000 (64%)]\tLoss: 0.137977, lr: 0.00273 lambd: 0.98087\n",
            "Train Epoch: \t\t\t\t\t5 [51200/60000 (85%)]\tLoss: 0.232367, lr: 0.00266 lambd: 0.98452\n",
            "\n",
            "Target Domain Test set: Average loss: 0.9560, Accuracy: 7393/10000 (74%)\n",
            "Source Domain Test set: Average loss: 0.0687, Accuracy: 9778/10000 (98%)\n",
            "Domains predictor:  Accuracy: 14327/20000 (72%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t6 [0/60000 (0%)]\tLoss: 0.200763, lr: 0.00261 lambd: 0.98661\n",
            "Train Epoch: \t\t\t\t\t6 [12800/60000 (21%)]\tLoss: 0.173522, lr: 0.00254 lambd: 0.98917\n",
            "Train Epoch: \t\t\t\t\t6 [25600/60000 (43%)]\tLoss: 0.108857, lr: 0.00248 lambd: 0.99124\n",
            "Train Epoch: \t\t\t\t\t6 [38400/60000 (64%)]\tLoss: 0.126563, lr: 0.00242 lambd: 0.99292\n",
            "Train Epoch: \t\t\t\t\t6 [51200/60000 (85%)]\tLoss: 0.141906, lr: 0.00236 lambd: 0.99428\n",
            "\n",
            "Target Domain Test set: Average loss: 0.8650, Accuracy: 7519/10000 (75%)\n",
            "Source Domain Test set: Average loss: 0.0599, Accuracy: 9809/10000 (98%)\n",
            "Domains predictor:  Accuracy: 12617/20000 (63%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t7 [0/60000 (0%)]\tLoss: 0.182191, lr: 0.00232 lambd: 0.99505\n",
            "Train Epoch: \t\t\t\t\t7 [12800/60000 (21%)]\tLoss: 0.201400, lr: 0.00227 lambd: 0.99600\n",
            "Train Epoch: \t\t\t\t\t7 [25600/60000 (43%)]\tLoss: 0.115925, lr: 0.00222 lambd: 0.99677\n",
            "Train Epoch: \t\t\t\t\t7 [38400/60000 (64%)]\tLoss: 0.166766, lr: 0.00218 lambd: 0.99739\n",
            "Train Epoch: \t\t\t\t\t7 [51200/60000 (85%)]\tLoss: 0.189622, lr: 0.00213 lambd: 0.99789\n",
            "\n",
            "Target Domain Test set: Average loss: 0.8678, Accuracy: 7332/10000 (73%)\n",
            "Source Domain Test set: Average loss: 0.0636, Accuracy: 9796/10000 (98%)\n",
            "Domains predictor:  Accuracy: 13513/20000 (68%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t8 [0/60000 (0%)]\tLoss: 0.107886, lr: 0.00210 lambd: 0.99818\n",
            "Train Epoch: \t\t\t\t\t8 [12800/60000 (21%)]\tLoss: 0.134360, lr: 0.00206 lambd: 0.99853\n",
            "Train Epoch: \t\t\t\t\t8 [25600/60000 (43%)]\tLoss: 0.202102, lr: 0.00202 lambd: 0.99881\n",
            "Train Epoch: \t\t\t\t\t8 [38400/60000 (64%)]\tLoss: 0.237934, lr: 0.00198 lambd: 0.99904\n",
            "Train Epoch: \t\t\t\t\t8 [51200/60000 (85%)]\tLoss: 0.116310, lr: 0.00195 lambd: 0.99922\n",
            "\n",
            "Target Domain Test set: Average loss: 0.8703, Accuracy: 7438/10000 (74%)\n",
            "Source Domain Test set: Average loss: 0.0566, Accuracy: 9810/10000 (98%)\n",
            "Domains predictor:  Accuracy: 13463/20000 (67%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t9 [0/60000 (0%)]\tLoss: 0.117980, lr: 0.00192 lambd: 0.99933\n",
            "Train Epoch: \t\t\t\t\t9 [12800/60000 (21%)]\tLoss: 0.236234, lr: 0.00189 lambd: 0.99946\n",
            "Train Epoch: \t\t\t\t\t9 [25600/60000 (43%)]\tLoss: 0.235230, lr: 0.00186 lambd: 0.99956\n",
            "Train Epoch: \t\t\t\t\t9 [38400/60000 (64%)]\tLoss: 0.136297, lr: 0.00183 lambd: 0.99965\n",
            "Train Epoch: \t\t\t\t\t9 [51200/60000 (85%)]\tLoss: 0.181172, lr: 0.00180 lambd: 0.99971\n",
            "\n",
            "Target Domain Test set: Average loss: 0.7648, Accuracy: 7665/10000 (77%)\n",
            "Source Domain Test set: Average loss: 0.0642, Accuracy: 9814/10000 (98%)\n",
            "Domains predictor:  Accuracy: 13079/20000 (65%)\n",
            "\n",
            "Train Epoch: \t\t\t\t\t10 [0/60000 (0%)]\tLoss: 0.093681, lr: 0.00178 lambd: 0.99975\n",
            "Train Epoch: \t\t\t\t\t10 [12800/60000 (21%)]\tLoss: 0.141431, lr: 0.00175 lambd: 0.99980\n",
            "Train Epoch: \t\t\t\t\t10 [25600/60000 (43%)]\tLoss: 0.117273, lr: 0.00172 lambd: 0.99984\n",
            "Train Epoch: \t\t\t\t\t10 [38400/60000 (64%)]\tLoss: 0.232385, lr: 0.00170 lambd: 0.99987\n",
            "Train Epoch: \t\t\t\t\t10 [51200/60000 (85%)]\tLoss: 0.148965, lr: 0.00167 lambd: 0.99989\n",
            "\n",
            "Target Domain Test set: Average loss: 0.7481, Accuracy: 7762/10000 (78%)\n",
            "Source Domain Test set: Average loss: 0.0525, Accuracy: 9838/10000 (98%)\n",
            "Domains predictor:  Accuracy: 13521/20000 (68%)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "bqwmQnVpNYt-",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}