VGG_16_CIFAR_10.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compressing a VGG-16 trained on CIFAR-10\n",
    "\n",
    "#### To replicate results with pretrained models please download the following models\n",
    "\n",
    "1. vgg16_cifar10.pth\n",
    "2. vgg16_cifar10_small_cup_t_point_9.pth\n",
    "3. vgg16_cifar10_cupSS_K_point2.pth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import sys; sys.argv=['']; del sys\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "plt.tight_layout()\n",
    "\n",
    "import argparse\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "from tensorboardX import SummaryWriter\n",
    "\n",
    "import numpy as np\n",
    "import random\n",
    "import os\n",
    "import time\n",
    "import copy\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = '4'\n",
    "\n",
    "from src.utils import plot_tsne,fancy_dendrogram,save_obj,load_obj\n",
    "from src.model import VGG,load_model\n",
    "from src.prune_model import prune_model\n",
    "from src.cluster_model import cluster_model\n",
    "from src.train_test import train,test,adjust_learning_rate_nips,adjust_learning_rate_iccv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train baseline VGG-16 model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "parser = argparse.ArgumentParser(description='PyTorch MNIST Example')\n",
    "parser.add_argument('--batch-size', type=int, default=64, metavar='N',\n",
    "                    help='input batch size for training (default: 128)')\n",
    "parser.add_argument('-j', '--workers', default=4, type=int, metavar='N',\n",
    "                    help='number of data loading workers (default: 4)')\n",
    "parser.add_argument('--epochs', default=160, type=int, metavar='N',\n",
    "                    help='number of total epochs to run')\n",
    "parser.add_argument('--lr', '--learning-rate', default=0.1, type=float,\n",
    "                    metavar='LR', help='initial learning rate')\n",
    "parser.add_argument('--momentum', default=0.9, type=float, metavar='M',\n",
    "                    help='momentum')\n",
    "parser.add_argument('--weight-decay', '--wd', default=1e-4, type=float,\n",
    "                    metavar='W', help='weight decay (default: 1e-4)')\n",
    "parser.add_argument('--no-cuda', action='store_true', default=False,\n",
    "                    help='disables CUDA training')\n",
    "parser.add_argument('--seed', type=int, default=12346, metavar='S',\n",
    "                    help='random seed (default: 12346)')\n",
    "parser.add_argument('--num_output', type=int, default=10, metavar='S',\n",
    "                    help='number of classes(default: 10)')\n",
    "parser.add_argument('--log-interval', type=int, default=100, metavar='N',\n",
    "                    help='how many batches to wait before logging training status')\n",
    "parser.add_argument('--checkpoint_path', type=str, default='./checkpoints/vgg16_cifar10.pth', metavar='S',\n",
    "                    help='path to store model training checkpoints')\n",
    "parser.add_argument('--gpu', type=int, default=[5], nargs='+', help='used gpu')\n",
    "\n",
    "args = parser.parse_args()\n",
    "\n",
    "\n",
    "#set device to CPU or GPU\n",
    "args = parser.parse_args()\n",
    "\n",
    "use_cuda = not args.no_cuda and torch.cuda.is_available()\n",
    "device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
    "\n",
    "writer = SummaryWriter('logs/vgg_16_cifar10/base_iccv_param/')\n",
    "\n",
    "#set all seeds for reproducability\n",
    "def set_random_seed(seed):    \n",
    "    random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed_all(seed)\n",
    "    np.random.seed(args.seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "\n",
    "set_random_seed(args.seed)\n",
    "\n",
    "\n",
    "kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}\n",
    "\n",
    "normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
    "                                     std=[0.229, 0.224, 0.225])\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "        datasets.CIFAR10(root='data/', train=True, transform=transforms.Compose([\n",
    "            transforms.RandomHorizontalFlip(),\n",
    "            transforms.RandomCrop(32, 4),\n",
    "            transforms.ToTensor(),\n",
    "            normalize,\n",
    "        ]), download=True),\n",
    "        batch_size=args.batch_size, shuffle=True,\n",
    "        num_workers=args.workers, pin_memory=True)\n",
    "\n",
    "val_loader = torch.utils.data.DataLoader(\n",
    "        datasets.CIFAR10(root='data/', train=False, transform=transforms.Compose([\n",
    "            transforms.ToTensor(),\n",
    "            normalize,\n",
    "        ])),\n",
    "        batch_size=args.batch_size, shuffle=False,\n",
    "        num_workers=args.workers, pin_memory=True)\n",
    "\n",
    "vgg16_bn = VGG('VGG16',num_output=args.num_output)\n",
    "vgg16_bn = vgg16_bn.to(device)\n",
    "        \n",
    "optimizer = optim.SGD(vgg16_bn.parameters(),lr=args.lr,momentum=args.momentum,weight_decay=args.weight_decay,nesterov=False)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading state from epoch 152 and test loss 0.3201139727592468\n",
      "original model accuracy is 93.64\n"
     ]
    }
   ],
   "source": [
    "test_loss = 0\n",
    "best_val_acc = 0\n",
    "\n",
    "if not os.path.isfile(args.checkpoint_path):\n",
    "    for epoch in range(1, args.epochs + 1):\n",
    "        start = time.time()\n",
    "        args.lr = adjust_learning_rate_iccv(args,optimizer,epoch)\n",
    "        train_loss,train_acc = train(args,vgg16_bn,device,train_loader,optimizer,epoch)\n",
    "        val_loss,val_acc = test(args,vgg16_bn,device,val_loader)\n",
    "\n",
    "        writer.add_scalars('base_model/loss',{'train_loss': train_loss,\n",
    "                                        'val_loss' : val_loss}, epoch)\n",
    "        writer.add_scalars('base_model/accuracy',{'train_acc': train_acc,\n",
    "                                            'val_acc' : val_acc}, epoch)    \n",
    "\n",
    "        print('Time taken for epoch : {}\\n'.format(time.time()-start))\n",
    "\n",
    "        if val_acc > best_val_acc: \n",
    "            best_val_acc = val_acc\n",
    "\n",
    "            torch.save({\n",
    "                        'epoch': epoch,\n",
    "                        'model_state_dict': vgg16_bn.state_dict(),\n",
    "                        'optimizer_state_dict': optimizer.state_dict(),\n",
    "                        'loss': val_loss,\n",
    "                        }, args.checkpoint_path, pickle_protocol=4)\n",
    "else:\n",
    "    vgg16_bn,optimizer = load_model('vgg16_bn','sgd',args)\n",
    "    orig_loss, orig_acc = test(args, vgg16_bn, device, val_loader,verbose=False)\n",
    "    best_val_acc = orig_acc\n",
    "    \n",
    "print('original model accuracy is {}'.format(best_val_acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "### Compress using CUP (T = 0.9)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load pre-trained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading state from epoch 152 and test loss 0.3201139727592468\n",
      "\n",
      "Test set: Average loss: 0.3201, Accuracy: 9364/10000 (94%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "set_random_seed(args.seed)\n",
    "vgg16_bn,optimizer = load_model('vgg16_bn','sgd',args)\n",
    "orig_loss, orig_acc = test(args, vgg16_bn, device, val_loader)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Prune using T=0.9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test set: Average loss: 2.3457, Accuracy: 1000/10000 (10%)\n",
      "\n",
      "Original accuracy 93.64, compressed model accuracy 93.55, accuracy drop 0.09000000000000341\n"
     ]
    }
   ],
   "source": [
    "cluster_args = {\n",
    "    'cluster_layers' : {0:0,3:0,7:0,10:0,14:0,17:0,20:0,24:0,27:0,30:0,34:0,37:0,40:0},\n",
    "    'conv_feature_size' : 1,\n",
    "    'reshape_exists' : True,\n",
    "    'features' : 'both',\n",
    "    'channel_reduction' : 'fro',\n",
    "    'use_bias' : True,\n",
    "    'linkage_method' : 'ward',\n",
    "    'distance_metric' : 'euclidean',\n",
    "    'cluster_criterion' : 'hierarchical',\n",
    "    'distance_threshold' : 0.90,\n",
    "    'merge_criterion' : 'max_l2_norm',    \n",
    "    'verbose' : False\n",
    "}\n",
    "\n",
    "path = args.checkpoint_path[:-4] + '_small_cup_t_point_9.pth' \n",
    "model_modifier = cluster_model(vgg16_bn,cluster_args)\n",
    "vgg16_bn_clustered = model_modifier.cluster_model()#[int(nodes*drop_percentage) for nodes in [500,300]])\n",
    "vgg16_bn_clustered.cuda()\n",
    "\n",
    "val_loss_no_retrain, val_accuracy_no_retrain = test(args, vgg16_bn_clustered, device, val_loader)\n",
    "\n",
    "args.epochs = 160\n",
    "args.lr = 0.1\n",
    "\n",
    "optimizer = optim.SGD(vgg16_bn_clustered.parameters(),lr=args.lr,momentum=args.momentum,weight_decay=args.weight_decay,nesterov=False)\n",
    "\n",
    "best_val_acc = 0\n",
    "\n",
    "if not os.path.isfile(path):\n",
    "    for epoch in range(1, args.epochs+1):\n",
    "        args.lr = adjust_learning_rate_iccv(args,optimizer,epoch)\n",
    "        start = time.time()\n",
    "        train_loss,train_acc = train(args,vgg16_bn_clustered,device,train_loader,optimizer,epoch)\n",
    "        val_loss,val_acc = test(args,vgg16_bn_clustered,device,val_loader)\n",
    "\n",
    "        if val_acc > best_val_acc:  \n",
    "                torch.save(vgg16_bn_clustered, path, pickle_protocol=4)            \n",
    "                best_val_acc = val_acc       \n",
    "\n",
    "        writer.add_scalars('clustered/0_point_90_loss',{'train_loss': train_loss,\n",
    "                                        'val_loss' : val_loss}, epoch)\n",
    "        writer.add_scalars('clustered/0_point_90_accuracy',{'train_acc': train_acc,\n",
    "                                            'val_acc' : val_acc}, epoch) \n",
    "        print('Time taken for epoch : {}\\n'.format(time.time()-start))\n",
    "else:\n",
    "    vgg16_bn_clustered = torch.load(path)\n",
    "    val_loss, val_acc = test(args, vgg16_bn_clustered, device, val_loader, verbose=False)\n",
    "    best_val_acc = val_acc\n",
    "    \n",
    "print('Original accuracy {}, compressed model accuracy {}, accuracy drop {}'.format(orig_acc,best_val_acc,orig_acc-best_val_acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### calculate #params and #flops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- stats for original model ---\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "--- stats for compressed model ---\n",
      "  + Number of params: 0.91M\n",
      "  + Number of FLOPs: 0.21G\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "214583746.0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from src.compute_flops import print_model_param_nums,print_model_param_flops\n",
    "\n",
    "print('--- stats for original model ---')\n",
    "print_model_param_nums(vgg16_bn)\n",
    "print_model_param_flops(vgg16_bn.cpu(),input_res=32)\n",
    "\n",
    "print('--- stats for compressed model ---')\n",
    "print_model_param_nums(vgg16_bn_clustered)\n",
    "print_model_param_flops(vgg16_bn_clustered.cpu(),input_res=32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compress using CUP (T = 0.95)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load pre-trained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading state from epoch 152 and test loss 0.3201139727592468\n",
      "\n",
      "Test set: Average loss: 0.3201, Accuracy: 9364/10000 (94%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "set_random_seed(args.seed)\n",
    "vgg16_bn,optimizer = load_model('vgg16_bn','sgd',args)\n",
    "orig_loss, orig_acc = test(args, vgg16_bn, device, val_loader)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Compress using CUP with T = 0.95"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test set: Average loss: 2.3335, Accuracy: 1000/10000 (10%)\n",
      "\n",
      "Original accuracy 93.64, compressed model accuracy 92.94, accuracy drop 0.7000000000000028\n"
     ]
    }
   ],
   "source": [
    "cluster_args = {\n",
    "    'cluster_layers' : {0:0,3:0,7:0,10:0,14:0,17:0,20:0,24:0,27:0,30:0,34:0,37:0,40:0},\n",
    "    'conv_feature_size' : 1,\n",
    "    'reshape_exists' : True,\n",
    "    'features' : 'both',\n",
    "    'channel_reduction' : 'fro',\n",
    "    'use_bias' : True,\n",
    "    'linkage_method' : 'ward',\n",
    "    'distance_metric' : 'euclidean',\n",
    "    'cluster_criterion' : 'hierarchical',\n",
    "    'distance_threshold' : 0.95,\n",
    "    'merge_criterion' : 'max_l2_norm',    \n",
    "    'verbose' : False\n",
    "}\n",
    "\n",
    "path = args.checkpoint_path[:-4] + '_small_cup_t_point_95.pth' \n",
    "model_modifier = cluster_model(vgg16_bn,cluster_args)\n",
    "vgg16_bn_clustered = model_modifier.cluster_model()#[int(nodes*drop_percentage) for nodes in [500,300]])\n",
    "vgg16_bn_clustered.cuda()\n",
    "\n",
    "val_loss_no_retrain, val_accuracy_no_retrain = test(args, vgg16_bn_clustered, device, val_loader)\n",
    "\n",
    "args.epochs = 160\n",
    "args.lr = 0.1\n",
    "\n",
    "optimizer = optim.SGD(vgg16_bn_clustered.parameters(),lr=args.lr,momentum=args.momentum,weight_decay=args.weight_decay,nesterov=False)\n",
    "\n",
    "best_val_acc = 0\n",
    "\n",
    "if not os.path.isfile(path):\n",
    "    for epoch in range(1, args.epochs+1):\n",
    "        args.lr = adjust_learning_rate_iccv(args,optimizer,epoch)\n",
    "        start = time.time()\n",
    "        train_loss,train_acc = train(args,vgg16_bn_clustered,device,train_loader,optimizer,epoch)\n",
    "        val_loss,val_acc = test(args,vgg16_bn_clustered,device,val_loader)\n",
    "\n",
    "        if val_acc > best_val_acc:  \n",
    "                torch.save(vgg16_bn_clustered, path, pickle_protocol=4)            \n",
    "                best_val_acc = val_acc       \n",
    "\n",
    "        writer.add_scalars('clustered/0_point_95_loss',{'train_loss': train_loss,\n",
    "                                        'val_loss' : val_loss}, epoch)\n",
    "        writer.add_scalars('clustered/0_point_95_accuracy',{'train_acc': train_acc,\n",
    "                                            'val_acc' : val_acc}, epoch) \n",
    "        print('Time taken for epoch : {}\\n'.format(time.time()-start))\n",
    "else:\n",
    "    vgg16_bn_clustered = torch.load(path)\n",
    "    val_loss, val_acc = test(args, vgg16_bn_clustered, device, val_loader, verbose=False)\n",
    "    best_val_acc = val_acc\n",
    "    \n",
    "print('Original accuracy {}, compressed model accuracy {}, accuracy drop {}'.format(orig_acc,best_val_acc,orig_acc-best_val_acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Calculate #params and #flops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- stats for original model ---\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "--- stats for compressed model ---\n",
      "  + Number of params: 0.63M\n",
      "  + Number of FLOPs: 0.17G\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "171726280.0"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from src.compute_flops import print_model_param_nums,print_model_param_flops\n",
    "\n",
    "print('--- stats for original model ---')\n",
    "print_model_param_nums(vgg16_bn)\n",
    "print_model_param_flops(vgg16_bn.cpu(),input_res=32)\n",
    "\n",
    "print('--- stats for compressed model ---')\n",
    "print_model_param_nums(vgg16_bn_clustered)\n",
    "print_model_param_flops(vgg16_bn_clustered.cpu(),input_res=32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get execution time versus T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test torch.Size([64, 3]) torch.Size([64, 64])\n",
      "test torch.Size([64, 64]) torch.Size([64, 128])\n",
      "test torch.Size([128, 64]) torch.Size([128, 128])\n",
      "test torch.Size([128, 128]) torch.Size([128, 256])\n",
      "test torch.Size([256, 128]) torch.Size([256, 256])\n",
      "test torch.Size([256, 256]) torch.Size([256, 256])\n",
      "test torch.Size([256, 256]) torch.Size([256, 512])\n",
      "test torch.Size([512, 256]) torch.Size([512, 512])\n",
      "test torch.Size([512, 512]) torch.Size([512, 512])\n",
      "test torch.Size([512, 512]) torch.Size([512, 512])\n",
      "test torch.Size([512, 512]) torch.Size([512, 512])\n",
      "test torch.Size([512, 512]) torch.Size([512, 512])\n",
      "\n",
      "Test set: Average loss: 0.3201, Accuracy: 9364/10000 (94%)\n",
      "\n",
      "Time taken for epoch : 205.01955676078796\n",
      "\n",
      "  + Number of params: 14.73M\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "  + Number of FLOPs: 0.63G\n"
     ]
    }
   ],
   "source": [
    "from compute_flops import print_model_param_nums,print_model_param_flops\n",
    "\n",
    "device = torch.device(\"cpu\")\n",
    "\n",
    "T_values = [0]\n",
    "\n",
    "for T in T_values:\n",
    "    \n",
    "    cluster_args['distance_threshold'] = T\n",
    "    \n",
    "    model_modifier = cluster_model(vgg16_bn,cluster_args)\n",
    "    vgg16_bn_clustered = model_modifier.cluster_model()\n",
    "    if device == torch.device(\"cuda\"):\n",
    "        vgg16_bn_clustered.cuda()\n",
    "    \n",
    "    args.epochs = 1\n",
    "    args.lr = 0.1\n",
    "    \n",
    "    optimizer = optim.SGD(vgg16_bn_clustered.parameters(),lr=args.lr,momentum=args.momentum,weight_decay=args.weight_decay,nesterov=False)\n",
    "\n",
    "    \n",
    "    start = time.time()\n",
    "    val_loss,val_acc = test(args,vgg16_bn_clustered,device,val_loader)        \n",
    "        \n",
    "       \n",
    "    print('Time taken for epoch : {}\\n'.format(time.time()-start))\n",
    "    \n",
    "   \n",
    "    print_model_param_nums(vgg16_bn)\n",
    "    print_model_param_nums(vgg16_bn_clustered)\n",
    "\n",
    "    print_model_param_flops(vgg16_bn.cpu(),input_res=32)\n",
    "    print_model_param_flops(vgg16_bn_clustered.cpu(),input_res=32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Try pruning alongside training the model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### CUP-SS (K = 0.02)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.compute_flops import print_model_param_nums,print_model_param_flops\n",
    "\n",
    "set_random_seed(args.seed)\n",
    "vgg16_bn_clustered = VGG('VGG16',num_output=args.num_output)\n",
    "vgg16_bn_clustered = vgg16_bn_clustered.to(device)\n",
    "\n",
    "cluster_args = {\n",
    "    'cluster_layers' : {0:0,3:0,7:0,10:0,14:0,17:0,20:0,24:0,27:0,30:0,34:0,37:0,40:0},\n",
    "    'conv_feature_size' : 1,\n",
    "    'reshape_exists' : True,\n",
    "    'features' : 'both',\n",
    "    'channel_reduction' : 'fro',\n",
    "    'use_bias' : True,\n",
    "    'linkage_method' : 'ward',\n",
    "    'distance_metric' : 'euclidean',\n",
    "    'cluster_criterion' : 'hierarchical',\n",
    "    'distance_threshold' : 0.95,\n",
    "    'merge_criterion' : 'max_l2_norm',    \n",
    "    'verbose' : False\n",
    "}\n",
    "\n",
    "path = args.checkpoint_path[:-4] + '_cupSS_K_point02.pth' \n",
    "\n",
    "T_values = {}\n",
    "slope = 0.02\n",
    "\n",
    "for epoch in range(args.epochs+1):\n",
    "    T_values[epoch] = slope * (epoch - 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best accuracy 93.15\n"
     ]
    }
   ],
   "source": [
    "args.epochs = 160\n",
    "args.lr = 0.1\n",
    "best_val_acc = 0\n",
    "flag = True\n",
    "\n",
    "if not os.path.isfile(path):\n",
    "    for epoch in range(1, args.epochs+1):          \n",
    "        if epoch in T_values.keys() and flag:\n",
    "            print('changing T value to {}'.format(T_values[epoch]))\n",
    "            cluster_args['distance_threshold'] = T_values[epoch]        \n",
    "            model_modifier = cluster_model(vgg16_bn_clustered,cluster_args)\n",
    "            vgg16_bn_clustered = model_modifier.cluster_model()\n",
    "            vgg16_bn_clustered.cuda()\n",
    "            optimizer = optim.SGD(vgg16_bn_clustered.parameters(),lr=args.lr,momentum=args.momentum,weight_decay=args.weight_decay,nesterov=False)\n",
    "                        \n",
    "            params = print_model_param_nums(copy.deepcopy(vgg16_bn_clustered))\n",
    "            flops = print_model_param_flops(copy.deepcopy(vgg16_bn_clustered).cpu(),input_res=32)\n",
    "            \n",
    "            if (params/1e6 <= 0.9):\n",
    "                print('stop filter pruning')\n",
    "                flag = False\n",
    "        \n",
    "        args.lr = adjust_learning_rate_iccv(args,optimizer,epoch)\n",
    "        start = time.time()\n",
    "        train_loss,train_acc = train(args,vgg16_bn_clustered,device,train_loader,optimizer,epoch)\n",
    "        val_loss,val_acc = test(args,vgg16_bn_clustered,device,val_loader)\n",
    "\n",
    "        if val_acc > best_val_acc:  \n",
    "                torch.save(vgg16_bn_clustered, path, pickle_protocol=4)            \n",
    "                best_val_acc = val_acc       \n",
    "\n",
    "        writer.add_scalars('vgg16_cifar10_small_cupSS_K_point2/loss',{'train_loss': train_loss,\n",
    "                                        'val_loss' : val_loss}, epoch)\n",
    "        writer.add_scalars('vgg16_cifar10_small_cupSS_K_point2/accuracy',{'train_acc': train_acc,\n",
    "                                            'val_acc' : val_acc}, epoch) \n",
    "        writer.add_scalars('vgg16_cifar10_small_cupSS_K_point2/params',{'params': params}, epoch)\n",
    "        writer.add_scalars('vgg16_cifar10_small_cupSS_K_point2/flops',{'flops': flops}, epoch) \n",
    "        print('Time taken for epoch : {}\\n'.format(time.time()-start))\n",
    "else:\n",
    "    vgg16_bn_clustered = torch.load(path)\n",
    "    val_loss, val_acc = test(args, vgg16_bn_clustered, device, val_loader, verbose=False)\n",
    "    best_val_acc = val_acc\n",
    "    \n",
    "print('Best accuracy {}'.format(best_val_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- stats for original model ---\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "--- stats for compressed model ---\n",
      "  + Number of params: 0.87M\n",
      "  + Number of FLOPs: 0.20G\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "204803302.0"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from src.compute_flops import print_model_param_nums,print_model_param_flops\n",
    "\n",
    "print('--- stats for original model ---')\n",
    "print_model_param_nums(vgg16_bn)\n",
    "print_model_param_flops(vgg16_bn.cpu(),input_res=32)\n",
    "\n",
    "print('--- stats for compressed model ---')\n",
    "print_model_param_nums(vgg16_bn_clustered)\n",
    "print_model_param_flops(vgg16_bn_clustered.cpu(),input_res=32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### CUP-SS (K = 0.035)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.compute_flops import print_model_param_nums,print_model_param_flops\n",
    "\n",
    "set_random_seed(args.seed)\n",
    "vgg16_bn_clustered = VGG('VGG16',num_output=args.num_output)\n",
    "vgg16_bn_clustered = vgg16_bn_clustered.to(device)\n",
    "\n",
    "cluster_args = {\n",
    "    'cluster_layers' : {0:0,3:0,7:0,10:0,14:0,17:0,20:0,24:0,27:0,30:0,34:0,37:0,40:0},\n",
    "    'conv_feature_size' : 1,\n",
    "    'reshape_exists' : True,\n",
    "    'features' : 'both',\n",
    "    'channel_reduction' : 'fro',\n",
    "    'use_bias' : True,\n",
    "    'linkage_method' : 'ward',\n",
    "    'distance_metric' : 'euclidean',\n",
    "    'cluster_criterion' : 'hierarchical',\n",
    "    'distance_threshold' : 0.95,\n",
    "    'merge_criterion' : 'max_l2_norm',    \n",
    "    'verbose' : False\n",
    "}\n",
    "\n",
    "path = args.checkpoint_path[:-4] + '_cupSS_K_point035.pth' \n",
    "\n",
    "T_values = {}\n",
    "slope = 0.035\n",
    "\n",
    "for epoch in range(args.epochs+1):\n",
    "    T_values[epoch] = slope * (epoch - 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best accuracy 93.24\n"
     ]
    }
   ],
   "source": [
    "args.epochs = 160\n",
    "args.lr = 0.1\n",
    "best_val_acc = 0\n",
    "flag = True\n",
    "\n",
    "if not os.path.isfile(path):\n",
    "    for epoch in range(1, args.epochs+1):          \n",
    "        if epoch in T_values.keys() and flag:\n",
    "            print('changing T value to {}'.format(T_values[epoch]))\n",
    "            cluster_args['distance_threshold'] = T_values[epoch]        \n",
    "            model_modifier = cluster_model(vgg16_bn_clustered,cluster_args)\n",
    "            vgg16_bn_clustered = model_modifier.cluster_model()\n",
    "            vgg16_bn_clustered.cuda()\n",
    "            optimizer = optim.SGD(vgg16_bn_clustered.parameters(),lr=args.lr,momentum=args.momentum,weight_decay=args.weight_decay,nesterov=False)\n",
    "                        \n",
    "            params = print_model_param_nums(copy.deepcopy(vgg16_bn_clustered))\n",
    "            flops = print_model_param_flops(copy.deepcopy(vgg16_bn_clustered).cpu(),input_res=32)\n",
    "            \n",
    "            if (params/1e6 <= 0.9):\n",
    "                print('stop filter pruning')\n",
    "                flag = False\n",
    "        \n",
    "        args.lr = adjust_learning_rate_iccv(args,optimizer,epoch)\n",
    "        start = time.time()\n",
    "        train_loss,train_acc = train(args,vgg16_bn_clustered,device,train_loader,optimizer,epoch)\n",
    "        val_loss,val_acc = test(args,vgg16_bn_clustered,device,val_loader)\n",
    "\n",
    "        if val_acc > best_val_acc:  \n",
    "                torch.save(vgg16_bn_clustered, path, pickle_protocol=4)            \n",
    "                best_val_acc = val_acc       \n",
    "\n",
    "        writer.add_scalars('vgg16_cifar10_small_cupSS_K_point035/loss',{'train_loss': train_loss,\n",
    "                                        'val_loss' : val_loss}, epoch)\n",
    "        writer.add_scalars('vgg16_cifar10_small_cupSS_K_point035/accuracy',{'train_acc': train_acc,\n",
    "                                            'val_acc' : val_acc}, epoch) \n",
    "        writer.add_scalars('vgg16_cifar10_small_cupSS_K_point035/params',{'params': params}, epoch)\n",
    "        writer.add_scalars('vgg16_cifar10_small_cupSS_K_point035/flops',{'flops': flops}, epoch) \n",
    "        print('Time taken for epoch : {}\\n'.format(time.time()-start))\n",
    "else:\n",
    "    vgg16_bn_clustered = torch.load(path)\n",
    "    val_loss, val_acc = test(args, vgg16_bn_clustered, device, val_loader, verbose=False)\n",
    "    best_val_acc = val_acc\n",
    "    \n",
    "print('Best accuracy {}'.format(best_val_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- stats for original model ---\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "--- stats for compressed model ---\n",
      "  + Number of params: 0.86M\n",
      "  + Number of FLOPs: 0.20G\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "199400292.0"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from src.compute_flops import print_model_param_nums,print_model_param_flops\n",
    "\n",
    "print('--- stats for original model ---')\n",
    "print_model_param_nums(vgg16_bn)\n",
    "print_model_param_flops(vgg16_bn.cpu(),input_res=32)\n",
    "\n",
    "print('--- stats for compressed model ---')\n",
    "print_model_param_nums(vgg16_bn_clustered)\n",
    "print_model_param_flops(vgg16_bn_clustered.cpu(),input_res=32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run Ablation studies on K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from joblib import Parallel, delayed, wrap_non_picklable_objects\n",
    "from src.compute_flops import print_model_param_nums,print_model_param_flops\n",
    "\n",
    "slopes = [0.005,0.010,0.015,0.02,0.025,0.03,0.035,0.04,0.045,0.05]\n",
    "metrics = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "changing T value to 0.0\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 1 [0/50000 (0%)]\tLoss: 2.443508\n",
      "Train Epoch: 1 [6400/50000 (13%)]\tLoss: 2.260955\n",
      "Train Epoch: 1 [12800/50000 (26%)]\tLoss: 2.251731\n",
      "Train Epoch: 1 [19200/50000 (38%)]\tLoss: 1.990405\n",
      "Train Epoch: 1 [25600/50000 (51%)]\tLoss: 2.111107\n",
      "Train Epoch: 1 [32000/50000 (64%)]\tLoss: 2.069531\n",
      "Train Epoch: 1 [38400/50000 (77%)]\tLoss: 1.839504\n",
      "Train Epoch: 1 [44800/50000 (90%)]\tLoss: 1.543399\n",
      "\n",
      "Test set: Average loss: 1.7962, Accuracy: 3125/10000 (31%)\n",
      "\n",
      "Time taken for epoch : 23.688902616500854\n",
      "\n",
      "changing T value to 0.005\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 2 [0/50000 (0%)]\tLoss: 1.733816\n",
      "Train Epoch: 2 [6400/50000 (13%)]\tLoss: 1.833429\n",
      "Train Epoch: 2 [12800/50000 (26%)]\tLoss: 1.595596\n",
      "Train Epoch: 2 [19200/50000 (38%)]\tLoss: 1.501862\n",
      "Train Epoch: 2 [25600/50000 (51%)]\tLoss: 1.613213\n",
      "Train Epoch: 2 [32000/50000 (64%)]\tLoss: 1.662730\n",
      "Train Epoch: 2 [38400/50000 (77%)]\tLoss: 1.263088\n",
      "Train Epoch: 2 [44800/50000 (90%)]\tLoss: 1.278486\n",
      "\n",
      "Test set: Average loss: 1.2761, Accuracy: 5280/10000 (53%)\n",
      "\n",
      "Time taken for epoch : 23.266464233398438\n",
      "\n",
      "changing T value to 0.01\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 3 [0/50000 (0%)]\tLoss: 1.219441\n",
      "Train Epoch: 3 [6400/50000 (13%)]\tLoss: 1.076018\n",
      "Train Epoch: 3 [12800/50000 (26%)]\tLoss: 1.206103\n",
      "Train Epoch: 3 [19200/50000 (38%)]\tLoss: 1.397383\n",
      "Train Epoch: 3 [25600/50000 (51%)]\tLoss: 1.502324\n",
      "Train Epoch: 3 [32000/50000 (64%)]\tLoss: 1.036352\n",
      "Train Epoch: 3 [38400/50000 (77%)]\tLoss: 1.231936\n",
      "Train Epoch: 3 [44800/50000 (90%)]\tLoss: 0.886776\n",
      "\n",
      "Test set: Average loss: 1.0013, Accuracy: 6473/10000 (65%)\n",
      "\n",
      "Time taken for epoch : 23.438434839248657\n",
      "\n",
      "changing T value to 0.015\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 4 [0/50000 (0%)]\tLoss: 1.207456\n",
      "Train Epoch: 4 [6400/50000 (13%)]\tLoss: 1.172813\n",
      "Train Epoch: 4 [12800/50000 (26%)]\tLoss: 1.045074\n",
      "Train Epoch: 4 [19200/50000 (38%)]\tLoss: 0.678464\n",
      "Train Epoch: 4 [25600/50000 (51%)]\tLoss: 0.799135\n",
      "Train Epoch: 4 [32000/50000 (64%)]\tLoss: 0.791394\n",
      "Train Epoch: 4 [38400/50000 (77%)]\tLoss: 0.650092\n",
      "Train Epoch: 4 [44800/50000 (90%)]\tLoss: 1.037316\n",
      "\n",
      "Test set: Average loss: 1.0297, Accuracy: 6521/10000 (65%)\n",
      "\n",
      "Time taken for epoch : 23.64414954185486\n",
      "\n",
      "changing T value to 0.02\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 5 [0/50000 (0%)]\tLoss: 0.712438\n",
      "Train Epoch: 5 [6400/50000 (13%)]\tLoss: 0.824609\n",
      "Train Epoch: 5 [12800/50000 (26%)]\tLoss: 0.743877\n",
      "Train Epoch: 5 [19200/50000 (38%)]\tLoss: 0.639310\n",
      "Train Epoch: 5 [25600/50000 (51%)]\tLoss: 0.725990\n",
      "Train Epoch: 5 [32000/50000 (64%)]\tLoss: 0.842255\n",
      "Train Epoch: 5 [38400/50000 (77%)]\tLoss: 0.689120\n",
      "Train Epoch: 5 [44800/50000 (90%)]\tLoss: 0.908393\n",
      "\n",
      "Test set: Average loss: 0.8783, Accuracy: 7058/10000 (71%)\n",
      "\n",
      "Time taken for epoch : 23.21260643005371\n",
      "\n",
      "changing T value to 0.025\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 6 [0/50000 (0%)]\tLoss: 0.757933\n",
      "Train Epoch: 6 [6400/50000 (13%)]\tLoss: 0.724827\n",
      "Train Epoch: 6 [12800/50000 (26%)]\tLoss: 0.808995\n",
      "Train Epoch: 6 [19200/50000 (38%)]\tLoss: 0.673538\n",
      "Train Epoch: 6 [25600/50000 (51%)]\tLoss: 0.712111\n",
      "Train Epoch: 6 [32000/50000 (64%)]\tLoss: 0.504620\n",
      "Train Epoch: 6 [38400/50000 (77%)]\tLoss: 0.578694\n",
      "Train Epoch: 6 [44800/50000 (90%)]\tLoss: 0.677296\n",
      "\n",
      "Test set: Average loss: 0.7683, Accuracy: 7435/10000 (74%)\n",
      "\n",
      "Time taken for epoch : 23.403637409210205\n",
      "\n",
      "changing T value to 0.03\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 7 [0/50000 (0%)]\tLoss: 0.558249\n",
      "Train Epoch: 7 [6400/50000 (13%)]\tLoss: 0.752271\n",
      "Train Epoch: 7 [12800/50000 (26%)]\tLoss: 0.638111\n",
      "Train Epoch: 7 [19200/50000 (38%)]\tLoss: 0.514772\n",
      "Train Epoch: 7 [25600/50000 (51%)]\tLoss: 0.637916\n",
      "Train Epoch: 7 [32000/50000 (64%)]\tLoss: 0.683136\n",
      "Train Epoch: 7 [38400/50000 (77%)]\tLoss: 0.598730\n",
      "Train Epoch: 7 [44800/50000 (90%)]\tLoss: 0.587872\n",
      "\n",
      "Test set: Average loss: 0.8077, Accuracy: 7378/10000 (74%)\n",
      "\n",
      "Time taken for epoch : 21.181530714035034\n",
      "\n",
      "changing T value to 0.035\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 8 [0/50000 (0%)]\tLoss: 0.828549\n",
      "Train Epoch: 8 [6400/50000 (13%)]\tLoss: 0.528166\n",
      "Train Epoch: 8 [12800/50000 (26%)]\tLoss: 0.592211\n",
      "Train Epoch: 8 [19200/50000 (38%)]\tLoss: 0.407185\n",
      "Train Epoch: 8 [25600/50000 (51%)]\tLoss: 0.760300\n",
      "Train Epoch: 8 [32000/50000 (64%)]\tLoss: 0.612696\n",
      "Train Epoch: 8 [38400/50000 (77%)]\tLoss: 0.728544\n",
      "Train Epoch: 8 [44800/50000 (90%)]\tLoss: 0.699615\n",
      "\n",
      "Test set: Average loss: 0.6445, Accuracy: 7854/10000 (79%)\n",
      "\n",
      "Time taken for epoch : 23.748313665390015\n",
      "\n",
      "changing T value to 0.04\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 9 [0/50000 (0%)]\tLoss: 0.632457\n",
      "Train Epoch: 9 [6400/50000 (13%)]\tLoss: 0.755933\n",
      "Train Epoch: 9 [12800/50000 (26%)]\tLoss: 0.686469\n",
      "Train Epoch: 9 [19200/50000 (38%)]\tLoss: 0.414108\n",
      "Train Epoch: 9 [25600/50000 (51%)]\tLoss: 0.551906\n",
      "Train Epoch: 9 [32000/50000 (64%)]\tLoss: 0.358694\n",
      "Train Epoch: 9 [38400/50000 (77%)]\tLoss: 0.402664\n",
      "Train Epoch: 9 [44800/50000 (90%)]\tLoss: 0.564057\n",
      "\n",
      "Test set: Average loss: 0.5710, Accuracy: 8155/10000 (82%)\n",
      "\n",
      "Time taken for epoch : 23.983129501342773\n",
      "\n",
      "changing T value to 0.045\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 10 [0/50000 (0%)]\tLoss: 0.673575\n",
      "Train Epoch: 10 [6400/50000 (13%)]\tLoss: 0.621606\n",
      "Train Epoch: 10 [12800/50000 (26%)]\tLoss: 0.497783\n",
      "Train Epoch: 10 [19200/50000 (38%)]\tLoss: 0.636915\n",
      "Train Epoch: 10 [25600/50000 (51%)]\tLoss: 0.561688\n",
      "Train Epoch: 10 [32000/50000 (64%)]\tLoss: 0.430808\n",
      "Train Epoch: 10 [38400/50000 (77%)]\tLoss: 0.437550\n",
      "Train Epoch: 10 [44800/50000 (90%)]\tLoss: 0.423739\n",
      "\n",
      "Test set: Average loss: 0.5821, Accuracy: 8122/10000 (81%)\n",
      "\n",
      "Time taken for epoch : 21.053393125534058\n",
      "\n",
      "changing T value to 0.05\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 11 [0/50000 (0%)]\tLoss: 0.579760\n",
      "Train Epoch: 11 [6400/50000 (13%)]\tLoss: 0.425478\n",
      "Train Epoch: 11 [12800/50000 (26%)]\tLoss: 0.697174\n",
      "Train Epoch: 11 [19200/50000 (38%)]\tLoss: 0.504442\n",
      "Train Epoch: 11 [25600/50000 (51%)]\tLoss: 0.749582\n",
      "Train Epoch: 11 [32000/50000 (64%)]\tLoss: 0.576172\n",
      "Train Epoch: 11 [38400/50000 (77%)]\tLoss: 0.517303\n",
      "Train Epoch: 11 [44800/50000 (90%)]\tLoss: 0.471195\n",
      "\n",
      "Test set: Average loss: 0.5738, Accuracy: 8056/10000 (81%)\n",
      "\n",
      "Time taken for epoch : 21.051047325134277\n",
      "\n",
      "changing T value to 0.055\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 12 [0/50000 (0%)]\tLoss: 0.421539\n",
      "Train Epoch: 12 [6400/50000 (13%)]\tLoss: 0.453289\n",
      "Train Epoch: 12 [12800/50000 (26%)]\tLoss: 0.348544\n",
      "Train Epoch: 12 [19200/50000 (38%)]\tLoss: 0.286096\n",
      "Train Epoch: 12 [25600/50000 (51%)]\tLoss: 0.670627\n",
      "Train Epoch: 12 [32000/50000 (64%)]\tLoss: 0.452027\n",
      "Train Epoch: 12 [38400/50000 (77%)]\tLoss: 0.491687\n",
      "Train Epoch: 12 [44800/50000 (90%)]\tLoss: 0.549055\n",
      "\n",
      "Test set: Average loss: 0.6190, Accuracy: 8065/10000 (81%)\n",
      "\n",
      "Time taken for epoch : 21.07546615600586\n",
      "\n",
      "changing T value to 0.06\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 13 [0/50000 (0%)]\tLoss: 0.506843\n",
      "Train Epoch: 13 [6400/50000 (13%)]\tLoss: 0.274806\n",
      "Train Epoch: 13 [12800/50000 (26%)]\tLoss: 0.501148\n",
      "Train Epoch: 13 [19200/50000 (38%)]\tLoss: 0.337362\n",
      "Train Epoch: 13 [25600/50000 (51%)]\tLoss: 0.450214\n",
      "Train Epoch: 13 [32000/50000 (64%)]\tLoss: 0.461584\n",
      "Train Epoch: 13 [38400/50000 (77%)]\tLoss: 0.343834\n",
      "Train Epoch: 13 [44800/50000 (90%)]\tLoss: 0.441280\n",
      "\n",
      "Test set: Average loss: 0.5034, Accuracy: 8266/10000 (83%)\n",
      "\n",
      "Time taken for epoch : 23.546892166137695\n",
      "\n",
      "changing T value to 0.065\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 14 [0/50000 (0%)]\tLoss: 0.493883\n",
      "Train Epoch: 14 [6400/50000 (13%)]\tLoss: 0.433645\n",
      "Train Epoch: 14 [12800/50000 (26%)]\tLoss: 0.470091\n",
      "Train Epoch: 14 [19200/50000 (38%)]\tLoss: 0.243466\n",
      "Train Epoch: 14 [25600/50000 (51%)]\tLoss: 0.549322\n",
      "Train Epoch: 14 [32000/50000 (64%)]\tLoss: 0.485480\n",
      "Train Epoch: 14 [38400/50000 (77%)]\tLoss: 0.833367\n",
      "Train Epoch: 14 [44800/50000 (90%)]\tLoss: 0.512234\n",
      "\n",
      "Test set: Average loss: 0.4531, Accuracy: 8466/10000 (85%)\n",
      "\n",
      "Time taken for epoch : 23.332207679748535\n",
      "\n",
      "changing T value to 0.07\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 15 [0/50000 (0%)]\tLoss: 0.574426\n",
      "Train Epoch: 15 [6400/50000 (13%)]\tLoss: 0.467998\n",
      "Train Epoch: 15 [12800/50000 (26%)]\tLoss: 0.263782\n",
      "Train Epoch: 15 [19200/50000 (38%)]\tLoss: 0.383285\n",
      "Train Epoch: 15 [25600/50000 (51%)]\tLoss: 0.276063\n",
      "Train Epoch: 15 [32000/50000 (64%)]\tLoss: 0.408345\n",
      "Train Epoch: 15 [38400/50000 (77%)]\tLoss: 0.198436\n",
      "Train Epoch: 15 [44800/50000 (90%)]\tLoss: 0.476374\n",
      "\n",
      "Test set: Average loss: 0.4641, Accuracy: 8496/10000 (85%)\n",
      "\n",
      "Time taken for epoch : 23.653558492660522\n",
      "\n",
      "changing T value to 0.075\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 16 [0/50000 (0%)]\tLoss: 0.389648\n",
      "Train Epoch: 16 [6400/50000 (13%)]\tLoss: 0.282125\n",
      "Train Epoch: 16 [12800/50000 (26%)]\tLoss: 0.378985\n",
      "Train Epoch: 16 [19200/50000 (38%)]\tLoss: 0.303575\n",
      "Train Epoch: 16 [25600/50000 (51%)]\tLoss: 0.370752\n",
      "Train Epoch: 16 [32000/50000 (64%)]\tLoss: 0.460292\n",
      "Train Epoch: 16 [38400/50000 (77%)]\tLoss: 0.456103\n",
      "Train Epoch: 16 [44800/50000 (90%)]\tLoss: 0.543738\n",
      "\n",
      "Test set: Average loss: 0.5036, Accuracy: 8340/10000 (83%)\n",
      "\n",
      "Time taken for epoch : 21.09345030784607\n",
      "\n",
      "changing T value to 0.08\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 17 [0/50000 (0%)]\tLoss: 0.442368\n",
      "Train Epoch: 17 [6400/50000 (13%)]\tLoss: 0.339838\n",
      "Train Epoch: 17 [12800/50000 (26%)]\tLoss: 0.474178\n",
      "Train Epoch: 17 [19200/50000 (38%)]\tLoss: 0.517513\n",
      "Train Epoch: 17 [25600/50000 (51%)]\tLoss: 0.331806\n",
      "Train Epoch: 17 [32000/50000 (64%)]\tLoss: 0.498432\n",
      "Train Epoch: 17 [38400/50000 (77%)]\tLoss: 0.446878\n",
      "Train Epoch: 17 [44800/50000 (90%)]\tLoss: 0.387217\n",
      "\n",
      "Test set: Average loss: 0.4837, Accuracy: 8421/10000 (84%)\n",
      "\n",
      "Time taken for epoch : 21.060237407684326\n",
      "\n",
      "changing T value to 0.085\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 18 [0/50000 (0%)]\tLoss: 0.386948\n",
      "Train Epoch: 18 [6400/50000 (13%)]\tLoss: 0.338695\n",
      "Train Epoch: 18 [12800/50000 (26%)]\tLoss: 0.623139\n",
      "Train Epoch: 18 [19200/50000 (38%)]\tLoss: 0.483280\n",
      "Train Epoch: 18 [25600/50000 (51%)]\tLoss: 0.480040\n",
      "Train Epoch: 18 [32000/50000 (64%)]\tLoss: 0.614535\n",
      "Train Epoch: 18 [38400/50000 (77%)]\tLoss: 0.358957\n",
      "Train Epoch: 18 [44800/50000 (90%)]\tLoss: 0.274687\n",
      "\n",
      "Test set: Average loss: 0.4254, Accuracy: 8582/10000 (86%)\n",
      "\n",
      "Time taken for epoch : 23.575181007385254\n",
      "\n",
      "changing T value to 0.09\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 19 [0/50000 (0%)]\tLoss: 0.327024\n",
      "Train Epoch: 19 [6400/50000 (13%)]\tLoss: 0.409375\n",
      "Train Epoch: 19 [12800/50000 (26%)]\tLoss: 0.281848\n",
      "Train Epoch: 19 [19200/50000 (38%)]\tLoss: 0.577531\n",
      "Train Epoch: 19 [25600/50000 (51%)]\tLoss: 0.373231\n",
      "Train Epoch: 19 [32000/50000 (64%)]\tLoss: 0.425387\n",
      "Train Epoch: 19 [38400/50000 (77%)]\tLoss: 0.403228\n",
      "Train Epoch: 19 [44800/50000 (90%)]\tLoss: 0.433499\n",
      "\n",
      "Test set: Average loss: 0.4906, Accuracy: 8440/10000 (84%)\n",
      "\n",
      "Time taken for epoch : 21.126172304153442\n",
      "\n",
      "changing T value to 0.095\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 20 [0/50000 (0%)]\tLoss: 0.316497\n",
      "Train Epoch: 20 [6400/50000 (13%)]\tLoss: 0.278230\n",
      "Train Epoch: 20 [12800/50000 (26%)]\tLoss: 0.258624\n",
      "Train Epoch: 20 [19200/50000 (38%)]\tLoss: 0.335888\n",
      "Train Epoch: 20 [25600/50000 (51%)]\tLoss: 0.526255\n",
      "Train Epoch: 20 [32000/50000 (64%)]\tLoss: 0.334985\n",
      "Train Epoch: 20 [38400/50000 (77%)]\tLoss: 0.371729\n",
      "Train Epoch: 20 [44800/50000 (90%)]\tLoss: 0.286948\n",
      "\n",
      "Test set: Average loss: 0.4376, Accuracy: 8535/10000 (85%)\n",
      "\n",
      "Time taken for epoch : 21.021864414215088\n",
      "\n",
      "changing T value to 0.1\n",
      "  + Number of params: 14.35M\n",
      "  + Number of FLOPs: 0.62G\n",
      "Train Epoch: 21 [0/50000 (0%)]\tLoss: 0.395780\n",
      "Train Epoch: 21 [6400/50000 (13%)]\tLoss: 0.281592\n",
      "Train Epoch: 21 [12800/50000 (26%)]\tLoss: 0.211788\n",
      "Train Epoch: 21 [19200/50000 (38%)]\tLoss: 0.237219\n",
      "Train Epoch: 21 [25600/50000 (51%)]\tLoss: 0.235969\n",
      "Train Epoch: 21 [32000/50000 (64%)]\tLoss: 0.381477\n",
      "Train Epoch: 21 [38400/50000 (77%)]\tLoss: 0.371700\n",
      "Train Epoch: 21 [44800/50000 (90%)]\tLoss: 0.269213\n",
      "\n",
      "Test set: Average loss: 0.4633, Accuracy: 8523/10000 (85%)\n",
      "\n",
      "Time taken for epoch : 21.006037950515747\n",
      "\n",
      "changing T value to 0.105\n",
      "  + Number of params: 13.90M\n",
      "  + Number of FLOPs: 0.61G\n",
      "Train Epoch: 22 [0/50000 (0%)]\tLoss: 0.280469\n",
      "Train Epoch: 22 [6400/50000 (13%)]\tLoss: 0.130713\n",
      "Train Epoch: 22 [12800/50000 (26%)]\tLoss: 0.545742\n",
      "Train Epoch: 22 [19200/50000 (38%)]\tLoss: 0.344439\n",
      "Train Epoch: 22 [25600/50000 (51%)]\tLoss: 0.455099\n",
      "Train Epoch: 22 [32000/50000 (64%)]\tLoss: 0.182311\n",
      "Train Epoch: 22 [38400/50000 (77%)]\tLoss: 0.549683\n",
      "Train Epoch: 22 [44800/50000 (90%)]\tLoss: 0.611736\n",
      "\n",
      "Test set: Average loss: 0.4688, Accuracy: 8431/10000 (84%)\n",
      "\n",
      "Time taken for epoch : 20.629908323287964\n",
      "\n",
      "changing T value to 0.11\n",
      "  + Number of params: 13.59M\n",
      "  + Number of FLOPs: 0.60G\n",
      "Train Epoch: 23 [0/50000 (0%)]\tLoss: 0.251287\n",
      "Train Epoch: 23 [6400/50000 (13%)]\tLoss: 0.258312\n",
      "Train Epoch: 23 [12800/50000 (26%)]\tLoss: 0.280401\n",
      "Train Epoch: 23 [19200/50000 (38%)]\tLoss: 0.234659\n",
      "Train Epoch: 23 [25600/50000 (51%)]\tLoss: 0.405497\n",
      "Train Epoch: 23 [32000/50000 (64%)]\tLoss: 0.467313\n",
      "Train Epoch: 23 [38400/50000 (77%)]\tLoss: 0.417925\n",
      "Train Epoch: 23 [44800/50000 (90%)]\tLoss: 0.422981\n",
      "\n",
      "Test set: Average loss: 0.4346, Accuracy: 8559/10000 (86%)\n",
      "\n",
      "Time taken for epoch : 20.562881231307983\n",
      "\n",
      "changing T value to 0.115\n",
      "  + Number of params: 12.66M\n",
      "  + Number of FLOPs: 0.58G\n",
      "Train Epoch: 24 [0/50000 (0%)]\tLoss: 0.364582\n",
      "Train Epoch: 24 [6400/50000 (13%)]\tLoss: 0.304349\n",
      "Train Epoch: 24 [12800/50000 (26%)]\tLoss: 0.343283\n",
      "Train Epoch: 24 [19200/50000 (38%)]\tLoss: 0.706874\n",
      "Train Epoch: 24 [25600/50000 (51%)]\tLoss: 0.253995\n",
      "Train Epoch: 24 [32000/50000 (64%)]\tLoss: 0.372720\n",
      "Train Epoch: 24 [38400/50000 (77%)]\tLoss: 0.312231\n",
      "Train Epoch: 24 [44800/50000 (90%)]\tLoss: 0.314852\n",
      "\n",
      "Test set: Average loss: 0.5460, Accuracy: 8323/10000 (83%)\n",
      "\n",
      "Time taken for epoch : 20.44442391395569\n",
      "\n",
      "changing T value to 0.12\n",
      "  + Number of params: 11.58M\n",
      "  + Number of FLOPs: 0.56G\n",
      "Train Epoch: 25 [0/50000 (0%)]\tLoss: 0.335744\n",
      "Train Epoch: 25 [6400/50000 (13%)]\tLoss: 0.248156\n",
      "Train Epoch: 25 [12800/50000 (26%)]\tLoss: 0.643582\n",
      "Train Epoch: 25 [19200/50000 (38%)]\tLoss: 0.306744\n",
      "Train Epoch: 25 [25600/50000 (51%)]\tLoss: 0.409466\n",
      "Train Epoch: 25 [32000/50000 (64%)]\tLoss: 0.281217\n",
      "Train Epoch: 25 [38400/50000 (77%)]\tLoss: 0.406306\n",
      "Train Epoch: 25 [44800/50000 (90%)]\tLoss: 0.373198\n",
      "\n",
      "Test set: Average loss: 0.4045, Accuracy: 8661/10000 (87%)\n",
      "\n",
      "Time taken for epoch : 22.14571976661682\n",
      "\n",
      "changing T value to 0.125\n",
      "  + Number of params: 10.73M\n",
      "  + Number of FLOPs: 0.54G\n",
      "Train Epoch: 26 [0/50000 (0%)]\tLoss: 0.363718\n",
      "Train Epoch: 26 [6400/50000 (13%)]\tLoss: 0.233661\n",
      "Train Epoch: 26 [12800/50000 (26%)]\tLoss: 0.282246\n",
      "Train Epoch: 26 [19200/50000 (38%)]\tLoss: 0.404631\n",
      "Train Epoch: 26 [25600/50000 (51%)]\tLoss: 0.306849\n",
      "Train Epoch: 26 [32000/50000 (64%)]\tLoss: 0.202789\n",
      "Train Epoch: 26 [38400/50000 (77%)]\tLoss: 0.248490\n",
      "Train Epoch: 26 [44800/50000 (90%)]\tLoss: 0.275787\n",
      "\n",
      "Test set: Average loss: 0.4539, Accuracy: 8557/10000 (86%)\n",
      "\n",
      "Time taken for epoch : 19.53439712524414\n",
      "\n",
      "changing T value to 0.13\n",
      "  + Number of params: 10.05M\n",
      "  + Number of FLOPs: 0.52G\n",
      "Train Epoch: 27 [0/50000 (0%)]\tLoss: 0.202136\n",
      "Train Epoch: 27 [6400/50000 (13%)]\tLoss: 0.472579\n",
      "Train Epoch: 27 [12800/50000 (26%)]\tLoss: 0.162614\n",
      "Train Epoch: 27 [19200/50000 (38%)]\tLoss: 0.305944\n",
      "Train Epoch: 27 [25600/50000 (51%)]\tLoss: 0.356743\n",
      "Train Epoch: 27 [32000/50000 (64%)]\tLoss: 0.376626\n",
      "Train Epoch: 27 [38400/50000 (77%)]\tLoss: 0.262856\n",
      "Train Epoch: 27 [44800/50000 (90%)]\tLoss: 0.239663\n",
      "\n",
      "Test set: Average loss: 0.4430, Accuracy: 8597/10000 (86%)\n",
      "\n",
      "Time taken for epoch : 19.015876054763794\n",
      "\n",
      "changing T value to 0.135\n",
      "  + Number of params: 9.48M\n",
      "  + Number of FLOPs: 0.51G\n",
      "Train Epoch: 28 [0/50000 (0%)]\tLoss: 0.224118\n",
      "Train Epoch: 28 [6400/50000 (13%)]\tLoss: 0.195039\n",
      "Train Epoch: 28 [12800/50000 (26%)]\tLoss: 0.501474\n",
      "Train Epoch: 28 [19200/50000 (38%)]\tLoss: 0.369220\n",
      "Train Epoch: 28 [25600/50000 (51%)]\tLoss: 0.407730\n",
      "Train Epoch: 28 [32000/50000 (64%)]\tLoss: 0.252996\n",
      "Train Epoch: 28 [38400/50000 (77%)]\tLoss: 0.246476\n",
      "Train Epoch: 28 [44800/50000 (90%)]\tLoss: 0.415457\n",
      "\n",
      "Test set: Average loss: 0.7329, Accuracy: 7736/10000 (77%)\n",
      "\n",
      "Time taken for epoch : 18.607688426971436\n",
      "\n",
      "changing T value to 0.14\n",
      "  + Number of params: 8.98M\n",
      "  + Number of FLOPs: 0.51G\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 29 [0/50000 (0%)]\tLoss: 0.321369\n",
      "Train Epoch: 29 [6400/50000 (13%)]\tLoss: 0.337672\n",
      "Train Epoch: 29 [12800/50000 (26%)]\tLoss: 0.174678\n",
      "Train Epoch: 29 [19200/50000 (38%)]\tLoss: 0.281255\n",
      "Train Epoch: 29 [25600/50000 (51%)]\tLoss: 0.570636\n",
      "Train Epoch: 29 [32000/50000 (64%)]\tLoss: 0.411844\n",
      "Train Epoch: 29 [38400/50000 (77%)]\tLoss: 0.514067\n",
      "Train Epoch: 29 [44800/50000 (90%)]\tLoss: 0.232840\n",
      "\n",
      "Test set: Average loss: 0.4423, Accuracy: 8602/10000 (86%)\n",
      "\n",
      "Time taken for epoch : 18.119868516921997\n",
      "\n",
      "changing T value to 0.145\n",
      "  + Number of params: 8.59M\n",
      "  + Number of FLOPs: 0.50G\n",
      "Train Epoch: 30 [0/50000 (0%)]\tLoss: 0.281020\n",
      "Train Epoch: 30 [6400/50000 (13%)]\tLoss: 0.209687\n",
      "Train Epoch: 30 [12800/50000 (26%)]\tLoss: 0.331443\n",
      "Train Epoch: 30 [19200/50000 (38%)]\tLoss: 0.361232\n",
      "Train Epoch: 30 [25600/50000 (51%)]\tLoss: 0.240579\n",
      "Train Epoch: 30 [32000/50000 (64%)]\tLoss: 0.153424\n",
      "Train Epoch: 30 [38400/50000 (77%)]\tLoss: 0.219642\n",
      "Train Epoch: 30 [44800/50000 (90%)]\tLoss: 0.248688\n",
      "\n",
      "Test set: Average loss: 0.4756, Accuracy: 8533/10000 (85%)\n",
      "\n",
      "Time taken for epoch : 17.99648904800415\n",
      "\n",
      "changing T value to 0.15\n",
      "  + Number of params: 8.11M\n",
      "  + Number of FLOPs: 0.49G\n",
      "Train Epoch: 31 [0/50000 (0%)]\tLoss: 0.295033\n",
      "Train Epoch: 31 [6400/50000 (13%)]\tLoss: 0.249774\n",
      "Train Epoch: 31 [12800/50000 (26%)]\tLoss: 0.125139\n",
      "Train Epoch: 31 [19200/50000 (38%)]\tLoss: 0.520538\n",
      "Train Epoch: 31 [25600/50000 (51%)]\tLoss: 0.164483\n",
      "Train Epoch: 31 [32000/50000 (64%)]\tLoss: 0.347295\n",
      "Train Epoch: 31 [38400/50000 (77%)]\tLoss: 0.178715\n",
      "Train Epoch: 31 [44800/50000 (90%)]\tLoss: 0.428080\n",
      "\n",
      "Test set: Average loss: 0.4423, Accuracy: 8625/10000 (86%)\n",
      "\n",
      "Time taken for epoch : 17.490803718566895\n",
      "\n",
      "changing T value to 0.155\n",
      "  + Number of params: 7.75M\n",
      "  + Number of FLOPs: 0.48G\n",
      "Train Epoch: 32 [0/50000 (0%)]\tLoss: 0.195281\n",
      "Train Epoch: 32 [6400/50000 (13%)]\tLoss: 0.182281\n",
      "Train Epoch: 32 [12800/50000 (26%)]\tLoss: 0.394623\n",
      "Train Epoch: 32 [19200/50000 (38%)]\tLoss: 0.215178\n",
      "Train Epoch: 32 [25600/50000 (51%)]\tLoss: 0.312827\n",
      "Train Epoch: 32 [32000/50000 (64%)]\tLoss: 0.307175\n",
      "Train Epoch: 32 [38400/50000 (77%)]\tLoss: 0.335825\n",
      "Train Epoch: 32 [44800/50000 (90%)]\tLoss: 0.209361\n",
      "\n",
      "Test set: Average loss: 0.3987, Accuracy: 8746/10000 (87%)\n",
      "\n",
      "Time taken for epoch : 19.381008863449097\n",
      "\n",
      "changing T value to 0.16\n",
      "  + Number of params: 7.49M\n",
      "  + Number of FLOPs: 0.48G\n",
      "Train Epoch: 33 [0/50000 (0%)]\tLoss: 0.466464\n",
      "Train Epoch: 33 [6400/50000 (13%)]\tLoss: 0.139397\n",
      "Train Epoch: 33 [12800/50000 (26%)]\tLoss: 0.407588\n",
      "Train Epoch: 33 [19200/50000 (38%)]\tLoss: 0.125484\n",
      "Train Epoch: 33 [25600/50000 (51%)]\tLoss: 0.326026\n",
      "Train Epoch: 33 [32000/50000 (64%)]\tLoss: 0.513860\n",
      "Train Epoch: 33 [38400/50000 (77%)]\tLoss: 0.196335\n",
      "Train Epoch: 33 [44800/50000 (90%)]\tLoss: 0.240078\n",
      "\n",
      "Test set: Average loss: 0.3767, Accuracy: 8729/10000 (87%)\n",
      "\n",
      "Time taken for epoch : 17.42280673980713\n",
      "\n",
      "changing T value to 0.165\n",
      "  + Number of params: 7.19M\n",
      "  + Number of FLOPs: 0.47G\n",
      "Train Epoch: 34 [0/50000 (0%)]\tLoss: 0.224274\n",
      "Train Epoch: 34 [6400/50000 (13%)]\tLoss: 0.373071\n",
      "Train Epoch: 34 [12800/50000 (26%)]\tLoss: 0.215590\n",
      "Train Epoch: 34 [19200/50000 (38%)]\tLoss: 0.162748\n",
      "Train Epoch: 34 [25600/50000 (51%)]\tLoss: 0.279088\n",
      "Train Epoch: 34 [32000/50000 (64%)]\tLoss: 0.370581\n",
      "Train Epoch: 34 [38400/50000 (77%)]\tLoss: 0.300571\n",
      "Train Epoch: 34 [44800/50000 (90%)]\tLoss: 0.313588\n",
      "\n",
      "Test set: Average loss: 0.4692, Accuracy: 8557/10000 (86%)\n",
      "\n",
      "Time taken for epoch : 16.741147994995117\n",
      "\n",
      "changing T value to 0.17\n",
      "  + Number of params: 6.97M\n",
      "  + Number of FLOPs: 0.47G\n",
      "Train Epoch: 35 [0/50000 (0%)]\tLoss: 0.223951\n",
      "Train Epoch: 35 [6400/50000 (13%)]\tLoss: 0.165744\n",
      "Train Epoch: 35 [12800/50000 (26%)]\tLoss: 0.299903\n",
      "Train Epoch: 35 [19200/50000 (38%)]\tLoss: 0.411134\n",
      "Train Epoch: 35 [25600/50000 (51%)]\tLoss: 0.333434\n",
      "Train Epoch: 35 [32000/50000 (64%)]\tLoss: 0.346384\n",
      "Train Epoch: 35 [38400/50000 (77%)]\tLoss: 0.582334\n",
      "Train Epoch: 35 [44800/50000 (90%)]\tLoss: 0.319577\n",
      "\n",
      "Test set: Average loss: 0.4920, Accuracy: 8502/10000 (85%)\n",
      "\n",
      "Time taken for epoch : 16.753982067108154\n",
      "\n",
      "changing T value to 0.17500000000000002\n",
      "  + Number of params: 6.75M\n",
      "  + Number of FLOPs: 0.46G\n",
      "Train Epoch: 36 [0/50000 (0%)]\tLoss: 0.186832\n",
      "Train Epoch: 36 [6400/50000 (13%)]\tLoss: 0.311171\n",
      "Train Epoch: 36 [12800/50000 (26%)]\tLoss: 0.183034\n",
      "Train Epoch: 36 [19200/50000 (38%)]\tLoss: 0.299923\n",
      "Train Epoch: 36 [25600/50000 (51%)]\tLoss: 0.272801\n",
      "Train Epoch: 36 [32000/50000 (64%)]\tLoss: 0.495463\n",
      "Train Epoch: 36 [38400/50000 (77%)]\tLoss: 0.177443\n",
      "Train Epoch: 36 [44800/50000 (90%)]\tLoss: 0.387578\n",
      "\n",
      "Test set: Average loss: 0.4695, Accuracy: 8510/10000 (85%)\n",
      "\n",
      "Time taken for epoch : 16.654040813446045\n",
      "\n",
      "changing T value to 0.18\n",
      "  + Number of params: 6.52M\n",
      "  + Number of FLOPs: 0.46G\n",
      "Train Epoch: 37 [0/50000 (0%)]\tLoss: 0.423924\n",
      "Train Epoch: 37 [6400/50000 (13%)]\tLoss: 0.297085\n",
      "Train Epoch: 37 [12800/50000 (26%)]\tLoss: 0.314869\n",
      "Train Epoch: 37 [19200/50000 (38%)]\tLoss: 0.104188\n",
      "Train Epoch: 37 [25600/50000 (51%)]\tLoss: 0.331915\n",
      "Train Epoch: 37 [32000/50000 (64%)]\tLoss: 0.401656\n",
      "Train Epoch: 37 [38400/50000 (77%)]\tLoss: 0.284604\n",
      "Train Epoch: 37 [44800/50000 (90%)]\tLoss: 0.256413\n",
      "\n",
      "Test set: Average loss: 0.4596, Accuracy: 8567/10000 (86%)\n",
      "\n",
      "Time taken for epoch : 16.435799598693848\n",
      "\n",
      "changing T value to 0.185\n",
      "  + Number of params: 6.32M\n",
      "  + Number of FLOPs: 0.46G\n",
      "Train Epoch: 38 [0/50000 (0%)]\tLoss: 0.235377\n",
      "Train Epoch: 38 [6400/50000 (13%)]\tLoss: 0.252369\n",
      "Train Epoch: 38 [12800/50000 (26%)]\tLoss: 0.350607\n",
      "Train Epoch: 38 [19200/50000 (38%)]\tLoss: 0.393385\n",
      "Train Epoch: 38 [25600/50000 (51%)]\tLoss: 0.298409\n",
      "Train Epoch: 38 [32000/50000 (64%)]\tLoss: 0.225718\n",
      "Train Epoch: 38 [38400/50000 (77%)]\tLoss: 0.277213\n",
      "Train Epoch: 38 [44800/50000 (90%)]\tLoss: 0.264113\n",
      "\n",
      "Test set: Average loss: 0.4553, Accuracy: 8561/10000 (86%)\n",
      "\n",
      "Time taken for epoch : 16.48786211013794\n",
      "\n",
      "changing T value to 0.19\n",
      "  + Number of params: 6.13M\n",
      "  + Number of FLOPs: 0.45G\n",
      "Train Epoch: 39 [0/50000 (0%)]\tLoss: 0.351827\n",
      "Train Epoch: 39 [6400/50000 (13%)]\tLoss: 0.204652\n",
      "Train Epoch: 39 [12800/50000 (26%)]\tLoss: 0.399514\n",
      "Train Epoch: 39 [19200/50000 (38%)]\tLoss: 0.405023\n",
      "Train Epoch: 39 [25600/50000 (51%)]\tLoss: 0.252239\n",
      "Train Epoch: 39 [32000/50000 (64%)]\tLoss: 0.190353\n",
      "Train Epoch: 39 [38400/50000 (77%)]\tLoss: 0.369602\n",
      "Train Epoch: 39 [44800/50000 (90%)]\tLoss: 0.451688\n",
      "\n",
      "Test set: Average loss: 0.4961, Accuracy: 8511/10000 (85%)\n",
      "\n",
      "Time taken for epoch : 16.422980546951294\n",
      "\n",
      "changing T value to 0.195\n",
      "  + Number of params: 5.92M\n",
      "  + Number of FLOPs: 0.45G\n",
      "Train Epoch: 40 [0/50000 (0%)]\tLoss: 0.311182\n",
      "Train Epoch: 40 [6400/50000 (13%)]\tLoss: 0.413435\n",
      "Train Epoch: 40 [12800/50000 (26%)]\tLoss: 0.133332\n",
      "Train Epoch: 40 [19200/50000 (38%)]\tLoss: 0.183781\n",
      "Train Epoch: 40 [25600/50000 (51%)]\tLoss: 0.243243\n",
      "Train Epoch: 40 [32000/50000 (64%)]\tLoss: 0.465143\n",
      "Train Epoch: 40 [38400/50000 (77%)]\tLoss: 0.151788\n",
      "Train Epoch: 40 [44800/50000 (90%)]\tLoss: 0.105605\n",
      "\n",
      "Test set: Average loss: 0.4292, Accuracy: 8640/10000 (86%)\n",
      "\n",
      "Time taken for epoch : 16.097512006759644\n",
      "\n",
      "changing T value to 0.2\n",
      "  + Number of params: 5.79M\n",
      "  + Number of FLOPs: 0.44G\n",
      "Train Epoch: 41 [0/50000 (0%)]\tLoss: 0.351248\n",
      "Train Epoch: 41 [6400/50000 (13%)]\tLoss: 0.288018\n",
      "Train Epoch: 41 [12800/50000 (26%)]\tLoss: 0.399030\n",
      "Train Epoch: 41 [19200/50000 (38%)]\tLoss: 0.103285\n",
      "Train Epoch: 41 [25600/50000 (51%)]\tLoss: 0.293551\n",
      "Train Epoch: 41 [32000/50000 (64%)]\tLoss: 0.439835\n",
      "Train Epoch: 41 [38400/50000 (77%)]\tLoss: 0.305329\n",
      "Train Epoch: 41 [44800/50000 (90%)]\tLoss: 0.408328\n",
      "\n",
      "Test set: Average loss: 0.4080, Accuracy: 8720/10000 (87%)\n",
      "\n",
      "Time taken for epoch : 15.774065256118774\n",
      "\n",
      "changing T value to 0.20500000000000002\n",
      "  + Number of params: 5.60M\n",
      "  + Number of FLOPs: 0.44G\n",
      "Train Epoch: 42 [0/50000 (0%)]\tLoss: 0.159536\n",
      "Train Epoch: 42 [6400/50000 (13%)]\tLoss: 0.368819\n",
      "Train Epoch: 42 [12800/50000 (26%)]\tLoss: 0.133229\n",
      "Train Epoch: 42 [19200/50000 (38%)]\tLoss: 0.255200\n",
      "Train Epoch: 42 [25600/50000 (51%)]\tLoss: 0.257113\n",
      "Train Epoch: 42 [32000/50000 (64%)]\tLoss: 0.261488\n",
      "Train Epoch: 42 [38400/50000 (77%)]\tLoss: 0.146161\n",
      "Train Epoch: 42 [44800/50000 (90%)]\tLoss: 0.290210\n",
      "\n",
      "Test set: Average loss: 0.4261, Accuracy: 8659/10000 (87%)\n",
      "\n",
      "Time taken for epoch : 15.970852375030518\n",
      "\n",
      "changing T value to 0.21\n",
      "  + Number of params: 5.44M\n",
      "  + Number of FLOPs: 0.43G\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 43 [0/50000 (0%)]\tLoss: 0.253460\n",
      "Train Epoch: 43 [6400/50000 (13%)]\tLoss: 0.454776\n",
      "Train Epoch: 43 [12800/50000 (26%)]\tLoss: 0.397594\n",
      "Train Epoch: 43 [19200/50000 (38%)]\tLoss: 0.323823\n",
      "Train Epoch: 43 [25600/50000 (51%)]\tLoss: 0.243249\n",
      "Train Epoch: 43 [32000/50000 (64%)]\tLoss: 0.352057\n",
      "Train Epoch: 43 [38400/50000 (77%)]\tLoss: 0.460611\n",
      "Train Epoch: 43 [44800/50000 (90%)]\tLoss: 0.138245\n",
      "\n",
      "Test set: Average loss: 0.4777, Accuracy: 8515/10000 (85%)\n",
      "\n",
      "Time taken for epoch : 15.869643211364746\n",
      "\n",
      "changing T value to 0.215\n",
      "  + Number of params: 5.33M\n",
      "  + Number of FLOPs: 0.43G\n",
      "Train Epoch: 44 [0/50000 (0%)]\tLoss: 0.219222\n",
      "Train Epoch: 44 [6400/50000 (13%)]\tLoss: 0.309198\n",
      "Train Epoch: 44 [12800/50000 (26%)]\tLoss: 0.149276\n",
      "Train Epoch: 44 [19200/50000 (38%)]\tLoss: 0.103480\n",
      "Train Epoch: 44 [25600/50000 (51%)]\tLoss: 0.243506\n",
      "Train Epoch: 44 [32000/50000 (64%)]\tLoss: 0.270514\n",
      "Train Epoch: 44 [38400/50000 (77%)]\tLoss: 0.454484\n",
      "Train Epoch: 44 [44800/50000 (90%)]\tLoss: 0.210017\n",
      "\n",
      "Test set: Average loss: 0.4534, Accuracy: 8599/10000 (86%)\n",
      "\n",
      "Time taken for epoch : 15.791066884994507\n",
      "\n",
      "changing T value to 0.22\n",
      "  + Number of params: 5.23M\n",
      "  + Number of FLOPs: 0.43G\n",
      "Train Epoch: 45 [0/50000 (0%)]\tLoss: 0.298038\n",
      "Train Epoch: 45 [6400/50000 (13%)]\tLoss: 0.214360\n",
      "Train Epoch: 45 [12800/50000 (26%)]\tLoss: 0.253157\n",
      "Train Epoch: 45 [19200/50000 (38%)]\tLoss: 0.290978\n",
      "Train Epoch: 45 [25600/50000 (51%)]\tLoss: 0.221122\n",
      "Train Epoch: 45 [32000/50000 (64%)]\tLoss: 0.275041\n",
      "Train Epoch: 45 [38400/50000 (77%)]\tLoss: 0.286779\n",
      "Train Epoch: 45 [44800/50000 (90%)]\tLoss: 0.195077\n",
      "\n",
      "Test set: Average loss: 0.3933, Accuracy: 8738/10000 (87%)\n",
      "\n",
      "Time taken for epoch : 15.818120956420898\n",
      "\n",
      "changing T value to 0.225\n",
      "  + Number of params: 5.09M\n",
      "  + Number of FLOPs: 0.43G\n",
      "Train Epoch: 46 [0/50000 (0%)]\tLoss: 0.297269\n",
      "Train Epoch: 46 [6400/50000 (13%)]\tLoss: 0.098562\n",
      "Train Epoch: 46 [12800/50000 (26%)]\tLoss: 0.137004\n",
      "Train Epoch: 46 [19200/50000 (38%)]\tLoss: 0.171018\n",
      "Train Epoch: 46 [25600/50000 (51%)]\tLoss: 0.253591\n",
      "Train Epoch: 46 [32000/50000 (64%)]\tLoss: 0.326029\n",
      "Train Epoch: 46 [38400/50000 (77%)]\tLoss: 0.215056\n",
      "Train Epoch: 46 [44800/50000 (90%)]\tLoss: 0.328969\n",
      "\n",
      "Test set: Average loss: 0.3838, Accuracy: 8764/10000 (88%)\n",
      "\n",
      "Time taken for epoch : 18.00476884841919\n",
      "\n",
      "changing T value to 0.23\n",
      "  + Number of params: 4.95M\n",
      "  + Number of FLOPs: 0.42G\n",
      "Train Epoch: 47 [0/50000 (0%)]\tLoss: 0.105773\n",
      "Train Epoch: 47 [6400/50000 (13%)]\tLoss: 0.304520\n",
      "Train Epoch: 47 [12800/50000 (26%)]\tLoss: 0.390396\n",
      "Train Epoch: 47 [19200/50000 (38%)]\tLoss: 0.394667\n",
      "Train Epoch: 47 [25600/50000 (51%)]\tLoss: 0.309131\n",
      "Train Epoch: 47 [32000/50000 (64%)]\tLoss: 0.287185\n",
      "Train Epoch: 47 [38400/50000 (77%)]\tLoss: 0.174920\n",
      "Train Epoch: 47 [44800/50000 (90%)]\tLoss: 0.175508\n",
      "\n",
      "Test set: Average loss: 0.4428, Accuracy: 8642/10000 (86%)\n",
      "\n",
      "Time taken for epoch : 15.383449077606201\n",
      "\n",
      "changing T value to 0.23500000000000001\n",
      "  + Number of params: 4.82M\n",
      "  + Number of FLOPs: 0.42G\n",
      "Train Epoch: 48 [0/50000 (0%)]\tLoss: 0.188715\n",
      "Train Epoch: 48 [6400/50000 (13%)]\tLoss: 0.091386\n",
      "Train Epoch: 48 [12800/50000 (26%)]\tLoss: 0.177213\n",
      "Train Epoch: 48 [19200/50000 (38%)]\tLoss: 0.213331\n",
      "Train Epoch: 48 [25600/50000 (51%)]\tLoss: 0.309067\n",
      "Train Epoch: 48 [32000/50000 (64%)]\tLoss: 0.291252\n",
      "Train Epoch: 48 [38400/50000 (77%)]\tLoss: 0.138605\n",
      "Train Epoch: 48 [44800/50000 (90%)]\tLoss: 0.275313\n",
      "\n",
      "Test set: Average loss: 0.5269, Accuracy: 8343/10000 (83%)\n",
      "\n",
      "Time taken for epoch : 15.187529802322388\n",
      "\n",
      "changing T value to 0.24\n",
      "  + Number of params: 4.72M\n",
      "  + Number of FLOPs: 0.42G\n",
      "Train Epoch: 49 [0/50000 (0%)]\tLoss: 0.183645\n",
      "Train Epoch: 49 [6400/50000 (13%)]\tLoss: 0.305714\n",
      "Train Epoch: 49 [12800/50000 (26%)]\tLoss: 0.246537\n",
      "Train Epoch: 49 [19200/50000 (38%)]\tLoss: 0.552705\n",
      "Train Epoch: 49 [25600/50000 (51%)]\tLoss: 0.229748\n",
      "Train Epoch: 49 [32000/50000 (64%)]\tLoss: 0.412683\n",
      "Train Epoch: 49 [38400/50000 (77%)]\tLoss: 0.275556\n",
      "Train Epoch: 49 [44800/50000 (90%)]\tLoss: 0.248262\n",
      "\n",
      "Test set: Average loss: 0.5195, Accuracy: 8355/10000 (84%)\n",
      "\n",
      "Time taken for epoch : 15.174768924713135\n",
      "\n",
      "changing T value to 0.245\n",
      "  + Number of params: 4.62M\n",
      "  + Number of FLOPs: 0.41G\n",
      "Train Epoch: 50 [0/50000 (0%)]\tLoss: 0.230752\n",
      "Train Epoch: 50 [6400/50000 (13%)]\tLoss: 0.168738\n",
      "Train Epoch: 50 [12800/50000 (26%)]\tLoss: 0.215781\n",
      "Train Epoch: 50 [19200/50000 (38%)]\tLoss: 0.120971\n",
      "Train Epoch: 50 [25600/50000 (51%)]\tLoss: 0.200504\n",
      "Train Epoch: 50 [32000/50000 (64%)]\tLoss: 0.363919\n",
      "Train Epoch: 50 [38400/50000 (77%)]\tLoss: 0.219881\n",
      "Train Epoch: 50 [44800/50000 (90%)]\tLoss: 0.263318\n",
      "\n",
      "Test set: Average loss: 0.3744, Accuracy: 8785/10000 (88%)\n",
      "\n",
      "Time taken for epoch : 17.54606795310974\n",
      "\n",
      "changing T value to 0.25\n",
      "  + Number of params: 4.52M\n",
      "  + Number of FLOPs: 0.41G\n",
      "Train Epoch: 51 [0/50000 (0%)]\tLoss: 0.223223\n",
      "Train Epoch: 51 [6400/50000 (13%)]\tLoss: 0.356117\n",
      "Train Epoch: 51 [12800/50000 (26%)]\tLoss: 0.156665\n",
      "Train Epoch: 51 [19200/50000 (38%)]\tLoss: 0.206486\n",
      "Train Epoch: 51 [25600/50000 (51%)]\tLoss: 0.227934\n",
      "Train Epoch: 51 [32000/50000 (64%)]\tLoss: 0.204553\n",
      "Train Epoch: 51 [38400/50000 (77%)]\tLoss: 0.347487\n",
      "Train Epoch: 51 [44800/50000 (90%)]\tLoss: 0.223796\n",
      "\n",
      "Test set: Average loss: 0.4429, Accuracy: 8672/10000 (87%)\n",
      "\n",
      "Time taken for epoch : 14.923396825790405\n",
      "\n",
      "changing T value to 0.255\n",
      "  + Number of params: 4.45M\n",
      "  + Number of FLOPs: 0.41G\n",
      "Train Epoch: 52 [0/50000 (0%)]\tLoss: 0.224770\n",
      "Train Epoch: 52 [6400/50000 (13%)]\tLoss: 0.184420\n",
      "Train Epoch: 52 [12800/50000 (26%)]\tLoss: 0.181244\n",
      "Train Epoch: 52 [19200/50000 (38%)]\tLoss: 0.265837\n",
      "Train Epoch: 52 [25600/50000 (51%)]\tLoss: 0.170589\n",
      "Train Epoch: 52 [32000/50000 (64%)]\tLoss: 0.290195\n",
      "Train Epoch: 52 [38400/50000 (77%)]\tLoss: 0.306958\n",
      "Train Epoch: 52 [44800/50000 (90%)]\tLoss: 0.405678\n",
      "\n",
      "Test set: Average loss: 0.4462, Accuracy: 8653/10000 (87%)\n",
      "\n",
      "Time taken for epoch : 14.603601455688477\n",
      "\n",
      "changing T value to 0.26\n",
      "  + Number of params: 4.37M\n",
      "  + Number of FLOPs: 0.41G\n",
      "Train Epoch: 53 [0/50000 (0%)]\tLoss: 0.191151\n",
      "Train Epoch: 53 [6400/50000 (13%)]\tLoss: 0.233983\n",
      "Train Epoch: 53 [12800/50000 (26%)]\tLoss: 0.269487\n",
      "Train Epoch: 53 [19200/50000 (38%)]\tLoss: 0.401862\n",
      "Train Epoch: 53 [25600/50000 (51%)]\tLoss: 0.263180\n",
      "Train Epoch: 53 [32000/50000 (64%)]\tLoss: 0.214822\n",
      "Train Epoch: 53 [38400/50000 (77%)]\tLoss: 0.259604\n",
      "Train Epoch: 53 [44800/50000 (90%)]\tLoss: 0.162306\n",
      "\n",
      "Test set: Average loss: 0.4667, Accuracy: 8559/10000 (86%)\n",
      "\n",
      "Time taken for epoch : 14.663226127624512\n",
      "\n",
      "changing T value to 0.265\n",
      "  + Number of params: 4.28M\n",
      "  + Number of FLOPs: 0.41G\n",
      "Train Epoch: 54 [0/50000 (0%)]\tLoss: 0.201516\n",
      "Train Epoch: 54 [6400/50000 (13%)]\tLoss: 0.080023\n",
      "Train Epoch: 54 [12800/50000 (26%)]\tLoss: 0.187502\n",
      "Train Epoch: 54 [19200/50000 (38%)]\tLoss: 0.164864\n",
      "Train Epoch: 54 [25600/50000 (51%)]\tLoss: 0.199424\n",
      "Train Epoch: 54 [32000/50000 (64%)]\tLoss: 0.220693\n",
      "Train Epoch: 54 [38400/50000 (77%)]\tLoss: 0.227276\n",
      "Train Epoch: 54 [44800/50000 (90%)]\tLoss: 0.147769\n",
      "\n",
      "Test set: Average loss: 0.3883, Accuracy: 8731/10000 (87%)\n",
      "\n",
      "Time taken for epoch : 14.210998773574829\n",
      "\n",
      "changing T value to 0.27\n",
      "  + Number of params: 4.18M\n",
      "  + Number of FLOPs: 0.40G\n",
      "Train Epoch: 55 [0/50000 (0%)]\tLoss: 0.148577\n",
      "Train Epoch: 55 [6400/50000 (13%)]\tLoss: 0.098080\n",
      "Train Epoch: 55 [12800/50000 (26%)]\tLoss: 0.237181\n",
      "Train Epoch: 55 [19200/50000 (38%)]\tLoss: 0.333245\n",
      "Train Epoch: 55 [25600/50000 (51%)]\tLoss: 0.196583\n",
      "Train Epoch: 55 [32000/50000 (64%)]\tLoss: 0.173760\n",
      "Train Epoch: 55 [38400/50000 (77%)]\tLoss: 0.221719\n",
      "Train Epoch: 55 [44800/50000 (90%)]\tLoss: 0.080165\n",
      "\n",
      "Test set: Average loss: 0.5022, Accuracy: 8461/10000 (85%)\n",
      "\n",
      "Time taken for epoch : 14.275905847549438\n",
      "\n",
      "changing T value to 0.275\n",
      "  + Number of params: 4.12M\n",
      "  + Number of FLOPs: 0.40G\n",
      "Train Epoch: 56 [0/50000 (0%)]\tLoss: 0.251294\n",
      "Train Epoch: 56 [6400/50000 (13%)]\tLoss: 0.499253\n",
      "Train Epoch: 56 [12800/50000 (26%)]\tLoss: 0.250497\n",
      "Train Epoch: 56 [19200/50000 (38%)]\tLoss: 0.259467\n",
      "Train Epoch: 56 [25600/50000 (51%)]\tLoss: 0.217711\n",
      "Train Epoch: 56 [32000/50000 (64%)]\tLoss: 0.206742\n",
      "Train Epoch: 56 [38400/50000 (77%)]\tLoss: 0.395330\n",
      "Train Epoch: 56 [44800/50000 (90%)]\tLoss: 0.302606\n",
      "\n",
      "Test set: Average loss: 0.4858, Accuracy: 8446/10000 (84%)\n",
      "\n",
      "Time taken for epoch : 14.175929069519043\n",
      "\n",
      "changing T value to 0.28\n",
      "  + Number of params: 4.03M\n",
      "  + Number of FLOPs: 0.40G\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 57 [0/50000 (0%)]\tLoss: 0.304276\n",
      "Train Epoch: 57 [6400/50000 (13%)]\tLoss: 0.400413\n",
      "Train Epoch: 57 [12800/50000 (26%)]\tLoss: 0.114983\n",
      "Train Epoch: 57 [19200/50000 (38%)]\tLoss: 0.165705\n",
      "Train Epoch: 57 [25600/50000 (51%)]\tLoss: 0.271891\n",
      "Train Epoch: 57 [32000/50000 (64%)]\tLoss: 0.201439\n",
      "Train Epoch: 57 [38400/50000 (77%)]\tLoss: 0.213348\n",
      "Train Epoch: 57 [44800/50000 (90%)]\tLoss: 0.204691\n",
      "\n",
      "Test set: Average loss: 0.4050, Accuracy: 8761/10000 (88%)\n",
      "\n",
      "Time taken for epoch : 14.036344528198242\n",
      "\n",
      "changing T value to 0.28500000000000003\n",
      "  + Number of params: 3.97M\n",
      "  + Number of FLOPs: 0.40G\n",
      "Train Epoch: 58 [0/50000 (0%)]\tLoss: 0.158576\n",
      "Train Epoch: 58 [6400/50000 (13%)]\tLoss: 0.264300\n",
      "Train Epoch: 58 [12800/50000 (26%)]\tLoss: 0.212655\n",
      "Train Epoch: 58 [19200/50000 (38%)]\tLoss: 0.402436\n",
      "Train Epoch: 58 [25600/50000 (51%)]\tLoss: 0.474452\n",
      "Train Epoch: 58 [32000/50000 (64%)]\tLoss: 0.351490\n",
      "Train Epoch: 58 [38400/50000 (77%)]\tLoss: 0.168483\n",
      "Train Epoch: 58 [44800/50000 (90%)]\tLoss: 0.412887\n",
      "\n",
      "Test set: Average loss: 0.4538, Accuracy: 8622/10000 (86%)\n",
      "\n",
      "Time taken for epoch : 13.899519681930542\n",
      "\n",
      "changing T value to 0.29\n",
      "  + Number of params: 3.90M\n",
      "  + Number of FLOPs: 0.40G\n",
      "Train Epoch: 59 [0/50000 (0%)]\tLoss: 0.154032\n",
      "Train Epoch: 59 [6400/50000 (13%)]\tLoss: 0.366693\n",
      "Train Epoch: 59 [12800/50000 (26%)]\tLoss: 0.420680\n",
      "Train Epoch: 59 [19200/50000 (38%)]\tLoss: 0.209155\n",
      "Train Epoch: 59 [25600/50000 (51%)]\tLoss: 0.197980\n",
      "Train Epoch: 59 [32000/50000 (64%)]\tLoss: 0.159803\n",
      "Train Epoch: 59 [38400/50000 (77%)]\tLoss: 0.415525\n",
      "Train Epoch: 59 [44800/50000 (90%)]\tLoss: 0.158099\n",
      "\n",
      "Test set: Average loss: 0.4518, Accuracy: 8609/10000 (86%)\n",
      "\n",
      "Time taken for epoch : 13.967195749282837\n",
      "\n",
      "changing T value to 0.295\n",
      "  + Number of params: 3.84M\n",
      "  + Number of FLOPs: 0.40G\n",
      "Train Epoch: 60 [0/50000 (0%)]\tLoss: 0.179028\n",
      "Train Epoch: 60 [6400/50000 (13%)]\tLoss: 0.140910\n",
      "Train Epoch: 60 [12800/50000 (26%)]\tLoss: 0.327053\n",
      "Train Epoch: 60 [19200/50000 (38%)]\tLoss: 0.299848\n",
      "Train Epoch: 60 [25600/50000 (51%)]\tLoss: 0.228251\n",
      "Train Epoch: 60 [32000/50000 (64%)]\tLoss: 0.151601\n",
      "Train Epoch: 60 [38400/50000 (77%)]\tLoss: 0.157459\n",
      "Train Epoch: 60 [44800/50000 (90%)]\tLoss: 0.216490\n",
      "\n",
      "Test set: Average loss: 0.4761, Accuracy: 8568/10000 (86%)\n",
      "\n",
      "Time taken for epoch : 13.789786577224731\n",
      "\n",
      "changing T value to 0.3\n",
      "  + Number of params: 3.79M\n",
      "  + Number of FLOPs: 0.40G\n",
      "Train Epoch: 61 [0/50000 (0%)]\tLoss: 0.240688\n",
      "Train Epoch: 61 [6400/50000 (13%)]\tLoss: 0.172693\n",
      "Train Epoch: 61 [12800/50000 (26%)]\tLoss: 0.090274\n",
      "Train Epoch: 61 [19200/50000 (38%)]\tLoss: 0.276819\n",
      "Train Epoch: 61 [25600/50000 (51%)]\tLoss: 0.181525\n",
      "Train Epoch: 61 [32000/50000 (64%)]\tLoss: 0.183128\n",
      "Train Epoch: 61 [38400/50000 (77%)]\tLoss: 0.378159\n",
      "Train Epoch: 61 [44800/50000 (90%)]\tLoss: 0.201445\n",
      "\n",
      "Test set: Average loss: 0.4155, Accuracy: 8679/10000 (87%)\n",
      "\n",
      "Time taken for epoch : 13.676010608673096\n",
      "\n",
      "changing T value to 0.305\n",
      "  + Number of params: 3.75M\n",
      "  + Number of FLOPs: 0.39G\n",
      "Train Epoch: 62 [0/50000 (0%)]\tLoss: 0.329231\n",
      "Train Epoch: 62 [6400/50000 (13%)]\tLoss: 0.166763\n",
      "Train Epoch: 62 [12800/50000 (26%)]\tLoss: 0.234524\n",
      "Train Epoch: 62 [19200/50000 (38%)]\tLoss: 0.223211\n",
      "Train Epoch: 62 [25600/50000 (51%)]\tLoss: 0.417918\n",
      "Train Epoch: 62 [32000/50000 (64%)]\tLoss: 0.262462\n",
      "Train Epoch: 62 [38400/50000 (77%)]\tLoss: 0.203131\n",
      "Train Epoch: 62 [44800/50000 (90%)]\tLoss: 0.162902\n",
      "\n",
      "Test set: Average loss: 0.4575, Accuracy: 8530/10000 (85%)\n",
      "\n",
      "Time taken for epoch : 13.711112976074219\n",
      "\n",
      "changing T value to 0.31\n",
      "  + Number of params: 3.69M\n",
      "  + Number of FLOPs: 0.39G\n",
      "Train Epoch: 63 [0/50000 (0%)]\tLoss: 0.430784\n",
      "Train Epoch: 63 [6400/50000 (13%)]\tLoss: 0.185602\n",
      "Train Epoch: 63 [12800/50000 (26%)]\tLoss: 0.194635\n",
      "Train Epoch: 63 [19200/50000 (38%)]\tLoss: 0.209850\n",
      "Train Epoch: 63 [25600/50000 (51%)]\tLoss: 0.266659\n",
      "Train Epoch: 63 [32000/50000 (64%)]\tLoss: 0.308011\n",
      "Train Epoch: 63 [38400/50000 (77%)]\tLoss: 0.198461\n",
      "Train Epoch: 63 [44800/50000 (90%)]\tLoss: 0.119757\n",
      "\n",
      "Test set: Average loss: 0.4350, Accuracy: 8675/10000 (87%)\n",
      "\n",
      "Time taken for epoch : 13.627105712890625\n",
      "\n",
      "changing T value to 0.315\n",
      "  + Number of params: 3.65M\n",
      "  + Number of FLOPs: 0.39G\n",
      "Train Epoch: 64 [0/50000 (0%)]\tLoss: 0.136927\n",
      "Train Epoch: 64 [6400/50000 (13%)]\tLoss: 0.264854\n",
      "Train Epoch: 64 [12800/50000 (26%)]\tLoss: 0.219129\n",
      "Train Epoch: 64 [19200/50000 (38%)]\tLoss: 0.175713\n",
      "Train Epoch: 64 [25600/50000 (51%)]\tLoss: 0.439625\n",
      "Train Epoch: 64 [32000/50000 (64%)]\tLoss: 0.231262\n",
      "Train Epoch: 64 [38400/50000 (77%)]\tLoss: 0.105352\n",
      "Train Epoch: 64 [44800/50000 (90%)]\tLoss: 0.249604\n",
      "\n",
      "Test set: Average loss: 0.4753, Accuracy: 8530/10000 (85%)\n",
      "\n",
      "Time taken for epoch : 13.669300317764282\n",
      "\n",
      "changing T value to 0.32\n",
      "  + Number of params: 3.58M\n",
      "  + Number of FLOPs: 0.39G\n",
      "Train Epoch: 65 [0/50000 (0%)]\tLoss: 0.290412\n",
      "Train Epoch: 65 [6400/50000 (13%)]\tLoss: 0.194388\n",
      "Train Epoch: 65 [12800/50000 (26%)]\tLoss: 0.068408\n",
      "Train Epoch: 65 [19200/50000 (38%)]\tLoss: 0.180941\n",
      "Train Epoch: 65 [25600/50000 (51%)]\tLoss: 0.165722\n",
      "Train Epoch: 65 [32000/50000 (64%)]\tLoss: 0.297441\n",
      "Train Epoch: 65 [38400/50000 (77%)]\tLoss: 0.318696\n",
      "Train Epoch: 65 [44800/50000 (90%)]\tLoss: 0.194555\n",
      "\n",
      "Test set: Average loss: 0.3992, Accuracy: 8787/10000 (88%)\n",
      "\n",
      "Time taken for epoch : 16.147183179855347\n",
      "\n",
      "changing T value to 0.325\n",
      "  + Number of params: 3.52M\n",
      "  + Number of FLOPs: 0.39G\n",
      "Train Epoch: 66 [0/50000 (0%)]\tLoss: 0.294890\n",
      "Train Epoch: 66 [6400/50000 (13%)]\tLoss: 0.292108\n",
      "Train Epoch: 66 [12800/50000 (26%)]\tLoss: 0.377157\n",
      "Train Epoch: 66 [19200/50000 (38%)]\tLoss: 0.127445\n",
      "Train Epoch: 66 [25600/50000 (51%)]\tLoss: 0.381597\n",
      "Train Epoch: 66 [32000/50000 (64%)]\tLoss: 0.210978\n",
      "Train Epoch: 66 [38400/50000 (77%)]\tLoss: 0.372519\n",
      "Train Epoch: 66 [44800/50000 (90%)]\tLoss: 0.106021\n",
      "\n",
      "Test set: Average loss: 0.4293, Accuracy: 8665/10000 (87%)\n",
      "\n",
      "Time taken for epoch : 13.655772686004639\n",
      "\n",
      "changing T value to 0.33\n",
      "  + Number of params: 3.50M\n",
      "  + Number of FLOPs: 0.39G\n",
      "Train Epoch: 67 [0/50000 (0%)]\tLoss: 0.412928\n",
      "Train Epoch: 67 [6400/50000 (13%)]\tLoss: 0.229540\n",
      "Train Epoch: 67 [12800/50000 (26%)]\tLoss: 0.230878\n",
      "Train Epoch: 67 [19200/50000 (38%)]\tLoss: 0.384961\n",
      "Train Epoch: 67 [25600/50000 (51%)]\tLoss: 0.161109\n",
      "Train Epoch: 67 [32000/50000 (64%)]\tLoss: 0.105243\n",
      "Train Epoch: 67 [38400/50000 (77%)]\tLoss: 0.370662\n",
      "Train Epoch: 67 [44800/50000 (90%)]\tLoss: 0.328184\n",
      "\n",
      "Test set: Average loss: 0.3900, Accuracy: 8752/10000 (88%)\n",
      "\n",
      "Time taken for epoch : 13.594546556472778\n",
      "\n",
      "changing T value to 0.335\n",
      "  + Number of params: 3.47M\n",
      "  + Number of FLOPs: 0.39G\n",
      "Train Epoch: 68 [0/50000 (0%)]\tLoss: 0.397714\n",
      "Train Epoch: 68 [6400/50000 (13%)]\tLoss: 0.273154\n",
      "Train Epoch: 68 [12800/50000 (26%)]\tLoss: 0.190917\n",
      "Train Epoch: 68 [19200/50000 (38%)]\tLoss: 0.191118\n",
      "Train Epoch: 68 [25600/50000 (51%)]\tLoss: 0.219813\n",
      "Train Epoch: 68 [32000/50000 (64%)]\tLoss: 0.258176\n",
      "Train Epoch: 68 [38400/50000 (77%)]\tLoss: 0.317062\n",
      "Train Epoch: 68 [44800/50000 (90%)]\tLoss: 0.239446\n",
      "\n",
      "Test set: Average loss: 0.4303, Accuracy: 8680/10000 (87%)\n",
      "\n",
      "Time taken for epoch : 13.608627319335938\n",
      "\n",
      "changing T value to 0.34\n",
      "  + Number of params: 3.43M\n",
      "  + Number of FLOPs: 0.39G\n",
      "Train Epoch: 69 [0/50000 (0%)]\tLoss: 0.254015\n",
      "Train Epoch: 69 [6400/50000 (13%)]\tLoss: 0.224048\n",
      "Train Epoch: 69 [12800/50000 (26%)]\tLoss: 0.238973\n",
      "Train Epoch: 69 [19200/50000 (38%)]\tLoss: 0.123189\n",
      "Train Epoch: 69 [25600/50000 (51%)]\tLoss: 0.338661\n",
      "Train Epoch: 69 [32000/50000 (64%)]\tLoss: 0.282111\n",
      "Train Epoch: 69 [38400/50000 (77%)]\tLoss: 0.204627\n",
      "Train Epoch: 69 [44800/50000 (90%)]\tLoss: 0.162042\n",
      "\n",
      "Test set: Average loss: 0.6147, Accuracy: 8199/10000 (82%)\n",
      "\n",
      "Time taken for epoch : 13.507100105285645\n",
      "\n",
      "changing T value to 0.34500000000000003\n",
      "  + Number of params: 3.40M\n",
      "  + Number of FLOPs: 0.38G\n",
      "Train Epoch: 70 [0/50000 (0%)]\tLoss: 0.373099\n",
      "Train Epoch: 70 [6400/50000 (13%)]\tLoss: 0.458639\n",
      "Train Epoch: 70 [12800/50000 (26%)]\tLoss: 0.178676\n",
      "Train Epoch: 70 [19200/50000 (38%)]\tLoss: 0.118076\n",
      "Train Epoch: 70 [25600/50000 (51%)]\tLoss: 0.261201\n",
      "Train Epoch: 70 [32000/50000 (64%)]\tLoss: 0.522919\n",
      "Train Epoch: 70 [38400/50000 (77%)]\tLoss: 0.410915\n",
      "Train Epoch: 70 [44800/50000 (90%)]\tLoss: 0.286516\n",
      "\n",
      "Test set: Average loss: 0.3809, Accuracy: 8801/10000 (88%)\n",
      "\n",
      "Time taken for epoch : 15.558411598205566\n",
      "\n",
      "changing T value to 0.35000000000000003\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  + Number of params: 3.35M\n",
      "  + Number of FLOPs: 0.38G\n",
      "Train Epoch: 71 [0/50000 (0%)]\tLoss: 0.127882\n",
      "Train Epoch: 71 [6400/50000 (13%)]\tLoss: 0.290028\n",
      "Train Epoch: 71 [12800/50000 (26%)]\tLoss: 0.111749\n",
      "Train Epoch: 71 [19200/50000 (38%)]\tLoss: 0.221156\n",
      "Train Epoch: 71 [25600/50000 (51%)]\tLoss: 0.121421\n",
      "Train Epoch: 71 [32000/50000 (64%)]\tLoss: 0.146315\n",
      "Train Epoch: 71 [38400/50000 (77%)]\tLoss: 0.287158\n",
      "Train Epoch: 71 [44800/50000 (90%)]\tLoss: 0.195196\n",
      "\n",
      "Test set: Average loss: 0.4447, Accuracy: 8686/10000 (87%)\n",
      "\n",
      "Time taken for epoch : 13.574462175369263\n",
      "\n",
      "changing T value to 0.355\n",
      "  + Number of params: 3.30M\n",
      "  + Number of FLOPs: 0.38G\n",
      "Train Epoch: 72 [0/50000 (0%)]\tLoss: 0.152774\n",
      "Train Epoch: 72 [6400/50000 (13%)]\tLoss: 0.153075\n",
      "Train Epoch: 72 [12800/50000 (26%)]\tLoss: 0.177606\n",
      "Train Epoch: 72 [19200/50000 (38%)]\tLoss: 0.253988\n",
      "Train Epoch: 72 [25600/50000 (51%)]\tLoss: 0.207237\n",
      "Train Epoch: 72 [32000/50000 (64%)]\tLoss: 0.340186\n",
      "Train Epoch: 72 [38400/50000 (77%)]\tLoss: 0.264682\n",
      "Train Epoch: 72 [44800/50000 (90%)]\tLoss: 0.340947\n",
      "\n",
      "Test set: Average loss: 0.4243, Accuracy: 8686/10000 (87%)\n",
      "\n",
      "Time taken for epoch : 13.325214385986328\n",
      "\n",
      "changing T value to 0.36\n",
      "  + Number of params: 3.26M\n",
      "  + Number of FLOPs: 0.38G\n",
      "Train Epoch: 73 [0/50000 (0%)]\tLoss: 0.154428\n",
      "Train Epoch: 73 [6400/50000 (13%)]\tLoss: 0.260663\n",
      "Train Epoch: 73 [12800/50000 (26%)]\tLoss: 0.141341\n",
      "Train Epoch: 73 [19200/50000 (38%)]\tLoss: 0.145921\n",
      "Train Epoch: 73 [25600/50000 (51%)]\tLoss: 0.335754\n",
      "Train Epoch: 73 [32000/50000 (64%)]\tLoss: 0.399621\n",
      "Train Epoch: 73 [38400/50000 (77%)]\tLoss: 0.134945\n",
      "Train Epoch: 73 [44800/50000 (90%)]\tLoss: 0.362722\n",
      "\n",
      "Test set: Average loss: 0.3778, Accuracy: 8837/10000 (88%)\n",
      "\n",
      "Time taken for epoch : 15.431414365768433\n",
      "\n",
      "changing T value to 0.365\n",
      "  + Number of params: 3.24M\n",
      "  + Number of FLOPs: 0.38G\n",
      "Train Epoch: 74 [0/50000 (0%)]\tLoss: 0.098222\n",
      "Train Epoch: 74 [6400/50000 (13%)]\tLoss: 0.103434\n",
      "Train Epoch: 74 [12800/50000 (26%)]\tLoss: 0.384931\n",
      "Train Epoch: 74 [19200/50000 (38%)]\tLoss: 0.193364\n",
      "Train Epoch: 74 [25600/50000 (51%)]\tLoss: 0.187186\n",
      "Train Epoch: 74 [32000/50000 (64%)]\tLoss: 0.187318\n",
      "Train Epoch: 74 [38400/50000 (77%)]\tLoss: 0.196541\n",
      "Train Epoch: 74 [44800/50000 (90%)]\tLoss: 0.282659\n",
      "\n",
      "Test set: Average loss: 0.4470, Accuracy: 8645/10000 (86%)\n",
      "\n",
      "Time taken for epoch : 13.33427095413208\n",
      "\n",
      "changing T value to 0.37\n",
      "  + Number of params: 3.21M\n",
      "  + Number of FLOPs: 0.38G\n",
      "Train Epoch: 75 [0/50000 (0%)]\tLoss: 0.171895\n",
      "Train Epoch: 75 [6400/50000 (13%)]\tLoss: 0.127953\n",
      "Train Epoch: 75 [12800/50000 (26%)]\tLoss: 0.356222\n",
      "Train Epoch: 75 [19200/50000 (38%)]\tLoss: 0.183004\n",
      "Train Epoch: 75 [25600/50000 (51%)]\tLoss: 0.203141\n",
      "Train Epoch: 75 [32000/50000 (64%)]\tLoss: 0.110962\n",
      "Train Epoch: 75 [38400/50000 (77%)]\tLoss: 0.274051\n",
      "Train Epoch: 75 [44800/50000 (90%)]\tLoss: 0.304521\n",
      "\n",
      "Test set: Average loss: 0.3230, Accuracy: 8984/10000 (90%)\n",
      "\n",
      "Time taken for epoch : 15.697187900543213\n",
      "\n",
      "changing T value to 0.375\n",
      "  + Number of params: 3.17M\n",
      "  + Number of FLOPs: 0.38G\n",
      "Train Epoch: 76 [0/50000 (0%)]\tLoss: 0.280063\n",
      "Train Epoch: 76 [6400/50000 (13%)]\tLoss: 0.083426\n",
      "Train Epoch: 76 [12800/50000 (26%)]\tLoss: 0.261196\n",
      "Train Epoch: 76 [19200/50000 (38%)]\tLoss: 0.194377\n",
      "Train Epoch: 76 [25600/50000 (51%)]\tLoss: 0.212288\n",
      "Train Epoch: 76 [32000/50000 (64%)]\tLoss: 0.417521\n",
      "Train Epoch: 76 [38400/50000 (77%)]\tLoss: 0.149652\n",
      "Train Epoch: 76 [44800/50000 (90%)]\tLoss: 0.214803\n",
      "\n",
      "Test set: Average loss: 0.3843, Accuracy: 8799/10000 (88%)\n",
      "\n",
      "Time taken for epoch : 13.243665933609009\n",
      "\n",
      "changing T value to 0.38\n",
      "  + Number of params: 3.13M\n",
      "  + Number of FLOPs: 0.38G\n",
      "Train Epoch: 77 [0/50000 (0%)]\tLoss: 0.237935\n",
      "Train Epoch: 77 [6400/50000 (13%)]\tLoss: 0.309042\n",
      "Train Epoch: 77 [12800/50000 (26%)]\tLoss: 0.452984\n",
      "Train Epoch: 77 [19200/50000 (38%)]\tLoss: 0.218875\n",
      "Train Epoch: 77 [25600/50000 (51%)]\tLoss: 0.435554\n",
      "Train Epoch: 77 [32000/50000 (64%)]\tLoss: 0.300917\n",
      "Train Epoch: 77 [38400/50000 (77%)]\tLoss: 0.223930\n",
      "Train Epoch: 77 [44800/50000 (90%)]\tLoss: 0.211839\n",
      "\n",
      "Test set: Average loss: 0.3872, Accuracy: 8768/10000 (88%)\n",
      "\n",
      "Time taken for epoch : 13.215089082717896\n",
      "\n",
      "changing T value to 0.385\n",
      "  + Number of params: 3.08M\n",
      "  + Number of FLOPs: 0.38G\n",
      "Train Epoch: 78 [0/50000 (0%)]\tLoss: 0.320004\n",
      "Train Epoch: 78 [6400/50000 (13%)]\tLoss: 0.235326\n",
      "Train Epoch: 78 [12800/50000 (26%)]\tLoss: 0.223674\n",
      "Train Epoch: 78 [19200/50000 (38%)]\tLoss: 0.096865\n",
      "Train Epoch: 78 [25600/50000 (51%)]\tLoss: 0.256274\n",
      "Train Epoch: 78 [32000/50000 (64%)]\tLoss: 0.321121\n",
      "Train Epoch: 78 [38400/50000 (77%)]\tLoss: 0.179272\n",
      "Train Epoch: 78 [44800/50000 (90%)]\tLoss: 0.366760\n",
      "\n",
      "Test set: Average loss: 0.4064, Accuracy: 8762/10000 (88%)\n",
      "\n",
      "Time taken for epoch : 13.21291732788086\n",
      "\n",
      "changing T value to 0.39\n",
      "  + Number of params: 3.04M\n",
      "  + Number of FLOPs: 0.37G\n",
      "Train Epoch: 79 [0/50000 (0%)]\tLoss: 0.198827\n",
      "Train Epoch: 79 [6400/50000 (13%)]\tLoss: 0.200745\n",
      "Train Epoch: 79 [12800/50000 (26%)]\tLoss: 0.380468\n",
      "Train Epoch: 79 [19200/50000 (38%)]\tLoss: 0.222302\n",
      "Train Epoch: 79 [25600/50000 (51%)]\tLoss: 0.377327\n",
      "Train Epoch: 79 [32000/50000 (64%)]\tLoss: 0.252694\n",
      "Train Epoch: 79 [38400/50000 (77%)]\tLoss: 0.221459\n",
      "Train Epoch: 79 [44800/50000 (90%)]\tLoss: 0.277718\n",
      "\n",
      "Test set: Average loss: 0.4711, Accuracy: 8563/10000 (86%)\n",
      "\n",
      "Time taken for epoch : 13.395303964614868\n",
      "\n",
      "changing T value to 0.395\n",
      "  + Number of params: 3.02M\n",
      "  + Number of FLOPs: 0.37G\n",
      "Changing Learning Rate to 0.010000000000000002\n",
      "Train Epoch: 80 [0/50000 (0%)]\tLoss: 0.183819\n",
      "Train Epoch: 80 [6400/50000 (13%)]\tLoss: 0.108225\n",
      "Train Epoch: 80 [12800/50000 (26%)]\tLoss: 0.073102\n",
      "Train Epoch: 80 [19200/50000 (38%)]\tLoss: 0.070331\n",
      "Train Epoch: 80 [25600/50000 (51%)]\tLoss: 0.080220\n",
      "Train Epoch: 80 [32000/50000 (64%)]\tLoss: 0.021379\n",
      "Train Epoch: 80 [38400/50000 (77%)]\tLoss: 0.101478\n",
      "Train Epoch: 80 [44800/50000 (90%)]\tLoss: 0.016462\n",
      "\n",
      "Test set: Average loss: 0.2617, Accuracy: 9244/10000 (92%)\n",
      "\n",
      "Time taken for epoch : 15.332973957061768\n",
      "\n",
      "changing T value to 0.4\n",
      "  + Number of params: 2.98M\n",
      "  + Number of FLOPs: 0.37G\n",
      "Train Epoch: 81 [0/50000 (0%)]\tLoss: 0.132018\n",
      "Train Epoch: 81 [6400/50000 (13%)]\tLoss: 0.104618\n",
      "Train Epoch: 81 [12800/50000 (26%)]\tLoss: 0.022424\n",
      "Train Epoch: 81 [19200/50000 (38%)]\tLoss: 0.028157\n",
      "Train Epoch: 81 [25600/50000 (51%)]\tLoss: 0.034734\n",
      "Train Epoch: 81 [32000/50000 (64%)]\tLoss: 0.033938\n",
      "Train Epoch: 81 [38400/50000 (77%)]\tLoss: 0.088089\n",
      "Train Epoch: 81 [44800/50000 (90%)]\tLoss: 0.008851\n",
      "\n",
      "Test set: Average loss: 0.2611, Accuracy: 9283/10000 (93%)\n",
      "\n",
      "Time taken for epoch : 15.684466361999512\n",
      "\n",
      "changing T value to 0.405\n",
      "  + Number of params: 2.91M\n",
      "  + Number of FLOPs: 0.37G\n",
      "Train Epoch: 82 [0/50000 (0%)]\tLoss: 0.067166\n",
      "Train Epoch: 82 [6400/50000 (13%)]\tLoss: 0.032908\n",
      "Train Epoch: 82 [12800/50000 (26%)]\tLoss: 0.010603\n",
      "Train Epoch: 82 [19200/50000 (38%)]\tLoss: 0.079550\n",
      "Train Epoch: 82 [25600/50000 (51%)]\tLoss: 0.091439\n",
      "Train Epoch: 82 [32000/50000 (64%)]\tLoss: 0.068790\n",
      "Train Epoch: 82 [38400/50000 (77%)]\tLoss: 0.031240\n",
      "Train Epoch: 82 [44800/50000 (90%)]\tLoss: 0.008775\n",
      "\n",
      "Test set: Average loss: 0.2657, Accuracy: 9296/10000 (93%)\n",
      "\n",
      "Time taken for epoch : 15.498997926712036\n",
      "\n",
      "changing T value to 0.41000000000000003\n",
      "  + Number of params: 2.82M\n",
      "  + Number of FLOPs: 0.37G\n",
      "Train Epoch: 83 [0/50000 (0%)]\tLoss: 0.152885\n",
      "Train Epoch: 83 [6400/50000 (13%)]\tLoss: 0.085764\n",
      "Train Epoch: 83 [12800/50000 (26%)]\tLoss: 0.071181\n",
      "Train Epoch: 83 [19200/50000 (38%)]\tLoss: 0.082596\n",
      "Train Epoch: 83 [25600/50000 (51%)]\tLoss: 0.031730\n",
      "Train Epoch: 83 [32000/50000 (64%)]\tLoss: 0.010850\n",
      "Train Epoch: 83 [38400/50000 (77%)]\tLoss: 0.141375\n",
      "Train Epoch: 83 [44800/50000 (90%)]\tLoss: 0.026061\n",
      "\n",
      "Test set: Average loss: 0.2640, Accuracy: 9281/10000 (93%)\n",
      "\n",
      "Time taken for epoch : 12.926945924758911\n",
      "\n",
      "changing T value to 0.41500000000000004\n",
      "  + Number of params: 2.71M\n",
      "  + Number of FLOPs: 0.36G\n",
      "Train Epoch: 84 [0/50000 (0%)]\tLoss: 0.046854\n",
      "Train Epoch: 84 [6400/50000 (13%)]\tLoss: 0.003433\n",
      "Train Epoch: 84 [12800/50000 (26%)]\tLoss: 0.026321\n",
      "Train Epoch: 84 [19200/50000 (38%)]\tLoss: 0.105775\n",
      "Train Epoch: 84 [25600/50000 (51%)]\tLoss: 0.062894\n",
      "Train Epoch: 84 [32000/50000 (64%)]\tLoss: 0.027273\n",
      "Train Epoch: 84 [38400/50000 (77%)]\tLoss: 0.019848\n",
      "Train Epoch: 84 [44800/50000 (90%)]\tLoss: 0.058331\n",
      "\n",
      "Test set: Average loss: 0.2739, Accuracy: 9293/10000 (93%)\n",
      "\n",
      "Time taken for epoch : 12.765605449676514\n",
      "\n",
      "changing T value to 0.42\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  + Number of params: 2.61M\n",
      "  + Number of FLOPs: 0.36G\n",
      "Train Epoch: 85 [0/50000 (0%)]\tLoss: 0.016496\n",
      "Train Epoch: 85 [6400/50000 (13%)]\tLoss: 0.058460\n",
      "Train Epoch: 85 [12800/50000 (26%)]\tLoss: 0.060709\n",
      "Train Epoch: 85 [19200/50000 (38%)]\tLoss: 0.079058\n",
      "Train Epoch: 85 [25600/50000 (51%)]\tLoss: 0.014127\n",
      "Train Epoch: 85 [32000/50000 (64%)]\tLoss: 0.027676\n",
      "Train Epoch: 85 [38400/50000 (77%)]\tLoss: 0.007848\n",
      "Train Epoch: 85 [44800/50000 (90%)]\tLoss: 0.078514\n",
      "\n",
      "Test set: Average loss: 0.2699, Accuracy: 9290/10000 (93%)\n",
      "\n",
      "Time taken for epoch : 12.748064279556274\n",
      "\n",
      "changing T value to 0.425\n",
      "  + Number of params: 2.49M\n",
      "  + Number of FLOPs: 0.36G\n",
      "Train Epoch: 86 [0/50000 (0%)]\tLoss: 0.007262\n",
      "Train Epoch: 86 [6400/50000 (13%)]\tLoss: 0.090014\n",
      "Train Epoch: 86 [12800/50000 (26%)]\tLoss: 0.007856\n",
      "Train Epoch: 86 [19200/50000 (38%)]\tLoss: 0.010201\n",
      "Train Epoch: 86 [25600/50000 (51%)]\tLoss: 0.056313\n",
      "Train Epoch: 86 [32000/50000 (64%)]\tLoss: 0.013075\n",
      "Train Epoch: 86 [38400/50000 (77%)]\tLoss: 0.003207\n",
      "Train Epoch: 86 [44800/50000 (90%)]\tLoss: 0.008982\n",
      "\n",
      "Test set: Average loss: 0.2827, Accuracy: 9266/10000 (93%)\n",
      "\n",
      "Time taken for epoch : 12.596033573150635\n",
      "\n",
      "changing T value to 0.43\n",
      "  + Number of params: 2.37M\n",
      "  + Number of FLOPs: 0.35G\n",
      "Train Epoch: 87 [0/50000 (0%)]\tLoss: 0.047369\n",
      "Train Epoch: 87 [6400/50000 (13%)]\tLoss: 0.009832\n",
      "Train Epoch: 87 [12800/50000 (26%)]\tLoss: 0.010119\n",
      "Train Epoch: 87 [19200/50000 (38%)]\tLoss: 0.014508\n",
      "Train Epoch: 87 [25600/50000 (51%)]\tLoss: 0.063401\n",
      "Train Epoch: 87 [32000/50000 (64%)]\tLoss: 0.033603\n",
      "Train Epoch: 87 [38400/50000 (77%)]\tLoss: 0.032894\n",
      "Train Epoch: 87 [44800/50000 (90%)]\tLoss: 0.062338\n",
      "\n",
      "Test set: Average loss: 0.2946, Accuracy: 9273/10000 (93%)\n",
      "\n",
      "Time taken for epoch : 12.352541446685791\n",
      "\n",
      "changing T value to 0.435\n",
      "  + Number of params: 2.28M\n",
      "  + Number of FLOPs: 0.35G\n",
      "Train Epoch: 88 [0/50000 (0%)]\tLoss: 0.028872\n",
      "Train Epoch: 88 [6400/50000 (13%)]\tLoss: 0.043964\n",
      "Train Epoch: 88 [12800/50000 (26%)]\tLoss: 0.045844\n",
      "Train Epoch: 88 [19200/50000 (38%)]\tLoss: 0.009269\n",
      "Train Epoch: 88 [25600/50000 (51%)]\tLoss: 0.021431\n",
      "Train Epoch: 88 [32000/50000 (64%)]\tLoss: 0.005189\n",
      "Train Epoch: 88 [38400/50000 (77%)]\tLoss: 0.021826\n",
      "Train Epoch: 88 [44800/50000 (90%)]\tLoss: 0.025271\n",
      "\n",
      "Test set: Average loss: 0.2855, Accuracy: 9286/10000 (93%)\n",
      "\n",
      "Time taken for epoch : 12.228326559066772\n",
      "\n",
      "changing T value to 0.44\n",
      "  + Number of params: 2.21M\n",
      "  + Number of FLOPs: 0.35G\n",
      "Train Epoch: 89 [0/50000 (0%)]\tLoss: 0.114715\n",
      "Train Epoch: 89 [6400/50000 (13%)]\tLoss: 0.045729\n",
      "Train Epoch: 89 [12800/50000 (26%)]\tLoss: 0.049979\n",
      "Train Epoch: 89 [19200/50000 (38%)]\tLoss: 0.135952\n",
      "Train Epoch: 89 [25600/50000 (51%)]\tLoss: 0.007176\n",
      "Train Epoch: 89 [32000/50000 (64%)]\tLoss: 0.027831\n",
      "Train Epoch: 89 [38400/50000 (77%)]\tLoss: 0.084510\n",
      "Train Epoch: 89 [44800/50000 (90%)]\tLoss: 0.026556\n",
      "\n",
      "Test set: Average loss: 0.2969, Accuracy: 9235/10000 (92%)\n",
      "\n",
      "Time taken for epoch : 12.205741882324219\n",
      "\n",
      "changing T value to 0.445\n",
      "  + Number of params: 2.14M\n",
      "  + Number of FLOPs: 0.35G\n",
      "Train Epoch: 90 [0/50000 (0%)]\tLoss: 0.097653\n",
      "Train Epoch: 90 [6400/50000 (13%)]\tLoss: 0.013816\n",
      "Train Epoch: 90 [12800/50000 (26%)]\tLoss: 0.083405\n",
      "Train Epoch: 90 [19200/50000 (38%)]\tLoss: 0.008338\n",
      "Train Epoch: 90 [25600/50000 (51%)]\tLoss: 0.032815\n",
      "Train Epoch: 90 [32000/50000 (64%)]\tLoss: 0.004931\n",
      "Train Epoch: 90 [38400/50000 (77%)]\tLoss: 0.019873\n",
      "Train Epoch: 90 [44800/50000 (90%)]\tLoss: 0.132611\n",
      "\n",
      "Test set: Average loss: 0.2959, Accuracy: 9235/10000 (92%)\n",
      "\n",
      "Time taken for epoch : 12.225059986114502\n",
      "\n",
      "changing T value to 0.45\n",
      "  + Number of params: 2.08M\n",
      "  + Number of FLOPs: 0.34G\n",
      "Train Epoch: 91 [0/50000 (0%)]\tLoss: 0.127559\n",
      "Train Epoch: 91 [6400/50000 (13%)]\tLoss: 0.042942\n",
      "Train Epoch: 91 [12800/50000 (26%)]\tLoss: 0.030806\n",
      "Train Epoch: 91 [19200/50000 (38%)]\tLoss: 0.010125\n",
      "Train Epoch: 91 [25600/50000 (51%)]\tLoss: 0.030055\n",
      "Train Epoch: 91 [32000/50000 (64%)]\tLoss: 0.059626\n",
      "Train Epoch: 91 [38400/50000 (77%)]\tLoss: 0.050530\n",
      "Train Epoch: 91 [44800/50000 (90%)]\tLoss: 0.075340\n",
      "\n",
      "Test set: Average loss: 0.3166, Accuracy: 9257/10000 (93%)\n",
      "\n",
      "Time taken for epoch : 12.092686414718628\n",
      "\n",
      "changing T value to 0.455\n",
      "  + Number of params: 2.04M\n",
      "  + Number of FLOPs: 0.34G\n",
      "Train Epoch: 92 [0/50000 (0%)]\tLoss: 0.135361\n",
      "Train Epoch: 92 [6400/50000 (13%)]\tLoss: 0.136635\n",
      "Train Epoch: 92 [12800/50000 (26%)]\tLoss: 0.006394\n",
      "Train Epoch: 92 [19200/50000 (38%)]\tLoss: 0.008024\n",
      "Train Epoch: 92 [25600/50000 (51%)]\tLoss: 0.024981\n",
      "Train Epoch: 92 [32000/50000 (64%)]\tLoss: 0.015592\n",
      "Train Epoch: 92 [38400/50000 (77%)]\tLoss: 0.006323\n",
      "Train Epoch: 92 [44800/50000 (90%)]\tLoss: 0.085693\n",
      "\n",
      "Test set: Average loss: 0.2957, Accuracy: 9245/10000 (92%)\n",
      "\n",
      "Time taken for epoch : 12.010449886322021\n",
      "\n",
      "changing T value to 0.46\n",
      "  + Number of params: 2.02M\n",
      "  + Number of FLOPs: 0.34G\n",
      "Train Epoch: 93 [0/50000 (0%)]\tLoss: 0.026312\n",
      "Train Epoch: 93 [6400/50000 (13%)]\tLoss: 0.039699\n",
      "Train Epoch: 93 [12800/50000 (26%)]\tLoss: 0.012161\n",
      "Train Epoch: 93 [19200/50000 (38%)]\tLoss: 0.109931\n",
      "Train Epoch: 93 [25600/50000 (51%)]\tLoss: 0.010618\n",
      "Train Epoch: 93 [32000/50000 (64%)]\tLoss: 0.012351\n",
      "Train Epoch: 93 [38400/50000 (77%)]\tLoss: 0.038010\n",
      "Train Epoch: 93 [44800/50000 (90%)]\tLoss: 0.048460\n",
      "\n",
      "Test set: Average loss: 0.3321, Accuracy: 9221/10000 (92%)\n",
      "\n",
      "Time taken for epoch : 11.941216945648193\n",
      "\n",
      "changing T value to 0.465\n",
      "  + Number of params: 1.99M\n",
      "  + Number of FLOPs: 0.34G\n",
      "Train Epoch: 94 [0/50000 (0%)]\tLoss: 0.007825\n",
      "Train Epoch: 94 [6400/50000 (13%)]\tLoss: 0.011230\n",
      "Train Epoch: 94 [12800/50000 (26%)]\tLoss: 0.002499\n",
      "Train Epoch: 94 [19200/50000 (38%)]\tLoss: 0.008257\n",
      "Train Epoch: 94 [25600/50000 (51%)]\tLoss: 0.021749\n",
      "Train Epoch: 94 [32000/50000 (64%)]\tLoss: 0.061408\n",
      "Train Epoch: 94 [38400/50000 (77%)]\tLoss: 0.064312\n",
      "Train Epoch: 94 [44800/50000 (90%)]\tLoss: 0.166283\n",
      "\n",
      "Test set: Average loss: 0.3330, Accuracy: 9195/10000 (92%)\n",
      "\n",
      "Time taken for epoch : 12.13210129737854\n",
      "\n",
      "changing T value to 0.47000000000000003\n",
      "  + Number of params: 1.96M\n",
      "  + Number of FLOPs: 0.34G\n",
      "Train Epoch: 95 [0/50000 (0%)]\tLoss: 0.022538\n",
      "Train Epoch: 95 [6400/50000 (13%)]\tLoss: 0.044120\n",
      "Train Epoch: 95 [12800/50000 (26%)]\tLoss: 0.028743\n",
      "Train Epoch: 95 [19200/50000 (38%)]\tLoss: 0.069111\n",
      "Train Epoch: 95 [25600/50000 (51%)]\tLoss: 0.022037\n",
      "Train Epoch: 95 [32000/50000 (64%)]\tLoss: 0.010316\n",
      "Train Epoch: 95 [38400/50000 (77%)]\tLoss: 0.036871\n",
      "Train Epoch: 95 [44800/50000 (90%)]\tLoss: 0.006352\n",
      "\n",
      "Test set: Average loss: 0.3313, Accuracy: 9208/10000 (92%)\n",
      "\n",
      "Time taken for epoch : 11.716838359832764\n",
      "\n",
      "changing T value to 0.47500000000000003\n",
      "  + Number of params: 1.95M\n",
      "  + Number of FLOPs: 0.34G\n",
      "Train Epoch: 96 [0/50000 (0%)]\tLoss: 0.158657\n",
      "Train Epoch: 96 [6400/50000 (13%)]\tLoss: 0.110846\n",
      "Train Epoch: 96 [12800/50000 (26%)]\tLoss: 0.015671\n",
      "Train Epoch: 96 [19200/50000 (38%)]\tLoss: 0.037720\n",
      "Train Epoch: 96 [25600/50000 (51%)]\tLoss: 0.040425\n",
      "Train Epoch: 96 [32000/50000 (64%)]\tLoss: 0.018198\n",
      "Train Epoch: 96 [38400/50000 (77%)]\tLoss: 0.007241\n",
      "Train Epoch: 96 [44800/50000 (90%)]\tLoss: 0.059416\n",
      "\n",
      "Test set: Average loss: 0.3443, Accuracy: 9201/10000 (92%)\n",
      "\n",
      "Time taken for epoch : 11.825178146362305\n",
      "\n",
      "changing T value to 0.48\n",
      "  + Number of params: 1.92M\n",
      "  + Number of FLOPs: 0.34G\n",
      "Train Epoch: 97 [0/50000 (0%)]\tLoss: 0.060352\n",
      "Train Epoch: 97 [6400/50000 (13%)]\tLoss: 0.022224\n",
      "Train Epoch: 97 [12800/50000 (26%)]\tLoss: 0.018130\n",
      "Train Epoch: 97 [19200/50000 (38%)]\tLoss: 0.010590\n",
      "Train Epoch: 97 [25600/50000 (51%)]\tLoss: 0.028570\n",
      "Train Epoch: 97 [32000/50000 (64%)]\tLoss: 0.057033\n",
      "Train Epoch: 97 [38400/50000 (77%)]\tLoss: 0.043600\n",
      "Train Epoch: 97 [44800/50000 (90%)]\tLoss: 0.043573\n",
      "\n",
      "Test set: Average loss: 0.3258, Accuracy: 9226/10000 (92%)\n",
      "\n",
      "Time taken for epoch : 11.587865114212036\n",
      "\n",
      "changing T value to 0.485\n",
      "  + Number of params: 1.90M\n",
      "  + Number of FLOPs: 0.33G\n",
      "Train Epoch: 98 [0/50000 (0%)]\tLoss: 0.055877\n",
      "Train Epoch: 98 [6400/50000 (13%)]\tLoss: 0.051225\n",
      "Train Epoch: 98 [12800/50000 (26%)]\tLoss: 0.028958\n",
      "Train Epoch: 98 [19200/50000 (38%)]\tLoss: 0.069848\n",
      "Train Epoch: 98 [25600/50000 (51%)]\tLoss: 0.017632\n",
      "Train Epoch: 98 [32000/50000 (64%)]\tLoss: 0.006807\n",
      "Train Epoch: 98 [38400/50000 (77%)]\tLoss: 0.008182\n",
      "Train Epoch: 98 [44800/50000 (90%)]\tLoss: 0.007816\n",
      "\n",
      "Test set: Average loss: 0.3206, Accuracy: 9264/10000 (93%)\n",
      "\n",
      "Time taken for epoch : 11.543086051940918\n",
      "\n",
      "changing T value to 0.49\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  + Number of params: 1.89M\n",
      "  + Number of FLOPs: 0.33G\n",
      "Train Epoch: 99 [0/50000 (0%)]\tLoss: 0.137662\n",
      "Train Epoch: 99 [6400/50000 (13%)]\tLoss: 0.024192\n",
      "Train Epoch: 99 [12800/50000 (26%)]\tLoss: 0.020283\n",
      "Train Epoch: 99 [19200/50000 (38%)]\tLoss: 0.011861\n",
      "Train Epoch: 99 [25600/50000 (51%)]\tLoss: 0.006766\n",
      "Train Epoch: 99 [32000/50000 (64%)]\tLoss: 0.019855\n",
      "Train Epoch: 99 [38400/50000 (77%)]\tLoss: 0.004913\n",
      "Train Epoch: 99 [44800/50000 (90%)]\tLoss: 0.088335\n",
      "\n",
      "Test set: Average loss: 0.3434, Accuracy: 9218/10000 (92%)\n",
      "\n",
      "Time taken for epoch : 11.827802419662476\n",
      "\n",
      "changing T value to 0.495\n",
      "  + Number of params: 1.87M\n",
      "  + Number of FLOPs: 0.33G\n",
      "Train Epoch: 100 [0/50000 (0%)]\tLoss: 0.005777\n",
      "Train Epoch: 100 [6400/50000 (13%)]\tLoss: 0.012229\n",
      "Train Epoch: 100 [12800/50000 (26%)]\tLoss: 0.039917\n",
      "Train Epoch: 100 [19200/50000 (38%)]\tLoss: 0.011385\n",
      "Train Epoch: 100 [25600/50000 (51%)]\tLoss: 0.011112\n",
      "Train Epoch: 100 [32000/50000 (64%)]\tLoss: 0.001961\n",
      "Train Epoch: 100 [38400/50000 (77%)]\tLoss: 0.002975\n",
      "Train Epoch: 100 [44800/50000 (90%)]\tLoss: 0.069537\n",
      "\n",
      "Test set: Average loss: 0.3518, Accuracy: 9196/10000 (92%)\n",
      "\n",
      "Time taken for epoch : 11.969708919525146\n",
      "\n",
      "changing T value to 0.5\n",
      "  + Number of params: 1.85M\n",
      "  + Number of FLOPs: 0.33G\n",
      "Train Epoch: 101 [0/50000 (0%)]\tLoss: 0.007637\n",
      "Train Epoch: 101 [6400/50000 (13%)]\tLoss: 0.077842\n",
      "Train Epoch: 101 [12800/50000 (26%)]\tLoss: 0.005971\n",
      "Train Epoch: 101 [19200/50000 (38%)]\tLoss: 0.094469\n",
      "Train Epoch: 101 [25600/50000 (51%)]\tLoss: 0.032030\n",
      "Train Epoch: 101 [32000/50000 (64%)]\tLoss: 0.006481\n",
      "Train Epoch: 101 [38400/50000 (77%)]\tLoss: 0.077580\n",
      "Train Epoch: 101 [44800/50000 (90%)]\tLoss: 0.003616\n",
      "\n",
      "Test set: Average loss: 0.3102, Accuracy: 9238/10000 (92%)\n",
      "\n",
      "Time taken for epoch : 11.87393069267273\n",
      "\n",
      "changing T value to 0.505\n",
      "  + Number of params: 1.83M\n",
      "  + Number of FLOPs: 0.33G\n",
      "Train Epoch: 102 [0/50000 (0%)]\tLoss: 0.049631\n",
      "Train Epoch: 102 [6400/50000 (13%)]\tLoss: 0.010410\n",
      "Train Epoch: 102 [12800/50000 (26%)]\tLoss: 0.018821\n",
      "Train Epoch: 102 [19200/50000 (38%)]\tLoss: 0.006345\n",
      "Train Epoch: 102 [25600/50000 (51%)]\tLoss: 0.014972\n",
      "Train Epoch: 102 [32000/50000 (64%)]\tLoss: 0.006032\n",
      "Train Epoch: 102 [38400/50000 (77%)]\tLoss: 0.085981\n",
      "Train Epoch: 102 [44800/50000 (90%)]\tLoss: 0.018895\n",
      "\n",
      "Test set: Average loss: 0.3600, Accuracy: 9176/10000 (92%)\n",
      "\n",
      "Time taken for epoch : 11.588574409484863\n",
      "\n",
      "changing T value to 0.51\n",
      "  + Number of params: 1.81M\n",
      "  + Number of FLOPs: 0.33G\n",
      "Train Epoch: 103 [0/50000 (0%)]\tLoss: 0.003399\n",
      "Train Epoch: 103 [6400/50000 (13%)]\tLoss: 0.043201\n",
      "Train Epoch: 103 [12800/50000 (26%)]\tLoss: 0.052701\n",
      "Train Epoch: 103 [19200/50000 (38%)]\tLoss: 0.021351\n",
      "Train Epoch: 103 [25600/50000 (51%)]\tLoss: 0.065230\n",
      "Train Epoch: 103 [32000/50000 (64%)]\tLoss: 0.079665\n",
      "Train Epoch: 103 [38400/50000 (77%)]\tLoss: 0.025725\n",
      "Train Epoch: 103 [44800/50000 (90%)]\tLoss: 0.041952\n",
      "\n",
      "Test set: Average loss: 0.3576, Accuracy: 9206/10000 (92%)\n",
      "\n",
      "Time taken for epoch : 11.634021759033203\n",
      "\n",
      "changing T value to 0.515\n",
      "  + Number of params: 1.80M\n",
      "  + Number of FLOPs: 0.33G\n",
      "Train Epoch: 104 [0/50000 (0%)]\tLoss: 0.009129\n",
      "Train Epoch: 104 [6400/50000 (13%)]\tLoss: 0.016904\n",
      "Train Epoch: 104 [12800/50000 (26%)]\tLoss: 0.044375\n",
      "Train Epoch: 104 [19200/50000 (38%)]\tLoss: 0.009669\n",
      "Train Epoch: 104 [25600/50000 (51%)]\tLoss: 0.001436\n",
      "Train Epoch: 104 [32000/50000 (64%)]\tLoss: 0.023772\n",
      "Train Epoch: 104 [38400/50000 (77%)]\tLoss: 0.030335\n",
      "Train Epoch: 104 [44800/50000 (90%)]\tLoss: 0.030143\n",
      "\n",
      "Test set: Average loss: 0.3297, Accuracy: 9253/10000 (93%)\n",
      "\n",
      "Time taken for epoch : 11.384353399276733\n",
      "\n",
      "changing T value to 0.52\n",
      "  + Number of params: 1.78M\n",
      "  + Number of FLOPs: 0.33G\n",
      "Train Epoch: 105 [0/50000 (0%)]\tLoss: 0.066616\n",
      "Train Epoch: 105 [6400/50000 (13%)]\tLoss: 0.010500\n",
      "Train Epoch: 105 [12800/50000 (26%)]\tLoss: 0.027386\n",
      "Train Epoch: 105 [19200/50000 (38%)]\tLoss: 0.002994\n",
      "Train Epoch: 105 [25600/50000 (51%)]\tLoss: 0.120448\n",
      "Train Epoch: 105 [32000/50000 (64%)]\tLoss: 0.012269\n",
      "Train Epoch: 105 [38400/50000 (77%)]\tLoss: 0.002097\n",
      "Train Epoch: 105 [44800/50000 (90%)]\tLoss: 0.065950\n",
      "\n",
      "Test set: Average loss: 0.3383, Accuracy: 9213/10000 (92%)\n",
      "\n",
      "Time taken for epoch : 11.703673362731934\n",
      "\n",
      "changing T value to 0.525\n",
      "  + Number of params: 1.77M\n",
      "  + Number of FLOPs: 0.33G\n",
      "Train Epoch: 106 [0/50000 (0%)]\tLoss: 0.045195\n",
      "Train Epoch: 106 [6400/50000 (13%)]\tLoss: 0.007487\n",
      "Train Epoch: 106 [12800/50000 (26%)]\tLoss: 0.014581\n",
      "Train Epoch: 106 [19200/50000 (38%)]\tLoss: 0.010585\n",
      "Train Epoch: 106 [25600/50000 (51%)]\tLoss: 0.008141\n",
      "Train Epoch: 106 [32000/50000 (64%)]\tLoss: 0.014655\n",
      "Train Epoch: 106 [38400/50000 (77%)]\tLoss: 0.059212\n",
      "Train Epoch: 106 [44800/50000 (90%)]\tLoss: 0.018591\n",
      "\n",
      "Test set: Average loss: 0.3189, Accuracy: 9283/10000 (93%)\n",
      "\n",
      "Time taken for epoch : 11.469794511795044\n",
      "\n",
      "changing T value to 0.53\n",
      "  + Number of params: 1.76M\n",
      "  + Number of FLOPs: 0.33G\n",
      "Train Epoch: 107 [0/50000 (0%)]\tLoss: 0.066069\n",
      "Train Epoch: 107 [6400/50000 (13%)]\tLoss: 0.014525\n",
      "Train Epoch: 107 [12800/50000 (26%)]\tLoss: 0.006997\n",
      "Train Epoch: 107 [19200/50000 (38%)]\tLoss: 0.071424\n",
      "Train Epoch: 107 [25600/50000 (51%)]\tLoss: 0.002330\n",
      "Train Epoch: 107 [32000/50000 (64%)]\tLoss: 0.103452\n",
      "Train Epoch: 107 [38400/50000 (77%)]\tLoss: 0.071420\n",
      "Train Epoch: 107 [44800/50000 (90%)]\tLoss: 0.071150\n",
      "\n",
      "Test set: Average loss: 0.3430, Accuracy: 9217/10000 (92%)\n",
      "\n",
      "Time taken for epoch : 11.958000183105469\n",
      "\n",
      "changing T value to 0.535\n",
      "  + Number of params: 1.75M\n",
      "  + Number of FLOPs: 0.33G\n",
      "Train Epoch: 108 [0/50000 (0%)]\tLoss: 0.007496\n",
      "Train Epoch: 108 [6400/50000 (13%)]\tLoss: 0.009107\n",
      "Train Epoch: 108 [12800/50000 (26%)]\tLoss: 0.003043\n",
      "Train Epoch: 108 [19200/50000 (38%)]\tLoss: 0.013735\n",
      "Train Epoch: 108 [25600/50000 (51%)]\tLoss: 0.011574\n",
      "Train Epoch: 108 [32000/50000 (64%)]\tLoss: 0.056189\n",
      "Train Epoch: 108 [38400/50000 (77%)]\tLoss: 0.161421\n",
      "Train Epoch: 108 [44800/50000 (90%)]\tLoss: 0.000901\n",
      "\n",
      "Test set: Average loss: 0.3292, Accuracy: 9251/10000 (93%)\n",
      "\n",
      "Time taken for epoch : 11.836896896362305\n",
      "\n",
      "changing T value to 0.54\n",
      "  + Number of params: 1.74M\n",
      "  + Number of FLOPs: 0.32G\n",
      "Train Epoch: 109 [0/50000 (0%)]\tLoss: 0.006592\n",
      "Train Epoch: 109 [6400/50000 (13%)]\tLoss: 0.001686\n",
      "Train Epoch: 109 [12800/50000 (26%)]\tLoss: 0.002130\n",
      "Train Epoch: 109 [19200/50000 (38%)]\tLoss: 0.009348\n",
      "Train Epoch: 109 [25600/50000 (51%)]\tLoss: 0.041829\n",
      "Train Epoch: 109 [32000/50000 (64%)]\tLoss: 0.051121\n",
      "Train Epoch: 109 [38400/50000 (77%)]\tLoss: 0.004596\n",
      "Train Epoch: 109 [44800/50000 (90%)]\tLoss: 0.020198\n",
      "\n",
      "Test set: Average loss: 0.3411, Accuracy: 9262/10000 (93%)\n",
      "\n",
      "Time taken for epoch : 11.581540822982788\n",
      "\n",
      "changing T value to 0.545\n",
      "  + Number of params: 1.73M\n",
      "  + Number of FLOPs: 0.32G\n",
      "Train Epoch: 110 [0/50000 (0%)]\tLoss: 0.027448\n",
      "Train Epoch: 110 [6400/50000 (13%)]\tLoss: 0.022070\n",
      "Train Epoch: 110 [12800/50000 (26%)]\tLoss: 0.049529\n",
      "Train Epoch: 110 [19200/50000 (38%)]\tLoss: 0.003032\n",
      "Train Epoch: 110 [25600/50000 (51%)]\tLoss: 0.007148\n",
      "Train Epoch: 110 [32000/50000 (64%)]\tLoss: 0.011603\n",
      "Train Epoch: 110 [38400/50000 (77%)]\tLoss: 0.046979\n",
      "Train Epoch: 110 [44800/50000 (90%)]\tLoss: 0.063286\n",
      "\n",
      "Test set: Average loss: 0.3146, Accuracy: 9283/10000 (93%)\n",
      "\n",
      "Time taken for epoch : 12.163393020629883\n",
      "\n",
      "changing T value to 0.55\n",
      "  + Number of params: 1.72M\n",
      "  + Number of FLOPs: 0.32G\n",
      "Train Epoch: 111 [0/50000 (0%)]\tLoss: 0.013386\n",
      "Train Epoch: 111 [6400/50000 (13%)]\tLoss: 0.046814\n",
      "Train Epoch: 111 [12800/50000 (26%)]\tLoss: 0.008516\n",
      "Train Epoch: 111 [19200/50000 (38%)]\tLoss: 0.001919\n",
      "Train Epoch: 111 [25600/50000 (51%)]\tLoss: 0.047866\n",
      "Train Epoch: 111 [32000/50000 (64%)]\tLoss: 0.085729\n",
      "Train Epoch: 111 [38400/50000 (77%)]\tLoss: 0.005065\n",
      "Train Epoch: 111 [44800/50000 (90%)]\tLoss: 0.051283\n",
      "\n",
      "Test set: Average loss: 0.3191, Accuracy: 9257/10000 (93%)\n",
      "\n",
      "Time taken for epoch : 11.859538555145264\n",
      "\n",
      "changing T value to 0.555\n",
      "  + Number of params: 1.71M\n",
      "  + Number of FLOPs: 0.32G\n",
      "Train Epoch: 112 [0/50000 (0%)]\tLoss: 0.030114\n",
      "Train Epoch: 112 [6400/50000 (13%)]\tLoss: 0.049468\n",
      "Train Epoch: 112 [12800/50000 (26%)]\tLoss: 0.039833\n",
      "Train Epoch: 112 [19200/50000 (38%)]\tLoss: 0.021878\n",
      "Train Epoch: 112 [25600/50000 (51%)]\tLoss: 0.008677\n",
      "Train Epoch: 112 [32000/50000 (64%)]\tLoss: 0.009198\n",
      "Train Epoch: 112 [38400/50000 (77%)]\tLoss: 0.063296\n",
      "Train Epoch: 112 [44800/50000 (90%)]\tLoss: 0.003782\n",
      "\n",
      "Test set: Average loss: 0.3309, Accuracy: 9234/10000 (92%)\n",
      "\n",
      "Time taken for epoch : 11.667529821395874\n",
      "\n",
      "changing T value to 0.56\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  + Number of params: 1.71M\n",
      "  + Number of FLOPs: 0.32G\n",
      "Train Epoch: 113 [0/50000 (0%)]\tLoss: 0.063430\n",
      "Train Epoch: 113 [6400/50000 (13%)]\tLoss: 0.004403\n",
      "Train Epoch: 113 [12800/50000 (26%)]\tLoss: 0.006074\n",
      "Train Epoch: 113 [19200/50000 (38%)]\tLoss: 0.010410\n",
      "Train Epoch: 113 [25600/50000 (51%)]\tLoss: 0.037249\n",
      "Train Epoch: 113 [32000/50000 (64%)]\tLoss: 0.005917\n",
      "Train Epoch: 113 [38400/50000 (77%)]\tLoss: 0.051743\n",
      "Train Epoch: 113 [44800/50000 (90%)]\tLoss: 0.086719\n",
      "\n",
      "Test set: Average loss: 0.3252, Accuracy: 9248/10000 (92%)\n",
      "\n",
      "Time taken for epoch : 11.354389429092407\n",
      "\n",
      "changing T value to 0.5650000000000001\n",
      "  + Number of params: 1.68M\n",
      "  + Number of FLOPs: 0.32G\n",
      "Train Epoch: 114 [0/50000 (0%)]\tLoss: 0.010718\n",
      "Train Epoch: 114 [6400/50000 (13%)]\tLoss: 0.096781\n",
      "Train Epoch: 114 [12800/50000 (26%)]\tLoss: 0.004217\n",
      "Train Epoch: 114 [19200/50000 (38%)]\tLoss: 0.044474\n",
      "Train Epoch: 114 [25600/50000 (51%)]\tLoss: 0.088521\n",
      "Train Epoch: 114 [32000/50000 (64%)]\tLoss: 0.009684\n",
      "Train Epoch: 114 [38400/50000 (77%)]\tLoss: 0.003274\n",
      "Train Epoch: 114 [44800/50000 (90%)]\tLoss: 0.047165\n",
      "\n",
      "Test set: Average loss: 0.3363, Accuracy: 9253/10000 (93%)\n",
      "\n",
      "Time taken for epoch : 11.38552713394165\n",
      "\n",
      "changing T value to 0.5700000000000001\n",
      "  + Number of params: 1.67M\n",
      "  + Number of FLOPs: 0.32G\n",
      "Train Epoch: 115 [0/50000 (0%)]\tLoss: 0.003685\n",
      "Train Epoch: 115 [6400/50000 (13%)]\tLoss: 0.069280\n",
      "Train Epoch: 115 [12800/50000 (26%)]\tLoss: 0.008248\n",
      "Train Epoch: 115 [19200/50000 (38%)]\tLoss: 0.024718\n",
      "Train Epoch: 115 [25600/50000 (51%)]\tLoss: 0.050367\n",
      "Train Epoch: 115 [32000/50000 (64%)]\tLoss: 0.072544\n",
      "Train Epoch: 115 [38400/50000 (77%)]\tLoss: 0.020276\n",
      "Train Epoch: 115 [44800/50000 (90%)]\tLoss: 0.039049\n",
      "\n",
      "Test set: Average loss: 0.3266, Accuracy: 9256/10000 (93%)\n",
      "\n",
      "Time taken for epoch : 11.27479100227356\n",
      "\n",
      "changing T value to 0.5750000000000001\n",
      "  + Number of params: 1.66M\n",
      "  + Number of FLOPs: 0.31G\n",
      "Train Epoch: 116 [0/50000 (0%)]\tLoss: 0.135784\n",
      "Train Epoch: 116 [6400/50000 (13%)]\tLoss: 0.011390\n",
      "Train Epoch: 116 [12800/50000 (26%)]\tLoss: 0.010430\n",
      "Train Epoch: 116 [19200/50000 (38%)]\tLoss: 0.005300\n",
      "Train Epoch: 116 [25600/50000 (51%)]\tLoss: 0.007831\n",
      "Train Epoch: 116 [32000/50000 (64%)]\tLoss: 0.007176\n",
      "Train Epoch: 116 [38400/50000 (77%)]\tLoss: 0.011601\n",
      "Train Epoch: 116 [44800/50000 (90%)]\tLoss: 0.009552\n",
      "\n",
      "Test set: Average loss: 0.3327, Accuracy: 9238/10000 (92%)\n",
      "\n",
      "Time taken for epoch : 11.508784770965576\n",
      "\n",
      "changing T value to 0.58\n",
      "  + Number of params: 1.65M\n",
      "  + Number of FLOPs: 0.31G\n",
      "Train Epoch: 117 [0/50000 (0%)]\tLoss: 0.023890\n",
      "Train Epoch: 117 [6400/50000 (13%)]\tLoss: 0.038151\n",
      "Train Epoch: 117 [12800/50000 (26%)]\tLoss: 0.007058\n",
      "Train Epoch: 117 [19200/50000 (38%)]\tLoss: 0.023770\n",
      "Train Epoch: 117 [25600/50000 (51%)]\tLoss: 0.037341\n",
      "Train Epoch: 117 [32000/50000 (64%)]\tLoss: 0.002242\n",
      "Train Epoch: 117 [38400/50000 (77%)]\tLoss: 0.043929\n",
      "Train Epoch: 117 [44800/50000 (90%)]\tLoss: 0.011416\n",
      "\n",
      "Test set: Average loss: 0.3395, Accuracy: 9212/10000 (92%)\n",
      "\n",
      "Time taken for epoch : 12.10411810874939\n",
      "\n",
      "changing T value to 0.585\n",
      "  + Number of params: 1.64M\n",
      "  + Number of FLOPs: 0.31G\n",
      "Train Epoch: 118 [0/50000 (0%)]\tLoss: 0.081815\n",
      "Train Epoch: 118 [6400/50000 (13%)]\tLoss: 0.005038\n",
      "Train Epoch: 118 [12800/50000 (26%)]\tLoss: 0.001905\n",
      "Train Epoch: 118 [19200/50000 (38%)]\tLoss: 0.029146\n",
      "Train Epoch: 118 [25600/50000 (51%)]\tLoss: 0.003999\n",
      "Train Epoch: 118 [32000/50000 (64%)]\tLoss: 0.004354\n",
      "Train Epoch: 118 [38400/50000 (77%)]\tLoss: 0.046897\n",
      "Train Epoch: 118 [44800/50000 (90%)]\tLoss: 0.065354\n",
      "\n",
      "Test set: Average loss: 0.3195, Accuracy: 9231/10000 (92%)\n",
      "\n",
      "Time taken for epoch : 11.394246339797974\n",
      "\n",
      "changing T value to 0.59\n",
      "  + Number of params: 1.63M\n",
      "  + Number of FLOPs: 0.31G\n",
      "Train Epoch: 119 [0/50000 (0%)]\tLoss: 0.039357\n",
      "Train Epoch: 119 [6400/50000 (13%)]\tLoss: 0.011071\n",
      "Train Epoch: 119 [12800/50000 (26%)]\tLoss: 0.005738\n",
      "Train Epoch: 119 [19200/50000 (38%)]\tLoss: 0.047270\n",
      "Train Epoch: 119 [25600/50000 (51%)]\tLoss: 0.069222\n",
      "Train Epoch: 119 [32000/50000 (64%)]\tLoss: 0.017132\n",
      "Train Epoch: 119 [38400/50000 (77%)]\tLoss: 0.024637\n",
      "Train Epoch: 119 [44800/50000 (90%)]\tLoss: 0.012419\n",
      "\n",
      "Test set: Average loss: 0.3444, Accuracy: 9231/10000 (92%)\n",
      "\n",
      "Time taken for epoch : 11.26201605796814\n",
      "\n",
      "changing T value to 0.595\n",
      "  + Number of params: 1.62M\n",
      "  + Number of FLOPs: 0.31G\n",
      "Changing Learning Rate to 0.0010000000000000002\n",
      "Train Epoch: 120 [0/50000 (0%)]\tLoss: 0.111730\n",
      "Train Epoch: 120 [6400/50000 (13%)]\tLoss: 0.006468\n",
      "Train Epoch: 120 [12800/50000 (26%)]\tLoss: 0.013925\n",
      "Train Epoch: 120 [19200/50000 (38%)]\tLoss: 0.001116\n",
      "Train Epoch: 120 [25600/50000 (51%)]\tLoss: 0.019297\n",
      "Train Epoch: 120 [32000/50000 (64%)]\tLoss: 0.072176\n",
      "Train Epoch: 120 [38400/50000 (77%)]\tLoss: 0.026763\n",
      "Train Epoch: 120 [44800/50000 (90%)]\tLoss: 0.032284\n",
      "\n",
      "Test set: Average loss: 0.3090, Accuracy: 9326/10000 (93%)\n",
      "\n",
      "Time taken for epoch : 13.702202558517456\n",
      "\n",
      "changing T value to 0.6\n",
      "  + Number of params: 1.61M\n",
      "  + Number of FLOPs: 0.31G\n",
      "Train Epoch: 121 [0/50000 (0%)]\tLoss: 0.003922\n",
      "Train Epoch: 121 [6400/50000 (13%)]\tLoss: 0.010073\n",
      "Train Epoch: 121 [12800/50000 (26%)]\tLoss: 0.001764\n",
      "Train Epoch: 121 [19200/50000 (38%)]\tLoss: 0.003215\n",
      "Train Epoch: 121 [25600/50000 (51%)]\tLoss: 0.001328\n",
      "Train Epoch: 121 [32000/50000 (64%)]\tLoss: 0.001685\n",
      "Train Epoch: 121 [38400/50000 (77%)]\tLoss: 0.001310\n",
      "Train Epoch: 121 [44800/50000 (90%)]\tLoss: 0.008324\n",
      "\n",
      "Test set: Average loss: 0.3091, Accuracy: 9315/10000 (93%)\n",
      "\n",
      "Time taken for epoch : 11.485147476196289\n",
      "\n",
      "changing T value to 0.605\n",
      "  + Number of params: 1.60M\n",
      "  + Number of FLOPs: 0.31G\n",
      "Train Epoch: 122 [0/50000 (0%)]\tLoss: 0.016646\n",
      "Train Epoch: 122 [6400/50000 (13%)]\tLoss: 0.001693\n",
      "Train Epoch: 122 [12800/50000 (26%)]\tLoss: 0.004257\n",
      "Train Epoch: 122 [19200/50000 (38%)]\tLoss: 0.008714\n",
      "Train Epoch: 122 [25600/50000 (51%)]\tLoss: 0.002382\n",
      "Train Epoch: 122 [32000/50000 (64%)]\tLoss: 0.002623\n",
      "Train Epoch: 122 [38400/50000 (77%)]\tLoss: 0.010145\n",
      "Train Epoch: 122 [44800/50000 (90%)]\tLoss: 0.001329\n",
      "\n",
      "Test set: Average loss: 0.3103, Accuracy: 9316/10000 (93%)\n",
      "\n",
      "Time taken for epoch : 11.401124954223633\n",
      "\n",
      "changing T value to 0.61\n",
      "  + Number of params: 1.58M\n",
      "  + Number of FLOPs: 0.31G\n",
      "Train Epoch: 123 [0/50000 (0%)]\tLoss: 0.018471\n",
      "Train Epoch: 123 [6400/50000 (13%)]\tLoss: 0.015140\n",
      "Train Epoch: 123 [12800/50000 (26%)]\tLoss: 0.001258\n",
      "Train Epoch: 123 [19200/50000 (38%)]\tLoss: 0.034852\n",
      "Train Epoch: 123 [25600/50000 (51%)]\tLoss: 0.009397\n",
      "Train Epoch: 123 [32000/50000 (64%)]\tLoss: 0.001088\n",
      "Train Epoch: 123 [38400/50000 (77%)]\tLoss: 0.032192\n",
      "Train Epoch: 123 [44800/50000 (90%)]\tLoss: 0.002577\n",
      "\n",
      "Test set: Average loss: 0.3008, Accuracy: 9322/10000 (93%)\n",
      "\n",
      "Time taken for epoch : 11.209616661071777\n",
      "\n",
      "changing T value to 0.615\n",
      "  + Number of params: 1.55M\n",
      "  + Number of FLOPs: 0.30G\n",
      "Train Epoch: 124 [0/50000 (0%)]\tLoss: 0.119783\n",
      "Train Epoch: 124 [6400/50000 (13%)]\tLoss: 0.063818\n",
      "Train Epoch: 124 [12800/50000 (26%)]\tLoss: 0.028140\n",
      "Train Epoch: 124 [19200/50000 (38%)]\tLoss: 0.015966\n",
      "Train Epoch: 124 [25600/50000 (51%)]\tLoss: 0.033664\n",
      "Train Epoch: 124 [32000/50000 (64%)]\tLoss: 0.041726\n",
      "Train Epoch: 124 [38400/50000 (77%)]\tLoss: 0.002549\n",
      "Train Epoch: 124 [44800/50000 (90%)]\tLoss: 0.006032\n",
      "\n",
      "Test set: Average loss: 0.2954, Accuracy: 9278/10000 (93%)\n",
      "\n",
      "Time taken for epoch : 11.113167762756348\n",
      "\n",
      "changing T value to 0.62\n",
      "  + Number of params: 1.52M\n",
      "  + Number of FLOPs: 0.30G\n",
      "Train Epoch: 125 [0/50000 (0%)]\tLoss: 0.163161\n",
      "Train Epoch: 125 [6400/50000 (13%)]\tLoss: 0.043910\n",
      "Train Epoch: 125 [12800/50000 (26%)]\tLoss: 0.002933\n",
      "Train Epoch: 125 [19200/50000 (38%)]\tLoss: 0.101814\n",
      "Train Epoch: 125 [25600/50000 (51%)]\tLoss: 0.033846\n",
      "Train Epoch: 125 [32000/50000 (64%)]\tLoss: 0.108308\n",
      "Train Epoch: 125 [38400/50000 (77%)]\tLoss: 0.023086\n",
      "Train Epoch: 125 [44800/50000 (90%)]\tLoss: 0.031726\n",
      "\n",
      "Test set: Average loss: 0.3089, Accuracy: 9269/10000 (93%)\n",
      "\n",
      "Time taken for epoch : 11.257841110229492\n",
      "\n",
      "changing T value to 0.625\n",
      "  + Number of params: 1.50M\n",
      "  + Number of FLOPs: 0.30G\n",
      "Train Epoch: 126 [0/50000 (0%)]\tLoss: 0.058844\n",
      "Train Epoch: 126 [6400/50000 (13%)]\tLoss: 0.099462\n",
      "Train Epoch: 126 [12800/50000 (26%)]\tLoss: 0.042583\n",
      "Train Epoch: 126 [19200/50000 (38%)]\tLoss: 0.013757\n",
      "Train Epoch: 126 [25600/50000 (51%)]\tLoss: 0.032076\n",
      "Train Epoch: 126 [32000/50000 (64%)]\tLoss: 0.032155\n",
      "Train Epoch: 126 [38400/50000 (77%)]\tLoss: 0.003266\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 126 [44800/50000 (90%)]\tLoss: 0.039737\n",
      "\n",
      "Test set: Average loss: 0.3177, Accuracy: 9255/10000 (93%)\n",
      "\n",
      "Time taken for epoch : 11.236500978469849\n",
      "\n",
      "changing T value to 0.63\n",
      "  + Number of params: 1.46M\n",
      "  + Number of FLOPs: 0.30G\n",
      "Train Epoch: 127 [0/50000 (0%)]\tLoss: 0.132901\n",
      "Train Epoch: 127 [6400/50000 (13%)]\tLoss: 0.044522\n",
      "Train Epoch: 127 [12800/50000 (26%)]\tLoss: 0.023068\n",
      "Train Epoch: 127 [19200/50000 (38%)]\tLoss: 0.026074\n",
      "Train Epoch: 127 [25600/50000 (51%)]\tLoss: 0.179529\n",
      "Train Epoch: 127 [32000/50000 (64%)]\tLoss: 0.063958\n",
      "Train Epoch: 127 [38400/50000 (77%)]\tLoss: 0.055441\n",
      "Train Epoch: 127 [44800/50000 (90%)]\tLoss: 0.025978\n",
      "\n",
      "Test set: Average loss: 0.3205, Accuracy: 9208/10000 (92%)\n",
      "\n",
      "Time taken for epoch : 11.156548500061035\n",
      "\n",
      "changing T value to 0.635\n",
      "  + Number of params: 1.43M\n",
      "  + Number of FLOPs: 0.29G\n",
      "Train Epoch: 128 [0/50000 (0%)]\tLoss: 0.483534\n",
      "Train Epoch: 128 [6400/50000 (13%)]\tLoss: 0.085539\n",
      "Train Epoch: 128 [12800/50000 (26%)]\tLoss: 0.192663\n",
      "Train Epoch: 128 [19200/50000 (38%)]\tLoss: 0.107056\n",
      "Train Epoch: 128 [25600/50000 (51%)]\tLoss: 0.051636\n",
      "Train Epoch: 128 [32000/50000 (64%)]\tLoss: 0.010839\n",
      "Train Epoch: 128 [38400/50000 (77%)]\tLoss: 0.021818\n",
      "Train Epoch: 128 [44800/50000 (90%)]\tLoss: 0.050999\n",
      "\n",
      "Test set: Average loss: 0.3373, Accuracy: 9151/10000 (92%)\n",
      "\n",
      "Time taken for epoch : 11.023577213287354\n",
      "\n",
      "changing T value to 0.64\n",
      "  + Number of params: 1.40M\n",
      "  + Number of FLOPs: 0.29G\n",
      "Train Epoch: 129 [0/50000 (0%)]\tLoss: 1.597865\n",
      "Train Epoch: 129 [6400/50000 (13%)]\tLoss: 0.129392\n",
      "Train Epoch: 129 [12800/50000 (26%)]\tLoss: 0.135483\n",
      "Train Epoch: 129 [19200/50000 (38%)]\tLoss: 0.137689\n",
      "Train Epoch: 129 [25600/50000 (51%)]\tLoss: 0.047360\n",
      "Train Epoch: 129 [32000/50000 (64%)]\tLoss: 0.094684\n",
      "Train Epoch: 129 [38400/50000 (77%)]\tLoss: 0.069071\n",
      "Train Epoch: 129 [44800/50000 (90%)]\tLoss: 0.033636\n",
      "\n",
      "Test set: Average loss: 0.3272, Accuracy: 9139/10000 (91%)\n",
      "\n",
      "Time taken for epoch : 11.196488857269287\n",
      "\n",
      "changing T value to 0.645\n",
      "  + Number of params: 1.37M\n",
      "  + Number of FLOPs: 0.28G\n",
      "Train Epoch: 130 [0/50000 (0%)]\tLoss: 0.613422\n",
      "Train Epoch: 130 [6400/50000 (13%)]\tLoss: 0.148881\n",
      "Train Epoch: 130 [12800/50000 (26%)]\tLoss: 0.123488\n",
      "Train Epoch: 130 [19200/50000 (38%)]\tLoss: 0.122741\n",
      "Train Epoch: 130 [25600/50000 (51%)]\tLoss: 0.174548\n",
      "Train Epoch: 130 [32000/50000 (64%)]\tLoss: 0.045353\n",
      "Train Epoch: 130 [38400/50000 (77%)]\tLoss: 0.054960\n",
      "Train Epoch: 130 [44800/50000 (90%)]\tLoss: 0.085709\n",
      "\n",
      "Test set: Average loss: 0.3414, Accuracy: 9085/10000 (91%)\n",
      "\n",
      "Time taken for epoch : 11.495752334594727\n",
      "\n",
      "changing T value to 0.65\n",
      "  + Number of params: 1.34M\n",
      "  + Number of FLOPs: 0.28G\n",
      "Train Epoch: 131 [0/50000 (0%)]\tLoss: 1.585097\n",
      "Train Epoch: 131 [6400/50000 (13%)]\tLoss: 0.357224\n",
      "Train Epoch: 131 [12800/50000 (26%)]\tLoss: 0.178525\n",
      "Train Epoch: 131 [19200/50000 (38%)]\tLoss: 0.079633\n",
      "Train Epoch: 131 [25600/50000 (51%)]\tLoss: 0.088497\n",
      "Train Epoch: 131 [32000/50000 (64%)]\tLoss: 0.135944\n",
      "Train Epoch: 131 [38400/50000 (77%)]\tLoss: 0.268679\n",
      "Train Epoch: 131 [44800/50000 (90%)]\tLoss: 0.300627\n",
      "\n",
      "Test set: Average loss: 0.3782, Accuracy: 8979/10000 (90%)\n",
      "\n",
      "Time taken for epoch : 11.034712791442871\n",
      "\n",
      "changing T value to 0.655\n",
      "  + Number of params: 1.33M\n",
      "  + Number of FLOPs: 0.28G\n",
      "Train Epoch: 132 [0/50000 (0%)]\tLoss: 0.160957\n",
      "Train Epoch: 132 [6400/50000 (13%)]\tLoss: 0.192990\n",
      "Train Epoch: 132 [12800/50000 (26%)]\tLoss: 0.124275\n",
      "Train Epoch: 132 [19200/50000 (38%)]\tLoss: 0.165948\n",
      "Train Epoch: 132 [25600/50000 (51%)]\tLoss: 0.086535\n",
      "Train Epoch: 132 [32000/50000 (64%)]\tLoss: 0.114481\n",
      "Train Epoch: 132 [38400/50000 (77%)]\tLoss: 0.072038\n",
      "Train Epoch: 132 [44800/50000 (90%)]\tLoss: 0.191583\n",
      "\n",
      "Test set: Average loss: 0.3476, Accuracy: 9064/10000 (91%)\n",
      "\n",
      "Time taken for epoch : 10.289975881576538\n",
      "\n",
      "changing T value to 0.66\n",
      "  + Number of params: 1.31M\n",
      "  + Number of FLOPs: 0.28G\n",
      "Train Epoch: 133 [0/50000 (0%)]\tLoss: 0.046091\n",
      "Train Epoch: 133 [6400/50000 (13%)]\tLoss: 0.091257\n",
      "Train Epoch: 133 [12800/50000 (26%)]\tLoss: 0.124689\n",
      "Train Epoch: 133 [19200/50000 (38%)]\tLoss: 0.035647\n",
      "Train Epoch: 133 [25600/50000 (51%)]\tLoss: 0.148778\n",
      "Train Epoch: 133 [32000/50000 (64%)]\tLoss: 0.070547\n",
      "Train Epoch: 133 [38400/50000 (77%)]\tLoss: 0.144622\n",
      "Train Epoch: 133 [44800/50000 (90%)]\tLoss: 0.076019\n",
      "\n",
      "Test set: Average loss: 0.3463, Accuracy: 9109/10000 (91%)\n",
      "\n",
      "Time taken for epoch : 10.297252178192139\n",
      "\n",
      "changing T value to 0.665\n",
      "  + Number of params: 1.29M\n",
      "  + Number of FLOPs: 0.27G\n",
      "Train Epoch: 134 [0/50000 (0%)]\tLoss: 0.085368\n",
      "Train Epoch: 134 [6400/50000 (13%)]\tLoss: 0.171273\n",
      "Train Epoch: 134 [12800/50000 (26%)]\tLoss: 0.162728\n",
      "Train Epoch: 134 [19200/50000 (38%)]\tLoss: 0.124842\n",
      "Train Epoch: 134 [25600/50000 (51%)]\tLoss: 0.111389\n",
      "Train Epoch: 134 [32000/50000 (64%)]\tLoss: 0.199695\n",
      "Train Epoch: 134 [38400/50000 (77%)]\tLoss: 0.054342\n",
      "Train Epoch: 134 [44800/50000 (90%)]\tLoss: 0.092265\n",
      "\n",
      "Test set: Average loss: 0.3433, Accuracy: 9116/10000 (91%)\n",
      "\n",
      "Time taken for epoch : 11.062390327453613\n",
      "\n",
      "changing T value to 0.67\n",
      "  + Number of params: 1.28M\n",
      "  + Number of FLOPs: 0.27G\n",
      "Train Epoch: 135 [0/50000 (0%)]\tLoss: 0.166587\n",
      "Train Epoch: 135 [6400/50000 (13%)]\tLoss: 0.023113\n",
      "Train Epoch: 135 [12800/50000 (26%)]\tLoss: 0.089554\n",
      "Train Epoch: 135 [19200/50000 (38%)]\tLoss: 0.043009\n",
      "Train Epoch: 135 [25600/50000 (51%)]\tLoss: 0.153465\n",
      "Train Epoch: 135 [32000/50000 (64%)]\tLoss: 0.125191\n",
      "Train Epoch: 135 [38400/50000 (77%)]\tLoss: 0.099152\n",
      "Train Epoch: 135 [44800/50000 (90%)]\tLoss: 0.030291\n",
      "\n",
      "Test set: Average loss: 0.3560, Accuracy: 9131/10000 (91%)\n",
      "\n",
      "Time taken for epoch : 10.984417915344238\n",
      "\n",
      "changing T value to 0.675\n",
      "  + Number of params: 1.26M\n",
      "  + Number of FLOPs: 0.27G\n",
      "Train Epoch: 136 [0/50000 (0%)]\tLoss: 0.047652\n",
      "Train Epoch: 136 [6400/50000 (13%)]\tLoss: 0.024675\n",
      "Train Epoch: 136 [12800/50000 (26%)]\tLoss: 0.013459\n",
      "Train Epoch: 136 [19200/50000 (38%)]\tLoss: 0.007627\n",
      "Train Epoch: 136 [25600/50000 (51%)]\tLoss: 0.054046\n",
      "Train Epoch: 136 [32000/50000 (64%)]\tLoss: 0.062691\n",
      "Train Epoch: 136 [38400/50000 (77%)]\tLoss: 0.028524\n",
      "Train Epoch: 136 [44800/50000 (90%)]\tLoss: 0.051792\n",
      "\n",
      "Test set: Average loss: 0.3411, Accuracy: 9141/10000 (91%)\n",
      "\n",
      "Time taken for epoch : 11.309128999710083\n",
      "\n",
      "changing T value to 0.68\n",
      "  + Number of params: 1.25M\n",
      "  + Number of FLOPs: 0.27G\n",
      "Train Epoch: 137 [0/50000 (0%)]\tLoss: 0.026496\n",
      "Train Epoch: 137 [6400/50000 (13%)]\tLoss: 0.034556\n",
      "Train Epoch: 137 [12800/50000 (26%)]\tLoss: 0.104301\n",
      "Train Epoch: 137 [19200/50000 (38%)]\tLoss: 0.181071\n",
      "Train Epoch: 137 [25600/50000 (51%)]\tLoss: 0.101212\n",
      "Train Epoch: 137 [32000/50000 (64%)]\tLoss: 0.060243\n",
      "Train Epoch: 137 [38400/50000 (77%)]\tLoss: 0.063622\n",
      "Train Epoch: 137 [44800/50000 (90%)]\tLoss: 0.112903\n",
      "\n",
      "Test set: Average loss: 0.3405, Accuracy: 9132/10000 (91%)\n",
      "\n",
      "Time taken for epoch : 10.865131616592407\n",
      "\n",
      "changing T value to 0.685\n",
      "  + Number of params: 1.24M\n",
      "  + Number of FLOPs: 0.27G\n",
      "Train Epoch: 138 [0/50000 (0%)]\tLoss: 1.046590\n",
      "Train Epoch: 138 [6400/50000 (13%)]\tLoss: 0.268367\n",
      "Train Epoch: 138 [12800/50000 (26%)]\tLoss: 0.296884\n",
      "Train Epoch: 138 [19200/50000 (38%)]\tLoss: 0.309692\n",
      "Train Epoch: 138 [25600/50000 (51%)]\tLoss: 0.127674\n",
      "Train Epoch: 138 [32000/50000 (64%)]\tLoss: 0.072563\n",
      "Train Epoch: 138 [38400/50000 (77%)]\tLoss: 0.144072\n",
      "Train Epoch: 138 [44800/50000 (90%)]\tLoss: 0.122653\n",
      "\n",
      "Test set: Average loss: 0.3611, Accuracy: 9034/10000 (90%)\n",
      "\n",
      "Time taken for epoch : 10.846550464630127\n",
      "\n",
      "changing T value to 0.6900000000000001\n",
      "  + Number of params: 1.21M\n",
      "  + Number of FLOPs: 0.26G\n",
      "Train Epoch: 139 [0/50000 (0%)]\tLoss: 0.297509\n",
      "Train Epoch: 139 [6400/50000 (13%)]\tLoss: 0.117830\n",
      "Train Epoch: 139 [12800/50000 (26%)]\tLoss: 0.128970\n",
      "Train Epoch: 139 [19200/50000 (38%)]\tLoss: 0.110860\n",
      "Train Epoch: 139 [25600/50000 (51%)]\tLoss: 0.162436\n",
      "Train Epoch: 139 [32000/50000 (64%)]\tLoss: 0.186439\n",
      "Train Epoch: 139 [38400/50000 (77%)]\tLoss: 0.315495\n",
      "Train Epoch: 139 [44800/50000 (90%)]\tLoss: 0.136468\n",
      "\n",
      "Test set: Average loss: 0.3414, Accuracy: 9080/10000 (91%)\n",
      "\n",
      "Time taken for epoch : 10.87151551246643\n",
      "\n",
      "changing T value to 0.6950000000000001\n",
      "  + Number of params: 1.19M\n",
      "  + Number of FLOPs: 0.26G\n",
      "Train Epoch: 140 [0/50000 (0%)]\tLoss: 0.075010\n",
      "Train Epoch: 140 [6400/50000 (13%)]\tLoss: 0.093017\n",
      "Train Epoch: 140 [12800/50000 (26%)]\tLoss: 0.073298\n",
      "Train Epoch: 140 [19200/50000 (38%)]\tLoss: 0.019887\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 140 [25600/50000 (51%)]\tLoss: 0.090194\n",
      "Train Epoch: 140 [32000/50000 (64%)]\tLoss: 0.222369\n",
      "Train Epoch: 140 [38400/50000 (77%)]\tLoss: 0.059848\n",
      "Train Epoch: 140 [44800/50000 (90%)]\tLoss: 0.104594\n",
      "\n",
      "Test set: Average loss: 0.3454, Accuracy: 9041/10000 (90%)\n",
      "\n",
      "Time taken for epoch : 10.780612468719482\n",
      "\n",
      "changing T value to 0.7000000000000001\n",
      "  + Number of params: 1.17M\n",
      "  + Number of FLOPs: 0.26G\n",
      "Train Epoch: 141 [0/50000 (0%)]\tLoss: 0.139486\n",
      "Train Epoch: 141 [6400/50000 (13%)]\tLoss: 0.317588\n",
      "Train Epoch: 141 [12800/50000 (26%)]\tLoss: 0.075545\n",
      "Train Epoch: 141 [19200/50000 (38%)]\tLoss: 0.066698\n",
      "Train Epoch: 141 [25600/50000 (51%)]\tLoss: 0.067721\n",
      "Train Epoch: 141 [32000/50000 (64%)]\tLoss: 0.067560\n",
      "Train Epoch: 141 [38400/50000 (77%)]\tLoss: 0.087778\n",
      "Train Epoch: 141 [44800/50000 (90%)]\tLoss: 0.102672\n",
      "\n",
      "Test set: Average loss: 0.3351, Accuracy: 9095/10000 (91%)\n",
      "\n",
      "Time taken for epoch : 10.658896446228027\n",
      "\n",
      "changing T value to 0.705\n",
      "  + Number of params: 1.14M\n",
      "  + Number of FLOPs: 0.26G\n",
      "Train Epoch: 142 [0/50000 (0%)]\tLoss: 0.052125\n",
      "Train Epoch: 142 [6400/50000 (13%)]\tLoss: 0.091566\n",
      "Train Epoch: 142 [12800/50000 (26%)]\tLoss: 0.111924\n",
      "Train Epoch: 142 [19200/50000 (38%)]\tLoss: 0.092353\n",
      "Train Epoch: 142 [25600/50000 (51%)]\tLoss: 0.037320\n",
      "Train Epoch: 142 [32000/50000 (64%)]\tLoss: 0.124073\n",
      "Train Epoch: 142 [38400/50000 (77%)]\tLoss: 0.094643\n",
      "Train Epoch: 142 [44800/50000 (90%)]\tLoss: 0.037381\n",
      "\n",
      "Test set: Average loss: 0.3364, Accuracy: 9103/10000 (91%)\n",
      "\n",
      "Time taken for epoch : 10.741985321044922\n",
      "\n",
      "changing T value to 0.71\n",
      "  + Number of params: 1.11M\n",
      "  + Number of FLOPs: 0.25G\n",
      "Train Epoch: 143 [0/50000 (0%)]\tLoss: 0.123889\n",
      "Train Epoch: 143 [6400/50000 (13%)]\tLoss: 0.065409\n",
      "Train Epoch: 143 [12800/50000 (26%)]\tLoss: 0.097188\n",
      "Train Epoch: 143 [19200/50000 (38%)]\tLoss: 0.041279\n",
      "Train Epoch: 143 [25600/50000 (51%)]\tLoss: 0.021636\n",
      "Train Epoch: 143 [32000/50000 (64%)]\tLoss: 0.098255\n",
      "Train Epoch: 143 [38400/50000 (77%)]\tLoss: 0.018829\n",
      "Train Epoch: 143 [44800/50000 (90%)]\tLoss: 0.025954\n",
      "\n",
      "Test set: Average loss: 0.3408, Accuracy: 9116/10000 (91%)\n",
      "\n",
      "Time taken for epoch : 10.6717209815979\n",
      "\n",
      "changing T value to 0.715\n",
      "  + Number of params: 1.08M\n",
      "  + Number of FLOPs: 0.25G\n",
      "Train Epoch: 144 [0/50000 (0%)]\tLoss: 0.179375\n",
      "Train Epoch: 144 [6400/50000 (13%)]\tLoss: 0.175530\n",
      "Train Epoch: 144 [12800/50000 (26%)]\tLoss: 0.056428\n",
      "Train Epoch: 144 [19200/50000 (38%)]\tLoss: 0.029712\n",
      "Train Epoch: 144 [25600/50000 (51%)]\tLoss: 0.027257\n",
      "Train Epoch: 144 [32000/50000 (64%)]\tLoss: 0.039685\n",
      "Train Epoch: 144 [38400/50000 (77%)]\tLoss: 0.034200\n",
      "Train Epoch: 144 [44800/50000 (90%)]\tLoss: 0.020879\n",
      "\n",
      "Test set: Average loss: 0.3376, Accuracy: 9107/10000 (91%)\n",
      "\n",
      "Time taken for epoch : 10.619156837463379\n",
      "\n",
      "changing T value to 0.72\n",
      "  + Number of params: 1.02M\n",
      "  + Number of FLOPs: 0.24G\n",
      "Train Epoch: 145 [0/50000 (0%)]\tLoss: 0.355854\n",
      "Train Epoch: 145 [6400/50000 (13%)]\tLoss: 0.280671\n",
      "Train Epoch: 145 [12800/50000 (26%)]\tLoss: 0.125920\n",
      "Train Epoch: 145 [19200/50000 (38%)]\tLoss: 0.115026\n",
      "Train Epoch: 145 [25600/50000 (51%)]\tLoss: 0.031849\n",
      "Train Epoch: 145 [32000/50000 (64%)]\tLoss: 0.124947\n",
      "Train Epoch: 145 [38400/50000 (77%)]\tLoss: 0.164116\n",
      "Train Epoch: 145 [44800/50000 (90%)]\tLoss: 0.047227\n",
      "\n",
      "Test set: Average loss: 0.3719, Accuracy: 9015/10000 (90%)\n",
      "\n",
      "Time taken for epoch : 10.583009243011475\n",
      "\n",
      "changing T value to 0.725\n",
      "  + Number of params: 0.97M\n",
      "  + Number of FLOPs: 0.23G\n",
      "Train Epoch: 146 [0/50000 (0%)]\tLoss: 0.100167\n",
      "Train Epoch: 146 [6400/50000 (13%)]\tLoss: 0.325844\n",
      "Train Epoch: 146 [12800/50000 (26%)]\tLoss: 0.157985\n",
      "Train Epoch: 146 [19200/50000 (38%)]\tLoss: 0.201590\n",
      "Train Epoch: 146 [25600/50000 (51%)]\tLoss: 0.072192\n",
      "Train Epoch: 146 [32000/50000 (64%)]\tLoss: 0.228038\n",
      "Train Epoch: 146 [38400/50000 (77%)]\tLoss: 0.109201\n",
      "Train Epoch: 146 [44800/50000 (90%)]\tLoss: 0.111657\n",
      "\n",
      "Test set: Average loss: 0.3804, Accuracy: 9016/10000 (90%)\n",
      "\n",
      "Time taken for epoch : 10.426148414611816\n",
      "\n",
      "changing T value to 0.73\n",
      "  + Number of params: 0.93M\n",
      "  + Number of FLOPs: 0.23G\n",
      "Train Epoch: 147 [0/50000 (0%)]\tLoss: 0.347380\n",
      "Train Epoch: 147 [6400/50000 (13%)]\tLoss: 0.085072\n",
      "Train Epoch: 147 [12800/50000 (26%)]\tLoss: 0.112447\n",
      "Train Epoch: 147 [19200/50000 (38%)]\tLoss: 0.201967\n",
      "Train Epoch: 147 [25600/50000 (51%)]\tLoss: 0.155032\n",
      "Train Epoch: 147 [32000/50000 (64%)]\tLoss: 0.239934\n",
      "Train Epoch: 147 [38400/50000 (77%)]\tLoss: 0.121540\n",
      "Train Epoch: 147 [44800/50000 (90%)]\tLoss: 0.100145\n",
      "\n",
      "Test set: Average loss: 0.3630, Accuracy: 9001/10000 (90%)\n",
      "\n",
      "Time taken for epoch : 11.20406699180603\n",
      "\n",
      "changing T value to 0.735\n",
      "  + Number of params: 0.89M\n",
      "  + Number of FLOPs: 0.22G\n",
      "stop filter pruning\n",
      "Train Epoch: 148 [0/50000 (0%)]\tLoss: 3.079386\n",
      "Train Epoch: 148 [6400/50000 (13%)]\tLoss: 0.489638\n",
      "Train Epoch: 148 [12800/50000 (26%)]\tLoss: 0.348316\n",
      "Train Epoch: 148 [19200/50000 (38%)]\tLoss: 0.301187\n",
      "Train Epoch: 148 [25600/50000 (51%)]\tLoss: 0.257348\n",
      "Train Epoch: 148 [32000/50000 (64%)]\tLoss: 0.557438\n",
      "Train Epoch: 148 [38400/50000 (77%)]\tLoss: 0.410557\n",
      "Train Epoch: 148 [44800/50000 (90%)]\tLoss: 0.314085\n",
      "\n",
      "Test set: Average loss: 0.4715, Accuracy: 8673/10000 (87%)\n",
      "\n",
      "Time taken for epoch : 10.304870367050171\n",
      "\n",
      "Train Epoch: 149 [0/50000 (0%)]\tLoss: 0.233638\n",
      "Train Epoch: 149 [6400/50000 (13%)]\tLoss: 0.230848\n",
      "Train Epoch: 149 [12800/50000 (26%)]\tLoss: 0.293381\n",
      "Train Epoch: 149 [19200/50000 (38%)]\tLoss: 0.138195\n",
      "Train Epoch: 149 [25600/50000 (51%)]\tLoss: 0.182626\n",
      "Train Epoch: 149 [32000/50000 (64%)]\tLoss: 0.126045\n",
      "Train Epoch: 149 [38400/50000 (77%)]\tLoss: 0.199365\n",
      "Train Epoch: 149 [44800/50000 (90%)]\tLoss: 0.561957\n",
      "\n",
      "Test set: Average loss: 0.4232, Accuracy: 8836/10000 (88%)\n",
      "\n",
      "Time taken for epoch : 10.4924635887146\n",
      "\n",
      "Train Epoch: 150 [0/50000 (0%)]\tLoss: 0.485578\n",
      "Train Epoch: 150 [6400/50000 (13%)]\tLoss: 0.301279\n",
      "Train Epoch: 150 [12800/50000 (26%)]\tLoss: 0.132418\n",
      "Train Epoch: 150 [19200/50000 (38%)]\tLoss: 0.175904\n",
      "Train Epoch: 150 [25600/50000 (51%)]\tLoss: 0.065300\n",
      "Train Epoch: 150 [32000/50000 (64%)]\tLoss: 0.193115\n",
      "Train Epoch: 150 [38400/50000 (77%)]\tLoss: 0.373514\n",
      "Train Epoch: 150 [44800/50000 (90%)]\tLoss: 0.324086\n",
      "\n",
      "Test set: Average loss: 0.4015, Accuracy: 8889/10000 (89%)\n",
      "\n",
      "Time taken for epoch : 10.638098955154419\n",
      "\n",
      "Train Epoch: 151 [0/50000 (0%)]\tLoss: 0.160074\n",
      "Train Epoch: 151 [6400/50000 (13%)]\tLoss: 0.294670\n",
      "Train Epoch: 151 [12800/50000 (26%)]\tLoss: 0.126113\n",
      "Train Epoch: 151 [19200/50000 (38%)]\tLoss: 0.282758\n",
      "Train Epoch: 151 [25600/50000 (51%)]\tLoss: 0.336621\n",
      "Train Epoch: 151 [32000/50000 (64%)]\tLoss: 0.141816\n",
      "Train Epoch: 151 [38400/50000 (77%)]\tLoss: 0.176004\n",
      "Train Epoch: 151 [44800/50000 (90%)]\tLoss: 0.152828\n",
      "\n",
      "Test set: Average loss: 0.3986, Accuracy: 8938/10000 (89%)\n",
      "\n",
      "Time taken for epoch : 10.448115348815918\n",
      "\n",
      "Train Epoch: 152 [0/50000 (0%)]\tLoss: 0.061895\n",
      "Train Epoch: 152 [6400/50000 (13%)]\tLoss: 0.233617\n",
      "Train Epoch: 152 [12800/50000 (26%)]\tLoss: 0.259159\n",
      "Train Epoch: 152 [19200/50000 (38%)]\tLoss: 0.069311\n",
      "Train Epoch: 152 [25600/50000 (51%)]\tLoss: 0.218960\n",
      "Train Epoch: 152 [32000/50000 (64%)]\tLoss: 0.146608\n",
      "Train Epoch: 152 [38400/50000 (77%)]\tLoss: 0.123292\n",
      "Train Epoch: 152 [44800/50000 (90%)]\tLoss: 0.337719\n",
      "\n",
      "Test set: Average loss: 0.3948, Accuracy: 8951/10000 (90%)\n",
      "\n",
      "Time taken for epoch : 10.455233335494995\n",
      "\n",
      "Train Epoch: 153 [0/50000 (0%)]\tLoss: 0.153430\n",
      "Train Epoch: 153 [6400/50000 (13%)]\tLoss: 0.087580\n",
      "Train Epoch: 153 [12800/50000 (26%)]\tLoss: 0.060020\n",
      "Train Epoch: 153 [19200/50000 (38%)]\tLoss: 0.188400\n",
      "Train Epoch: 153 [25600/50000 (51%)]\tLoss: 0.058599\n",
      "Train Epoch: 153 [32000/50000 (64%)]\tLoss: 0.135447\n",
      "Train Epoch: 153 [38400/50000 (77%)]\tLoss: 0.119970\n",
      "Train Epoch: 153 [44800/50000 (90%)]\tLoss: 0.080528\n",
      "\n",
      "Test set: Average loss: 0.3859, Accuracy: 8980/10000 (90%)\n",
      "\n",
      "Time taken for epoch : 10.530332088470459\n",
      "\n",
      "Train Epoch: 154 [0/50000 (0%)]\tLoss: 0.080009\n",
      "Train Epoch: 154 [6400/50000 (13%)]\tLoss: 0.278483\n",
      "Train Epoch: 154 [12800/50000 (26%)]\tLoss: 0.138319\n",
      "Train Epoch: 154 [19200/50000 (38%)]\tLoss: 0.088314\n",
      "Train Epoch: 154 [25600/50000 (51%)]\tLoss: 0.234032\n",
      "Train Epoch: 154 [32000/50000 (64%)]\tLoss: 0.100693\n",
      "Train Epoch: 154 [38400/50000 (77%)]\tLoss: 0.060626\n",
      "Train Epoch: 154 [44800/50000 (90%)]\tLoss: 0.192241\n",
      "\n",
      "Test set: Average loss: 0.4052, Accuracy: 8903/10000 (89%)\n",
      "\n",
      "Time taken for epoch : 10.358656406402588\n",
      "\n",
      "Train Epoch: 155 [0/50000 (0%)]\tLoss: 0.069254\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 155 [6400/50000 (13%)]\tLoss: 0.149575\n",
      "Train Epoch: 155 [12800/50000 (26%)]\tLoss: 0.030344\n",
      "Train Epoch: 155 [19200/50000 (38%)]\tLoss: 0.171971\n",
      "Train Epoch: 155 [25600/50000 (51%)]\tLoss: 0.033338\n",
      "Train Epoch: 155 [32000/50000 (64%)]\tLoss: 0.109925\n",
      "Train Epoch: 155 [38400/50000 (77%)]\tLoss: 0.024307\n",
      "Train Epoch: 155 [44800/50000 (90%)]\tLoss: 0.117576\n",
      "\n",
      "Test set: Average loss: 0.3935, Accuracy: 8980/10000 (90%)\n",
      "\n",
      "Time taken for epoch : 10.353330850601196\n",
      "\n",
      "Train Epoch: 156 [0/50000 (0%)]\tLoss: 0.128935\n",
      "Train Epoch: 156 [6400/50000 (13%)]\tLoss: 0.043870\n",
      "Train Epoch: 156 [12800/50000 (26%)]\tLoss: 0.162693\n",
      "Train Epoch: 156 [19200/50000 (38%)]\tLoss: 0.130986\n",
      "Train Epoch: 156 [25600/50000 (51%)]\tLoss: 0.041155\n",
      "Train Epoch: 156 [32000/50000 (64%)]\tLoss: 0.044763\n",
      "Train Epoch: 156 [38400/50000 (77%)]\tLoss: 0.060326\n",
      "Train Epoch: 156 [44800/50000 (90%)]\tLoss: 0.219512\n",
      "\n",
      "Test set: Average loss: 0.3907, Accuracy: 8993/10000 (90%)\n",
      "\n",
      "Time taken for epoch : 10.470640182495117\n",
      "\n",
      "Train Epoch: 157 [0/50000 (0%)]\tLoss: 0.068851\n",
      "Train Epoch: 157 [6400/50000 (13%)]\tLoss: 0.090909\n",
      "Train Epoch: 157 [12800/50000 (26%)]\tLoss: 0.035658\n",
      "Train Epoch: 157 [19200/50000 (38%)]\tLoss: 0.134750\n",
      "Train Epoch: 157 [25600/50000 (51%)]\tLoss: 0.100603\n",
      "Train Epoch: 157 [32000/50000 (64%)]\tLoss: 0.081469\n",
      "Train Epoch: 157 [38400/50000 (77%)]\tLoss: 0.060268\n",
      "Train Epoch: 157 [44800/50000 (90%)]\tLoss: 0.033416\n",
      "\n",
      "Test set: Average loss: 0.3871, Accuracy: 8981/10000 (90%)\n",
      "\n",
      "Time taken for epoch : 10.4944007396698\n",
      "\n",
      "Train Epoch: 158 [0/50000 (0%)]\tLoss: 0.116467\n",
      "Train Epoch: 158 [6400/50000 (13%)]\tLoss: 0.050365\n",
      "Train Epoch: 158 [12800/50000 (26%)]\tLoss: 0.113392\n",
      "Train Epoch: 158 [19200/50000 (38%)]\tLoss: 0.090136\n",
      "Train Epoch: 158 [25600/50000 (51%)]\tLoss: 0.057341\n",
      "Train Epoch: 158 [32000/50000 (64%)]\tLoss: 0.046466\n",
      "Train Epoch: 158 [38400/50000 (77%)]\tLoss: 0.137853\n",
      "Train Epoch: 158 [44800/50000 (90%)]\tLoss: 0.132884\n",
      "\n",
      "Test set: Average loss: 0.3759, Accuracy: 8998/10000 (90%)\n",
      "\n",
      "Time taken for epoch : 10.526099681854248\n",
      "\n",
      "Train Epoch: 159 [0/50000 (0%)]\tLoss: 0.070811\n",
      "Train Epoch: 159 [6400/50000 (13%)]\tLoss: 0.094218\n",
      "Train Epoch: 159 [12800/50000 (26%)]\tLoss: 0.159133\n",
      "Train Epoch: 159 [19200/50000 (38%)]\tLoss: 0.073262\n",
      "Train Epoch: 159 [25600/50000 (51%)]\tLoss: 0.039624\n",
      "Train Epoch: 159 [32000/50000 (64%)]\tLoss: 0.171970\n",
      "Train Epoch: 159 [38400/50000 (77%)]\tLoss: 0.174919\n",
      "Train Epoch: 159 [44800/50000 (90%)]\tLoss: 0.036723\n",
      "\n",
      "Test set: Average loss: 0.3950, Accuracy: 9009/10000 (90%)\n",
      "\n",
      "Time taken for epoch : 10.458432912826538\n",
      "\n",
      "Train Epoch: 160 [0/50000 (0%)]\tLoss: 0.116634\n",
      "Train Epoch: 160 [6400/50000 (13%)]\tLoss: 0.220602\n",
      "Train Epoch: 160 [12800/50000 (26%)]\tLoss: 0.064771\n",
      "Train Epoch: 160 [19200/50000 (38%)]\tLoss: 0.081533\n",
      "Train Epoch: 160 [25600/50000 (51%)]\tLoss: 0.021848\n",
      "Train Epoch: 160 [32000/50000 (64%)]\tLoss: 0.148420\n",
      "Train Epoch: 160 [38400/50000 (77%)]\tLoss: 0.055539\n",
      "Train Epoch: 160 [44800/50000 (90%)]\tLoss: 0.111275\n",
      "\n",
      "Test set: Average loss: 0.3968, Accuracy: 9019/10000 (90%)\n",
      "\n",
      "Time taken for epoch : 10.423452377319336\n",
      "\n",
      "Best accuracy 93.26\n",
      "changing T value to 0.0\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 1 [0/50000 (0%)]\tLoss: 2.443508\n",
      "Train Epoch: 1 [6400/50000 (13%)]\tLoss: 2.260955\n",
      "Train Epoch: 1 [12800/50000 (26%)]\tLoss: 2.251731\n",
      "Train Epoch: 1 [19200/50000 (38%)]\tLoss: 1.990405\n",
      "Train Epoch: 1 [25600/50000 (51%)]\tLoss: 2.111107\n",
      "Train Epoch: 1 [32000/50000 (64%)]\tLoss: 2.069531\n",
      "Train Epoch: 1 [38400/50000 (77%)]\tLoss: 1.839504\n",
      "Train Epoch: 1 [44800/50000 (90%)]\tLoss: 1.543399\n",
      "\n",
      "Test set: Average loss: 1.7962, Accuracy: 3125/10000 (31%)\n",
      "\n",
      "Time taken for epoch : 23.157761096954346\n",
      "\n",
      "changing T value to 0.01\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 2 [0/50000 (0%)]\tLoss: 1.733816\n",
      "Train Epoch: 2 [6400/50000 (13%)]\tLoss: 1.833429\n",
      "Train Epoch: 2 [12800/50000 (26%)]\tLoss: 1.595596\n",
      "Train Epoch: 2 [19200/50000 (38%)]\tLoss: 1.501862\n",
      "Train Epoch: 2 [25600/50000 (51%)]\tLoss: 1.613213\n",
      "Train Epoch: 2 [32000/50000 (64%)]\tLoss: 1.662730\n",
      "Train Epoch: 2 [38400/50000 (77%)]\tLoss: 1.263088\n",
      "Train Epoch: 2 [44800/50000 (90%)]\tLoss: 1.278486\n",
      "\n",
      "Test set: Average loss: 1.2761, Accuracy: 5280/10000 (53%)\n",
      "\n",
      "Time taken for epoch : 23.47551441192627\n",
      "\n",
      "changing T value to 0.02\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 3 [0/50000 (0%)]\tLoss: 1.219441\n",
      "Train Epoch: 3 [6400/50000 (13%)]\tLoss: 1.076018\n",
      "Train Epoch: 3 [12800/50000 (26%)]\tLoss: 1.206103\n",
      "Train Epoch: 3 [19200/50000 (38%)]\tLoss: 1.397383\n",
      "Train Epoch: 3 [25600/50000 (51%)]\tLoss: 1.502324\n",
      "Train Epoch: 3 [32000/50000 (64%)]\tLoss: 1.036352\n",
      "Train Epoch: 3 [38400/50000 (77%)]\tLoss: 1.231936\n",
      "Train Epoch: 3 [44800/50000 (90%)]\tLoss: 0.886776\n",
      "\n",
      "Test set: Average loss: 1.0013, Accuracy: 6473/10000 (65%)\n",
      "\n",
      "Time taken for epoch : 23.94604182243347\n",
      "\n",
      "changing T value to 0.03\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 4 [0/50000 (0%)]\tLoss: 1.207456\n",
      "Train Epoch: 4 [6400/50000 (13%)]\tLoss: 1.172813\n",
      "Train Epoch: 4 [12800/50000 (26%)]\tLoss: 1.045074\n",
      "Train Epoch: 4 [19200/50000 (38%)]\tLoss: 0.678464\n",
      "Train Epoch: 4 [25600/50000 (51%)]\tLoss: 0.799135\n",
      "Train Epoch: 4 [32000/50000 (64%)]\tLoss: 0.791394\n",
      "Train Epoch: 4 [38400/50000 (77%)]\tLoss: 0.650092\n",
      "Train Epoch: 4 [44800/50000 (90%)]\tLoss: 1.037316\n",
      "\n",
      "Test set: Average loss: 1.0297, Accuracy: 6521/10000 (65%)\n",
      "\n",
      "Time taken for epoch : 23.880462169647217\n",
      "\n",
      "changing T value to 0.04\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 5 [0/50000 (0%)]\tLoss: 0.712438\n",
      "Train Epoch: 5 [6400/50000 (13%)]\tLoss: 0.824609\n",
      "Train Epoch: 5 [12800/50000 (26%)]\tLoss: 0.743877\n",
      "Train Epoch: 5 [19200/50000 (38%)]\tLoss: 0.639310\n",
      "Train Epoch: 5 [25600/50000 (51%)]\tLoss: 0.725990\n",
      "Train Epoch: 5 [32000/50000 (64%)]\tLoss: 0.842255\n",
      "Train Epoch: 5 [38400/50000 (77%)]\tLoss: 0.689120\n",
      "Train Epoch: 5 [44800/50000 (90%)]\tLoss: 0.908393\n",
      "\n",
      "Test set: Average loss: 0.8783, Accuracy: 7058/10000 (71%)\n",
      "\n",
      "Time taken for epoch : 23.759278535842896\n",
      "\n",
      "changing T value to 0.05\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 6 [0/50000 (0%)]\tLoss: 0.757933\n",
      "Train Epoch: 6 [6400/50000 (13%)]\tLoss: 0.724827\n",
      "Train Epoch: 6 [12800/50000 (26%)]\tLoss: 0.808995\n",
      "Train Epoch: 6 [19200/50000 (38%)]\tLoss: 0.673538\n",
      "Train Epoch: 6 [25600/50000 (51%)]\tLoss: 0.712111\n",
      "Train Epoch: 6 [32000/50000 (64%)]\tLoss: 0.504620\n",
      "Train Epoch: 6 [38400/50000 (77%)]\tLoss: 0.578694\n",
      "Train Epoch: 6 [44800/50000 (90%)]\tLoss: 0.677296\n",
      "\n",
      "Test set: Average loss: 0.7683, Accuracy: 7435/10000 (74%)\n",
      "\n",
      "Time taken for epoch : 24.12608790397644\n",
      "\n",
      "changing T value to 0.06\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 7 [0/50000 (0%)]\tLoss: 0.558249\n",
      "Train Epoch: 7 [6400/50000 (13%)]\tLoss: 0.752271\n",
      "Train Epoch: 7 [12800/50000 (26%)]\tLoss: 0.638111\n",
      "Train Epoch: 7 [19200/50000 (38%)]\tLoss: 0.514772\n",
      "Train Epoch: 7 [25600/50000 (51%)]\tLoss: 0.637916\n",
      "Train Epoch: 7 [32000/50000 (64%)]\tLoss: 0.683136\n",
      "Train Epoch: 7 [38400/50000 (77%)]\tLoss: 0.598730\n",
      "Train Epoch: 7 [44800/50000 (90%)]\tLoss: 0.587872\n",
      "\n",
      "Test set: Average loss: 0.8077, Accuracy: 7378/10000 (74%)\n",
      "\n",
      "Time taken for epoch : 21.446449518203735\n",
      "\n",
      "changing T value to 0.07\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 8 [0/50000 (0%)]\tLoss: 0.828549\n",
      "Train Epoch: 8 [6400/50000 (13%)]\tLoss: 0.528166\n",
      "Train Epoch: 8 [12800/50000 (26%)]\tLoss: 0.592211\n",
      "Train Epoch: 8 [19200/50000 (38%)]\tLoss: 0.407185\n",
      "Train Epoch: 8 [25600/50000 (51%)]\tLoss: 0.760300\n",
      "Train Epoch: 8 [32000/50000 (64%)]\tLoss: 0.612696\n",
      "Train Epoch: 8 [38400/50000 (77%)]\tLoss: 0.728544\n",
      "Train Epoch: 8 [44800/50000 (90%)]\tLoss: 0.699615\n",
      "\n",
      "Test set: Average loss: 0.6445, Accuracy: 7854/10000 (79%)\n",
      "\n",
      "Time taken for epoch : 23.665515422821045\n",
      "\n",
      "changing T value to 0.08\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 9 [0/50000 (0%)]\tLoss: 0.632457\n",
      "Train Epoch: 9 [6400/50000 (13%)]\tLoss: 0.755933\n",
      "Train Epoch: 9 [12800/50000 (26%)]\tLoss: 0.686469\n",
      "Train Epoch: 9 [19200/50000 (38%)]\tLoss: 0.414108\n",
      "Train Epoch: 9 [25600/50000 (51%)]\tLoss: 0.551906\n",
      "Train Epoch: 9 [32000/50000 (64%)]\tLoss: 0.358694\n",
      "Train Epoch: 9 [38400/50000 (77%)]\tLoss: 0.402664\n",
      "Train Epoch: 9 [44800/50000 (90%)]\tLoss: 0.564057\n",
      "\n",
      "Test set: Average loss: 0.5710, Accuracy: 8155/10000 (82%)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time taken for epoch : 24.134310245513916\n",
      "\n",
      "changing T value to 0.09\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 10 [0/50000 (0%)]\tLoss: 0.673575\n",
      "Train Epoch: 10 [6400/50000 (13%)]\tLoss: 0.621606\n",
      "Train Epoch: 10 [12800/50000 (26%)]\tLoss: 0.497783\n",
      "Train Epoch: 10 [19200/50000 (38%)]\tLoss: 0.636915\n",
      "Train Epoch: 10 [25600/50000 (51%)]\tLoss: 0.561688\n",
      "Train Epoch: 10 [32000/50000 (64%)]\tLoss: 0.430808\n",
      "Train Epoch: 10 [38400/50000 (77%)]\tLoss: 0.437550\n",
      "Train Epoch: 10 [44800/50000 (90%)]\tLoss: 0.423739\n",
      "\n",
      "Test set: Average loss: 0.5821, Accuracy: 8122/10000 (81%)\n",
      "\n",
      "Time taken for epoch : 21.46430993080139\n",
      "\n",
      "changing T value to 0.1\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 11 [0/50000 (0%)]\tLoss: 0.579760\n",
      "Train Epoch: 11 [6400/50000 (13%)]\tLoss: 0.425478\n",
      "Train Epoch: 11 [12800/50000 (26%)]\tLoss: 0.697174\n",
      "Train Epoch: 11 [19200/50000 (38%)]\tLoss: 0.504442\n",
      "Train Epoch: 11 [25600/50000 (51%)]\tLoss: 0.749582\n",
      "Train Epoch: 11 [32000/50000 (64%)]\tLoss: 0.576172\n",
      "Train Epoch: 11 [38400/50000 (77%)]\tLoss: 0.517303\n",
      "Train Epoch: 11 [44800/50000 (90%)]\tLoss: 0.471195\n",
      "\n",
      "Test set: Average loss: 0.5738, Accuracy: 8056/10000 (81%)\n",
      "\n",
      "Time taken for epoch : 21.61017346382141\n",
      "\n",
      "changing T value to 0.11\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 12 [0/50000 (0%)]\tLoss: 0.421539\n",
      "Train Epoch: 12 [6400/50000 (13%)]\tLoss: 0.453289\n",
      "Train Epoch: 12 [12800/50000 (26%)]\tLoss: 0.348544\n",
      "Train Epoch: 12 [19200/50000 (38%)]\tLoss: 0.286096\n",
      "Train Epoch: 12 [25600/50000 (51%)]\tLoss: 0.670627\n",
      "Train Epoch: 12 [32000/50000 (64%)]\tLoss: 0.452027\n",
      "Train Epoch: 12 [38400/50000 (77%)]\tLoss: 0.491687\n",
      "Train Epoch: 12 [44800/50000 (90%)]\tLoss: 0.549055\n",
      "\n",
      "Test set: Average loss: 0.6190, Accuracy: 8065/10000 (81%)\n",
      "\n",
      "Time taken for epoch : 21.655084133148193\n",
      "\n",
      "changing T value to 0.12\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 13 [0/50000 (0%)]\tLoss: 0.506843\n",
      "Train Epoch: 13 [6400/50000 (13%)]\tLoss: 0.274806\n",
      "Train Epoch: 13 [12800/50000 (26%)]\tLoss: 0.501148\n",
      "Train Epoch: 13 [19200/50000 (38%)]\tLoss: 0.337362\n",
      "Train Epoch: 13 [25600/50000 (51%)]\tLoss: 0.450214\n",
      "Train Epoch: 13 [32000/50000 (64%)]\tLoss: 0.461584\n",
      "Train Epoch: 13 [38400/50000 (77%)]\tLoss: 0.343834\n",
      "Train Epoch: 13 [44800/50000 (90%)]\tLoss: 0.441280\n",
      "\n",
      "Test set: Average loss: 0.5034, Accuracy: 8266/10000 (83%)\n",
      "\n",
      "Time taken for epoch : 24.13441491127014\n",
      "\n",
      "changing T value to 0.13\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 14 [0/50000 (0%)]\tLoss: 0.493883\n",
      "Train Epoch: 14 [6400/50000 (13%)]\tLoss: 0.433645\n",
      "Train Epoch: 14 [12800/50000 (26%)]\tLoss: 0.470091\n",
      "Train Epoch: 14 [19200/50000 (38%)]\tLoss: 0.243466\n",
      "Train Epoch: 14 [25600/50000 (51%)]\tLoss: 0.549322\n",
      "Train Epoch: 14 [32000/50000 (64%)]\tLoss: 0.485480\n",
      "Train Epoch: 14 [38400/50000 (77%)]\tLoss: 0.833367\n",
      "Train Epoch: 14 [44800/50000 (90%)]\tLoss: 0.512234\n",
      "\n",
      "Test set: Average loss: 0.4531, Accuracy: 8466/10000 (85%)\n",
      "\n",
      "Time taken for epoch : 23.886394500732422\n",
      "\n",
      "changing T value to 0.14\n",
      "  + Number of params: 14.73M\n",
      "  + Number of FLOPs: 0.63G\n",
      "Train Epoch: 15 [0/50000 (0%)]\tLoss: 0.574426\n",
      "Train Epoch: 15 [6400/50000 (13%)]\tLoss: 0.467998\n",
      "Train Epoch: 15 [12800/50000 (26%)]\tLoss: 0.263782\n",
      "Train Epoch: 15 [19200/50000 (38%)]\tLoss: 0.383285\n",
      "Train Epoch: 15 [25600/50000 (51%)]\tLoss: 0.276063\n",
      "Train Epoch: 15 [32000/50000 (64%)]\tLoss: 0.408345\n",
      "Train Epoch: 15 [38400/50000 (77%)]\tLoss: 0.198436\n",
      "Train Epoch: 15 [44800/50000 (90%)]\tLoss: 0.476374\n",
      "\n",
      "Test set: Average loss: 0.4641, Accuracy: 8496/10000 (85%)\n",
      "\n",
      "Time taken for epoch : 26.02077627182007\n",
      "\n",
      "changing T value to 0.15\n",
      "  + Number of params: 14.26M\n",
      "  + Number of FLOPs: 0.62G\n",
      "Train Epoch: 16 [0/50000 (0%)]\tLoss: 0.293175\n",
      "Train Epoch: 16 [6400/50000 (13%)]\tLoss: 0.278790\n",
      "Train Epoch: 16 [12800/50000 (26%)]\tLoss: 0.445464\n",
      "Train Epoch: 16 [19200/50000 (38%)]\tLoss: 0.445448\n",
      "Train Epoch: 16 [25600/50000 (51%)]\tLoss: 0.361800\n",
      "Train Epoch: 16 [32000/50000 (64%)]\tLoss: 0.510755\n",
      "Train Epoch: 16 [38400/50000 (77%)]\tLoss: 0.321295\n",
      "Train Epoch: 16 [44800/50000 (90%)]\tLoss: 0.330366\n",
      "\n",
      "Test set: Average loss: 0.5562, Accuracy: 8066/10000 (81%)\n",
      "\n",
      "Time taken for epoch : 21.403506755828857\n",
      "\n",
      "changing T value to 0.16\n",
      "  + Number of params: 13.81M\n",
      "  + Number of FLOPs: 0.61G\n",
      "Train Epoch: 17 [0/50000 (0%)]\tLoss: 0.456865\n",
      "Train Epoch: 17 [6400/50000 (13%)]\tLoss: 0.302435\n",
      "Train Epoch: 17 [12800/50000 (26%)]\tLoss: 0.465358\n",
      "Train Epoch: 17 [19200/50000 (38%)]\tLoss: 0.543697\n",
      "Train Epoch: 17 [25600/50000 (51%)]\tLoss: 0.385239\n",
      "Train Epoch: 17 [32000/50000 (64%)]\tLoss: 0.303399\n",
      "Train Epoch: 17 [38400/50000 (77%)]\tLoss: 0.385291\n",
      "Train Epoch: 17 [44800/50000 (90%)]\tLoss: 0.486682\n",
      "\n",
      "Test set: Average loss: 0.5155, Accuracy: 8386/10000 (84%)\n",
      "\n",
      "Time taken for epoch : 21.196157217025757\n",
      "\n",
      "changing T value to 0.17\n",
      "  + Number of params: 12.90M\n",
      "  + Number of FLOPs: 0.59G\n",
      "Train Epoch: 18 [0/50000 (0%)]\tLoss: 0.410275\n",
      "Train Epoch: 18 [6400/50000 (13%)]\tLoss: 0.391082\n",
      "Train Epoch: 18 [12800/50000 (26%)]\tLoss: 0.349084\n",
      "Train Epoch: 18 [19200/50000 (38%)]\tLoss: 0.401489\n",
      "Train Epoch: 18 [25600/50000 (51%)]\tLoss: 0.508953\n",
      "Train Epoch: 18 [32000/50000 (64%)]\tLoss: 0.662891\n",
      "Train Epoch: 18 [38400/50000 (77%)]\tLoss: 0.522874\n",
      "Train Epoch: 18 [44800/50000 (90%)]\tLoss: 0.248829\n",
      "\n",
      "Test set: Average loss: 0.7267, Accuracy: 7783/10000 (78%)\n",
      "\n",
      "Time taken for epoch : 20.95412278175354\n",
      "\n",
      "changing T value to 0.18\n",
      "  + Number of params: 10.16M\n",
      "  + Number of FLOPs: 0.53G\n",
      "Train Epoch: 19 [0/50000 (0%)]\tLoss: 0.705472\n",
      "Train Epoch: 19 [6400/50000 (13%)]\tLoss: 0.506893\n",
      "Train Epoch: 19 [12800/50000 (26%)]\tLoss: 0.149671\n",
      "Train Epoch: 19 [19200/50000 (38%)]\tLoss: 0.272529\n",
      "Train Epoch: 19 [25600/50000 (51%)]\tLoss: 0.572152\n",
      "Train Epoch: 19 [32000/50000 (64%)]\tLoss: 0.365978\n",
      "Train Epoch: 19 [38400/50000 (77%)]\tLoss: 0.155356\n",
      "Train Epoch: 19 [44800/50000 (90%)]\tLoss: 0.230672\n",
      "\n",
      "Test set: Average loss: 0.4888, Accuracy: 8458/10000 (85%)\n",
      "\n",
      "Time taken for epoch : 19.566625356674194\n",
      "\n",
      "changing T value to 0.19\n",
      "  + Number of params: 8.03M\n",
      "  + Number of FLOPs: 0.49G\n",
      "Train Epoch: 20 [0/50000 (0%)]\tLoss: 0.348373\n",
      "Train Epoch: 20 [6400/50000 (13%)]\tLoss: 0.219086\n",
      "Train Epoch: 20 [12800/50000 (26%)]\tLoss: 0.537252\n",
      "Train Epoch: 20 [19200/50000 (38%)]\tLoss: 0.334706\n",
      "Train Epoch: 20 [25600/50000 (51%)]\tLoss: 0.622421\n",
      "Train Epoch: 20 [32000/50000 (64%)]\tLoss: 0.542725\n",
      "Train Epoch: 20 [38400/50000 (77%)]\tLoss: 0.550758\n",
      "Train Epoch: 20 [44800/50000 (90%)]\tLoss: 0.318044\n",
      "\n",
      "Test set: Average loss: 0.4706, Accuracy: 8474/10000 (85%)\n",
      "\n",
      "Time taken for epoch : 18.28241515159607\n",
      "\n",
      "changing T value to 0.2\n",
      "  + Number of params: 6.60M\n",
      "  + Number of FLOPs: 0.46G\n",
      "Train Epoch: 21 [0/50000 (0%)]\tLoss: 0.523777\n",
      "Train Epoch: 21 [6400/50000 (13%)]\tLoss: 0.237791\n",
      "Train Epoch: 21 [12800/50000 (26%)]\tLoss: 0.497108\n",
      "Train Epoch: 21 [19200/50000 (38%)]\tLoss: 0.344608\n",
      "Train Epoch: 21 [25600/50000 (51%)]\tLoss: 0.506214\n",
      "Train Epoch: 21 [32000/50000 (64%)]\tLoss: 0.234457\n",
      "Train Epoch: 21 [38400/50000 (77%)]\tLoss: 0.518609\n",
      "Train Epoch: 21 [44800/50000 (90%)]\tLoss: 0.440227\n",
      "\n",
      "Test set: Average loss: 0.3817, Accuracy: 8704/10000 (87%)\n",
      "\n",
      "Time taken for epoch : 19.653210639953613\n",
      "\n",
      "changing T value to 0.21\n",
      "  + Number of params: 5.70M\n",
      "  + Number of FLOPs: 0.44G\n",
      "Train Epoch: 22 [0/50000 (0%)]\tLoss: 0.569966\n",
      "Train Epoch: 22 [6400/50000 (13%)]\tLoss: 0.419019\n",
      "Train Epoch: 22 [12800/50000 (26%)]\tLoss: 0.435670\n",
      "Train Epoch: 22 [19200/50000 (38%)]\tLoss: 0.288544\n",
      "Train Epoch: 22 [25600/50000 (51%)]\tLoss: 0.251495\n",
      "Train Epoch: 22 [32000/50000 (64%)]\tLoss: 0.364832\n",
      "Train Epoch: 22 [38400/50000 (77%)]\tLoss: 0.408261\n",
      "Train Epoch: 22 [44800/50000 (90%)]\tLoss: 0.316191\n",
      "\n",
      "Test set: Average loss: 0.4385, Accuracy: 8563/10000 (86%)\n",
      "\n",
      "Time taken for epoch : 16.325645208358765\n",
      "\n",
      "changing T value to 0.22\n",
      "  + Number of params: 5.17M\n",
      "  + Number of FLOPs: 0.43G\n",
      "Train Epoch: 23 [0/50000 (0%)]\tLoss: 0.375218\n",
      "Train Epoch: 23 [6400/50000 (13%)]\tLoss: 0.232634\n",
      "Train Epoch: 23 [12800/50000 (26%)]\tLoss: 0.227962\n",
      "Train Epoch: 23 [19200/50000 (38%)]\tLoss: 0.188691\n",
      "Train Epoch: 23 [25600/50000 (51%)]\tLoss: 0.356482\n",
      "Train Epoch: 23 [32000/50000 (64%)]\tLoss: 0.197759\n",
      "Train Epoch: 23 [38400/50000 (77%)]\tLoss: 0.344778\n",
      "Train Epoch: 23 [44800/50000 (90%)]\tLoss: 0.726692\n",
      "\n",
      "Test set: Average loss: 0.4372, Accuracy: 8608/10000 (86%)\n",
      "\n",
      "Time taken for epoch : 16.171064138412476\n",
      "\n",
      "changing T value to 0.23\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  + Number of params: 4.75M\n",
      "  + Number of FLOPs: 0.41G\n",
      "Train Epoch: 24 [0/50000 (0%)]\tLoss: 0.244586\n",
      "Train Epoch: 24 [6400/50000 (13%)]\tLoss: 0.424029\n",
      "Train Epoch: 24 [12800/50000 (26%)]\tLoss: 0.337751\n",
      "Train Epoch: 24 [19200/50000 (38%)]\tLoss: 0.369609\n",
      "Train Epoch: 24 [25600/50000 (51%)]\tLoss: 0.463381\n",
      "Train Epoch: 24 [32000/50000 (64%)]\tLoss: 0.326591\n",
      "Train Epoch: 24 [38400/50000 (77%)]\tLoss: 0.442864\n",
      "Train Epoch: 24 [44800/50000 (90%)]\tLoss: 0.327329\n",
      "\n",
      "Test set: Average loss: 0.4236, Accuracy: 8653/10000 (87%)\n",
      "\n",
      "Time taken for epoch : 15.695441484451294\n",
      "\n",
      "changing T value to 0.24\n",
      "  + Number of params: 4.50M\n",
      "  + Number of FLOPs: 0.41G\n",
      "Train Epoch: 25 [0/50000 (0%)]\tLoss: 0.171481\n",
      "Train Epoch: 25 [6400/50000 (13%)]\tLoss: 0.339639\n",
      "Train Epoch: 25 [12800/50000 (26%)]\tLoss: 0.389844\n",
      "Train Epoch: 25 [19200/50000 (38%)]\tLoss: 0.434020\n",
      "Train Epoch: 25 [25600/50000 (51%)]\tLoss: 0.231618\n",
      "Train Epoch: 25 [32000/50000 (64%)]\tLoss: 0.303142\n",
      "Train Epoch: 25 [38400/50000 (77%)]\tLoss: 0.428371\n",
      "Train Epoch: 25 [44800/50000 (90%)]\tLoss: 0.220332\n",
      "\n",
      "Test set: Average loss: 0.4223, Accuracy: 8629/10000 (86%)\n",
      "\n",
      "Time taken for epoch : 15.40560269355774\n",
      "\n",
      "changing T value to 0.25\n",
      "  + Number of params: 4.29M\n",
      "  + Number of FLOPs: 0.40G\n",
      "Train Epoch: 26 [0/50000 (0%)]\tLoss: 0.273966\n",
      "Train Epoch: 26 [6400/50000 (13%)]\tLoss: 0.304922\n",
      "Train Epoch: 26 [12800/50000 (26%)]\tLoss: 0.158535\n",
      "Train Epoch: 26 [19200/50000 (38%)]\tLoss: 0.327042\n",
      "Train Epoch: 26 [25600/50000 (51%)]\tLoss: 0.244849\n",
      "Train Epoch: 26 [32000/50000 (64%)]\tLoss: 0.347522\n",
      "Train Epoch: 26 [38400/50000 (77%)]\tLoss: 0.278996\n",
      "Train Epoch: 26 [44800/50000 (90%)]\tLoss: 0.332655\n",
      "\n",
      "Test set: Average loss: 0.4482, Accuracy: 8573/10000 (86%)\n",
      "\n",
      "Time taken for epoch : 14.973002672195435\n",
      "\n",
      "changing T value to 0.26\n",
      "  + Number of params: 4.12M\n",
      "  + Number of FLOPs: 0.39G\n",
      "Train Epoch: 27 [0/50000 (0%)]\tLoss: 0.305604\n",
      "Train Epoch: 27 [6400/50000 (13%)]\tLoss: 0.612894\n",
      "Train Epoch: 27 [12800/50000 (26%)]\tLoss: 0.421600\n",
      "Train Epoch: 27 [19200/50000 (38%)]\tLoss: 0.240359\n",
      "Train Epoch: 27 [25600/50000 (51%)]\tLoss: 0.341044\n",
      "Train Epoch: 27 [32000/50000 (64%)]\tLoss: 0.702436\n",
      "Train Epoch: 27 [38400/50000 (77%)]\tLoss: 0.190826\n",
      "Train Epoch: 27 [44800/50000 (90%)]\tLoss: 0.183929\n",
      "\n",
      "Test set: Average loss: 0.4715, Accuracy: 8559/10000 (86%)\n",
      "\n",
      "Time taken for epoch : 15.128970384597778\n",
      "\n",
      "changing T value to 0.27\n",
      "  + Number of params: 3.96M\n",
      "  + Number of FLOPs: 0.39G\n",
      "Train Epoch: 28 [0/50000 (0%)]\tLoss: 0.264141\n",
      "Train Epoch: 28 [6400/50000 (13%)]\tLoss: 0.161125\n",
      "Train Epoch: 28 [12800/50000 (26%)]\tLoss: 0.451167\n",
      "Train Epoch: 28 [19200/50000 (38%)]\tLoss: 0.342181\n",
      "Train Epoch: 28 [25600/50000 (51%)]\tLoss: 0.279591\n",
      "Train Epoch: 28 [32000/50000 (64%)]\tLoss: 0.388751\n",
      "Train Epoch: 28 [38400/50000 (77%)]\tLoss: 0.670108\n",
      "Train Epoch: 28 [44800/50000 (90%)]\tLoss: 0.291181\n",
      "\n",
      "Test set: Average loss: 0.4734, Accuracy: 8525/10000 (85%)\n",
      "\n",
      "Time taken for epoch : 14.806230545043945\n",
      "\n",
      "changing T value to 0.28\n",
      "  + Number of params: 3.82M\n",
      "  + Number of FLOPs: 0.39G\n",
      "Train Epoch: 29 [0/50000 (0%)]\tLoss: 0.343433\n",
      "Train Epoch: 29 [6400/50000 (13%)]\tLoss: 0.233129\n",
      "Train Epoch: 29 [12800/50000 (26%)]\tLoss: 0.176324\n",
      "Train Epoch: 29 [19200/50000 (38%)]\tLoss: 0.407920\n",
      "Train Epoch: 29 [25600/50000 (51%)]\tLoss: 0.258279\n",
      "Train Epoch: 29 [32000/50000 (64%)]\tLoss: 0.311652\n",
      "Train Epoch: 29 [38400/50000 (77%)]\tLoss: 0.243151\n",
      "Train Epoch: 29 [44800/50000 (90%)]\tLoss: 0.282172\n",
      "\n",
      "Test set: Average loss: 0.4460, Accuracy: 8633/10000 (86%)\n",
      "\n",
      "Time taken for epoch : 14.683746099472046\n",
      "\n",
      "changing T value to 0.29\n",
      "  + Number of params: 3.67M\n",
      "  + Number of FLOPs: 0.38G\n",
      "Train Epoch: 30 [0/50000 (0%)]\tLoss: 0.294672\n",
      "Train Epoch: 30 [6400/50000 (13%)]\tLoss: 0.482673\n",
      "Train Epoch: 30 [12800/50000 (26%)]\tLoss: 0.369312\n",
      "Train Epoch: 30 [19200/50000 (38%)]\tLoss: 0.137028\n",
      "Train Epoch: 30 [25600/50000 (51%)]\tLoss: 0.495504\n",
      "Train Epoch: 30 [32000/50000 (64%)]\tLoss: 0.225258\n",
      "Train Epoch: 30 [38400/50000 (77%)]\tLoss: 0.245827\n",
      "Train Epoch: 30 [44800/50000 (90%)]\tLoss: 0.279070\n",
      "\n",
      "Test set: Average loss: 0.3989, Accuracy: 8720/10000 (87%)\n",
      "\n",
      "Time taken for epoch : 16.574926376342773\n",
      "\n",
      "changing T value to 0.3\n",
      "  + Number of params: 3.53M\n",
      "  + Number of FLOPs: 0.38G\n",
      "Train Epoch: 31 [0/50000 (0%)]\tLoss: 0.147793\n",
      "Train Epoch: 31 [6400/50000 (13%)]\tLoss: 0.190511\n",
      "Train Epoch: 31 [12800/50000 (26%)]\tLoss: 0.303894\n",
      "Train Epoch: 31 [19200/50000 (38%)]\tLoss: 0.154067\n",
      "Train Epoch: 31 [25600/50000 (51%)]\tLoss: 0.315209\n",
      "Train Epoch: 31 [32000/50000 (64%)]\tLoss: 0.277714\n",
      "Train Epoch: 31 [38400/50000 (77%)]\tLoss: 0.517208\n",
      "Train Epoch: 31 [44800/50000 (90%)]\tLoss: 0.545493\n",
      "\n",
      "Test set: Average loss: 0.4041, Accuracy: 8683/10000 (87%)\n",
      "\n",
      "Time taken for epoch : 14.467824697494507\n",
      "\n",
      "changing T value to 0.31\n",
      "  + Number of params: 3.44M\n",
      "  + Number of FLOPs: 0.38G\n",
      "Train Epoch: 32 [0/50000 (0%)]\tLoss: 0.175265\n",
      "Train Epoch: 32 [6400/50000 (13%)]\tLoss: 0.469183\n",
      "Train Epoch: 32 [12800/50000 (26%)]\tLoss: 0.138451\n",
      "Train Epoch: 32 [19200/50000 (38%)]\tLoss: 0.378330\n",
      "Train Epoch: 32 [25600/50000 (51%)]\tLoss: 0.293755\n",
      "Train Epoch: 32 [32000/50000 (64%)]\tLoss: 0.247429\n",
      "Train Epoch: 32 [38400/50000 (77%)]\tLoss: 0.406407\n",
      "Train Epoch: 32 [44800/50000 (90%)]\tLoss: 0.449455\n",
      "\n",
      "Test set: Average loss: 0.4226, Accuracy: 8654/10000 (87%)\n",
      "\n",
      "Time taken for epoch : 14.402207136154175\n",
      "\n",
      "changing T value to 0.32\n",
      "  + Number of params: 3.33M\n",
      "  + Number of FLOPs: 0.37G\n",
      "Train Epoch: 33 [0/50000 (0%)]\tLoss: 0.266692\n",
      "Train Epoch: 33 [6400/50000 (13%)]\tLoss: 0.342470\n",
      "Train Epoch: 33 [12800/50000 (26%)]\tLoss: 0.259632\n",
      "Train Epoch: 33 [19200/50000 (38%)]\tLoss: 0.257665\n",
      "Train Epoch: 33 [25600/50000 (51%)]\tLoss: 0.303850\n",
      "Train Epoch: 33 [32000/50000 (64%)]\tLoss: 0.331936\n",
      "Train Epoch: 33 [38400/50000 (77%)]\tLoss: 0.424182\n",
      "Train Epoch: 33 [44800/50000 (90%)]\tLoss: 0.338538\n",
      "\n",
      "Test set: Average loss: 0.4636, Accuracy: 8504/10000 (85%)\n",
      "\n",
      "Time taken for epoch : 14.406094551086426\n",
      "\n",
      "changing T value to 0.33\n",
      "  + Number of params: 3.24M\n",
      "  + Number of FLOPs: 0.37G\n",
      "Train Epoch: 34 [0/50000 (0%)]\tLoss: 0.201507\n",
      "Train Epoch: 34 [6400/50000 (13%)]\tLoss: 0.410812\n",
      "Train Epoch: 34 [12800/50000 (26%)]\tLoss: 0.234762\n",
      "Train Epoch: 34 [19200/50000 (38%)]\tLoss: 0.187214\n",
      "Train Epoch: 34 [25600/50000 (51%)]\tLoss: 0.418998\n",
      "Train Epoch: 34 [32000/50000 (64%)]\tLoss: 0.374483\n",
      "Train Epoch: 34 [38400/50000 (77%)]\tLoss: 0.318154\n",
      "Train Epoch: 34 [44800/50000 (90%)]\tLoss: 0.424494\n",
      "\n",
      "Test set: Average loss: 0.4765, Accuracy: 8507/10000 (85%)\n",
      "\n",
      "Time taken for epoch : 14.282011985778809\n",
      "\n",
      "changing T value to 0.34\n",
      "  + Number of params: 3.12M\n",
      "  + Number of FLOPs: 0.37G\n",
      "Train Epoch: 35 [0/50000 (0%)]\tLoss: 0.198205\n",
      "Train Epoch: 35 [6400/50000 (13%)]\tLoss: 0.296294\n",
      "Train Epoch: 35 [12800/50000 (26%)]\tLoss: 0.252728\n",
      "Train Epoch: 35 [19200/50000 (38%)]\tLoss: 0.347243\n",
      "Train Epoch: 35 [25600/50000 (51%)]\tLoss: 0.233621\n",
      "Train Epoch: 35 [32000/50000 (64%)]\tLoss: 0.295747\n",
      "Train Epoch: 35 [38400/50000 (77%)]\tLoss: 0.460062\n",
      "Train Epoch: 35 [44800/50000 (90%)]\tLoss: 0.336276\n",
      "\n",
      "Test set: Average loss: 0.4226, Accuracy: 8661/10000 (87%)\n",
      "\n",
      "Time taken for epoch : 14.13601040840149\n",
      "\n",
      "changing T value to 0.35000000000000003\n",
      "  + Number of params: 3.02M\n",
      "  + Number of FLOPs: 0.36G\n",
      "Train Epoch: 36 [0/50000 (0%)]\tLoss: 0.247097\n",
      "Train Epoch: 36 [6400/50000 (13%)]\tLoss: 0.309724\n",
      "Train Epoch: 36 [12800/50000 (26%)]\tLoss: 0.484042\n",
      "Train Epoch: 36 [19200/50000 (38%)]\tLoss: 0.247767\n",
      "Train Epoch: 36 [25600/50000 (51%)]\tLoss: 0.426962\n",
      "Train Epoch: 36 [32000/50000 (64%)]\tLoss: 0.340140\n",
      "Train Epoch: 36 [38400/50000 (77%)]\tLoss: 0.183967\n",
      "Train Epoch: 36 [44800/50000 (90%)]\tLoss: 0.277360\n",
      "\n",
      "Test set: Average loss: 0.4063, Accuracy: 8714/10000 (87%)\n",
      "\n",
      "Time taken for epoch : 14.30560564994812\n",
      "\n",
      "changing T value to 0.36\n",
      "  + Number of params: 2.95M\n",
      "  + Number of FLOPs: 0.36G\n",
      "Train Epoch: 37 [0/50000 (0%)]\tLoss: 0.134857\n",
      "Train Epoch: 37 [6400/50000 (13%)]\tLoss: 0.274917\n",
      "Train Epoch: 37 [12800/50000 (26%)]\tLoss: 0.223185\n",
      "Train Epoch: 37 [19200/50000 (38%)]\tLoss: 0.156236\n",
      "Train Epoch: 37 [25600/50000 (51%)]\tLoss: 0.307069\n",
      "Train Epoch: 37 [32000/50000 (64%)]\tLoss: 0.260832\n",
      "Train Epoch: 37 [38400/50000 (77%)]\tLoss: 0.219451\n",
      "Train Epoch: 37 [44800/50000 (90%)]\tLoss: 0.370813\n",
      "\n",
      "Test set: Average loss: 0.5183, Accuracy: 8369/10000 (84%)\n",
      "\n",
      "Time taken for epoch : 14.101009845733643\n",
      "\n",
      "changing T value to 0.37\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  + Number of params: 2.89M\n",
      "  + Number of FLOPs: 0.36G\n",
      "Train Epoch: 38 [0/50000 (0%)]\tLoss: 0.342714\n",
      "Train Epoch: 38 [6400/50000 (13%)]\tLoss: 0.208868\n",
      "Train Epoch: 38 [12800/50000 (26%)]\tLoss: 0.260117\n",
      "Train Epoch: 38 [19200/50000 (38%)]\tLoss: 0.250259\n",
      "Train Epoch: 38 [25600/50000 (51%)]\tLoss: 0.290218\n",
      "Train Epoch: 38 [32000/50000 (64%)]\tLoss: 0.213069\n",
      "Train Epoch: 38 [38400/50000 (77%)]\tLoss: 0.267450\n",
      "Train Epoch: 38 [44800/50000 (90%)]\tLoss: 0.286258\n",
      "\n",
      "Test set: Average loss: 0.5659, Accuracy: 8400/10000 (84%)\n",
      "\n",
      "Time taken for epoch : 13.873986005783081\n",
      "\n",
      "changing T value to 0.38\n",
      "  + Number of params: 2.84M\n",
      "  + Number of FLOPs: 0.36G\n",
      "Train Epoch: 39 [0/50000 (0%)]\tLoss: 0.281361\n",
      "Train Epoch: 39 [6400/50000 (13%)]\tLoss: 0.232035\n",
      "Train Epoch: 39 [12800/50000 (26%)]\tLoss: 0.382918\n",
      "Train Epoch: 39 [19200/50000 (38%)]\tLoss: 0.183424\n",
      "Train Epoch: 39 [25600/50000 (51%)]\tLoss: 0.339109\n",
      "Train Epoch: 39 [32000/50000 (64%)]\tLoss: 0.287540\n",
      "Train Epoch: 39 [38400/50000 (77%)]\tLoss: 0.370285\n",
      "Train Epoch: 39 [44800/50000 (90%)]\tLoss: 0.191439\n",
      "\n",
      "Test set: Average loss: 0.3938, Accuracy: 8677/10000 (87%)\n",
      "\n",
      "Time taken for epoch : 14.10519528388977\n",
      "\n",
      "changing T value to 0.39\n",
      "  + Number of params: 2.75M\n",
      "  + Number of FLOPs: 0.36G\n",
      "Train Epoch: 40 [0/50000 (0%)]\tLoss: 0.296906\n",
      "Train Epoch: 40 [6400/50000 (13%)]\tLoss: 0.191074\n",
      "Train Epoch: 40 [12800/50000 (26%)]\tLoss: 0.237214\n",
      "Train Epoch: 40 [19200/50000 (38%)]\tLoss: 0.182493\n",
      "Train Epoch: 40 [25600/50000 (51%)]\tLoss: 0.253002\n",
      "Train Epoch: 40 [32000/50000 (64%)]\tLoss: 0.389086\n",
      "Train Epoch: 40 [38400/50000 (77%)]\tLoss: 0.289115\n",
      "Train Epoch: 40 [44800/50000 (90%)]\tLoss: 0.432422\n",
      "\n",
      "Test set: Average loss: 0.4679, Accuracy: 8576/10000 (86%)\n",
      "\n",
      "Time taken for epoch : 13.861117362976074\n",
      "\n",
      "changing T value to 0.4\n",
      "  + Number of params: 2.70M\n",
      "  + Number of FLOPs: 0.36G\n",
      "Train Epoch: 41 [0/50000 (0%)]\tLoss: 0.178469\n",
      "Train Epoch: 41 [6400/50000 (13%)]\tLoss: 0.230996\n",
      "Train Epoch: 41 [12800/50000 (26%)]\tLoss: 0.152681\n",
      "Train Epoch: 41 [19200/50000 (38%)]\tLoss: 0.308889\n",
      "Train Epoch: 41 [25600/50000 (51%)]\tLoss: 0.236611\n",
      "Train Epoch: 41 [32000/50000 (64%)]\tLoss: 0.292298\n",
      "Train Epoch: 41 [38400/50000 (77%)]\tLoss: 0.067772\n",
      "Train Epoch: 41 [44800/50000 (90%)]\tLoss: 0.343589\n",
      "\n",
      "Test set: Average loss: 0.4823, Accuracy: 8462/10000 (85%)\n",
      "\n",
      "Time taken for epoch : 13.799217939376831\n",
      "\n",
      "changing T value to 0.41000000000000003\n",
      "  + Number of params: 2.64M\n",
      "  + Number of FLOPs: 0.35G\n",
      "Train Epoch: 42 [0/50000 (0%)]\tLoss: 0.186091\n",
      "Train Epoch: 42 [6400/50000 (13%)]\tLoss: 0.236326\n",
      "Train Epoch: 42 [12800/50000 (26%)]\tLoss: 0.169487\n",
      "Train Epoch: 42 [19200/50000 (38%)]\tLoss: 0.091174\n",
      "Train Epoch: 42 [25600/50000 (51%)]\tLoss: 0.209254\n",
      "Train Epoch: 42 [32000/50000 (64%)]\tLoss: 0.210159\n",
      "Train Epoch: 42 [38400/50000 (77%)]\tLoss: 0.208745\n"
     ]
    }
   ],
   "source": [
    "for slope in slopes:    \n",
    "    \n",
    "    set_random_seed(args.seed)\n",
    "    \n",
    "    metrics[slope] = {'best_val_acc':0, 'params':0, 'flops':0}\n",
    "    \n",
    "    vgg16_bn_clustered = VGG('VGG16',num_output=args.num_output)\n",
    "    vgg16_bn_clustered = vgg16_bn_clustered.to(device)  \n",
    "\n",
    "    optimizer = optim.SGD(vgg16_bn_clustered.parameters(),lr=args.lr,momentum=args.momentum,weight_decay=args.weight_decay,nesterov=False)\n",
    "\n",
    "    cluster_args = {\n",
    "        'cluster_layers' : {0:0,3:0,7:0,10:0,14:0,17:0,20:0,24:0,27:0,30:0,34:0,37:0,40:0},\n",
    "        'conv_feature_size' : 1,\n",
    "        'reshape_exists' : True,\n",
    "        'features' : 'both',\n",
    "        'channel_reduction' : 'fro',\n",
    "        'use_bias' : True,\n",
    "        'linkage_method' : 'ward',\n",
    "        'distance_metric' : 'euclidean',\n",
    "        'cluster_criterion' : 'hierarchical',\n",
    "        'distance_threshold' : 0.95,\n",
    "        'merge_criterion' : 'max_l2_norm',    \n",
    "        'verbose' : False\n",
    "    }\n",
    "    \n",
    "    T_values = {}\n",
    "    for epoch in range(args.epochs+1):\n",
    "        T_values[epoch] = slope * (epoch - 1)\n",
    "        \n",
    "    args.epochs = 160\n",
    "    args.lr = 0.1\n",
    "    flag = True\n",
    "    path = args.checkpoint_path[:-4] + '_cupSS_K_{}.pth'.format(str(slope)) \n",
    "\n",
    "    if not os.path.isfile(path):\n",
    "        for epoch in range(1, args.epochs+1):          \n",
    "            if epoch in T_values.keys() and flag:\n",
    "                print('changing T value to {}'.format(T_values[epoch]))\n",
    "                cluster_args['distance_threshold'] = T_values[epoch]        \n",
    "                model_modifier = cluster_model(vgg16_bn_clustered,cluster_args)\n",
    "                vgg16_bn_clustered = model_modifier.cluster_model()\n",
    "                vgg16_bn_clustered.cuda()\n",
    "                optimizer = optim.SGD(vgg16_bn_clustered.parameters(),lr=args.lr,momentum=args.momentum,weight_decay=args.weight_decay,nesterov=False)\n",
    "\n",
    "                metrics[slope]['params'] = print_model_param_nums(copy.deepcopy(vgg16_bn_clustered))\n",
    "                metrics[slope]['flops'] = print_model_param_flops(copy.deepcopy(vgg16_bn_clustered).cpu(),input_res=32)\n",
    "\n",
    "                if (metrics[slope]['params']/1e6 <= 0.9):\n",
    "                    print('stop filter pruning')\n",
    "                    flag = False\n",
    "\n",
    "            args.lr = adjust_learning_rate_iccv(args,optimizer,epoch)\n",
    "            start = time.time()\n",
    "            train_loss,train_acc = train(args,vgg16_bn_clustered,device,train_loader,optimizer,epoch)\n",
    "            val_loss,val_acc = test(args,vgg16_bn_clustered,device,val_loader)\n",
    "\n",
    "            if val_acc > metrics[slope]['best_val_acc']:  \n",
    "                    torch.save(vgg16_bn_clustered, path, pickle_protocol=4)            \n",
    "                    metrics[slope]['best_val_acc'] = val_acc       \n",
    "\n",
    "            print('Time taken for epoch : {}\\n'.format(time.time()-start))\n",
    "    \n",
    "    print('Best accuracy {}'.format(metrics[slope]['best_val_acc']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0.005: {'best_val_acc': 93.26, 'params': 886959, 'flops': 221926304.0}, 0.01: {'best_val_acc': 92.78, 'params': 871157, 'flops': 221455063.0}, 0.015: {'best_val_acc': 93.03, 'params': 870551, 'flops': 207975135.0}, 0.02: {'best_val_acc': 93.15, 'params': 870714, 'flops': 204803302.0}, 0.025: {'best_val_acc': 93.19, 'params': 879362, 'flops': 208217141.0}, 0.03: {'best_val_acc': 93.21, 'params': 869308, 'flops': 202863954.0}, 0.035: {'best_val_acc': 93.24, 'params': 864876, 'flops': 199400292.0}, 0.04: {'best_val_acc': 93.14, 'params': 876862, 'flops': 203072901.0}, 0.045: {'best_val_acc': 92.88, 'params': 861990, 'flops': 194892119.0}, 0.05: {'best_val_acc': 92.93, 'params': 883179, 'flops': 196780765.0}}\n"
     ]
    }
   ],
   "source": [
    "# save np.load\n",
    "np_load_old = np.load\n",
    "\n",
    "# modify the default parameters of np.load\n",
    "np.load = lambda *a,**k: np_load_old(*a, allow_pickle=True, **k)\n",
    "\n",
    "metrics_path = args.checkpoint_path[:-4]+'_cupSS_metrics.npy'\n",
    "\n",
    "if not os.path.isfile(metrics_path):\n",
    "    np.save(metrics_path,metrics)\n",
    "else:\n",
    "    metrics = np.load(metrics_path).item()\n",
    "    \n",
    "print(metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### plot best acc vs k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZsAAAFTCAYAAADvI6EhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdeXhU5fXA8e8JELKwyJYAboBoXSgVARXFBbCKC2hF1Kqg1rrgUitiobaiaK2WVltbFW3V8hNpcQEFBXElihuLW8UdQUQEwqJASCBAzu+P90bGYe7N3GTW5Hye5z5D7vvOvWeGZM7c976LqCrGGGNMMuWkOwBjjDH1nyUbY4wxSWfJxhhjTNJZsjHGGJN0lmyMMcYknSUbY4wxSWfJxiSNiHwpIutFZFXEViEiGmN/mYhMTHfMpuERkdkissb7vXwg3fHUV5ZsTLKdrqrtqzfgUZ/9f0l2ICLymohsT/Z5THZR1YFAn3THUd9ZsjHGGJN0jdMdgKnXBgAr46x7N9AkibEYY9LIrmxM0qjqF6paHmfdtar6fWISkVYicoeILPXu75SKyDQR6RH9XBE5WUReEZEVIvKNiCwSkX9U1xWRfiKyCjgUaBR1r+iooLhEJF9ErvWOv9yLZZmITBCRtj7P6SAi93r1VovIVyJSIiKjRaR1VN19ReRhL/bV3n2u50TkChEp8Op8ISIbvHsK50U89+5Y9xpE5Lio+2MHiMjfRWSxiGzz9vUVZ4R3z2KZiKz13r9HRKSzz2trKSK3icjnXrwrROQNEblZRHYXkYHecVREtnhx9POee5D381YR2SgiN/uc45SAY/wk6hhjvf17iciD3u/LShFZIiLTRWR40P9vEO/3b50XxxoR+V9tj2UAVbXNtpRtwERAgWMD6rQCPgYWAwd7+4qAZ4AKoG9E3X7ADuByoJG3rwewHHgg6rivAdtDxtvVi/e3QBNv3yHAZ8CnQEFU/b2BFcC7QFdvX3PgXu8450XU7QVsAF4A2nv72gFTvbqRr/OX0c+Piu+BGLE/4pWVACfhvlzu5Z2zL65lQ3FXlYURx3sTWA10iDrebsCHwDLgEG9fU+D33nF+H7FvDe6qtknUMarLetTwvlfX+xpoHFWWC5RGxNDE+794DmgTEesj8f5/x3ofvf+3EuDOdP/d1IfNrmxMJroN2B+4UlXfA1DVUuBCQIC7Iur+DPchOlFVd3h13wVuIf4mvCBbgWdU9TZV3eYd/x3gN8B+wM+j6t8HdAQuUNXFXv1NwJW4D2kARESA/wPygHNVdZVXdw1wPi6pJsoLqjpLVatU9SvgYuAL3IfrW8A1qrrZO/9iYAQuuV8edZzbgAOBX3nvAaq6VVX/AMytrqSqW4GHgPa4/59IQ4HF3v+RL+8YE4HdgVOiiocAX1bHAPwY938xXVXXec//DrgG+CToPH68K9AXgVdUdWRtjmF+yJKNySgi0gj3Ab4FeCmyzPsgXgIcIiIdvN2rvMeJIvKjiLr/VNUb6hqPqi5X1UExij7yHn8SEXtb4ARguaq+H3WcKtwHb/VrOgT3wT3PS6SRdcuA44BFdY3f80LU8R9T1ZWqukNV+1Qn0QixXlv1/8t2YHaMc1wC/Cfi5/txyWxEVL0RuKu8ePzTO8ZlUfsv845fbQ3u6vY6ERkkIo3B/b6oarc4z/U9EWmPu6J5QVVvDPt8E5t1EDCZpj3QAqgClrsLgB/IBzbjvvGuBP4GdAfOBIaKyIfAU8C/VfWLRAQkIsfjvuUfiGueqQIaecUFEVX3w115LY91nKhv89WJ0a/uG3UIOdoqvwIRORT4Na7psTXuw71a5GsrBloC33hXHT+gqp9E/bxERJ4DBorI/qr6iYh0x73ux+MJWlU/F5E5wPEi0llVl4rI/rgkOCWi3nIR+SXud2EGsF5EnsU1oz2nqmHWUdkTl2h+BHwrIjneFwVTR3ZlYzJN9QfDtxoxDidia6mqzVR1IYCqlqvq2bg29+tx37x/B3wiIpfWNRgRuQR3L2Ar0F9Vi9SNCwoal9E0xCnC1I0lnr/hmB+WInIi8AbQARiMu2/UHtgj4Fhh4p3gPVZfmYwAHlLVLSGOcR8ugV8ScaxHqpv9qqnqRFzz5bnAK7jmumeBF0QkN8T5+gM34a6qjgZGh3iuCWDJxmSaVbgb2G2qe2NFEpF2Xm+rRt7POSIiqrrEu69yMHAMUAb8TUTq2p36iupHVf26hrqf4ZLl7rEKRaRtRG+06isBv7rFItIyYld1U1f06ymuIaYgl+Gu0K5T1c9ruAKI/H/JixHvbiJSFLV7Ju7KbbiIFAPn4JJHGE/hOiv8wns/hvPDJjS8XnU53heP/6jq6biE+Ryu+/1ZIc43WVWn4O73fAqME5GeIWM2MViyMRnFa7L4r/fjGTGqjAVur+4MADwM/OAGrqq+CryKu/kembA2AzkikgMgIqd5Vy5Bou9nVNula7CqrsV9m24vIodHlnlJ70PgAm/Xu97PvUVkj6i67XAf0gMidlcnur2iTntMDfEHqX5t0Ukm1murAiZ7P54W41gv43qlRT5nB+4KoRUwDXhDVZeECdC7n/RvXIeFR4BPVPWDqGoDgOh7ZGvYmZRahTjldu/55bh7VApMjvXFx4SU7u5wtjWsjXBdn1cAR3j7GuO+iW8Gjo6o+wjwFdAzYt+RwLe43kmRx/2zd+6DcVcIc4E/1RDv1d5z/gu08vbtC7xHjC7H7Oz6/DbQxdu3Gy4pfgK0jKhb3fV5NlDs7euI+0Y+l4guv7jEuQrXHXxvb9/RuA4HNXV93sPntZ3mlb8M7B5x/he9/S9G1a/u+ryUnV3S84E/4a4+9o5xjvZApXe8QbX8nemMawpUYHiM8uO8squBXG9fO2AWsBHYK45zxOxCDlzn7b8v3X872b6lPQDbGsaG67a8CtelV4H13s/tfOrvBozHddFdA3wJTAcOjap3MK4r9AfAN94xP8CNi8mPqtsOdwN5jZcQngRaxxH7ZcD/cE1zX3sfzpd6r6PcO2fXiPodcPcrvvLKvsR9y+4Y49j7ApMiYl/sve6WMer+2Dv3d94x78bdLI+M4whvX+R7Xer9vGeMYw4F5nsfyt/gxtic4z1vq/e8fhH1WwK3e3Gu9l7jf4H9At6/x7x4c+rw+zPb+53Ji1HWEtfs9QruinCld75JwAFxHHuS9zsR+T7+CNddfbW3X739I9L9t5Stm3hvtjHGJIWITAY+VNU/pjsWkz6WbIwxCSUizYCtqrrNG3u0BHflV1rDU009Zh0EjDGJdh87B3OOBaZYojGWbIwxibYUuEXc5Kf74G6ymwbOmtGMMcYknU1XU0tt27bVTp061eq5mzdvprCwMLEBNXD2niaevaeJ1VDez7fffnutqraL3m/JppY6derEwoULa/XckpISjj322MQG1MDZe5p49p4mVkN5P0VkWaz9ds/GGGNM0lmyMcYYk3SWbIwxxiSdJRtjjDFJZ8nGGJNVVGHePDh/aDmtC7fQKKeK1oVbuODMcubPd+Um81iyMcZkjW3b4KJzKji7fyndpo1jUXkXtmoui8q7cNDUcZzVv5SLzqlgm9/CECZtLNkYY7KCKlw6vIJvZixgUXlnrqsaT0dW0pgddGQl11WNZ9HmzqyYvpBLh1fYFU6GsWRjjMkK8+fDnBmbmFp+IoWUx6xTSDnTKgYy5+lNLFiQ4gBNIEs2xpiMpgrvvAOXDivn0vI7fBNNtULKGVFxJxPuCK5nUsuSjTEmI331Fdx2G3TrBj17wief5zCcSXE999yqScyYaR9vmcSmqzHGZIyNG+GJJ2DSJHjllR/2LKsklyLiW6mgiFI2VOQmKUpTG5ZsjDFptW0bPP+8SzDTp8OWLbHr5VJJKUV0ZGWNxyyliJZ5lUBeYoM1tWbXmcaYlFOFhQvh6qth993hlFPg0Uf9Ew1ADlU8zLC4jv8ww9h7r6oERWsSwZKNMSZlli2DP/4RDjwQeveGv/8d1qyJ77kVFHAn17KZgsB6ZRRyJyN595MCJsV3i8ekgDWjGWOSasMGePxx10z26qt1O9YmmjOQ2cxmYMxeaWUUciLPUkZzAC6+GPbdFw4/vG7nNXVnVzbGmITbtg2efhrOPBOKi92Hfl0TzZ57wlXX5VN0Ui+6FS5lfM5oVtCRbTRmBR25jdF0ZgkL6cUW8gHYuhVOOw2WL0/AizJ1Ylc2xpiEUIUFC9wVzJQpsHZt3Y/ZvDmccQYMGwbHHAM5OaCaz4IF+dz7l7F0n3UTG8pzaayV5FBFRYwmttWrXcKZOxcKglvgTBJZsjHG1MmXX8Ijj7gk89lndT9eo0YwcCCcdx4MHrxrghCBQw+FQx/bWXDPPXlceaX/Md95By64wHVCEKl7jCY8SzbGmNC+/XbnfZjXXkvMMXv1clcwZ58NRUXhnnv55fDBB3D//f51Hn/cDRAdO7ZucZrasWRjjIlLZSU8+6xLME8/7X6uq732clcw550HBxxQ++OIwD/+AZ9+CiUl/vVuvBEOOgiGDKn9uUztWLIxxgDunsv8+XDvX8p5elYOGyqOpmX+Fo48vIqmrQqYMwfWr6/7eVq0gKFD3VXMUUe5+zCJ0KSJu3o59FBYutS/3vDhsM8+cPDBiTmviY/1RjPGBK4Tc+TL43h1ainl6ytqffzGjXcO3Fy1Ch54YOcN/0Rq29ZddTVr5l+nvNzdC1q9OrHnNsEs2RjTwKnChWdXsPzJ2OvEjGE8S+lMLxaSR7iEUz1w85tvdnaFzs9P0gvxHHQQ/Pe/wR0Bli+Hn/3MdY02qWHJxpgGaNUq1+R01VWw337w/LRNPLU1eJ2Y2QykOZtqPHanTvD738Mnn7hmuauugnbtEvwCanDKKXD77cF13nwTLr3UlpFOFbtnY0wDsGyZG1RZvUV2Uc6nnLHEt07MNdzJLYzdZTxLy5buqmXYMDjyyMQ3j9XGddfBokUETlnzf/8HP/4xXHtt6uJqqNKabEREgEuAXwD7AgJ8Dtytqg9H1e0BnAH0B/bATef6HfAScLuqfhnnOYuBC4AhwD5AI2At8Cxwq6ququvrMiadVF2vrMjkEjSCvor414kZziTGcRPg7sOcdJJLMKecAnkZNsGyCPzzn/D55/DWW/71rrvO9YQ76aTUxdYQpS3ZeIlmCu5D/yrgIa/oMmCiiOyvqtdHPOUW4DjgfOAJVd0hIn2AJ4AzReSQOBPOAqAFcC4wG1DgBOARYIiI9FHVZXV+gcakyI4dboxJZHKJd3JLCL9OzDZyuftuOOssd0M+k+XlwZNPuntHX38du44q/PznrlntwANTG19Dks6L3SHAmcBjqjpBVbd6213A08AYETkk6jn3qeqjqroDQFXfBG4FWgEXxXneHOBGVZ2pqjtUtUpVnwXGAh2AmxPw2oxJmspK9019/Hh3RdGmDfTo4abrnzo1XKKBnevExKOUInYrrOSKKzI/0VRr396tkxPUMWHjRtdDbd261MXV0KQz2ZzmPT4fo2wWrknt4oh9vwFui1H3K++xZZznvQuYGmP/XO+xT5zHMSYlysthzhwYNw4GDIBWraBPHxg9GmbOdLMq10WYdWIm5wxj8MnZt07MIYe4+zNBvvjC3Xfati01MTU06Uw2xd5jrO9h1fdN+lbvUNWPVDVWz/je3mNJPCdV1T+raqwL6uo1ZO27jUmrDRvcSP3f/tbdbN9tN+jfH266CV5+2SWfRAqzTsy9eSMZcW12zmY5dKibQSDIyy/Dr3+dmngamnQmm+pG4uIYZdUdJTv7PVlEWovIZbgrnr+q6rQ6xnOE9/hoHY9jzPdUYd48OH9oOa0Lt9Aop4rWhVu44Mxy5s935WvWuPsK11wDPXtC69buZvXtt8MbbyTvm3anTm40/b/+Bcec3JzT82f7JpwyChmS/yz9Bzend++YVbLC2LE1T1Vz770wYUJq4mlIRNPUyVxEzsXdlJ+mqkOiyh7H9TwDyNGoIEVkLdAGKANuAu5R1YAFZWuMpSnwP2Ab0MvvWCJyCa73HMXFxT2nTJlSq/OVlZXRLGiIswktE9/T7duFv97WiQ/eyOXKrX/jPJ1EEaWUUsQkGcZdOSOpbFrAuvLmKYln77030737Bn784+/o3n0DxcU7RzRGxnrF1rt+EOsjMox7cn/Nj4/cyjW//ZLGjbN7YEpFRQ6/+lUPFi/2f99zcpQ///l9Djnku4SdNxN/R5OhX79+b6tqr10KVDUtG67L8WxgBzAK10OsBTASWInrJbY54PmFuF5knwEfAwfUIZY7gfVhjtGzZ0+trTlz5tT6uSa2THtPq6pULzy7XE8oeEXLKFB1FzE/2Moo0L68qnmUxyqu0yai2qOH6tVXq06dqrp6dXwxz5unev7Qzdq6sEIbyXZtXVihF5y5WefPT/57lkrLlqkWFQW/h61bqy5enLhzZtrvaLIACzXGZ2bauj6r67o8CJdczgXGAJuAOUA/XAJZGfD8zcBzIjIEeA+YIiIHey82biJyNa4jwk9V9eNavRhjosyfD3Oe3sSi8ppH5Xdm6fcrS9ZW48aue+/RR7vtiCPcvZ4woteJKSkp4dhjj61TXJlqr71c02W/fv6zV69fD4MGuZ5/LVqkNr76KK2DOlV1G/Anb/ueiOzr/fPdOI7xgYgsA7oDXYAv4j2/iFwJjAMGqmrAsC9j4qcKN1xbzmWb6zYqP0h+vuuRdtRRLrkcfritQhnWEUe4QZ8XXOBf5+OP3RicGTPcom6m9jJgUomYDvMenwAQkd1E5CYRaeJTv/ovuk28JxCRkbgxNcer6usR+3uJSK7/M42JbccON99Yjx7w6us5DAsxKr+qhj/FFi1+2Gngu+/gpZdcD7X+/S3R1Nb558OoUcF1Zs1yPQNN3aR7uppFwNDI5itvZoHLgEXsHA+zG3Aj7h7PW1HH2AvYD5dwPowqaw+sV9XKqP1jcPeJBqhq9NXTAlwvuC/r8tpMw7F9u5tl+I9/dJNPAkjIUfmV/PD7Tdu2O5vEjj4aune3b9bJcvvt8NFHLqn4+fOf3WzS55+furjqm3RPxHkQcKeIDFfVNSLSDrgdN2dZP1XdHlX/IRH5JS7hKHAw8C+gCfAr7z4OACJyBvAY8AHwk4j9N+CuaKYCp4rIqUl7daZe27oVHn7YfVgtWfLDsupR+R39bzt+r5Qi8qSSn/087/vksv/+wVPkm8Rp1Mh9WTj8cNds5ueSS2DffV3zmwkv3cnmbtzAzU9FZCuwEZgJdFfVyMGey4GTcdPbPIhrLsvFdX2eB4xS1ZKoY68FNgDRa/ZVT0czxNuMCaWiwi3+NX68/3xb1aPyxzC+xuM9kjOMs86o4t+TExyoiVuLFu6+zGGH+a9GWlnp1sBZsMB1MDDhpLuDwFVx1tuBm8Im4EJ3l+eU4OZMi95v3xdNrWza5Ab73XEHlNbQQlY9Kv8q7g7sJFBGIRPyRvJYlo7Kr0+6doUnnoDjj3dNo7GUlsKpp8Jrr0FhYWrjy3aZ2kHAmIzx7bdw882w995uPrKaEk21TTRnIPV/VH590q+fW1k0yHvvuXs3Vdk3RVxaWbIxxseaNXD99S7J3HijSzph5LbIp+KgXhxUsJTxOaNZQUe20ZgVdGR8zmi6FSxh91N7cf/D+XZ/JoOMGAGXXx5cZ+pU9wXExM+SjTFRvvkGRo50Sea221zzWRht2sCtt8JXX8GCD/J5bE4RHw0ZS/fCL8jP2Ur3wi/4+IyxPF5SxEP/zaeJX4d+kzZ/+5u7ygkybpzr6m7ik+4OAsZkjC+/hD/9CR56yH9UeZD27d2qj5dcApFTYEWOyncybElLs4smTVwiOewwt/SAn/PPh332cUsYmGB2ZWMavM8+gwsvdN1a77svfKLZay+45x5YutRdETWAuRYbhDZt4Omng6eqqahwHQZW2WLyNbJkYxqsDz5wU5EccABMnOjfA8lP167w4INujfvLL3dLEJv65YAD3BicoHtqX3/tukRvqfW88w2DJRvT4Cxc6D4cuneHKVPC9yo66CCYPNkNAPzFLyDXJjeq1046yc0gEOStt1zzaZpWbMkKlmxMg/H663DiiW525KeeCv/8Qw6BadPgf/+Dc85xMy2bhmHkyOAJOwEmTYK//CUl4WQlSzamXlN1E1b26wd9+8Ls2eGP0aePmzer+ooox/5qGhwRdz+vT5/geqNHwzPPpCambGN/NqZeUnV/9EccAccdByUl4Y/Rv79bk776isjGwjRsTZu6NXD23NO/jqq76v3wQ/86DZU1BJispOoWKLv3L+U8PSuHDRVH0zJ/C4NOqmL/Qwp49FF4//3aHfukk+B3v7MJF82uiovdHGpHHgnlPrMQbdoEgwfDvHlu9m7j2JWNyTrbtsFF51Rwdv9Suk0bx6LyLmzVXBaVd+GAJ8bx1+tL+fT9itDHPf10ePttmDnTEo3xd/DBbrbvIEuWwNCh7nfVOJZsTFZRhUuHV/DNjAUsKu/MdVXj6chKGrODjqxkDONZSmd6sZA8ak44OTmu2WPRIjcFiQ3OM/EYMsTNIBCkpASuusp6qFWzZGOyyvz5MOfpTUwtP9F3NuVCypnNQJrjP89M48Zw0UXw6aeuG/NBByUrYlNf3XADnHlmcJ3774d7701NPJnOko3JKhPuKOfyijsCp+0Hl3Cu4U7yo+o1bQpXXAGLF7s1abp2TWa0pj4TgX//u+ar4auvdj0iGzpLNiarzJiZw7lVk+KqO5xJVHm/4gUFcO21bkqZu+92k2waU1cFBTB9upsXz0/ujnJOOW4LA/odTevCLVxwZjnz5ze85jVLNiarbKjIpYj4FpQpopRt5PK738GyZW7AXYcOSQ7QNDh77OEGCTdt+sP9eVTQjlLGMo4v6MJWXCeWg6aO46z+pVx0TkWD6kBgycZkleZNKymlKK66pRSxW0Elf/iDdUE1yXXYYa5ZtloeFfRmAUvpzBh+2InluqrxLNrcmRXTF3Lp8IoGc4UTd7IRkb4icqOI3CciT4rI4yJyt4iMFpFuyQzSGHATHeblVvEww+KqPzlnGINPseUUTWqcd56bQQCgOZt4luBOLNMqBjLn6U0sWJDCINOoxmQjIleIyHLgVWAscCZwGNAXGAbcBrwvIotE5ORkBmsathtugNUbC7iTa32XWq5WRiH35o1kxLXB9YxJpFtvhX06lDOS+DqxjKi4kwl3BNerLwKTjYhMBkYC44FuQFNVba2qHVW1g6q2BAqBQ4EngIkiMirZQZuG55VX4I473L830ZyBzPZNOGUUMiT/WfoPbk7v3ikM0jR4jRrB+u9yGE58nVjOrZrEjJkN426G76sUkZHANmA/Vf2Hqn6kqrus+KGqFar6tqreBHQFBovIsckK2DQ8Gze6GXer27a3kM9CetGZpdzGaFbQkW00ZgUdGZ8zmm4FS9j91F7c/3C+zWdmUm7DlnCdWDZUNIw1KoLmRnsU+Ktq/LevVHWDiPwUaF7nyIzxjBzplmyOtIV8tpDPLYzlFrmJSprQsmAbg0+u4vFRBXZFY9KmZX4lpeVFdGRljXVLKaJlfiUNYalw3ysbVV0RJtFEPG+rqq6tW1jGODNmuNUw/WzNKeDlN/J48eW5rCvL49+PWqIx6TX45Com54ToxHJyw+jEUqvGQhE5S0Rmisj7IvKciPxSxBosTGKtWQMXXxxc57e/hcMPT008xsRjxLUF3JsfXyeWe5o2nE4soZONdy/nduAd4D7gNeBqb58xCaEKl14KpQFN3z16wNixqYvJmHgceij0G9Sc0/ODO7GcyLPs3a3hdGKpqTdarNFzw4BeqnqDqk5Q1VuA3oB1ezYJM2mSW6jKT9Omrk5uw7i3arKICNz/cD67n9qLboVLuT2qE8ttjKYzS1hIL/JbNZxOLDVd2XwsIpdHNZFVArtH1WuX2LBMQ/bVV25q9iC33mozNZvM1aQJPPiffB59uYiXjhjLPnxBU7ayD19wC2NZSxFbyGfOHNiwId3RpkZNyeYE4AJgvoj09PbdBrwpIrNF5N8i8jTwEXBn8sI0DUVVFVx4oevu7Ofoo+HXv05dTMbUhohrUpv+QgE5+XkoOWwlj4qIprVt29xifQ1BYLJR1YW42QIeBGaLyD3AHOAQ4C1gO/A+cLSqPpTkWE0D8I9/wMsv+5c3awb/939u8Jwx2aCgAAYO9C8Pai6uT2rsIKDOfcCBQAHwCdBbVW9S1YtV9feq+m6yAzX138cfw5gxwXXuugs6dUpJOMYkzOmn+5fNmgUV4Vcxzzpx90ZT1TWqeiFwBjBKREpEZP/khWYakm3bYPhwN9mmn0GDXBObMdnm5JOhUaPY42nKy+GFF1IcUBrU1BtNROQiEZkiIpNF5GxVfR3oCUwHXhOR20QkPyXRmnrr1lth4UL/8rZt4V//osH03DH1S6tW0KPHd77lDaEpraYrmz8BvwFWAuuB8SJynaruUNW/At2BTrhea6cmNVJTby1YAH/4Q3Cdf/4TiotTE48xyXDUUf4Tq8yYAdt3mXmyfqkp2ZwI9FDVa1T1KqAXcF51oap+o6o/By7C9VIzJpSKChg2DHbs8K8zfDj87Gepi8mYZDjyyLW+V+br18Orr6Y2nlSrKdnkAE0ifi4AdpkvTVVfAn6SwLhMAzFmDHz6qX/5nnvC3/+euniMSZY2bSrp08e/fNq01MWSDjUlm0dxTWSTReQx4F3cFDW7UNXQq2l794QuFZF5IrJeRL4VkfkiMjxG3R4icquIvCkiy0VkjYh87q0c2insub1jXiAi60SkpDbPN3Xz0ks1J5KJE6Fly5SEY0zSBV2hP/WUG2dWX9U0zuZmYASwDlgBnOl1g64zb1aCKcA9wESgA9AemIxbhO2PUU+5BbgW+BvQSVXbAcOBQcA7YRKOiHQRkRdwK4+2rtMLMbXy3XdujZogV18N/funJBxjUiIo2axYEdxJJtvFM85muqr+yrtvk8gOekNwS0w/5s2xttXb7gKeBsaIyCFRz7lPVR9V1R1ebG8CtwKtcPeN4vWytx1f51dhauVXv4Kvv/Yv339/uM3uApp6Zp99oHt3//L63JQWtFLnoSISekUfESkSkR/FUfU07/H5GGWzAAEiJ86+c8sAACAASURBVJj/DbE7IXzlPYZpbDlaVW/DzYBgUmzqVDeJpp9GjVx5vnWoN/VQ0NXNtGk7V6Stb4KubDoBz/nM/ByTiByIu2IIWgG0WnVH1jUxylZ5j32rd3jLUq+OUbd6gu6SOMNEVb+quZZJhlWr3NIBQW64AXr1Sk08xqRaULL5/HP46KPUxZJKvklBVR8Tka7AEhGZgvswXwysBbbgrjzycUljf2AgrllqmKp+GMe5q1cqiTV6onoW6c5+TxaR1rhmuN/glq+uxxeg9YOqWwxt3Tr/Or17w/XXpy4mY1Kte3fo0gWWLIld/uST9XNG88ArEFX9o4i8gftAfzCg/mbcfZZeqhrQkfUHZgHn4NbBiZ7E8wTvsVBEJHp5ahFZC7QByoDf4zoZJJ2IXAJcAlBcXExJSUmtjlNWVlbr52azmTPb88wz/jMc5ebu4Mor3+b118tDH7uhvqfJZO9pYkW+n7167cOSJXvGrPfww5vo2/ftFEaWIqoa14a7J3Iy7kb8b3EJ6HxgANA03uNEHK8RMBvYAYwCWnjbSNyMBQpsDnh+IS4pfQZ8DBxQixg6eecpCfvcnj17am3NmTOn1s/NVkuWqDZrpuqub2Jvd91V++M3xPc02ew9TazI9/P114P/FpYuTVuYdQYs1BifmfHcW6lOShuAhK28oKo7RGSQl1zOBcYAm3BLGPTzEsjKgOdvxt1TGgK8B0wRkYO9F2syyI4dcP75UFbmX2fAALjyytTFZEw6HX44tG/v7mHG8tRT9W/NprhnfU4GVd2mqn9S1R6q2lZVO6vqL3BXO+AGkdZ0jA+AZbh52rokMVxTS3/9K8yd61/esiX8+9+Qk9bfRmNSJycHTg2YTbI+TsyZqX/eh3mPTwCIyG4icpOINPGpX93I3ybpkZlQFi2C3/0uuM4//uGmpTGmIQla42buXCgt9S/PRmlNNiKySEQOiNonwGXAImCqt3s34Ebc0gbRx9gL2A+XcD6MKmsvIrlJCN3EobISzjvPPfoZMsTVMaahOfZY/6mYVN1M0PVJuq9sDgLuFJF2AN7jA8A+wFBVjR50+ZCIHCEiOd68aj2AabjJQq/17uPgHesM4BtgQSpeiNnVuHHw/vv+5cXFMGGCrVFjGqbcXLcgoJ/6NptA3MlGRFol4fx34+ZD+1REVgKvARuA7qr6SUS95biecPNxXbBXAd/iulsvA/rprnO2rfWOtTT6pCIyVkRWsTMRHSEiq7ytHvZwT70334Tbbw+u88AD0K5dcB1j6rOgAZ4vvQQbN6YulmSLuzca8Crw40SeXN0aOfHU24EblzMrxLFLcHOmxSq7Gbg53mOZcDZvdmvQBM1g+8tfwimnpC4mYzLRCSdAXl7s5dArK2HWLDj77NTHlQxhmtH2EZHPvGWgD6i5ummorrsOFi/2L+/UCe68M2XhGJOxCgth4ED/8vrUlBYm2XyKm4dsKfBPb92Zq0TEeoCZ7z33nLsP40cEHn4YmjdPXUzGZLKgprRnn4191ZONwiSbQ1V1g6r+U1WPwk010xZ4TUSmi8jpAV2TTQOwfj1ceGFwnVGj4KijUhOPMdnglFPcTOexlJXBiy+mNp5kiTvZaNRKnKq6GDfD81vASbgxMV+JyASvl5hpYK64Alb6zvkA3brBzXanzJgfaN0a+vXzL68vTWlheqM94z3+yFue+UtcsjkR16usF26usZdw3ZlvTHi0JmNNmeI2P02auDVq8kKvkGRM/RfUlDZjBmyvBytvhWpGE5H5wEfANcCbuCWZd1e3iuc76lbafEJV++FW4jQNwIoVcPnlwXXGjYODD05NPMZkm9NO8y9bty54uqdsESbZtMEtJXAxUKyqP1fVWV635B8QkSvYuSaNqcdU4aKL4Ntv/ev06eN6qBljYuvY0U3O6ac+zJUWJtm8par9VPUhVd1UQ93vcFPOmHru/vtdDzQ/BQWu91njMCO6jGmAgprSnnwy+5eLDpNs+opIgYj8YGV4EdllNU1Vnayq0+scncloixfDtdcG17njDujaNTXxGJPNgpLN11/DwoWpiyUZwiSbEbiVMaNX4pwoInOSNJ2NyVDbt7tZAsoDFtU84QS49NLUxWRMNtt3X9dj00+2N6WFSTZnAaNUda+o/f2BEuCORAVlMt+f/+zmP/PTqhU89JBNsmlMGDU1pWWzMMmmparuMsmIqu5Q1XGAja1pIN57D26soWP7hAnupqcxJn5Ba9x88gl8/HHqYkm0MMmmWQ3lNgFJA7BlCwwbBtu2+dc5+2w466zUxWRMffGTn7i5A/1k89VNmGTztYjEbIEXkYuBFYkJyWSysWPd6pt+OnaEe+5JXTzG1CciwVc32TybQJgOqb8HXhSRq3HrwKzHTeHfC+gKDEh8eCaTzJ0Lf/lLcJ2HHnLTbxhjaudnP/OfFf3tt+Grr2Cv6DvnWSDM3GivAT/FLUh2HnA1MAzYCBynqq8nJUKTETZtgvPPD+7rP2KE64FmjKm9Pn2gqMi//KmnUhdLIoVaFlpV56pqH6Al0BnXaeAILxGZemzkSFi6y5qnO3Xt6nqoGWPqplGj4OlrsrUpLVSyqaaqZaq6TFXLqveJyC2JC8tkkmeecUs4+8nJcbMEFBamLiZj6rOgLtBz58KaNamLJVFCTyLiLR/QBWgaVXQecEMigjKZY80at4RzkDFj3KW/MSYx+veHFi1g48Zdy6qq3EzQF12U+rjqIu5kIyIdgOm4DgEKRA7Xy/JZe0wsqnDZZbB6tX+dgw+uecyNMSac3Fy3qNp//hO7/Mknsy/ZhGlGG49bKK078Amwr7cdDTwDjE54dCatJk8Obh/OzXVr1OTmpi4mYxqKoKa0F15wnXaySZhk0x34taouAraq6hfe9howFLB+SPWAKsybB0NPLueXw7YgVNGULeSz6yRot94aPJeTMab2Bg70X2ywshJmzUptPHUVJtlUqmqV9+8mIvL9c1W1Etg9oZGZlNu2DS46p4Kz+5fSa9Y4ltCFSnJZQhfGMo52lJJHBQBHHQXXXJPmgI2px5o1g+OP9y/PttkEwiSbKhE5wPv3F8AfRKTQ224AmiQ+PJMqqnDp8Aq+mbGAReWdGc14OrKSxuygIysZw3iW0pleLKRZowomTnRdNI0xyRPUlDZzpps+KluESTbPAG+KyH7An4GRuAGdG4GbvH0mS82fD3Oe3sTU8hMpjNFkBlBIObMZyG6NNrF2bYoDNKYBGjTI/0tdWRm89FJq46mLMDMI3KKqu6nqZ95sAUfglhX4B3Ciqv4zWUGa5JtwRzmXV9zhm2iqFVLOldvvZMIdwfWMMXXXpg0cc4x/eTY1pcWdbERkvLcVAajqO6r6G1X9tao+n7wQTSrMmJnDuVWT4qp7XtUkZsys1XhgY0xIQU1p06e7hQyzQZhPjJFAKW61TlPPbKjIpYjSuOoWUcqGCuvvbEwqBE1ds3YtvJ4ls1KGSTbvqepfVNXaT+qhlvmVlBIw+1+EUopomV+Z5IiMMQB77AGHHupfni1NaWGSzTsi8iO/QhF5JwHxmDQZeHwVDzMsrrqTc4Yx+OSqmisaYxIiaI2bJ58Mno09U4RJNguBqSJyp4hcIiLnRG64tW1Mltq3ewF3ci2bKQisV0Yh9+aNZMS1wfWMMYkTdN/mq6/gnSz4qh9mIs77vMcDfcqzILcaP2+/DZtozkBmM5uBMXullVHIkPxn6T+4Ob17pyFIYxqo/faDAw+Ejz6KXf7kk9CzZ2pjCivMlc3H7JwPLXrbDzdfmslCGza4uZa2kM9CetGZpdzGaFbQkW00ZgUdGZ8zmm4FS9j91F7c/3A+IjUf1xiTONm+XHSYZHNPxHxo0dti4NZkBWmS6+mn3VxL4BLOGoq4hbHswxc0ZSvdC77g4zPG8nhJEQ/9N58mNleEMSkX1JT28cfwSYZ/3Q8zqPPeGqq8X8dYTJo88cSu+yooYCt5nHpaDus25/HvRwus6cyYNOrRA/be278803ulJXJkXnwjAk1G2bgRZs/2Lz/jjNTFYozxJxJ8dVNvko2IfBa0AQfUeJBdjykicqmIzBOR9SLyrYjMF5HhMer2EJFbReRNEVkuImtE5HMRuU9EOtXi3INF5HURKRWR1SLylIgcFPY42e6ZZ2Dr1thlTZu6uZmMMZkhKNksWADLl6culrDCXNm0AeZFbYuAbUAz4PEwJxYRAaYA9wATgQ5Ae2AyMFFE/hj1lFuAa4G/AZ1UtR0wHBiEGwPUKcS5R+BWHZ3mnbMrUAHME5FeYV5HtovVhFbthBPc0rTGmMxw5JHQrp1/+VNPpS6WsMIkm1dUdVjUdrqqHgRcCYTt6T0EOBN4TFUnqOpWb7sLeBoYIyKHRD3nPlV9VFV3AKjqm7iOCa2AuBZJFZE9gTuB51X1DlWtUtVNwMVAJfCglwjrvbIyePZZ/3JrQjMmszRqBKee6l+eyU1pYToI+Ha8U9VpuNU6w6ie8SfWJJ6zAMElgGq/AW6LUfcr77FlnOf9BZAH/Ddyp6qW4ZJcd+CoOI+V1Z55xn89jCZNYPDg1MZjjKlZUFPaq6+Ssct/JKSDgIh0BQL6ScRU7D2uiVG2ynvsW71DVT9S1dUx6lb3kSqJ87wDvMe3Y5S9HVWnXgtqQjv+eGgZb/o2xqTMgAHQvHnssh073FCGTBT3DAIiEusKRHBNWN0I3xuteorh4hhl1a2SnQPiaY1rhvsN8Ffv6ioe+3uPK2KUVe/znQOuvti8OXgN86Fhr1ONMSnRtCmcfDJMmRK7/Mkn4cILUxtTPMJMV9MHeC9q3w5gCfBv4IGQ554FnAOcDDwUVXaC91goIqL6w2nmRGQtrsNCGfB7XCeDeO3mPW6OUVa9L+Y8byJyCXAJQHFxMSUlJSFOu1NZWVmtn5soc+a0o6Iidue7xo2raN36DUpKsmShDDLjPa1v7D1NrES+n/vt1w6I/fc7e3YVs2a9TkHBjoScK2FUNa4NeDfeunEerxEwG5ewRgEtvG0ksBI319rmgOcX4pLSZ7ipdA6I87yV3rGbxigb6JU9V9NxevbsqbU1Z86cWj83UYYOVXVzxe66nXhiuqMLLxPe0/rG3tPESuT7uXGjatOm/n/Djz2WsFOFBizUGJ+ZYe7ZJPSmuboeZYOA64FzcVdI7+Oa5Pp51VYGPH+zqj6H69W2HzAlzl5k33mPhTHKqvd9G8dxslZ5Ocyc6V9uTWjGZLbmzeGnP/Uvz8ReaWGSTSsRudxrSvqeiPxKREIP6ARQ1W2q+idV7aGqbVW1s6r+Ane1A/BuHMf4AFiG60XWJY7TVs8gtHuMsup9n8ZxnKw1a5ZLOLE0bhzctdIYkxmCJuYMGqydLmGSza+BcUCnqP3NgRdFJJFXPod5j08AiMhuInKTiPhNAVn90dkmjmO/5D3GmpC7Z1SdeimoF9qAAdC6depiMcbUzqBBkOPzCb5pE7z8cmrjqUmYZHMccKSqXh+5U1VvBU6iFrM+i8ii6KsirynsMtzsBFO93bsBNxIjQYjIXrhmtHLgw6iy9iKSG/WUh4AtwM+j6jbDNet9ALwW9rVki4oK963HjzWhGZMd2raFo4/2L8+0ZQfCJJsqVf0sVoGqvo+7wgnrIOBOEWkH4D0+AOwDDFXV6O5QD4nIESKS482r1gM35UwT4FpV/b6HmYicAXwDLIiKdTmuE8LxIjLSO1Yz4F9AU+AXqlpv1zyePdt1e46lUSM47bTYZcaYzBPUlDZ9uht3kynC3rPJi1UgIvns7FIcxt24uck+FZGVuCuKDUB3VY1cnWE5rov0fOBB3KDPb3Ej/pcB/VT1Pn5orXespdEnVdUJuBkMzvCOtQQoAA5T1YW1eB1Z4/GAGez694c28TREGmMyQtCXwzVr4I03UhdLTcKMs3kOmCkiY4C3VbVKRHKAQ4A/euWhqOpVcdbbgRuXEzAMcZfnlOAzXsYrn46bjLPBqKgIHl1sTWjGZJc994Tevd2Mz7FMmwZHZcjkW2GubMYArYG3gEoR+RY3ZmUe7sb8mMSHZxLp+efd5JuxWBOaMdmppjVufjgkPn3CTMT5La6X2OXAk7jZBJ4ERgBHqOp3AU83GSCoCe3YY4OnLjfGZKagZLNsGbwXPe9LmoRpRkNVK4H7vc1kkS1bYMYM/3JrQjMmO+2/PxxwAHz8cezyadPcktLpFmalzj0TPajTpM4LL7i+97Hk5AR/OzLGZLZsWC46Uwd1mgQLakI75hgoKkpdLMaYxApKNh9+CJ/FHLSSWmkd1GlSY+tW1+fejzWhGZPdevZ0PdP8ZMLVTboHdZoUePFF2LgxdpmINaEZk+1q+jvOtmSTjEGdJgWCmtCOPhrat09dLMaY5AiaTWDePFgRa7nIFAqTbKoHdfb2BnPiTfXSCzc4MvSgTpN8lZXWhGZMQ9C3r5svzc9TT6UullhsUGc999JL8J3PCCiR4G9Dxpjs0agRDB7sX57upjQb1FnPBTWh9e0LHTqkLhZjTHIFfXksKYF161IWyi4SNqhTRMRbEtRkiG3bgi+dzzgjdbEYY5JvwABo1iz2tFQ7drjlRc4/P/VxQbhmtJq8ncBjmQR4+WX4NmCB6yFDUheLMSb58vLgpJP8y9O5xk2oKxsR+QnwC9zyy02jivdJVFAmMYKa0I48EnaPtTC2MSarnX46PPZY7LLnn3frWRUWpjYmCDddzQnAG8ARwLFAvrd1xg34/DwJ8Zla2rYt+IagNaEZUz+deCLkRq9P7NmyxS2gmA5hmtFuBI5X1d7AYlU9ytv2Bc4F5iQlQlMrJSWwfr1/uSUbY+qnFi3gpz/1L09XU1qYZJOvqq97/5bIAlX9L9AzYVGZOgtqQuvTB/bYI3WxGGNSK2g2gZkz3fi7VAuTbCJ7mm0XkeLqH0SkJfCjhEVl6mT7dmtCM6YhGzzYzeYey4YNMCcN7VBhks0KEblRRJoCc4HnReQKEbkCeBn4NCkRmtBeeQXWrvUvt2RjTP3Wrl3wctDpaEoLk2z+CuwBFAF/ALYA//C2VsBVCY/O1EpQE9phh8Fee6UuFmNMegQ1pU2f7sbdpFKYGQReVtWLVXW5qq4D+gAHAYcA+6vqh8kK0sRvx47gby12VWNMw3Daaf5lq1fDm2+mLhaow6BOVa1S1Y9V9T1vZgGTAV59Fdas8S+3ZGNMw7D33m6dGz+pnistkTMImAwQ1ITWqxd06pSyUIwxaVbTGjepnGDMkk09UlMTmi0nYEzDEjQx59Kl8P77qYvFkk098tprri3WjzWhGdOwHHAA/ChgUEoqm9Is2dQjTzzhX3bIIdClS+piMcZkhkxZLjrM3GjH+Ow/TETuE5HOiQvLhFVVBVOn+pdbE5oxDVNQU9oHH8DixamJI+w4m1hWARuAR+oejqmt11+HlSv9y60JzZiGqVev4OmpUnV1EybZSKydqrpMVUcDzRITkqmNoCa0gw+Grl1TF4sxJnOIBI+5SdVsAoHr2YjIIGCQ9+MeIvLPWNVwMws0SXBsJk5VVcHJxprQjGnYTj8d7r47dtlbb8E330DHjsmNoabF07oCJ3r/bh7x70jbgKXAxQmMy4Tw5pvul8WPNaEZ07AddRS0aQPr1sUunz4dRoxIbgyBzWiq+ldV3VNV9wQ+rv531NZFVQdELD9gUizoqqZ7d9hvv9TFYozJPI0bu5mg/aSiKS3MPZuAVj+TLtaEZoyJR1AX6JoWW0yEMBNxLoveJyJHi8iVIvKTxIZl4jVvHnz9tX+5NaEZY8Ct3llYGLts+3Z45pnknj/MOJtRIrJVRG7xfr4AtxT034H5InJSckI0QYKuarp1g/33T10sxpjMlZcHJwV8Sie7C3SYZrTTgaGqeoP3843A60Ax8HPg9wmOzdRA1ZrQjDHxC2pKe+452Lw5eecOk2yaquoMABE5HNgbuElV16jqNKAgzInFuVRE5onIehH5VkTmi8jwGHX7iMi/RGSJiKwTkQ0i8qaIXCIioabcEZEWInKbiHzqnXediMwSkT5hjpMJ5s+Hr77yL7cmNGNMpJNOgiY+g1QqKlzCSZYwH9SR3aSHActV9eWIfVXxHkhEBJgC3ANMBDoA7YHJwEQR+WNE3f7AG0BnYICqtgF2x11V3U+ImQtEpDUwD/gFcBnQBte9ez3warY1BQZd1Rx4oNuMMaZay5Zw3HH+5clsSguTbD4RkXtEZBTuw/rB6gIROQU33iZeQ4AzgcdUdYKqbvW2u4CngTEickhEjNuBs1R1KYCqlqnqKGAB8HMR6RfneW8G9gduUNU56nwL/BJYBzwgIvkhXkfaqAavXWNNaMaYWIKa0p55BiqTtBRmmGQzEjgQuAV4FfgzgIhMAmYAAd+zd1Hdjfr5GGWzcLMSVA8SXQqM8paijvaa9xhvE1jM86rqFlxnhw5AQG/0zPH227Bsl/6BO1kTmjEmllNPdVPYxPLdd64bdDKE6fq8QlX7qWq+qp6gqhVe0XCgiar+OcR5i73HWAsYr/Ie+3rn/cK74okl13v0GRdb+/NmuqCrmv33h4MOSl0sxpjsUVQEfQM+5ZLVlFar9WxEpKOIdK/+UVV3hDxEqfdYHKOsnfcYz5IFRwBbgekpPm9axdOE5vfNxRhjgprSnnrKDRZPtJrmRvsBETkbd99jH9yVwO7AIyLyDXCdatwrWs8CzgFOBh6KKjvBeywUEfE7pnefpgfu/suqWHV8zvtL77z/iDhWLnCs96Pv7NUicglwCUBxcTEltbzeLCsrq/VzAT77rBlLl/byLd977wWUlCSxD2MGqut7anZl72liZdL72aFDHnB4zLJVq+Dee9+hW7eNiT2pqsa14cbSbAUeB24CPvH27wlMBX4b4liNgNnADmAU0MLbRgIrAQU2Bzy/FbAYeAbICXHeDsBXuGa3U3DNcB1wPdpWe+edGc+xevbsqbU1Z86cGusEGTNG1V3f7Lrtt59qVVWdDp+V6vqeml3Ze5pYmfZ+9ujh/zkyalTtjwss1BifmWGa0UYDx6nqUFW9CSj3ktVy3H2buG9Jq2t2GwRcD5wLLAHeB7oB1T3LYi4FJiIFuCTzDW6QadwXfKq6EugN/Ae3GNw3wEvAF8AVQefNFDU1oZ1xhjWhGWNqFtSUNm2a+6xJpDDJJldV58YqUNXNhGySU9VtqvonVe2hqm1VtbOq/gJ3tQPwbvRzRKQQ1xS2DThJd3ZSCHPe1ap6laru6533QFW9EXe1FPO8meT99+GLL/zLrcuzMSYeQctFL1niloxOpDDJJk9E2sUqEJEi3Ho3iXCY9/iDrtQi0hx4jp2Jpszb30ZEfpSg8+4AUrRIau0EXdV07Qo/sSlRjTFxOPBA2Hdf//JE90oLk2yeAuaKyDAR2QfIEZFiETkBmAk8FubEIrJIRA6I2ie4kf2LcPeBqve3xI2N2QgMUtXyiKcNws0kEH389t6N/8h9vUXkDe88kfvbAkOBB1Q1YBmy9LImNGNMoojsenWTTzlN2YJQxZ/GbeGCM8uZPz8xTWphks31wJfA/wGfAd1x9zxm4W6ujw157oOAO6uvlrzHB3A93Yaq6nZvfyvgReAnwHu42QVuqt6Isc6OiJzhxbYgqqgQNwD0D9WJSEQOxN0D+gDXWSFjffABfP65f7k1oRljwqi+b5NHBe0oZSzjWEIXKsllsXbhwKnjOKt/KRedU8G2MHPExBD3fRZ1o+wHishAYADQFlgLvKCqsWYCqMnduAGUn4rIVtxVy0ygu6pGDro8Faju5/tbn2O9EvXzWmADbvaBSEtxPc+GAleISBkuUT4C3K2qdXw7kyvoqqZLF+jRI3WxGGOyX69e0Cqvgm5bFvAsJ1LIzkajjqzkN1XjuWLz3Zw+fTaXDu/Fg//Jr3XrSWCyEZHqq5XFqvofAFWdjeu2XCeqelWc9SbiJusMc+wSdt7wj9y/DDeJaNaxJjRjTKItXAj52zftkmgiFVLOtIqBdHt6KQsW5HPoobU7V03NaCNw85TZx1iaffghfPqpf7k1oRljwppwRzlX77jDN9FUK6ScERV3MuGO4HpBako2q1R1nKpOrvUZTEIEXdV06gQ9e6YsFGNMPTFjZg7n6aS46p5bNYkZM2s1wxlQc7KJuw+CiPjdTzEJYE1oxphE21CRS9H3U0YGK6KUDRW5NVf0UVMHgVwR2ZP4mtHOBW6rdSTG10cfwccf+5dbE5oxpjZa5ldSWl5ExzgmTimliJb5lUBerc5VU7I5ENfd2aRR0FXNXntB796pi8UYU38MPrmKyVOHcV3V+BrrTs4ZxuCTaz8ddE3JZjVwXxzHEbzZkE3iBS3/bE1oxpjaGnFtAWfPupbLN98d2EmgjELuzRvJY9cW1PpcNSWbVao6Lp4DeeNvTIJ98gksWuRfbk1oxpjaOvRQ6DeoOadPn820ioExE04ZhQzJf5b+g5vXqRUlYR0EVDX24gimToKa0PbYg1r3eTfGGBG4/+F8dj+1F90KlzI+ZzQr6Mg2GrOCjozPGU23giXsfmov7n+49gM6If4OAluiRvWbFKmpCS2n9j0RjTGGJk3gwf/ks2BBPvf+ZSzdZ93EhopcWuZXMvjkKh4fVZCQ+8I1JZtmuKlgXgEurPvpTBiffQb/+59/uTWhGWMSQcS1khz6WOQ9mdr1OvMTmGxUtVNCz2ZCCWpC2313ONwaLo0xWcIaYTJYUBPakCHWhGaMyR72cZWhFi+G997zLz8j7kW4jTEm/SzZZKigJrQOHeDII1MXizHG1JUlmwxlTWjGmPrEPrIy0JIl8M47/uXWhGaMyTaWbDJQUBNacTH07Zu6WIwxJhEs2WSgmprQGjVKXSzGGJMIlmwyzNKlbqlWP9aEZozJRpZsMszUqf5lRUVw9NGpi8UYYxLFkk2GCbpfc/rp1oRmjMlOlmwyyLJlMH++f7k1oRljspUlmwwS1ITWti0cc0zqCGPmogAADfNJREFUYjHGmESyZJNBgprQfvYzaFzTHN3GGJOhLNlkiOXL4a23/MttOQFjTDazZJMhgprQ2rSBY49NWSjGGJNwlmwyRFAT2mmnudX0jDEmW1myyQBffw1vvOFfbk1oxphsZ8kmA0yb5l/WqhX075+6WIwxJhks2WQAa0IzxtR3lmzS7Jtv4PXX/cutCc0YUx9YskmzadNANXbZbrvBgAGpjccYY5LBkk2aBTWhnXoq5OamLhZjjEkWSzZptGoVzJ3rX25NaMaY+sKSTRoFNaG1aAHHHZfaeIwxJlks2aRRTU1oTZumLhZjjEkmSzZpsno1vPqqf7k1oRlj6pO0JhtxLhWReSKyXkS+FZH5IjI8Rt0+IvIvEVkiIutEZIOIvCkil4hIqNchIruJyM0iskhESkVkhYjMEZFBiXt1wZ58EqqqYpc1bw4//WmqIjHGmORLW7IREQGmAPcAE4EOQHtgMjBRRP4YUbc/8AbQGRigqm2A3YHXgfuBR0KcN8973khgFFAM7At8CMwQkdF1fW3xCGpCGzwY8vJSEYUxxqRGOq9shgBnAo+p6gRV3eptdwFPA2NE5BCvbg6wHThLVZcCqGqZqo4CFgA/F5F+Ic57IHC/qs5Wpxz4NVAK/F5Ekjpm/7vvmlBS4l9uTWjGmPomncnmNO/x+RhlswABLvZ+XgqMUtV1Meq+5j32ifO8e3iPiyN3qup24EugGdA6zmPVyty5bX2b0Jo1g+OPT+bZjTEm9dK59mOx97gmRtkq77EvgKp+Adzlc5zqYY+xElEsH3iP+0fuFJFcoAtQqqqr4zxWrbzySjvfskGDID8/mWc3xpjUS+eVTan3WByjrPrTuHMcxzkC2ApMj+ekqjoLmARcLCKni0gTEWmNu3dUCFwaz3Fqa+1aePfdVr7l1oRmjKmP0nllMws4BzgZeCiq7ATvsVBERDX20EfvPk0P4AZVXRWrjo8LgU+Ah4EmuKuj/+E6H7zp9yQRuQS4BKC4uJiSoBsvPmbO7EBV1Y9iluXl7aCg4HVKSnza2IyvsrKyWv1/GH/2niZWg38/VTUtG9AImA3swPUKa+FtI4GVgAKbA57fCnff5RkgJ8R52+F6tq0ETsQlmtbATcAW4FfxHKdnz55aG8cfr+rmDdh1O+usWh3SqOqcOXPSHUK9Y+9pYjWU9xNYqDE+M9PWjKaqO4BBwPXAucAS4H2gG1Dds2xlrOeKSAEuyXwDDFXVMJcCf8F1JrhGVZ9V1UpVXa+qNwEvAn8TkaNq8ZJqtG4dvPSSf/kZZyTjrMYYk35pHdSpqttU9U+q2kNV26pqZ9X/b+/+Y62u6ziOP1/8PlwQTWUGzYJWljJcibRlJtjaFAIUW/5glqYzsbUp2LRVDJrahj9yy4GmLoaaaZkiQ82ZkG41vJo/8kc4vWhF4i0sFO9VED798f0cODueyz33cr73c76H12P77sv5fL7fz/dz3pxz3vf76/MN3ybb2wF4unodSW1kh+B2ADNCCN193OxJcf5ojbq1ZFfB5XJz56pVsHNn7bqRI2HGjDy2amaWXrMOV/OFOP9tZaGk0cDv2ZNotsXygyXVPhHyYaPivNZ5oPIe0pi+dXfvQoD16+GKH3YxnPcQuxjOe5To2r3MzJlZwjEza0Wph6t5XtJnq8oEXAg8D9xTUT6G7J6ct4FZIbsRs2wW2UgC1e0fFi9prrQ+zmsdKju+apl9tmMHnHdWN6ef2MkFm5fQwUS2M4wOJrKIJRxKJyPo5pRTem/LzKyoUu/ZHAVcJ+lQgDi/Bfgk2bmYD2L5QWTnU44GniEbXWBxeWLPDaK7Sfo62Tmd9qqqy4Au4GeSjovjs42QtAA4FXiSPgx/szchwHe+2c2/7m/nha4JXM5SxvEGQ9jJON7gcpaykQlM4Ukeure7x8cNmJkVXcpLnwFuILtxc4Ok98n2WtYAk0MIlTd7zgGmxH//oIe2/lj1+j/AVrLRB3YLIbRLmhLbuZPsCrjBcbnFwDUhhO39fUOVnngC1q5+h+e7TqaNrprLtNHFQ5zEpAc30t5eYurURmzZzKy5JE02IYTv1bncCrLBOvvS9jqyy6Nr1b0EfGhk6UZbfm0XF3Vf22OiKWuji/nd17H82kVMvcsnbsys9aQ+jNbS7l8ziHm7bqtr2Xm7buP+Nf7vMLPW5F+3HG3tHsbY3aPy7N1YOtnaXX0tg5lZa3CyydGY0nY6GVvXsp2MZUypIaeKzMyajpNNjmbP3MXtg86ua9k7Bp3N7JkeE83MWpOTTY7mLxzJ8tJC3mXvJ/230cayEQuYv9AXB5hZa3KyydHUqTB91mjmlh7qMeFso43TSg9y4uzRHHvsAHfQzGyAONnkSIKbVpYYP2cKk9o2snTQZWxiHDsYwibGsXTQZUwa2cH4OVO4aWUJKXWPzczykfqmzpY3dCjc+qsS7e0lll2ziMkPLGZr11DGjNzB7Jm7+M2lI71HY2Ytz8lmAEjZIbWpd2eH0tatW8e0adPSdsrMbAD5MJqZmeXOycbMzHLnZGNmZrlzsjEzs9wp+CEq/SLp38Dr/Vz9ELJHIFjjOKaN55g21v4Sz4+HEA6tLnSySUDSkyGEKb0vafVyTBvPMW2s/T2ePoxmZma5c7IxM7PcOdmk8YvUHWhBjmnjOaaNtV/H0+dszMwsd96zMTOz3DnZmJlZ7pxs+kDSUZLuk/SmpE5Jf5I0p49tlCRdKakjttEh6SpJpb2sM1jS9yV1SVqxz2+kiQx0TCVNkHSFpGclbZG0VdJLcf0DGvfO0kgQz/GSFkh6WNLGuN3XJa2SdFzj3lk6qb73FeuOkfQPSUHStH6/kdRCCJ7qmIApwDbg18BoskS9AAjA/DrbGAY8BmwCjo5lR8fXjwHDaqzzOeApoCNua0XqWBQ1psBgYBfZzbhfAgQMBeYB7wEvAgemjktR4hnrLo3t/xRoi2WHA+uAncApqeNStJjWWH9l3F4ApqWOSb9jmboDRZjiB+w5YAswqqruYaAb+Fgd7ZS/mGdWlZ8VyxdWlY8GOoHzgemtlGxSxJTskRoBmFOjnatj3ZLUsSlKPCuWf7pGO0fG5R9PHZuixbRqmTlkow486GSzH0zACT390APnxrpFdbTTQfYX9PCq8hGx/NWq8qHAYfHf01os2Qx4TOOPxxXVPxyxbnbc5sOpY1OUeMby8cCna7QzKm7zudSxKVpMK+oPAd4EvgGsKHqy8Tmb+nwlzp+qUfdU1TI1SZoITABeCCG8X1kXQigfwpkoaUJF+Y4QwuZ+97q5DXhMQwi7Qgg/CiFsq9HcsDjfUmf/m02qz+imEMLLNZorP392Xe9db1pJYlphObA2hHB3n3rdpJxs6vOZON9Uo65cdsQ+tNGXdlpFs8X0i3F+Vx3LNqOmiKekkZJOBm4F1gKLetlmM0sWU0lnkZ1X/G4v7ReGk019Dozzd2vUlcsO2oc2+tJOq2iamEo6BPgW8AiwqpdtNqvk8ZR0L/A2sIYslmeEEP7XyzabWZKYSvoo8HPgghBCUfe0P8TJxvZrkgTcRPYjeXaIB8ut70IIpwIlsisoDwc2SDo1ba8K6RZgdQhhdeqONNKQ1B0oiPJfZ2016spl/92HNvrSTqtolpheR3YI7fiCnx9riniGEHYAz0qaS3Y+4jZJR4YQ/t7LtpvRgMdU0vnAZGBSnX0sDO/Z1OdvcT6+Rl25bMM+tNGXdlpF8phKuprsSp9pIYRXetlWs0sez0ohhC7gUbIf06/2tnyTShHTuWRX8m2QtLk8AafH+t/FssL9TjjZ1OcPcX5MjbpjqpapKYTQAbwGHCVpeGWdpBFk9yV0hBA27ltXCyNpTCVdT5ZovhxC2BDLhkkq6sOtksRT0sWSPtVDk11xfvDeu960BjymIYQZIYSDQgiHVU7suXBlbiwr3IVETjb1eRz4KzBL0qiqujPJrpX/ZbkgDi9T6y+ZZcBwsr9eKs2N5csb1uPmlySmyiwju6/mhBDCqxXV44D2fryXZpDqM3oxcFp1I5KGkN0bBvBEfW+h6fh730ipb/QpykQ2bMW7wJ3sGbbiErIbrS6qWnZ1LL+kqrzWsBWT4+vH2cuwFbTYTZ0pYko2PM3NsZ1bgcVV0/XZVyJ9bIoQz1j3GvAOcEa5jixp3x7bvzN1XIoW0x76sYKC39SZvANFmshO2q0iG0KmE/gztYc+uaH8BaxRVwKuAjbGNjbG16UetnkzsBl4K37YuuPrzcABqWNSpJgCn2DPGFM9TqljUpR4xmU/TzbUz1+AN8huit1Cdo/NucCg1DEpWkyr1muP3/Xu+Pl8K76+NHVc+jr54WlmZpY7n7MxM7PcOdmYmVnunGzMzCx3TjZmZpY7JxszM8udk42ZmeXOycbMzHLnZGNWMJIulPSipCDpnNT9MauHk41ZwYQQbgRmpO6HWV842ZiZWe6cbMzMLHdONmYtQNJ0SS9L+kDSM5Lmpe6TWSU/FtqsNawH/gksCSHckbozZtW8Z2NWcJJKZEPgr3SisWblZGNWbMOB+4DBIYQVifti1iMnG7NiuxLYDkyX9LXUnTHriZONWbHdDMwme+zwjZIOSNwfs5qcbMyKbUPIHrd7HvAR4JrE/TGrycnGrAWEEF4BfgycL2l66v6YVXOyMSsYSacDD8SXP5F0taQjgHMAAfdIeiRV/8xqUbYHbmZmlh/v2ZiZWe6cbMzMLHdONmZmljsnGzMzy52TjZmZ5c7JxszMcudkY2ZmuXOyMTOz3DnZmJlZ7v4P1tePFrW561AAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import rcParams\n",
    "\n",
    "\n",
    "plt.tight_layout()\n",
    "\n",
    "plt.rc('font', family='serif')\n",
    "plt.rc('xtick', labelsize='large')\n",
    "plt.rc('ytick', labelsize='large')\n",
    "plt.rcParams.update({'font.size': 16})\n",
    "rcParams.update({'figure.autolayout': True})\n",
    "\n",
    "\n",
    "best_acc = [metrics[k]['best_val_acc'] for k in metrics.keys()]\n",
    "k = list(metrics.keys())\n",
    "\n",
    "plt.figure(figsize=(6,5))\n",
    "plt.plot(k[1:-1],best_acc[1:-1],marker='o',color='blue',linewidth=7.0, markersize=12,mfc='red')\n",
    "plt.title('Test accuracy vs k')\n",
    "plt.xlabel('k')\n",
    "plt.ylabel('Test accuracy (%)')\n",
    "plt.grid(True)\n",
    "# plt.show()\n",
    "plt.savefig('figures/K_ablation/VGG16_cifar10_accuracy_VS_K.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([0.005, 0.01, 0.015, 0.02, 0.025, 0.03, 0.035, 0.04, 0.045, 0.05])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6 (kdd_final)",
   "language": "python",
   "name": "kdd_final"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}