MLP_MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compressing a multiplayer perceptron\n",
    "\n",
    "In this notebook, we compress a multiple layer perceptron with 784-500-300-10 neuron architecure considered in [1]. \n",
    "\n",
    "The notebook contains the following experiments -\n",
    "\n",
    "    1. We begin by training a baseline model (Table 1, row : base).\n",
    "    2. Different magnitude based pruning \n",
    "        a. Table 1, row : random\n",
    "        b. Table 1, row : L2\n",
    "        c. Table 1, row : L1\n",
    "    3. Cluster pruning\n",
    "        - Table 1, row : CUP (manual)\n",
    "    4. Plot of accuracy vs compression for input/output/both features (Fig 6 b) \n",
    " \n",
    "---\n",
    "    \n",
    " [1] Liu, Zhuang, et al. \"Learning efficient convolutional networks through network slimming.\" Proceedings of the IEEE International Conference on Computer Vision. 2017."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import sys; sys.argv=['']; del sys\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import argparse\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "\n",
    "import numpy as np\n",
    "import random\n",
    "import os\n",
    "import time\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = '0'\n",
    "\n",
    "from src.utils import plot_tsne,fancy_dendrogram,save_obj,load_obj\n",
    "from src.model import ANN,load_model\n",
    "from src.prune_model import prune_model\n",
    "from src.cluster_model import cluster_model\n",
    "from src.train_test import train,test,adjust_learning_rate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Train the baseline model (Table 2, row : base)\n",
    "\n",
    "We follow the same training hyperparameters as in [1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 1 [0/60000 (0%)]\tLoss: 2.296816\n",
      "Train Epoch: 1 [12800/60000 (21%)]\tLoss: 0.320204\n",
      "Train Epoch: 1 [25600/60000 (43%)]\tLoss: 0.331736\n",
      "Train Epoch: 1 [38400/60000 (64%)]\tLoss: 0.168032\n",
      "Train Epoch: 1 [51200/60000 (85%)]\tLoss: 0.124862\n",
      "\n",
      "Test set: Average loss: 0.1124, Accuracy: 9624/10000 (96%)\n",
      "\n",
      "Train Epoch: 2 [0/60000 (0%)]\tLoss: 0.015695\n",
      "Train Epoch: 2 [12800/60000 (21%)]\tLoss: 0.123365\n",
      "Train Epoch: 2 [25600/60000 (43%)]\tLoss: 0.062270\n",
      "Train Epoch: 2 [38400/60000 (64%)]\tLoss: 0.062941\n",
      "Train Epoch: 2 [51200/60000 (85%)]\tLoss: 0.083013\n",
      "\n",
      "Test set: Average loss: 0.0925, Accuracy: 9721/10000 (97%)\n",
      "\n",
      "Train Epoch: 3 [0/60000 (0%)]\tLoss: 0.065323\n",
      "Train Epoch: 3 [12800/60000 (21%)]\tLoss: 0.015946\n",
      "Train Epoch: 3 [25600/60000 (43%)]\tLoss: 0.063103\n",
      "Train Epoch: 3 [38400/60000 (64%)]\tLoss: 0.074530\n",
      "Train Epoch: 3 [51200/60000 (85%)]\tLoss: 0.092353\n",
      "\n",
      "Test set: Average loss: 0.0813, Accuracy: 9753/10000 (98%)\n",
      "\n",
      "Train Epoch: 4 [0/60000 (0%)]\tLoss: 0.025250\n",
      "Train Epoch: 4 [12800/60000 (21%)]\tLoss: 0.004034\n",
      "Train Epoch: 4 [25600/60000 (43%)]\tLoss: 0.008444\n",
      "Train Epoch: 4 [38400/60000 (64%)]\tLoss: 0.104451\n",
      "Train Epoch: 4 [51200/60000 (85%)]\tLoss: 0.032885\n",
      "\n",
      "Test set: Average loss: 0.0731, Accuracy: 9774/10000 (98%)\n",
      "\n",
      "Train Epoch: 5 [0/60000 (0%)]\tLoss: 0.041477\n",
      "Train Epoch: 5 [12800/60000 (21%)]\tLoss: 0.019378\n",
      "Train Epoch: 5 [25600/60000 (43%)]\tLoss: 0.103624\n",
      "Train Epoch: 5 [38400/60000 (64%)]\tLoss: 0.063768\n",
      "Train Epoch: 5 [51200/60000 (85%)]\tLoss: 0.013858\n",
      "\n",
      "Test set: Average loss: 0.0703, Accuracy: 9788/10000 (98%)\n",
      "\n",
      "Train Epoch: 6 [0/60000 (0%)]\tLoss: 0.036012\n",
      "Train Epoch: 6 [12800/60000 (21%)]\tLoss: 0.004163\n",
      "Train Epoch: 6 [25600/60000 (43%)]\tLoss: 0.061056\n",
      "Train Epoch: 6 [38400/60000 (64%)]\tLoss: 0.008670\n",
      "Train Epoch: 6 [51200/60000 (85%)]\tLoss: 0.076872\n",
      "\n",
      "Test set: Average loss: 0.0662, Accuracy: 9785/10000 (98%)\n",
      "\n",
      "Train Epoch: 7 [0/60000 (0%)]\tLoss: 0.016910\n",
      "Train Epoch: 7 [12800/60000 (21%)]\tLoss: 0.040723\n",
      "Train Epoch: 7 [25600/60000 (43%)]\tLoss: 0.037688\n",
      "Train Epoch: 7 [38400/60000 (64%)]\tLoss: 0.007459\n",
      "Train Epoch: 7 [51200/60000 (85%)]\tLoss: 0.020675\n",
      "\n",
      "Test set: Average loss: 0.0640, Accuracy: 9803/10000 (98%)\n",
      "\n",
      "Train Epoch: 8 [0/60000 (0%)]\tLoss: 0.003876\n",
      "Train Epoch: 8 [12800/60000 (21%)]\tLoss: 0.025357\n",
      "Train Epoch: 8 [25600/60000 (43%)]\tLoss: 0.023001\n",
      "Train Epoch: 8 [38400/60000 (64%)]\tLoss: 0.016859\n",
      "Train Epoch: 8 [51200/60000 (85%)]\tLoss: 0.043539\n",
      "\n",
      "Test set: Average loss: 0.0688, Accuracy: 9797/10000 (98%)\n",
      "\n",
      "Train Epoch: 9 [0/60000 (0%)]\tLoss: 0.003396\n",
      "Train Epoch: 9 [12800/60000 (21%)]\tLoss: 0.006455\n",
      "Train Epoch: 9 [25600/60000 (43%)]\tLoss: 0.013502\n",
      "Train Epoch: 9 [38400/60000 (64%)]\tLoss: 0.029040\n",
      "Train Epoch: 9 [51200/60000 (85%)]\tLoss: 0.005606\n",
      "\n",
      "Test set: Average loss: 0.0659, Accuracy: 9812/10000 (98%)\n",
      "\n",
      "Changing Learning Rate to 0.010000000000000002\n",
      "Train Epoch: 10 [0/60000 (0%)]\tLoss: 0.006661\n",
      "Train Epoch: 10 [12800/60000 (21%)]\tLoss: 0.006700\n",
      "Train Epoch: 10 [25600/60000 (43%)]\tLoss: 0.011791\n",
      "Train Epoch: 10 [38400/60000 (64%)]\tLoss: 0.003384\n",
      "Train Epoch: 10 [51200/60000 (85%)]\tLoss: 0.001372\n",
      "\n",
      "Test set: Average loss: 0.0511, Accuracy: 9857/10000 (99%)\n",
      "\n",
      "Train Epoch: 11 [0/60000 (0%)]\tLoss: 0.003199\n",
      "Train Epoch: 11 [12800/60000 (21%)]\tLoss: 0.002455\n",
      "Train Epoch: 11 [25600/60000 (43%)]\tLoss: 0.003293\n",
      "Train Epoch: 11 [38400/60000 (64%)]\tLoss: 0.002076\n",
      "Train Epoch: 11 [51200/60000 (85%)]\tLoss: 0.000641\n",
      "\n",
      "Test set: Average loss: 0.0503, Accuracy: 9853/10000 (99%)\n",
      "\n",
      "Train Epoch: 12 [0/60000 (0%)]\tLoss: 0.000904\n",
      "Train Epoch: 12 [12800/60000 (21%)]\tLoss: 0.002054\n",
      "Train Epoch: 12 [25600/60000 (43%)]\tLoss: 0.002253\n",
      "Train Epoch: 12 [38400/60000 (64%)]\tLoss: 0.000823\n",
      "Train Epoch: 12 [51200/60000 (85%)]\tLoss: 0.004327\n",
      "\n",
      "Test set: Average loss: 0.0495, Accuracy: 9857/10000 (99%)\n",
      "\n",
      "Train Epoch: 13 [0/60000 (0%)]\tLoss: 0.004120\n",
      "Train Epoch: 13 [12800/60000 (21%)]\tLoss: 0.003073\n",
      "Train Epoch: 13 [25600/60000 (43%)]\tLoss: 0.001488\n",
      "Train Epoch: 13 [38400/60000 (64%)]\tLoss: 0.001262\n",
      "Train Epoch: 13 [51200/60000 (85%)]\tLoss: 0.002951\n",
      "\n",
      "Test set: Average loss: 0.0493, Accuracy: 9860/10000 (99%)\n",
      "\n",
      "Train Epoch: 14 [0/60000 (0%)]\tLoss: 0.001615\n",
      "Train Epoch: 14 [12800/60000 (21%)]\tLoss: 0.000915\n",
      "Train Epoch: 14 [25600/60000 (43%)]\tLoss: 0.003803\n",
      "Train Epoch: 14 [38400/60000 (64%)]\tLoss: 0.002794\n",
      "Train Epoch: 14 [51200/60000 (85%)]\tLoss: 0.001506\n",
      "\n",
      "Test set: Average loss: 0.0489, Accuracy: 9861/10000 (99%)\n",
      "\n",
      "Train Epoch: 15 [0/60000 (0%)]\tLoss: 0.002917\n",
      "Train Epoch: 15 [12800/60000 (21%)]\tLoss: 0.001394\n",
      "Train Epoch: 15 [25600/60000 (43%)]\tLoss: 0.001368\n",
      "Train Epoch: 15 [38400/60000 (64%)]\tLoss: 0.010139\n",
      "Train Epoch: 15 [51200/60000 (85%)]\tLoss: 0.000684\n",
      "\n",
      "Test set: Average loss: 0.0489, Accuracy: 9862/10000 (99%)\n",
      "\n",
      "Train Epoch: 16 [0/60000 (0%)]\tLoss: 0.002817\n",
      "Train Epoch: 16 [12800/60000 (21%)]\tLoss: 0.001688\n",
      "Train Epoch: 16 [25600/60000 (43%)]\tLoss: 0.001811\n",
      "Train Epoch: 16 [38400/60000 (64%)]\tLoss: 0.001032\n",
      "Train Epoch: 16 [51200/60000 (85%)]\tLoss: 0.000991\n",
      "\n",
      "Test set: Average loss: 0.0486, Accuracy: 9862/10000 (99%)\n",
      "\n",
      "Train Epoch: 17 [0/60000 (0%)]\tLoss: 0.000699\n",
      "Train Epoch: 17 [12800/60000 (21%)]\tLoss: 0.001935\n",
      "Train Epoch: 17 [25600/60000 (43%)]\tLoss: 0.002670\n",
      "Train Epoch: 17 [38400/60000 (64%)]\tLoss: 0.000368\n",
      "Train Epoch: 17 [51200/60000 (85%)]\tLoss: 0.002331\n",
      "\n",
      "Test set: Average loss: 0.0488, Accuracy: 9866/10000 (99%)\n",
      "\n",
      "Train Epoch: 18 [0/60000 (0%)]\tLoss: 0.000754\n",
      "Train Epoch: 18 [12800/60000 (21%)]\tLoss: 0.002396\n",
      "Train Epoch: 18 [25600/60000 (43%)]\tLoss: 0.001046\n",
      "Train Epoch: 18 [38400/60000 (64%)]\tLoss: 0.001577\n",
      "Train Epoch: 18 [51200/60000 (85%)]\tLoss: 0.000613\n",
      "\n",
      "Test set: Average loss: 0.0486, Accuracy: 9863/10000 (99%)\n",
      "\n",
      "Train Epoch: 19 [0/60000 (0%)]\tLoss: 0.000914\n",
      "Train Epoch: 19 [12800/60000 (21%)]\tLoss: 0.001193\n",
      "Train Epoch: 19 [25600/60000 (43%)]\tLoss: 0.000973\n",
      "Train Epoch: 19 [38400/60000 (64%)]\tLoss: 0.000824\n",
      "Train Epoch: 19 [51200/60000 (85%)]\tLoss: 0.002081\n",
      "\n",
      "Test set: Average loss: 0.0484, Accuracy: 9864/10000 (99%)\n",
      "\n",
      "Changing Learning Rate to 0.0010000000000000002\n",
      "Train Epoch: 20 [0/60000 (0%)]\tLoss: 0.000917\n",
      "Train Epoch: 20 [12800/60000 (21%)]\tLoss: 0.003963\n",
      "Train Epoch: 20 [25600/60000 (43%)]\tLoss: 0.000685\n",
      "Train Epoch: 20 [38400/60000 (64%)]\tLoss: 0.002925\n",
      "Train Epoch: 20 [51200/60000 (85%)]\tLoss: 0.001178\n",
      "\n",
      "Test set: Average loss: 0.0484, Accuracy: 9865/10000 (99%)\n",
      "\n",
      "Train Epoch: 21 [0/60000 (0%)]\tLoss: 0.000624\n",
      "Train Epoch: 21 [12800/60000 (21%)]\tLoss: 0.001572\n",
      "Train Epoch: 21 [25600/60000 (43%)]\tLoss: 0.002182\n",
      "Train Epoch: 21 [38400/60000 (64%)]\tLoss: 0.002430\n",
      "Train Epoch: 21 [51200/60000 (85%)]\tLoss: 0.005723\n",
      "\n",
      "Test set: Average loss: 0.0484, Accuracy: 9862/10000 (99%)\n",
      "\n",
      "Train Epoch: 22 [0/60000 (0%)]\tLoss: 0.000442\n",
      "Train Epoch: 22 [12800/60000 (21%)]\tLoss: 0.001199\n",
      "Train Epoch: 22 [25600/60000 (43%)]\tLoss: 0.000974\n",
      "Train Epoch: 22 [38400/60000 (64%)]\tLoss: 0.002540\n",
      "Train Epoch: 22 [51200/60000 (85%)]\tLoss: 0.002267\n",
      "\n",
      "Test set: Average loss: 0.0484, Accuracy: 9862/10000 (99%)\n",
      "\n",
      "Train Epoch: 23 [0/60000 (0%)]\tLoss: 0.000566\n",
      "Train Epoch: 23 [12800/60000 (21%)]\tLoss: 0.002624\n",
      "Train Epoch: 23 [25600/60000 (43%)]\tLoss: 0.000388\n",
      "Train Epoch: 23 [38400/60000 (64%)]\tLoss: 0.001600\n",
      "Train Epoch: 23 [51200/60000 (85%)]\tLoss: 0.000855\n",
      "\n",
      "Test set: Average loss: 0.0484, Accuracy: 9863/10000 (99%)\n",
      "\n",
      "Train Epoch: 24 [0/60000 (0%)]\tLoss: 0.000720\n",
      "Train Epoch: 24 [12800/60000 (21%)]\tLoss: 0.000668\n",
      "Train Epoch: 24 [25600/60000 (43%)]\tLoss: 0.001867\n",
      "Train Epoch: 24 [38400/60000 (64%)]\tLoss: 0.000503\n",
      "Train Epoch: 24 [51200/60000 (85%)]\tLoss: 0.000799\n",
      "\n",
      "Test set: Average loss: 0.0484, Accuracy: 9863/10000 (99%)\n",
      "\n",
      "Train Epoch: 25 [0/60000 (0%)]\tLoss: 0.001550\n",
      "Train Epoch: 25 [12800/60000 (21%)]\tLoss: 0.002951\n",
      "Train Epoch: 25 [25600/60000 (43%)]\tLoss: 0.000622\n",
      "Train Epoch: 25 [38400/60000 (64%)]\tLoss: 0.001799\n",
      "Train Epoch: 25 [51200/60000 (85%)]\tLoss: 0.001191\n",
      "\n",
      "Test set: Average loss: 0.0484, Accuracy: 9863/10000 (99%)\n",
      "\n",
      "Train Epoch: 26 [0/60000 (0%)]\tLoss: 0.000417\n",
      "Train Epoch: 26 [12800/60000 (21%)]\tLoss: 0.000362\n",
      "Train Epoch: 26 [25600/60000 (43%)]\tLoss: 0.000497\n",
      "Train Epoch: 26 [38400/60000 (64%)]\tLoss: 0.001374\n",
      "Train Epoch: 26 [51200/60000 (85%)]\tLoss: 0.001438\n",
      "\n",
      "Test set: Average loss: 0.0484, Accuracy: 9863/10000 (99%)\n",
      "\n",
      "Train Epoch: 27 [0/60000 (0%)]\tLoss: 0.001922\n",
      "Train Epoch: 27 [12800/60000 (21%)]\tLoss: 0.000435\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 27 [25600/60000 (43%)]\tLoss: 0.000389\n",
      "Train Epoch: 27 [38400/60000 (64%)]\tLoss: 0.003448\n",
      "Train Epoch: 27 [51200/60000 (85%)]\tLoss: 0.002093\n",
      "\n",
      "Test set: Average loss: 0.0485, Accuracy: 9863/10000 (99%)\n",
      "\n",
      "Train Epoch: 28 [0/60000 (0%)]\tLoss: 0.002227\n",
      "Train Epoch: 28 [12800/60000 (21%)]\tLoss: 0.002412\n",
      "Train Epoch: 28 [25600/60000 (43%)]\tLoss: 0.001575\n",
      "Train Epoch: 28 [38400/60000 (64%)]\tLoss: 0.000707\n",
      "Train Epoch: 28 [51200/60000 (85%)]\tLoss: 0.001294\n",
      "\n",
      "Test set: Average loss: 0.0484, Accuracy: 9863/10000 (99%)\n",
      "\n",
      "Train Epoch: 29 [0/60000 (0%)]\tLoss: 0.003657\n",
      "Train Epoch: 29 [12800/60000 (21%)]\tLoss: 0.001111\n",
      "Train Epoch: 29 [25600/60000 (43%)]\tLoss: 0.000565\n",
      "Train Epoch: 29 [38400/60000 (64%)]\tLoss: 0.001567\n",
      "Train Epoch: 29 [51200/60000 (85%)]\tLoss: 0.001064\n",
      "\n",
      "Test set: Average loss: 0.0484, Accuracy: 9863/10000 (99%)\n",
      "\n",
      "Train Epoch: 30 [0/60000 (0%)]\tLoss: 0.001027\n",
      "Train Epoch: 30 [12800/60000 (21%)]\tLoss: 0.001275\n",
      "Train Epoch: 30 [25600/60000 (43%)]\tLoss: 0.001652\n",
      "Train Epoch: 30 [38400/60000 (64%)]\tLoss: 0.000738\n",
      "Train Epoch: 30 [51200/60000 (85%)]\tLoss: 0.000514\n",
      "\n",
      "Test set: Average loss: 0.0484, Accuracy: 9863/10000 (99%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Training settings\n",
    "parser = argparse.ArgumentParser(description='PyTorch MNIST Example')\n",
    "parser.add_argument('--batch-size', type=int, default=128, metavar='N',\n",
    "                    help='input batch size for training (default: 256)')\n",
    "parser.add_argument('--test-batch-size', type=int, default=256, metavar='N',\n",
    "                    help='input batch size for testing (default: 256)')\n",
    "parser.add_argument('--epochs', type=int, default=30, metavar='N',\n",
    "                    help='number of epochs to train (default: 1)')\n",
    "parser.add_argument('--lr', type=float, default=0.1, metavar='LR',\n",
    "                    help='learning rate (default: 0.1)')\n",
    "parser.add_argument('--momentum', type=float, default=0.9, metavar='M',\n",
    "                    help='SGD momentum (default: 0.9)')\n",
    "parser.add_argument('--weight_decay', type=float, default=1e-4, metavar='LR',\n",
    "                    help='learning rate (default: 0.0001)')\n",
    "parser.add_argument('--no-cuda', action='store_true', default=False,\n",
    "                    help='disables CUDA training')\n",
    "parser.add_argument('--seed', type=int, default=12346, metavar='S',\n",
    "                    help='random seed (default: 12346)')\n",
    "parser.add_argument('--log-interval', type=int, default=100, metavar='N',\n",
    "                    help='how many batches to wait before logging training status')\n",
    "parser.add_argument('--checkpoint_path', type=str, default='./checkpoints/ann.pth', metavar='S',\n",
    "                    help='path to store model training checkpoints')\n",
    "\n",
    "\n",
    "#set device to CPU or GPU\n",
    "args = parser.parse_args()\n",
    "use_cuda = not args.no_cuda and torch.cuda.is_available()\n",
    "device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
    "\n",
    "#set all seeds for reproducability\n",
    "def set_random_seed(seed):    \n",
    "    random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed_all(seed)\n",
    "    np.random.seed(args.seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "\n",
    "set_random_seed(args.seed)\n",
    "\n",
    "\n",
    "kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}\n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "    datasets.MNIST('../data', train=True, download=True,\n",
    "                   transform=transforms.Compose([\n",
    "                       transforms.ToTensor()#,\n",
    "                       #transforms.Normalize((0.1307,), (0.3081,))\n",
    "                   ])),\n",
    "    batch_size=args.batch_size, shuffle=True, **kwargs)\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "    datasets.MNIST('../data', train=False, transform=transforms.Compose([\n",
    "                       transforms.ToTensor()#,\n",
    "                       #transforms.Normalize((0.1307,), (0.3081,))\n",
    "                   ])),\n",
    "    batch_size=args.test_batch_size, shuffle=True, **kwargs)\n",
    "\n",
    "\n",
    "ann = ANN().to(device)\n",
    "        \n",
    "optimizer = optim.SGD(ann.parameters(),lr=args.lr,momentum=args.momentum,weight_decay=args.weight_decay,nesterov=False)\n",
    "\n",
    "if not os.path.isfile(args.checkpoint_path):\n",
    "    for epoch in range(1, args.epochs + 1):\n",
    "        adjust_learning_rate(args,optimizer,epoch)\n",
    "        train(args, ann, device, train_loader, optimizer, epoch)\n",
    "        test_loss,test_accuracy = test(args, ann, device, test_loader)\n",
    "        \n",
    "          \n",
    "    torch.save({\n",
    "                'epoch': epoch,\n",
    "                'model_state_dict': ann.state_dict(),\n",
    "                'optimizer_state_dict': optimizer.state_dict(),\n",
    "                'loss': test_loss,\n",
    "            }, args.checkpoint_path, pickle_protocol=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Test magnitude based pruning in [2]\n",
    "\n",
    "[2] Li, Hao, et al. \"Pruning filters for efficient convnets.\" arXiv preprint arXiv:1608.08710 (2016)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load pre-trained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "\n",
      "Test set: Average loss: 0.0484, Accuracy: 9863/10000 (99%)\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.04843035700321197, 98.63)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_random_seed(args.seed)\n",
    "ann,optimizer = load_model('ann','sgd',args)\n",
    "test(args, ann, device, test_loader)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Prune 80% filters from each layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "pruning_args = {\n",
    "    'criterion' : 'random',\n",
    "    'use_bias' : True,\n",
    "    'prune_layers' : {1:400, 3:240},\n",
    "    'conv_feature_size' : 4\n",
    "}\n",
    "\n",
    "model_modifier = prune_model(ann,pruning_args)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### a. row: random (Table 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pruning using :  random\n",
      "Accuracy post pruning : 41.59 (without retraining), 98.45 (with retraining)\n"
     ]
    }
   ],
   "source": [
    "set_random_seed(args.seed)\n",
    "\n",
    "path = args.checkpoint_path[:-4] + '_small_random.pth'\n",
    "pruned_ann = model_modifier.prune_model('random')\n",
    "pruned_ann.cuda()\n",
    "\n",
    "val_loss_no_retrain, val_accuracy_no_retrain = test(args, pruned_ann, device, test_loader,verbose=False)\n",
    "optimizer = optim.SGD(pruned_ann.parameters(),lr=args.lr,momentum=args.momentum,weight_decay=args.weight_decay,nesterov=False)\n",
    "\n",
    "best_val_accuracy_retrain = 0\n",
    "\n",
    "if not os.path.isfile(path):    \n",
    "    for epoch in range(1, args.epochs+1):\n",
    "        adjust_learning_rate(args,optimizer,epoch)\n",
    "        train(args, pruned_ann, device, train_loader, optimizer, epoch)\n",
    "        val_loss_retrain, val_accuracy_retrain = test(args, pruned_ann, device, test_loader)\n",
    "\n",
    "        if val_accuracy_retrain > best_val_accuracy_retrain:  \n",
    "            torch.save(pruned_ann, path, pickle_protocol=4)            \n",
    "            best_val_accuracy_retrain = val_accuracy_retrain   \n",
    "else:\n",
    "    pruned_ann = torch.load(path)\n",
    "    val_loss_retrain, val_accuracy_retrain = test(args, pruned_ann, device, test_loader,verbose=False)\n",
    "    best_val_accuracy_retrain = val_accuracy_retrain\n",
    "        \n",
    "print('Accuracy post pruning : {} (without retraining), {} (with retraining)'.format(val_accuracy_no_retrain,best_val_accuracy_retrain))    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### b. row: L2 (Table 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pruning using :  l2\n",
      "Accuracy post pruning : 79.59 (without retraining), 98.47 (with retraining)\n"
     ]
    }
   ],
   "source": [
    "set_random_seed(args.seed)\n",
    "\n",
    "path = args.checkpoint_path[:-4] + '_small_l2.pth'\n",
    "pruned_ann = model_modifier.prune_model('l2')\n",
    "pruned_ann.cuda()\n",
    "\n",
    "val_loss_no_retrain, val_accuracy_no_retrain = test(args, pruned_ann, device, test_loader,verbose=False)\n",
    "optimizer = optim.SGD(pruned_ann.parameters(),lr=args.lr,momentum=args.momentum,weight_decay=args.weight_decay,nesterov=False)\n",
    "\n",
    "best_val_accuracy_retrain = 0\n",
    "\n",
    "if not os.path.isfile(path):    \n",
    "    for epoch in range(1, args.epochs+1):\n",
    "        adjust_learning_rate(args,optimizer,epoch)\n",
    "        train(args, pruned_ann, device, train_loader, optimizer, epoch)\n",
    "        val_loss_retrain, val_accuracy_retrain = test(args, pruned_ann, device, test_loader)\n",
    "\n",
    "        if val_accuracy_retrain > best_val_accuracy_retrain:  \n",
    "            torch.save(pruned_ann, path, pickle_protocol=4)            \n",
    "            best_val_accuracy_retrain = val_accuracy_retrain   \n",
    "else:\n",
    "    pruned_ann = torch.load(path)\n",
    "    val_loss_retrain, val_accuracy_retrain = test(args, pruned_ann, device, test_loader,verbose=False)\n",
    "    best_val_accuracy_retrain = val_accuracy_retrain\n",
    "        \n",
    "print('Accuracy post pruning : {} (without retraining), {} (with retraining)'.format(val_accuracy_no_retrain,best_val_accuracy_retrain))    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### c. row: L1 (Table 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pruning using :  l1\n",
      "Accuracy post pruning : 80.15 (without retraining), 98.38 (with retraining)\n"
     ]
    }
   ],
   "source": [
    "set_random_seed(args.seed)\n",
    "\n",
    "path = args.checkpoint_path[:-4] + '_small_l1.pth'\n",
    "pruned_ann = model_modifier.prune_model('l1')\n",
    "pruned_ann.cuda()\n",
    "\n",
    "val_loss_no_retrain, val_accuracy_no_retrain = test(args, pruned_ann, device, test_loader, verbose=False)\n",
    "optimizer = optim.SGD(pruned_ann.parameters(),lr=args.lr,momentum=args.momentum,weight_decay=args.weight_decay,nesterov=False)\n",
    "\n",
    "best_val_accuracy_retrain = 0\n",
    "\n",
    "if not os.path.isfile(path):    \n",
    "    for epoch in range(1, args.epochs+1):\n",
    "        adjust_learning_rate(args,optimizer,epoch)\n",
    "        train(args, pruned_ann, device, train_loader, optimizer, epoch)\n",
    "        val_loss_retrain, val_accuracy_retrain = test(args, pruned_ann, device, test_loader)\n",
    "\n",
    "        if val_accuracy_retrain > best_val_accuracy_retrain:  \n",
    "            torch.save(pruned_ann, path, pickle_protocol=4)            \n",
    "            best_val_accuracy_retrain = val_accuracy_retrain   \n",
    "else:\n",
    "    pruned_ann = torch.load(path)\n",
    "    val_loss_retrain, val_accuracy_retrain = test(args, pruned_ann, device, test_loader, verbose=False)\n",
    "    best_val_accuracy_retrain = val_accuracy_retrain\n",
    "        \n",
    "print('Accuracy post pruning : {} (without retraining), {} (with retraining)'.format(val_accuracy_no_retrain,best_val_accuracy_retrain))    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Test proposed cluster pruning (CUP)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load pre-trained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "\n",
      "Test set: Average loss: 0.0484, Accuracy: 9863/10000 (99%)\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.04843035700321197, 98.63)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_random_seed(args.seed)\n",
    "ann,optimizer = load_model('ann','sgd',args)\n",
    "test(args, ann, device, test_loader)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Prune 80% of filters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy post pruning : 85.37 (without retraining), 98.63 (with retraining)\n"
     ]
    }
   ],
   "source": [
    "cluster_args = {\n",
    "    'cluster_layers' : {1:400, 3:240},\n",
    "    'conv_feature_size' : 1,\n",
    "    'reshape_exists' : False,\n",
    "    'features' : 'both',\n",
    "    'channel_reduction' : 'fro',\n",
    "    'use_bias' : False,\n",
    "    'linkage_method' : 'ward',\n",
    "    'distance_metric' : 'euclidean',\n",
    "    'cluster_criterion' : 'hierarchical_trunc',\n",
    "    'distance_threshold' : 1.60,\n",
    "    'merge_criterion' : 'max_l2_norm',    \n",
    "    'verbose' : False\n",
    "}\n",
    "\n",
    "path = args.checkpoint_path[:-4] + '_small_cup.pth' \n",
    "model_modifier = cluster_model(ann,cluster_args)\n",
    "compressed_model = model_modifier.cluster_model()\n",
    "compressed_model.cuda()\n",
    "\n",
    "val_loss_no_retrain, val_accuracy_no_retrain = test(args, compressed_model, device, test_loader,verbose=False)\n",
    "\n",
    "args.lr = 0.1\n",
    "args.epochs = 30\n",
    "optimizer = optim.SGD(compressed_model.parameters(),lr=args.lr,momentum=args.momentum,weight_decay=args.weight_decay,nesterov=False)\n",
    "\n",
    "best_val_accuracy_retrain = 0\n",
    "\n",
    "if not os.path.isfile(path):    \n",
    "    for epoch in range(1, args.epochs+1):\n",
    "        adjust_learning_rate(args,optimizer,epoch)\n",
    "        train(args, compressed_model, device, train_loader, optimizer, epoch)\n",
    "        val_loss_retrain, val_accuracy_retrain = test(args, compressed_model, device, test_loader)\n",
    "\n",
    "        if val_accuracy_retrain > best_val_accuracy_retrain:  \n",
    "            torch.save(compressed_model, path, pickle_protocol=4)            \n",
    "            best_val_accuracy_retrain = val_accuracy_retrain   \n",
    "else:\n",
    "    compressed_model = torch.load(path)\n",
    "    val_loss_retrain, val_accuracy_retrain = test(args, compressed_model, device, test_loader, verbose=False)\n",
    "    best_val_accuracy_retrain = val_accuracy_retrain\n",
    "        \n",
    "print('Accuracy post pruning : {} (without retraining), {} (with retraining)'.format(val_accuracy_no_retrain,best_val_accuracy_retrain))    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Plot of accuracy vs compression for input/output/both features (Fig 6 b) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n",
      "loading state from epoch 30 and test loss 0.04843035659790039\n"
     ]
    }
   ],
   "source": [
    "incoming_loss,incoming_acc = [],[]\n",
    "outgoing_loss,outgoing_acc = [],[]\n",
    "both_loss,both_acc = [],[]\n",
    "\n",
    "cluster_args = {\n",
    "    'cluster_layers' : {1:400, 3:240},\n",
    "    'conv_feature_size' : 1,\n",
    "    'reshape_exists' : False,\n",
    "    'features' : 'both',\n",
    "    'channel_reduction' : 'fro',\n",
    "    'use_bias' : False,\n",
    "    'linkage_method' : 'ward',\n",
    "    'distance_metric' : 'euclidean',\n",
    "    'cluster_criterion' : 'hierarchical_trunc',\n",
    "    'distance_threshold' : 1.60,\n",
    "    'merge_criterion' : 'max_l2_norm',    \n",
    "    'verbose' : False\n",
    "}\n",
    "\n",
    "    \n",
    "\n",
    "for drop_percentage in np.linspace(0.6,0.9,31):\n",
    "    \n",
    "    num_drop_nodes = [int(num_nodes * drop_percentage) for num_nodes in[500,300]]     \n",
    "    \n",
    "    cluster_args['cluster_layers'] = {1:int(500*drop_percentage),3:int(300*drop_percentage)}\n",
    "        \n",
    "    set_random_seed(args.seed)\n",
    "    ann,optimizer = load_model('ann','sgd',args)\n",
    "    cluster_args['features'] = 'incoming'\n",
    "    model_modifier = cluster_model(ann,cluster_args)\n",
    "    compressed_model = model_modifier.cluster_model()#[int(nodes*drop_percentage) for nodes in [500,300]])\n",
    "    compressed_model.cuda()\n",
    "    loss,acc = test(args, compressed_model, device, test_loader,verbose=False)\n",
    "    incoming_loss.append(loss)\n",
    "    incoming_acc.append(acc)\n",
    "    \n",
    "    set_random_seed(args.seed)\n",
    "    ann,optimizer = load_model('ann','sgd',args)\n",
    "    cluster_args['features'] = 'outgoing'\n",
    "    model_modifier = cluster_model(ann,cluster_args)\n",
    "    compressed_model = model_modifier.cluster_model()#[int(nodes*drop_percentage) for nodes in [500,300]])\n",
    "    compressed_model.cuda()\n",
    "    loss,acc = test(args, compressed_model, device, test_loader,verbose=False)\n",
    "    outgoing_loss.append(loss)\n",
    "    outgoing_acc.append(acc)\n",
    "    \n",
    "    set_random_seed(args.seed)\n",
    "    ann,optimizer = load_model('ann','sgd',args)\n",
    "    cluster_args['features'] = 'both'\n",
    "    model_modifier = cluster_model(ann,cluster_args)\n",
    "    compressed_model = model_modifier.cluster_model()#[int(nodes*drop_percentage) for nodes in [500,300]])\n",
    "    compressed_model.cuda()\n",
    "    loss,acc = test(args, compressed_model, device, test_loader,verbose=False)\n",
    "    both_loss.append(loss)\n",
    "    both_acc.append(acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'incoming_acc' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-14-e62f91d6b876>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfigure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfigsize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m6\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinspace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0.6\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0.9\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m31\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mincoming_acc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcolor\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'blue'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlinewidth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2.5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinspace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0.6\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0.9\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m31\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0moutgoing_acc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcolor\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'green'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlinewidth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2.5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinspace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0.6\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0.9\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m31\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mboth_acc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcolor\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'red'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlinewidth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2.5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'incoming_acc' is not defined"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x360 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import rcParams\n",
    "\n",
    "\n",
    "plt.tight_layout()\n",
    "\n",
    "plt.rc('font', family='serif')\n",
    "plt.rc('xtick', labelsize='large')\n",
    "plt.rc('ytick', labelsize='large')\n",
    "plt.rcParams.update({'font.size': 16})\n",
    "rcParams.update({'figure.autolayout': True})\n",
    "\n",
    "plt.figure(figsize=(6,5))\n",
    "plt.plot(np.linspace(0.6,0.9,31)[:],incoming_acc[:],color='blue',linewidth=2.5)\n",
    "plt.plot(np.linspace(0.6,0.9,31)[:],outgoing_acc[:],color='green',linewidth=2.5)\n",
    "plt.plot(np.linspace(0.6,0.9,31)[:],both_acc[:],color='red',linewidth=2.5)\n",
    "plt.legend(['incoming','outgoing','both'])\n",
    "plt.title('Test accuracy vs Percent pruned')\n",
    "plt.xlabel('Percent pruned')\n",
    "plt.ylabel('Test accuracy')\n",
    "plt.grid(True)\n",
    "# plt.show()\n",
    "plt.savefig('figures/features_acc_vs_compression.png')\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "plt.figure(figsize=(6,5))\n",
    "plt.plot(np.linspace(0.6,0.90,31)[:],incoming_loss[:])\n",
    "plt.plot(np.linspace(0.6,0.90,31)[:],outgoing_loss[:])\n",
    "plt.plot(np.linspace(0.6,0.90,31)[:],both_loss[:])\n",
    "\n",
    "plt.legend(['incoming + cluster + maxnorm','outgoing + cluster + maxnorm','both + cluster + maxnorm'])\n",
    "\n",
    "plt.title('test loss vs number of nodes')\n",
    "plt.xlabel('percentage compression')\n",
    "plt.ylabel('test loss')\n",
    "plt.grid(True)\n",
    "\n",
    "# plt.savefig('figures/features_loss_vs_compression.png')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6 (kdd_final)",
   "language": "python",
   "name": "kdd_final"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}