2.deep architectures.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://docs.google.com/presentation/d/1MFhet5q-SIPqc_54CXWiBvlT9OdSi6P8kpkm6IxuyEM/edit#slide=id.g522eca1928_0_11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "NUM_INPUTS = 100\n",
    "HIDDEN_SIZE = 1024\n",
    "NUM_OUTPUTS = 20"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1. Logistic Regresion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "lor = nn.Sequential(\n",
    "    nn.Linear(NUM_INPUTS,1),\n",
    "    nn.Sigmoid()\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. Linear Regresion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "lir = nn.Sequential(\n",
    "    nn.Linear(NUM_INPUTS,1)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3. Softmax classifer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "smx = nn.Sequential(\n",
    "    nn.Linear(NUM_INPUTS,NUM_OUTPUTS),\n",
    "    nn.LogSoftmax(dim=1)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4. MultiLayer Perceptron"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlp = nn.Sequential(\n",
    "    nn.Linear(NUM_INPUTS,HIDDEN_SIZE),\n",
    "    nn.Tanh(),\n",
    "    nn.Linear(HIDDEN_SIZE,HIDDEN_SIZE),\n",
    "    nn.Tanh(),\n",
    "    nn.Linear(HIDDEN_SIZE,NUM_OUTPUTS),\n",
    "    nn.LogSoftmax(dim=1)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 5. Embedding with fully connected layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input x tensor size:  torch.Size([10, 10000])\n",
      "Output y embedding size:  torch.Size([10, 100])\n"
     ]
    }
   ],
   "source": [
    "VOCAB_SIZE = 10000\n",
    "HIDDEN_SIZE = 100\n",
    "emb_1 = nn.Linear(VOCAB_SIZE,HIDDEN_SIZE)\n",
    "code = [1] + [0] * 9999\n",
    "x = torch.Tensor([code] * 10)\n",
    "print('Input x tensor size: ', x.size())\n",
    "y = emb_1(x)\n",
    "print('Output y embedding size: ', y.size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 6. Embedding with Embedding layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input x tensor size:  torch.Size([10, 1])\n",
      "Output y embedding size:  torch.Size([10, 1, 100])\n"
     ]
    }
   ],
   "source": [
    "VOCAB_SIZE = 10000\n",
    "HIDDEN_SIZE = 100\n",
    "emb_2 = nn.Embedding(VOCAB_SIZE,HIDDEN_SIZE)\n",
    "x = torch.zeros(10,1).long()\n",
    "print('Input x tensor size: ', x.size())\n",
    "y = emb_2(x)\n",
    "print('Output y embedding size: ', y.size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 7. Recurrent Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor size [seq_len, bsize, hidden_size]:  torch.Size([100, 1, 100])\n",
      "Output tensor h[t] size [seq_len, bsize, hidden_size]:  torch.Size([100, 1, 512])\n"
     ]
    }
   ],
   "source": [
    "NUM_INPUTS = 100\n",
    "HIDDEN_SIZE = 512\n",
    "NUM_LAYERS = 1\n",
    "rnn = nn.RNN(NUM_INPUTS,HIDDEN_SIZE,num_layers=NUM_LAYERS)\n",
    "SEQ_LEN = 100\n",
    "x = torch.randn(SEQ_LEN,1,NUM_INPUTS)\n",
    "print('Input tensor size [seq_len, bsize, hidden_size]: ', x.size())\n",
    "ht,state = rnn(x,None)\n",
    "print('Output tensor h[t] size [seq_len, bsize, hidden_size]: ', ht.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor size [bsize, seq_len, hidden_size]:  torch.Size([1, 100, 100])\n",
      "Output tensor h[t] size [bsize, seq_len, hidden_size]:  torch.Size([1, 100, 512])\n",
      "ht size:  torch.Size([1, 100, 512])\n",
      "state size:  torch.Size([1, 1, 512])\n"
     ]
    }
   ],
   "source": [
    "NUM_INPUTS = 100\n",
    "HIDDEN_SIZE = 512\n",
    "NUM_LAYERS = 1\n",
    "# define a recurrent layer, swapping batch and time axis\n",
    "rnn = nn.RNN(NUM_INPUTS, HIDDEN_SIZE, num_layers=NUM_LAYERS,batch_first=True)\n",
    "SEQ_LEN = 100\n",
    "x = torch.randn(1,SEQ_LEN,NUM_INPUTS)\n",
    "print('Input tensor size [bsize, seq_len, hidden_size]: ', x.size())\n",
    "ht,state = rnn(x,None)\n",
    "print('Output tensor h[t] size [bsize, seq_len, hidden_size]: ', ht.size())\n",
    "# let's check ht and state sizes\n",
    "print('ht size: ', ht.size())\n",
    "print('state size: ', state.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor size x:  torch.Size([1, 100, 100])\n",
      "Hidden tensor size ht:  torch.Size([1, 100, 512])\n",
      "Output tensor y size:  torch.Size([1, 100, 10])\n"
     ]
    }
   ],
   "source": [
    "NUM_INPUTS = 100\n",
    "NUM_OUTPUTS = 10\n",
    "HIDDEN_SIZE = 512\n",
    "SEQ_LEN = 100\n",
    "NUM_LAYERS = 1\n",
    "# define a recurrent layer, swapping batch and time axis and connect\n",
    "# an FC layer as an output layer to build a full network\n",
    "rnn = nn.RNN(NUM_INPUTS, HIDDEN_SIZE, num_layers=NUM_LAYERS,\n",
    "             batch_first=True)\n",
    "fc = nn.Sequential(\n",
    "    nn.Linear(HIDDEN_SIZE, NUM_OUTPUTS),\n",
    "    nn.LogSoftmax(dim=2)\n",
    ")\n",
    "\n",
    "x = torch.randn(1, SEQ_LEN, NUM_INPUTS)\n",
    "print('Input tensor size x: ', x.size())\n",
    "ht, state = rnn(x, None)\n",
    "print('Hidden tensor size ht: ', ht.size())\n",
    "y = fc(ht)\n",
    "print('Output tensor y size: ', y.size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 8. LSTM Recurrent Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor size x:  torch.Size([1, 100, 100])\n",
      "Output tensor ht size:  torch.Size([1, 100, 512])\n",
      "Last state h[T]:  torch.Size([1, 1, 512])\n",
      "Cell state c[T]:  torch.Size([1, 1, 512])\n"
     ]
    }
   ],
   "source": [
    "lstm = nn.LSTM(NUM_INPUTS,HIDDEN_SIZE,num_layers=NUM_LAYERS,batch_first=True)\n",
    "x = torch.randn(1,SEQ_LEN,NUM_INPUTS)\n",
    "print('Input tensor size x: ', x.size())\n",
    "ht,states = lstm(x,None)\n",
    "hT,cT = states[0],states[1]\n",
    "print('Output tensor ht size: ', ht.size())\n",
    "print('Last state h[T]: ', hT.size())\n",
    "print('Cell state c[T]: ', cT.size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 9. Convolutional Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor size x:  torch.Size([1, 1, 8])\n",
      "Output tensor y size:  torch.Size([1, 1024, 6])\n"
     ]
    }
   ],
   "source": [
    "NUM_CHANNELS_IN = 1\n",
    "HIDDEN_SIZE = 1024\n",
    "KERNEL_WIDTH = 3\n",
    "# Build a one-dimensional convolutional neural layer\n",
    "conv1d = nn.Conv1d(NUM_CHANNELS_IN,HIDDEN_SIZE,KERNEL_WIDTH)\n",
    "SEQ_LEN = 8\n",
    "x = torch.randn(1, NUM_CHANNELS_IN, SEQ_LEN)\n",
    "print('Input tensor size x: ', x.size())\n",
    "y = conv1d(x)\n",
    "print('Output tensor y size: ', y.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor size x:  torch.Size([1, 1, 8])\n",
      "Output tensor y size:  torch.Size([1, 1024, 8])\n"
     ]
    }
   ],
   "source": [
    "NUM_CHANNELS_IN = 1\n",
    "HIDDEN_SIZE = 1024\n",
    "KERNEL_WIDTH = 3\n",
    "PADDING = KERNEL_WIDTH // 2 # = 1\n",
    "# Build a one-dimensional convolutional neural layer\n",
    "conv1d = nn.Conv1d(NUM_CHANNELS_IN, HIDDEN_SIZE, KERNEL_WIDTH, \n",
    "                   padding=PADDING)\n",
    "\n",
    "SEQ_LEN = 8\n",
    "x = torch.randn(1, NUM_CHANNELS_IN, SEQ_LEN)\n",
    "print('Input tensor size x: ', x.size())\n",
    "y = conv1d(x)\n",
    "print('Output tensor y size: ', y.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor x size:  torch.Size([1, 1, 8])\n",
      "Input tensor after padding xpad size:  torch.Size([1, 1, 10])\n",
      "Output tensor y size:  torch.Size([1, 1024, 8])\n"
     ]
    }
   ],
   "source": [
    "# Causal Convolutional layer\n",
    "NUM_CHANNELS_IN = 1\n",
    "HIDDEN_SIZE = 1024\n",
    "KERNEL_WIDTH = 3\n",
    "# Build a one-dimensional convolutional neural layer\n",
    "conv1d = nn.Conv1d(NUM_CHANNELS_IN, HIDDEN_SIZE, KERNEL_WIDTH)\n",
    "                   \n",
    "SEQ_LEN = 8\n",
    "PADDING = KERNEL_WIDTH - 1 # = 2\n",
    "x = torch.randn(1, NUM_CHANNELS_IN, SEQ_LEN)\n",
    "print('Input tensor x size: ', x.size())\n",
    "xpad = F.pad(x, (PADDING, 0))\n",
    "print('Input tensor after padding xpad size: ', xpad.size())\n",
    "y = conv1d(xpad)\n",
    "print('Output tensor y size: ', y.size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 10. Convolutional Neural Network as an MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor x size:  torch.Size([1, 100, 1])\n",
      "Output tensor y size:  torch.Size([1, 20, 1])\n"
     ]
    }
   ],
   "source": [
    "NUM_INPUTS = 100\n",
    "HIDDEN_SIZE = 1024\n",
    "NUM_OUTPUTS= 20\n",
    "# MLP as a CNN\n",
    "mlp = nn.Sequential(\n",
    "    nn.Conv1d(NUM_INPUTS,HIDDEN_SIZE,1),\n",
    "    nn.Tanh(),\n",
    "    nn.Conv1d(HIDDEN_SIZE,HIDDEN_SIZE,1),\n",
    "    nn.Tanh(),\n",
    "    nn.Conv1d(HIDDEN_SIZE,NUM_OUTPUTS,1),\n",
    "    nn.LogSoftmax(dim=1)\n",
    ")\n",
    "x = torch.randn(1, 100, 1)\n",
    "print('Input tensor x size: ', x.size())\n",
    "y = mlp(x)\n",
    "print('Output tensor y size: ', y.size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 11. Deconvolutional Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor y size:  torch.Size([1, 1, 2])\n",
      "Output (interpolated) tensor x size:  torch.Size([1, 1, 12])\n"
     ]
    }
   ],
   "source": [
    "NUM_CHANNELS_IN = 1\n",
    "HIDDEN_SIZE = 1\n",
    "KERNEL_WIDTH = 8\n",
    "STRIDE = 4\n",
    "\n",
    "deconv = nn.ConvTranspose1d(NUM_CHANNELS_IN, HIDDEN_SIZE, KERNEL_WIDTH,\n",
    "                            stride=STRIDE)\n",
    "\n",
    "SEQ_LEN = 2\n",
    "y = torch.randn(1, NUM_CHANNELS_IN, SEQ_LEN)\n",
    "print('Input tensor y size: ', y.size())\n",
    "x = deconv(y)\n",
    "print('Output (interpolated) tensor x size: ', x.size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 12. Quasi Recurrent Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ht size:  torch.Size([1, 10, 100])\n",
      "state size:  torch.Size([1, 100])\n"
     ]
    }
   ],
   "source": [
    "class fQRNNLayer(nn.Module):\n",
    "\n",
    "    def __init__(self, num_inputs, num_outputs,\n",
    "                 kwidth=2):\n",
    "        super().__init__()\n",
    "        self.num_inputs = num_inputs\n",
    "        self.num_outputs = num_outputs\n",
    "        self.kwidth = kwidth\n",
    "        # double feature maps for zt and ft predictions with same conv layer\n",
    "        self.conv = nn.Conv1d(num_inputs, num_outputs * 2, kwidth)\n",
    "\n",
    "    def forward(self, x, state=None):\n",
    "        # x is [bsz, seq_len, num_inputs]\n",
    "        # state is [bsz, num_outputs] dimensional\n",
    "        # ---------- FEED FORWARD PART\n",
    "        # inference convolutional part\n",
    "        # transpose x axis first to work with CNN layer\n",
    "        x = x.transpose(1, 2)\n",
    "        pad = self.kwidth - 1\n",
    "        xp = F.pad(x, (pad, 0))\n",
    "        conv_h = self.conv(xp)\n",
    "        # split convolutional layer feature maps into zt (new state\n",
    "        # candidate) and forget activation ft\n",
    "        zt, ft = torch.chunk(conv_h, 2, dim=1)\n",
    "        # Convert forget gate into actual forget\n",
    "        ft = torch.sigmoid(ft)\n",
    "        # Convert zt into actual non-linear response\n",
    "        zt = torch.tanh(zt)\n",
    "        # ---------- SEQUENTIAL PART\n",
    "        # iterate through time now to make pooling\n",
    "        seqlen = ft.size(2)\n",
    "        if state is None:\n",
    "            # create the zero state\n",
    "            ht_1 = torch.zeros(ft.size(0), self.num_outputs, 1)\n",
    "        else:\n",
    "            # add the dim=2 to match 3D tensor shape\n",
    "            ht_1 = state.unsqueeze(2)\n",
    "        zts = torch.chunk(zt, zt.size(2), dim=2)\n",
    "        fts = torch.chunk(ft, ft.size(2), dim=2)\n",
    "        hts = []\n",
    "        for t in range(seqlen):\n",
    "            ht = ht_1 * fts[t] + (1 - fts[t]) * zts[t]\n",
    "            # transpose time, channels dims again to match RNN-like shape\n",
    "            hts.append(ht.transpose(1, 2))\n",
    "            # re-assign h[t-1] now\n",
    "            ht_1 = ht\n",
    "        # convert hts list into a 3D tensor [bsz, seq_len, num_outputs]\n",
    "        hts = torch.cat(hts, dim=1)\n",
    "        return hts, ht_1.squeeze(2)\n",
    "\n",
    "\n",
    "fqrnn = fQRNNLayer(1, 100, 2)\n",
    "x = torch.randn(1, 10, 1)\n",
    "ht, state = fqrnn(x)\n",
    "print('ht size: ', ht.size())\n",
    "print('state size: ', state.size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 13. AlexNet classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor x size:  torch.Size([1, 3, 224, 224])\n",
      "Output tensor y size:  torch.Size([1, 1000])\n"
     ]
    }
   ],
   "source": [
    "class AlexNet(nn.Module):\n",
    "\n",
    "    def __init__(self, num_classes=1000):\n",
    "        super(AlexNet, self).__init__()\n",
    "        self.features = nn.Sequential(\n",
    "            nn.Conv2d(3, 64, kernel_size=11, stride=4, padding=2),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(kernel_size=3, stride=2),\n",
    "            nn.Conv2d(64, 192, kernel_size=5, padding=2),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(kernel_size=3, stride=2),\n",
    "            nn.Conv2d(192, 384, kernel_size=3, padding=1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv2d(384, 256, kernel_size=3, padding=1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv2d(256, 256, kernel_size=3, padding=1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(kernel_size=3, stride=2),\n",
    "        )\n",
    "        self.avgpool = nn.AdaptiveAvgPool2d((6, 6))\n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Dropout(),\n",
    "            nn.Linear(256 * 6 * 6, 4096),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Dropout(),\n",
    "            nn.Linear(4096, 4096),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(4096, num_classes),\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.features(x)\n",
    "        x = self.avgpool(x)\n",
    "        x = x.view(x.size(0), 256 * 6 * 6)\n",
    "        x = self.classifier(x)\n",
    "        return x\n",
    "alexnet = AlexNet()\n",
    "x = torch.randn(1, 3, 224, 224)\n",
    "print('Input tensor x size: ', x.size())\n",
    "y = alexnet(x)\n",
    "print('Output tensor y size: ', y.size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 14. Time-Delayed Neural Network(TDNN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor x size:  torch.Size([1, 24, 10000])\n",
      "Output tensor y size:  torch.Size([1, 2000, 1])\n"
     ]
    }
   ],
   "source": [
    "class StatisticalPooling(nn.Module):\n",
    "\n",
    "    def forward(self, x):\n",
    "        # x is 3-D with axis [B, feats, T]\n",
    "        mu = x.mean(dim=2, keepdim=True)\n",
    "        std = x.std(dim=2, keepdim=True)\n",
    "        return torch.cat((mu, std), dim=1)\n",
    "\n",
    "\n",
    "class TDNN(nn.Module):\n",
    "    # Architecture taken from x-vectors extractor\n",
    "    # https://www.danielpovey.com/files/2018_icassp_xvectors.pdf\n",
    "    def __init__(self, num_inputs=24, num_outputs=2000):\n",
    "        super().__init__()\n",
    "        self.model = nn.Sequential(\n",
    "            nn.Conv1d(num_inputs, 512, 5, padding=2),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv1d(512, 512, 3, dilation=2, padding=2),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv1d(512, 512, 3, dilation=3, padding=3),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv1d(512, 512, 1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv1d(512, 1500, 1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            StatisticalPooling(),\n",
    "            nn.Conv1d(3000, 512, 1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv1d(512, 512, 1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv1d(512, num_outputs, 1),\n",
    "            nn.LogSoftmax(dim=1)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.model(x)\n",
    "    \n",
    "tdnn = TDNN()\n",
    "x = torch.randn(1, 24, 10000)\n",
    "print('Input tensor x size: ', x.size())\n",
    "# The output has to contain the final pooling through time with \n",
    "# 2000 class activations so [batch_size, num_classes, 1], being the \n",
    "# latter 1 the last time-step after pooling\n",
    "y = tdnn(x)\n",
    "print('Output tensor y size: ', y.size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 15. Residual connections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor x size:  torch.Size([1, 64, 100, 100])\n",
      "Output tensor y size:  torch.Size([1, 64, 100, 100])\n"
     ]
    }
   ],
   "source": [
    "class ResLayer(nn.Module):\n",
    "\n",
    "    def __init__(self, num_inputs):\n",
    "        super().__init__()\n",
    "        self.num_inputs = num_inputs\n",
    "        num_outputs = num_inputs\n",
    "        self.num_outputs = num_outputs\n",
    "        self.conv1 = nn.Sequential(\n",
    "            nn.Conv2d(num_inputs, num_outputs, 3, padding=1),\n",
    "            nn.BatchNorm2d(num_outputs),\n",
    "            nn.ReLU(inplace=True)\n",
    "        )\n",
    "        self.conv2 = nn.Sequential(\n",
    "            nn.Conv2d(num_outputs, num_outputs, 3, padding=1),\n",
    "            nn.BatchNorm2d(num_outputs),\n",
    "            nn.ReLU(inplace=True)\n",
    "        )\n",
    "        self.out_relu = nn.ReLU(inplace=True)\n",
    "\n",
    "    def forward(self, x):\n",
    "        # non-linear processing trunk\n",
    "        conv1_h = self.conv1(x)\n",
    "        conv2_h = self.conv2(conv1_h)\n",
    "        # output is result of res connection + non-linear processing\n",
    "        y = self.out_relu(x + conv2_h)\n",
    "        return y\n",
    "\n",
    "\n",
    "x = torch.randn(1, 64, 100, 100)\n",
    "print('Input tensor x size: ', x.size())\n",
    "reslayer = ResLayer(64)\n",
    "y = reslayer(x)\n",
    "print('Output tensor y size: ', y.size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 16. Auto-Encoder Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor x size:  torch.Size([10, 784])\n",
      "Output tensor y size:  torch.Size([10, 784])\n"
     ]
    }
   ],
   "source": [
    "class AE(nn.Module):\n",
    "    def __init__(self, num_inputs=784):\n",
    "        super().__init__()\n",
    "        self.encoder = nn.Sequential(\n",
    "            nn.Linear(num_inputs, 400),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(400, 400),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(400, 20)\n",
    "        )\n",
    "        self.decoder = nn.Sequential(\n",
    "            nn.Linear(20, 400),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(400, 400),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(400, num_inputs)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.decoder(self.encoder(x))\n",
    "\n",
    "\n",
    "ae = AE(784)\n",
    "x = torch.randn(10, 784)\n",
    "print('Input tensor x size: ', x.size())\n",
    "y = ae(x)\n",
    "print('Output tensor y size: ', y.size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 17. Variational Auto-Encoder Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor x size:  torch.Size([10, 784])\n",
      "Input tensor y size:  torch.Size([10, 784])\n",
      "Mean tensor mu size:  torch.Size([10, 20])\n",
      "Covariance tensor logvar size:  torch.Size([10, 20])\n"
     ]
    }
   ],
   "source": [
    "# from https://github.com/pytorch/examples/blob/master/vae/main.py\n",
    "class VAE(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(VAE, self).__init__()\n",
    "\n",
    "        self.fc1 = nn.Linear(784, 400)\n",
    "        self.fc21 = nn.Linear(400, 20)\n",
    "        self.fc22 = nn.Linear(400, 20)\n",
    "        self.fc3 = nn.Linear(20, 400)\n",
    "        self.fc4 = nn.Linear(400, 784)\n",
    "\n",
    "    def encode(self, x):\n",
    "        h1 = F.relu(self.fc1(x))\n",
    "        return self.fc21(h1), self.fc22(h1)\n",
    "\n",
    "    def reparameterize(self, mu, logvar):\n",
    "        std = torch.exp(0.5*logvar)\n",
    "        eps = torch.randn_like(std)\n",
    "        return mu + eps*std\n",
    "\n",
    "    def decode(self, z):\n",
    "        h3 = F.relu(self.fc3(z))\n",
    "        return torch.sigmoid(self.fc4(h3))\n",
    "\n",
    "    def forward(self, x):\n",
    "        mu, logvar = self.encode(x.view(-1, 784))\n",
    "        z = self.reparameterize(mu, logvar)\n",
    "        return self.decode(z), mu, logvar\n",
    "\n",
    "\n",
    "vae = VAE()\n",
    "x = torch.randn(10, 784)\n",
    "print('Input tensor x size: ', x.size())\n",
    "y, mu, logvar = vae(x)\n",
    "print('Input tensor y size: ', y.size())\n",
    "print('Mean tensor mu size: ', mu.size())\n",
    "print('Covariance tensor logvar size: ', logvar.size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 18. Deep Convolutional Auto-Encoder with skip connections(SEGAN G)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor x size:  torch.Size([1, 1, 4000])\n",
      "Output tensor y size:  torch.Size([1, 1, 1000])\n"
     ]
    }
   ],
   "source": [
    "class DownConv1dBlock(nn.Module):\n",
    "\n",
    "    def __init__(self, ninp, fmap, kwidth, stride):\n",
    "        super().__init__()\n",
    "        assert stride > 1, stride\n",
    "        self.kwidth = kwidth\n",
    "        self.conv = nn.Conv1d(ninp, fmap, kwidth, stride=stride)\n",
    "        self.act = nn.ReLU(inplace=True)\n",
    "\n",
    "    def forward(self, x):\n",
    "        # calculate padding with stride > 1\n",
    "        pad_left = self.kwidth // 2 - 1\n",
    "        pad_right = self.kwidth // 2\n",
    "        xp = F.pad(x, (pad_left, pad_right))\n",
    "        y = self.act(self.conv(xp))\n",
    "        return y\n",
    "\n",
    "block = DownConv1dBlock(1, 1, 31, 4)\n",
    "x = torch.randn(1, 1, 4000)\n",
    "print('Input tensor x size: ', x.size())\n",
    "y = block(x)\n",
    "print('Output tensor y size: ', y.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor x size:  torch.Size([1, 1, 1000])\n",
      "Output tensor y size:  torch.Size([1, 1, 4000])\n"
     ]
    }
   ],
   "source": [
    "class UpConv1dBlock(nn.Module):\n",
    "\n",
    "    def __init__(self, ninp, fmap, kwidth, stride, act=True):\n",
    "        super().__init__()\n",
    "        assert stride > 1, stride\n",
    "        self.kwidth = kwidth\n",
    "        pad = max(0, (stride - kwidth) // -2)\n",
    "        self.deconv = nn.ConvTranspose1d(ninp, fmap, kwidth,\n",
    "                                         stride=stride,\n",
    "                                         padding=pad)\n",
    "        if act:\n",
    "            self.act = nn.ReLU(inplace=True)\n",
    "\n",
    "    def forward(self, x):\n",
    "        h = self.deconv(x)\n",
    "        if self.kwidth % 2 != 0:\n",
    "            # drop last item for shape compatibility with TensorFlow deconvs\n",
    "            h = h[:, :, :-1]\n",
    "        if hasattr(self, 'act'):\n",
    "            y = self.act(h)\n",
    "        else:\n",
    "            y = h\n",
    "        return y\n",
    "\n",
    "\n",
    "block = UpConv1dBlock(1, 1, 31, 4)\n",
    "x = torch.randn(1, 1, 1000)\n",
    "print('Input tensor x size: ', x.size())\n",
    "y = block(x)\n",
    "print('Output tensor y size: ', y.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor x size:  torch.Size([1, 1, 8192])\n",
      "Output tensor y size:  torch.Size([1, 1, 8192])\n"
     ]
    }
   ],
   "source": [
    "class Conv1dGenerator(nn.Module):\n",
    "\n",
    "    def __init__(self, enc_fmaps=[64, 128, 256, 512], kwidth=31,\n",
    "                 pooling=4):\n",
    "        super().__init__()\n",
    "        self.enc = nn.ModuleList()\n",
    "        ninp = 1\n",
    "        for enc_fmap in enc_fmaps:\n",
    "            self.enc.append(DownConv1dBlock(ninp, enc_fmap, kwidth, pooling))\n",
    "            ninp = enc_fmap\n",
    "\n",
    "        self.dec = nn.ModuleList()\n",
    "        # revert encoder feature maps\n",
    "        dec_fmaps = enc_fmaps[::-1][1:] + [1]\n",
    "        act = True\n",
    "        for di, dec_fmap in enumerate(dec_fmaps, start=1):\n",
    "            if di >= len(dec_fmaps):\n",
    "                # last decoder layer has no activation\n",
    "                act = False\n",
    "            self.dec.append(UpConv1dBlock(\n",
    "                ninp, dec_fmap, kwidth, pooling, act=act))\n",
    "            ninp = dec_fmap\n",
    "\n",
    "    def forward(self, x):\n",
    "        skips = []\n",
    "        h = x\n",
    "        for ei, enc_layer in enumerate(self.enc, start=1):\n",
    "            h = enc_layer(h)\n",
    "            if ei < len(self.enc):\n",
    "                skips.append(h)\n",
    "        # now decode\n",
    "\n",
    "        for di, dec_layer in enumerate(self.dec, start=1):\n",
    "            if di > 1:\n",
    "                # sum skip connection\n",
    "                skip_h = skips.pop(-1)\n",
    "                h = h + skip_h\n",
    "            h = dec_layer(h)\n",
    "        y = h\n",
    "        return y\n",
    "\n",
    "\n",
    "G = Conv1dGenerator()\n",
    "x = torch.randn(1, 1, 8192)\n",
    "print('Input tensor x size: ', x.size())\n",
    "y = G(x)\n",
    "print('Output tensor y size: ', y.size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 19. DCGAN G and D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor z size:  torch.Size([1, 100, 1, 1])\n",
      "Output tensor x size:  torch.Size([1, 3, 64, 64])\n"
     ]
    }
   ],
   "source": [
    "# from https://github.com/pytorch/examples/blob/master/dcgan/main.py\n",
    "class Generator(nn.Module):\n",
    "    def __init__(self, nc=3):\n",
    "        super().__init__()\n",
    "        nz = 100\n",
    "        ngf = 64\n",
    "        self.main = nn.Sequential(\n",
    "            # input is Z, going into a convolution\n",
    "            nn.ConvTranspose2d(nz, ngf * 8, 4, 1, 0, bias=False),\n",
    "            nn.BatchNorm2d(ngf * 8),\n",
    "            nn.ReLU(True),\n",
    "            # state size. (ngf*8) x 4 x 4\n",
    "            nn.ConvTranspose2d(ngf * 8, ngf * 4, 4, 2, 1, bias=False),\n",
    "            nn.BatchNorm2d(ngf * 4),\n",
    "            nn.ReLU(True),\n",
    "            # state size. (ngf*4) x 8 x 8\n",
    "            nn.ConvTranspose2d(ngf * 4, ngf * 2, 4, 2, 1, bias=False),\n",
    "            nn.BatchNorm2d(ngf * 2),\n",
    "            nn.ReLU(True),\n",
    "            # state size. (ngf*2) x 16 x 16\n",
    "            nn.ConvTranspose2d(ngf * 2,     ngf, 4, 2, 1, bias=False),\n",
    "            nn.BatchNorm2d(ngf),\n",
    "            nn.ReLU(True),\n",
    "            # state size. (ngf) x 32 x 32\n",
    "            nn.ConvTranspose2d(ngf,      nc, 4, 2, 1, bias=False),\n",
    "            nn.Tanh()\n",
    "            # state size. (nc) x 64 x 64\n",
    "        )\n",
    "\n",
    "    def forward(self, input):\n",
    "        return self.main(input)\n",
    "\n",
    "\n",
    "z = torch.randn(1, 100, 1, 1)\n",
    "print('Input tensor z size: ', z.size())\n",
    "G = Generator()\n",
    "x = G(z)\n",
    "print('Output tensor x size: ', x.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input tensor x size:  torch.Size([1, 3, 64, 64])\n",
      "Output tensor y size:  torch.Size([1, 1, 1, 1])\n"
     ]
    }
   ],
   "source": [
    "class Discriminator(nn.Module):\n",
    "    def __init__(self, nc=3):\n",
    "        super(Discriminator, self).__init__()\n",
    "        ndf = 64\n",
    "        self.main = nn.Sequential(\n",
    "            # input is (nc) x 64 x 64\n",
    "            nn.Conv2d(nc, ndf, 4, 2, 1, bias=False),\n",
    "            nn.LeakyReLU(0.2, inplace=True),\n",
    "            # state size. (ndf) x 32 x 32\n",
    "            nn.Conv2d(ndf, ndf * 2, 4, 2, 1, bias=False),\n",
    "            nn.BatchNorm2d(ndf * 2),\n",
    "            nn.LeakyReLU(0.2, inplace=True),\n",
    "            # state size. (ndf*2) x 16 x 16\n",
    "            nn.Conv2d(ndf * 2, ndf * 4, 4, 2, 1, bias=False),\n",
    "            nn.BatchNorm2d(ndf * 4),\n",
    "            nn.LeakyReLU(0.2, inplace=True),\n",
    "            # state size. (ndf*4) x 8 x 8\n",
    "            nn.Conv2d(ndf * 4, ndf * 8, 4, 2, 1, bias=False),\n",
    "            nn.BatchNorm2d(ndf * 8),\n",
    "            nn.LeakyReLU(0.2, inplace=True),\n",
    "            # state size. (ndf*8) x 4 x 4\n",
    "            nn.Conv2d(ndf * 8, 1, 4, 1, 0, bias=False),\n",
    "            nn.Sigmoid()\n",
    "        )\n",
    "\n",
    "    def forward(self, input):\n",
    "        return self.main(input)\n",
    "\n",
    "\n",
    "x = torch.randn(1, 3, 64, 64)\n",
    "print('Input tensor x size: ', x.size())\n",
    "D = Discriminator()\n",
    "y = D(x)\n",
    "print('Output tensor y size: ', y.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:root] *",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}