convolution.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "matplotlib.use('Agg')\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convolution\n",
    "\n",
    "Convolutional layer define a kernel (weight matrix) which is then multiplied element by element with a section of the input of the same size. Sum all the resulting values. Move the kernel a number of pixels equal to a defined stride size and repeat until you go throught the whole image. Afterwards apply an activation function to each one of the values. Now repeat all of this for each filter in your layer."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![Convolution](https://raw.githubusercontent.com/UCIDataScienceInitiative/LearningWithPython/master/images/conv_layer.gif)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def convolution(input_image, conv_filter, stride):\n",
    "    \"\"\"\n",
    "    Perform convolution on input_image with conv_filter and return feature_map.\n",
    "    \n",
    "    Args\n",
    "        input_image: (d x d) numpy array\n",
    "        conv_filter: (s x s) square numpy array\n",
    "        stride: sliding width / height\n",
    "    \"\"\"\n",
    "    d = input_image.shape[0]\n",
    "    s = conv_filter.shape[0]\n",
    "    n = (d - s) / stride\n",
    "    feature_map = np.zeros((n, n))\n",
    "    for i in range(n):\n",
    "        for j in range(n):\n",
    "            feature_map[i, j] = 0\n",
    "    return feature_map"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convolution detects features in the image that match the filter, e.g. horizontal edges."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "input_image = np.zeros((20, 20))\n",
    "input_image[9:11, :] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQgAAAD8CAYAAACLgjpEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAADcRJREFUeJzt3X+oX/V9x/Hna/EXOp2m1tRfa6XL\nBC1dVkJcmRs6WxtFlnaULjI2tzniSoUVBsNtUEv3T8dwsqEotgva0aplW9ZA44+QDazQpkaJv1p/\nZGIxd9a0pjN1dtroe3/cE7m7+X68d9/zvfd779fnAy7fcz7n8z3nfbjwyjnne/N9p6qQpEF+ZtwF\nSFq6DAhJTQaEpCYDQlKTASGpyYCQ1GRASGoyICQ1GRCSmo4YdwGDHJWj6xiOG3cZ0sT6H/6b1+rV\nzDVvSQbEMRzHeblo3GVIE2tn7ZjXvF63GEnWJ3kyyZ4k1wzYfnSSO7vtO5O8p8/xJC2uoQMiyQrg\nRuAS4Bzg8iTnzJp2JfCjqvoF4Hrgr4c9nqTF1+cKYh2wp6qeqarXgDuADbPmbABu65b/CbgoyZz3\nPZKWhj4BcTrw3Iz1vd3YwDlVdRB4CXhHj2NKWkRL5iFlkk3AJoBjOHbM1UiCflcQU8CZM9bP6MYG\nzklyBPBzwIuDdlZVt1TV2qpaeyRH9yhL0qj0CYgHgNVJzkpyFLAR2Dprzlbgim7548C/lV9hJS0b\nQ99iVNXBJFcD9wArgM1V9XiSzwG7qmor8A/APybZA+xnOkQkLRNZiv+gn5CV5R9KSQtnZ+3gQO2f\n8xNF/y+GpCYDQlKTASGpyYCQ1GRASGoyICQ1GRCSmgwISU0GhKQmA0JSkwEhqcmAkNRkQEhqMiAk\nNRkQkpoMCElNBoSkJgNCUlOfzlpnJvn3JN9J8niSPxkw54IkLyXZ3f18pl+5khZTn74YB4E/raqH\nkhwPPJhke1V9Z9a8b1TVZT2OI2lMhr6CqKrnq+qhbvnHwHc5vLOWpGVsJM8guq7dvwzsHLD5g0ke\nTnJXknNHcTxJi6N3670kPwv8M/Dpqjowa/NDwLur6uUklwL/Cqxu7MfWe9IS0+sKIsmRTIfDl6vq\nX2Zvr6oDVfVyt7wNODLJyYP2Zes9aenp8ylGmO6c9d2q+tvGnHd180iyrjvewN6ckpaePrcYvwr8\nLvBokt3d2F8APw9QVTcz3Y/zk0kOAj8BNtqbU1o++vTmvB94y9ZdVXUDcMOwx5A0Xv4lpaQmA0JS\nkwEhqcmAkNRkQEhqMiAkNRkQkpoMCElNBoSkJgNCUpMBIanJgJDUZEBIajIgJDUZEJKaDAhJTQaE\npCYDQlJT74BI8mySR7vWersGbE+Sv0+yJ8kjST7Q95iSFkfvvhidC6vqh41tlzDdC2M1cB5wU/cq\naYlbjFuMDcCXatq3gBOTnLoIx5XU0ygCooB7kzzYdcea7XTguRnre7GHp7QsjOIW4/yqmkpyCrA9\nyRNVdd//dye23pOWnt5XEFU11b3uA7YA62ZNmQLOnLF+Rjc2ez+23pOWmL69OY9LcvyhZeBi4LFZ\n07YCv9d9mvErwEtV9Xyf40paHH1vMVYBW7r2m0cAX6mqu5P8MbzZfm8bcCmwB3gF+IOex5S0SHoF\nRFU9A/zSgPGbZywX8Kk+x5E0Hv4lpaQmA0JSkwEhqcmAkNRkQEhqMiAkNRkQkpoMCElNBoSkJgNC\nUpMBIanJgJDUZEBIajIgJDUZEJKaDAhJTQaEpCYDQlKTASGpaeiASHJ214/z0M+BJJ+eNeeCJC/N\nmPOZ/iVLWixDf2ltVT0JrAFIsoLpXhdbBkz9RlVdNuxxJI3PqG4xLgL+o6q+N6L9SVoCRhUQG4Hb\nG9s+mOThJHclObe1gySbkuxKsuunvDqisiT1kem2FT12kBwF/CdwblW9MGvbCcAbVfVykkuBv6uq\n1XPt84SsrPNyUa+6JLXtrB0cqP2Za94oriAuAR6aHQ4AVXWgql7ulrcBRyY5eQTHlLQIRhEQl9O4\nvUjyrnR9+ZKs64734giOKWkR9Gq91zXs/TBw1YyxmX05Pw58MslB4CfAxup7TyNp0fR+BrEQfAYh\nLazFfAYhaUIZEJKaDAhJTQaEpCYDQlJTr485F8ovvv8V7rln97jLkCbWuo+8Mq95XkFIajIgJDUZ\nEJKaDAhJTQaEpCYDQlKTASGpyYCQ1GRASGoyICQ1GRCSmgwISU3zCogkm5PsS/LYjLGVSbYnebp7\nPanx3iu6OU8nuWJUhUtaePO9grgVWD9r7BpgR9fnYke3/n8kWQlcC5wHrAOubQWJpKVnXgFRVfcB\n+2cNbwBu65ZvAz464K0fAbZX1f6q+hGwncODRtIS1ecZxKqqer5b/j6wasCc04HnZqzv7cYkLQMj\neUjZ9bro9f35M3tz/uDF10dRlqSe+gTEC0lOBehe9w2YMwWcOWP9jG7sMFV1S1Wtraq173zHih5l\nSRqVPgGxFTj0qcQVwNcGzLkHuDjJSd3DyYu7MUnLwHw/5rwd+CZwdpK9Sa4EPg98OMnTwIe6dZKs\nTfJFgKraD/wV8ED387luTNIyMK8vra2qyxubDuuPV1W7gD+asb4Z2DxUdZLGakl+q/VTjxzLR05b\nM+4ypIn1VL04r3n+qbWkJgNCUpMBIanJgJDUZEBIajIgJDUZEJKaDAhJTQaEpCYDQlKTASGpyYCQ\n1GRASGoyICQ1GRCSmgwISU0GhKSmOQOi0Xbvb5I8keSRJFuSnNh477NJHk2yO8muURYuaeHN5wri\nVg7vhrUdeF9VvR94Cvjzt3j/hVW1pqrWDleipHGZMyAGtd2rqnur6mC3+i2m+11ImjCjeAbxh8Bd\njW0F3JvkwSSbRnAsSYuo17daJ/lL4CDw5caU86tqKskpwPYkT3RXJIP2tQnYBHAMx/YpS9KIDH0F\nkeT3gcuA3+l6cx6mqqa6133AFmBda38zW+8dydHDliVphIYKiCTrgT8DfrOqXmnMOS7J8YeWmW67\n99iguZKWpvl8zDmo7d4NwPFM3zbsTnJzN/e0JNu6t64C7k/yMPBt4OtVdfeCnIWkBZHG3cFYnZCV\ndV4O6+onaUR21g4O1P7MNc+/pJTUZEBIajIgJDUZEJKaDAhJTQaEpCYDQlKTASGpyYCQ1GRASGoy\nICQ1GRCSmgwISU0GhKQmA0JSkwEhqcmAkNRkQEhqGrb13meTTHXfR7k7yaWN965P8mSSPUmuGWXh\nkhbesK33AK7vWuqtqaptszcmWQHcCFwCnANcnuScPsVKWlxDtd6bp3XAnqp6pqpeA+4ANgyxH0lj\n0ucZxNVdd+/NSU4asP104LkZ63u7MUnLxLABcRPwXmAN8DxwXd9CkmxKsivJrp/yat/dSRqBoQKi\nql6oqter6g3gCwxuqTcFnDlj/YxurLVPW+9JS8ywrfdOnbH6MQa31HsAWJ3krCRHARuBrcMcT9J4\nzNndu2u9dwFwcpK9wLXABUnWAAU8C1zVzT0N+GJVXVpVB5NcDdwDrAA2V9XjC3IWkhaErfektyFb\n70nqzYCQ1GRASGoyICQ1GRCSmgwISU0GhKQmA0JSkwEhqcmAkNRkQEhqMiAkNRkQkpoMCElNBoSk\nJgNCUpMBIanJgJDUNJ/vpNwMXAbsq6r3dWN3Amd3U04E/quq1gx477PAj4HXgYNVtXZEdUtaBHMG\nBNOt924AvnRooKp++9BykuuAl97i/RdW1Q+HLVDS+MwZEFV1X5L3DNqWJMAngN8YbVmSloK+zyB+\nDXihqp5ubC/g3iQPJtnU81iSFtl8bjHeyuXA7W+x/fyqmkpyCrA9yRNdM+DDdAGyCeAYju1ZlqRR\nGPoKIskRwG8Bd7bmVNVU97oP2MLgFn2H5tp6T1pi+txifAh4oqr2DtqY5Lgkxx9aBi5mcIs+SUvU\nnAHRtd77JnB2kr1Jruw2bWTW7UWS05Js61ZXAfcneRj4NvD1qrp7dKVLWmi23pPehmy9J6k3A0JS\nkwEhqcmAkNRkQEhqMiAkNRkQkpoMCElNBoSkJgNCUpMBIanJgJDUZEBIajIgJDUZEJKaDAhJTQaE\npKYl+Y1SSX4AfG/W8MnAJDbgmdTzgsk9t0k4r3dX1TvnmrQkA2KQJLsmsXXfpJ4XTO65Tep5DeIt\nhqQmA0JS03IKiFvGXcACmdTzgsk9t0k9r8Msm2cQkhbfcrqCkLTIlkVAJFmf5Mkke5JcM+56RiXJ\ns0keTbI7ya5x19NHks1J9iV5bMbYyiTbkzzdvZ40zhqH0TivzyaZ6n5vu5NcOs4aF9KSD4gkK4Ab\ngUuAc4DLk5wz3qpG6sKqWjMBH5vdCqyfNXYNsKOqVgM7uvXl5lYOPy+A67vf25qq2jZg+0RY8gHB\ndEfwPVX1TFW9BtwBbBhzTZqlqu4D9s8a3gDc1i3fBnx0UYsagcZ5vW0sh4A4HXhuxvrebmwSFHBv\nkgeTbBp3MQtgVVU93y1/n+mGzpPi6iSPdLcgy+7Wab6WQ0BMsvOr6gNM3z59Ksmvj7ughVLTH5dN\nykdmNwHvBdYAzwPXjbechbMcAmIKOHPG+hnd2LJXVVPd6z5gC9O3U5PkhSSnAnSv+8Zcz0hU1QtV\n9XpVvQF8gcn7vb1pOQTEA8DqJGclOQrYCGwdc029JTkuyfGHloGLgcfe+l3Lzlbgim75CuBrY6xl\nZA6FXudjTN7v7U1HjLuAuVTVwSRXA/cAK4DNVfX4mMsahVXAliQw/Xv4SlXdPd6ShpfkduAC4OQk\ne4Frgc8DX01yJdP/O/cT46twOI3zuiDJGqZvmZ4FrhpbgQvMv6SU1LQcbjEkjYkBIanJgJDUZEBI\najIgJDUZEJKaDAhJTQaEpKb/BTYL03J/XrABAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x112da06d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(input_image)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "conv_filter = np.zeros((4, 4))\n",
    "conv_filter[1:3, :] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.  0.  0.  0.]\n",
      " [ 1.  1.  1.  1.]\n",
      " [ 1.  1.  1.  1.]\n",
      " [ 0.  0.  0.  0.]]\n"
     ]
    }
   ],
   "source": [
    "print(conv_filter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "output = convolution(input_image, conv_filter, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAADPdJREFUeJzt3X2sZPVdx/H3x10eZIuwFKQUNgKV\nkGB9gGyQ1gYbV3FBwtakiRCrUJoQoig0GEIlsY1/tdbW+tCUIEVRCRApWNKAZaVtjIlsC+vyTMuC\nyEOXBymBSiOw7dc/5mxz93Lv7mXmnMNdf+9XMpkzc34z57u/uZ97zpw7O99UFZLa8yNvdgGS3hyG\nX2qU4ZcaZfilRhl+qVGGX2qU4ZcaZfilRhl+qVErx9zY3tmn9mXVmJuUmvK/vMyr9UqWMnbU8O/L\nKn4+68bcpNSUTXX7ksd62C81aqbwJ1mf5JtJtia5tK+iJA1v6vAnWQF8FjgVOA44K8lxfRUmaViz\n7PlPBLZW1aNV9SpwHbChn7IkDW2W8B8OPDHn9pPdfZL2AIOf7U9yHnAewL7sN/TmJC3RLHv+p4A1\nc24f0d23k6q6oqrWVtXavdhnhs1J6tMs4f8GcEySo5LsDZwJ3NxPWZKGNvVhf1VtT3IB8GVgBXBV\nVd3fW2WSBjXTe/6qugW4padaJI3IT/hJjTL8UqMMv9Qowy81yvBLjTL8UqMMv9Qowy81yvBLjTL8\nUqMMv9Qowy81yvBLjTL8UqMMv9Qowy81yvBLjTL8UqMMv9SoWdp1rUny1SQPJLk/yYV9FiZpWLN8\nged24OKq2pxkf+CuJBur6oGeapM0oKn3/FW1rao2d8vfBR7Edl3SHqOXdl1JjgSOBzYtsM52XdIy\nNPMJvyRvAb4AXFRVL81fb7suaXmaKfxJ9mIS/Guq6sZ+SpI0hlnO9gf4PPBgVX26v5IkjWGWPf8v\nAL8F/FKSLd3ltJ7qkjSwWRp1/huQHmuRNCI/4Sc1yvBLjTL8UqMMv9Qowy81yvBLjTL8UqMMv9Qo\nwy81yvBLjTL8UqMMv9Qowy81yvBLjTL8UqMMv9Qowy81yvBLjerjq7tXJPmPJF/qoyBJ4+hjz38h\nk249kvYgs35v/xHArwFX9lOOpLHMuuf/DHAJ8IMeapE0olmadpwOPFtVd+1m3HlJ7kxy52u8Mu3m\nJPVs1qYdZyR5DLiOSfOOf5g/yF590vI0S4vuj1TVEVV1JHAm8JWq+kBvlUkalH/nlxo1dbuuuarq\na8DX+nguSeNwzy81yvBLjTL8UqMMv9Qowy81yvBLjTL8UqMMv9Qowy81yvBLjTL8UqMMv9Qowy81\nyvBLjTL8UqMMv9Qowy81yvBLjZq1aceBSW5I8lCSB5O8q6/CJA1r1u/w+3Pgn6vq/Un2BvbroSZJ\nI5g6/EkOAE4GzgGoqleBV/spS9LQZjnsPwp4DvibrkvvlUlW9VSXpIHNEv6VwAnA56rqeOBl4NL5\ng2zXJS1Ps4T/SeDJqtrU3b6ByS+DndiuS1qeZmnX9TTwRJJju7vWAQ/0UpWkwc16tv/3gGu6M/2P\nAh+cvSRJY5gp/FW1BVjbUy2SRuQn/KRG9dKoc6leWbOKrRefNOYmpaa88qk7ljzWPb/UKMMvNcrw\nS40y/FKjDL/UKMMvNcrwS40y/FKjDL/UKMMvNcrwS40y/FKjDL/UqFH/V99Pr36Or//G5WNuUmrK\niVc9t+Sx7vmlRhl+qVGztuv6cJL7k9yX5Nok+/ZVmKRhTR3+JIcDvw+srap3AiuAM/sqTNKwZj3s\nXwn8aJKVTPr0fXv2kiSNYZbv7X8K+FPgcWAb8GJV3dZXYZKGNcth/2pgA5OefW8HViX5wALjftiu\n67nnvz99pZJ6Ncth/y8D/1lVz1XVa8CNwLvnD5rbruuQt66YYXOS+jRL+B8HTkqyX5Iwadf1YD9l\nSRraLO/5NzFpzrkZuLd7rit6qkvSwGZt1/VR4KM91SJpRH7CT2qU4ZcaNer/6rv3hUN4x/Xnj7lJ\nqSnffuEzSx7rnl9qlOGXGmX4pUYZfqlRhl9qlOGXGmX4pUYZfqlRhl9qlOGXGmX4pUYZfqlRo/7H\nnn2eeJmf/PAdY25Sasrz9fKSx7rnlxpl+KVG7Tb8Sa5K8myS++bcd1CSjUke7q5XD1umpL4tZc//\nt8D6efddCtxeVccAt3e3Je1Bdhv+qvpX4Dvz7t4AXN0tXw28r+e6JA1s2vf8h1bVtm75aeDQnuqR\nNJKZT/hVVQG12Pq57bpe45VZNyepJ9OG/5kkhwF0188uNnBuu6692GfKzUnq27Thvxk4u1s+G/hi\nP+VIGstS/tR3LfDvwLFJnkzyIeDjwK8keZhJw86PD1umpL7t9uO9VXXWIqvW9VyLpBH5CT+pUYZf\napThlxpl+KVGGX6pUYZfapThlxpl+KVGGX6pUYZfapThlxpl+KVGGX6pUYZfapThlxpl+KVGGX6p\nUYZfapThlxo1ba++TyZ5KMk9SW5KcuCwZUrq27S9+jYC76yqnwG+BXyk57okDWyqXn1VdVtVbe9u\n3gEcMUBtkgbUx3v+c4FbF1tpuy5peZop/EkuA7YD1yw2xnZd0vK026Ydi0lyDnA6sK5r1ilpDzJV\n+JOsBy4BfrGqvtdvSZLGMG2vvr8C9gc2JtmS5PKB65TUs2l79X1+gFokjchP+EmNMvxSowy/1CjD\nLzXK8EuNMvxSowy/1CjDLzXK8EuNMvxSowy/1CjDLzXK8EuNMvxSowy/1CjDLzXK8EuNMvxSo6Zq\n1zVn3cVJKsnBw5QnaSjTtusiyRrgFODxnmuSNIKp2nV1/ozJ13f7nf3SHmiq9/xJNgBPVdXdSxhr\nuy5pGXrDTTuS7Af8IZND/t2qqiuAKwB+LAd5lCAtE9Ps+d8BHAXcneQxJh16Nyd5W5+FSRrWG97z\nV9W9wI/vuN39AlhbVf/dY12SBjZtuy5Je7hp23XNXX9kb9VIGo2f8JMaZfilRhl+qVGGX2qU4Zca\nZfilRhl+qVGGX2qU4ZcaZfilRhl+qVGGX2qU4ZcaZfilRhl+qVGpGu9r9ZI8B/zXIqsPBpbDtwFZ\nx86sY2fLvY6fqKpDlvIEo4Z/V5LcWVVrrcM6rGOcOjzslxpl+KVGLafwX/FmF9Cxjp1Zx87+39Sx\nbN7zSxrXctrzSxrRqOFPsj7JN5NsTXLpAuv3SXJ9t35TkiMHqGFNkq8meSDJ/UkuXGDMe5O8mGRL\nd/mjvuuYs63HktzbbefOBdYnyV90c3JPkhN63v6xc/6dW5K8lOSieWMGm4+FWsAnOSjJxiQPd9er\nF3ns2d2Yh5OcPUAdn0zyUDfvNyU5cJHH7vI17KGOjyV5as78n7bIY3eZr9epqlEuwArgEeBoYG/g\nbuC4eWN+B7i8Wz4TuH6AOg4DTuiW9we+tUAd7wW+NNK8PAYcvIv1pwG3AgFOAjYN/Bo9zeRvxaPM\nB3AycAJw35z7/gS4tFu+FPjEAo87CHi0u17dLa/uuY5TgJXd8icWqmMpr2EPdXwM+IMlvHa7zNf8\ny5h7/hOBrVX1aFW9ClwHbJg3ZgNwdbd8A7AuSfosoqq2VdXmbvm7wIPA4X1uo2cbgL+riTuAA5Mc\nNtC21gGPVNViH8TqXS3cAn7uz8HVwPsWeOivAhur6jtV9QKwEVjfZx1VdVtVbe9u3sGkL+WgFpmP\npVhKvnYyZvgPB56Yc/tJXh+6H47pJv1F4K1DFdS9rTge2LTA6ncluTvJrUl+aqgagAJuS3JXkvMW\nWL+UeevLmcC1i6wbaz4ADq2qbd3y08ChC4wZc14AzmVyBLaQ3b2Gfbige/tx1SJvg97wfDR7wi/J\nW4AvABdV1UvzVm9mcuj7s8BfAv80YCnvqaoTgFOB301y8oDbWlSSvYEzgH9cYPWY87GTmhzTvql/\nkkpyGbAduGaRIUO/hp9j0h3754BtwKf6eNIxw/8UsGbO7SO6+xYck2QlcADwfN+FJNmLSfCvqaob\n56+vqpeq6n+65VuAvZIc3Hcd3fM/1V0/C9zE5PBtrqXMWx9OBTZX1TML1DjafHSe2fHWprt+doEx\no8xLknOA04Hf7H4Rvc4SXsOZVNUzVfX9qvoB8NeLPP8bno8xw/8N4JgkR3V7mTOBm+eNuRnYcdb2\n/cBXFpvwaXXnED4PPFhVn15kzNt2nGtIciKTeRril9CqJPvvWGZygum+ecNuBn67O+t/EvDinEPi\nPp3FIof8Y83HHHN/Ds4GvrjAmC8DpyRZ3R0Gn9Ld15sk64FLgDOq6nuLjFnKazhrHXPP8fz6Is+/\nlHztrI8zlG/gTOZpTM6uPwJc1t33x0wmF2BfJoedW4GvA0cPUMN7mBxG3gNs6S6nAecD53djLgDu\nZ3LG9A7g3QPNx9HdNu7utrdjTubWEuCz3ZzdC6wdoI5VTMJ8wJz7RpkPJr9wtgGvMXmf+iEm53lu\nBx4G/gU4qBu7FrhyzmPP7X5WtgIfHKCOrUzeR+/4Odnxl6i3A7fs6jXsuY6/7177e5gE+rD5dSyW\nr11d/ISf1KhmT/hJrTP8UqMMv9Qowy81yvBLjTL8UqMMv9Qowy816v8AYXq4N93bXRMAAAAASUVO\nRK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x112e5f310>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(output)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "More complex features such as a cross can be detected with a combination of different filters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "input_image = np.zeros((20, 20))\n",
    "input_image[9:11, :] = 1\n",
    "input_image[:, 9:11] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQgAAAD8CAYAAACLgjpEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAADfFJREFUeJzt3XuoZeV5x/Hvr+MNp9J4iRNvNZJO\nBRPiNAxjQ23RmnhDMklJ05HSTlrL2BChgUKxLWhI/0kpVmgVJZdBUxI1vUwykNFxsAUjJOoo4y3x\nMh0MMyfGSTTVGKNmzNM/zho5PbNf53Svfc7e5/j9wLDXete713oWB36stfae/aSqkKRBfmncBUia\nXAaEpCYDQlKTASGpyYCQ1GRASGoyICQ1GRCSmgwISU2HjLuAQQ7L4XUEy8ddxlver7/35XnZ75MP\nHzkv+9XcvcJPea1ezcHmTWRAHMFyzsp54y7jLW/r1h3zst8LTlw1L/vV3N1bd81pXq9bjCQXJnki\nyc4kVw7YfniS27rt9yZ5Z5/jSVpYQwdEkmXA9cBFwBnApUnOmDXtMuDHVfVrwLXA3w97PEkLr88V\nxBpgZ1XtqqrXgFuBtbPmrAVu7pb/DTgvyUHveyRNhj4BcRKwe8b6nm5s4Jyq2ge8ABzb45iSFtDE\nPKRMsgHYAHAEPuWWJkGfK4gp4JQZ6yd3YwPnJDkE+BXguUE7q6rPVdXqqlp9KIf3KEvSqPQJiPuB\nlUlOS3IYsA7YPGvOZmB9t/xR4D/Ln7CSFo2hbzGqal+SK4CtwDJgY1U9luQzwPaq2gx8EfiXJDuB\n55kOEUmLRK9nEFW1Bdgya+yqGcuvAL/f5xiSxsf/iyGpyYCQ1GRASGoyICQ1GRCSmgwISU0GhKQm\nA0JSkwEhqcmAkNRkQEhqMiAkNRkQkpoMCElNBoSkJgNCUpMBIanJgJDU1Kez1ilJ/ivJd5I8luQv\nBsw5J8kLSXZ0/64atC9Jk6nPb1LuA/6yqh5MchTwQJJtVfWdWfO+WVWX9DiOpDEZ+gqiqp6pqge7\n5Z8A3+XAzlqSFrGRPIPounb/BnDvgM3vT/JQktuTvHsUx5O0MHq33kvyy8C/A5+qqhdnbX4QOLWq\nXkpyMfA1YGVjP7bekyZMryuIJIcyHQ5frqr/mL29ql6sqpe65S3AoUmOG7QvW+9Jk6fPpxhhunPW\nd6vqHxtz3tHNI8ma7ngDe3NKmjx9bjF+C/gj4JEkO7qxvwF+FaCqbmS6H+cnkuwDfgasszentHj0\n6c15D5CDzLkOuG7YY0gaL79JKanJgJDUZEBIajIgJDUZEJKaDAhJTQaEpCYDQlKTASGpyYCQ1GRA\nSGoyICQ1GRCSmgwISU0GhKQmA0JSkwEhqcmAkNTUOyCSPJ3kka613vYB25Pkn5LsTPJwkvf1Paak\nhdG7L0bn3Kr6UWPbRUz3wlgJnAXc0L1KmnALcYuxFvhSTfs28LYkJyzAcSX1NIqAKODOJA903bFm\nOwnYPWN9D/bwlBaFUdxinF1VU0mOB7Ylebyq7v7/7sTWe9Lk6X0FUVVT3eteYBOwZtaUKeCUGesn\nd2Oz92PrPWnC9O3NuTzJUfuXgfOBR2dN2wz8cfdpxm8CL1TVM32OK2lh9L3FWAFs6tpvHgJ8paru\nSPLn8Eb7vS3AxcBO4GXgT3oeU9IC6RUQVbULOHPA+I0zlgv4ZJ/jSBoPv0kpqcmAkNRkQEhqMiAk\nNRkQkpoMCElNBoSkJgNCUpMBIanJgJDUZEBIajIgJDUZEJKaDAhJTQaEpCYDQlKTASGpyYCQ1GRA\nSGoaOiCSnN7149z/78Ukn5o155wkL8yYc1X/kiUtlKF/tLaqngBWASRZxnSvi00Dpn6zqi4Z9jiS\nxmdUtxjnAf9dVd8b0f4kTYBRBcQ64JbGtvcneSjJ7Une3dpBkg1JtifZ/nNeHVFZkvroHRBJDgM+\nBPzrgM0PAqdW1ZnAPwNfa+3H1nvS5BnFFcRFwINV9ezsDVX1YlW91C1vAQ5NctwIjilpAYwiIC6l\ncXuR5B3p+vIlWdMd77kRHFPSAujVeq9r2PtB4PIZYzP7cn4U+ESSfcDPgHVdKz5Ji0Df3pw/BY6d\nNTazL+d1wHV9jiFpfPwmpaQmA0JSkwEhqcmAkNRkQEhqyiR+6rj6zCPqvq2njLsMaclac8Futj/0\nSg42zysISU0GhKQmA0JSkwEhqcmAkNRkQEhqMiAkNRkQkpoMCElNBoSkJgNCUpMBIalpTgGRZGOS\nvUkenTF2TJJtSZ7qXo9uvHd9N+epJOtHVbik+TfXK4ibgAtnjV0J3FVVK4G7uvX/I8kxwNXAWcAa\n4OpWkEiaPHMKiKq6G3h+1vBa4OZu+WbgwwPeegGwraqer6ofA9s4MGgkTag+zyBWVNUz3fIPgBUD\n5pwE7J6xvqcbk7QIjOQhZdfrotcvz8zszfnD514fRVmSeuoTEM8mOQGge907YM4UMPOnoU7uxg4w\nszfn249d1qMsSaPSJyA2A/s/lVgPfH3AnK3A+UmO7h5Ont+NSVoE5vox5y3At4DTk+xJchnwWeCD\nSZ4CPtCtk2R1ki8AVNXzwN8B93f/PtONSVoE5tR6r6oubWw6b8Dc7cCfzVjfCGwcqjpJY9WrN+d8\nefLhI7ngxFXjLuMtb+v3d8zLfv3bjt+T9dyc5vlVa0lNBoSkJgNCUpMBIanJgJDUZEBIajIgJDUZ\nEJKaDAhJTQaEpCYDQlKTASGpyYCQ1GRASGoyICQ1GRCSmgwISU0HDYhG271/SPJ4koeTbErytsZ7\nn07ySJIdSbaPsnBJ828uVxA3cWA3rG3Ae6rqvcCTwF+/yfvPrapVVbV6uBIljctBA2JQ272qurOq\n9nWr32a634WkJWYUzyD+FLi9sa2AO5M8kGTDCI4laQH1+lXrJH8L7AO+3JhydlVNJTke2Jbk8e6K\nZNC+NgAbAI7gyD5lSRqRoa8gknwcuAT4w6435wGqaqp73QtsAta09jez9d6hHD5sWZJGaKiASHIh\n8FfAh6rq5cac5UmO2r/MdNu9RwfNlTSZ5vIx56C2e9cBRzF927AjyY3d3BOTbOneugK4J8lDwH3A\nN6rqjnk5C0nz4qDPIBpt977YmPt94OJueRdwZq/qJI2V36SU1GRASGoyICQ1GRCSmgwISU0GhKQm\nA0JSkwEhqcmAkNRkQEhqMiAkNRkQkpoMCElNBoSkJgNCUpMBIanJgJDUZEBIahq29d6nk0x1v0e5\nI8nFjfdemOSJJDuTXDnKwiXNv2Fb7wFc27XUW1VVW2ZvTLIMuB64CDgDuDTJGX2KlbSwhmq9N0dr\ngJ1VtauqXgNuBdYOsR9JY9LnGcQVXXfvjUmOHrD9JGD3jPU93ZikRWLYgLgBeBewCngGuKZvIUk2\nJNmeZPvPebXv7iSNwFABUVXPVtXrVfUL4PMMbqk3BZwyY/3kbqy1T1vvSRNm2NZ7J8xY/QiDW+rd\nD6xMclqSw4B1wOZhjidpPA7aWatrvXcOcFySPcDVwDlJVgEFPA1c3s09EfhCVV1cVfuSXAFsBZYB\nG6vqsXk5C0nzYt5a73XrW4ADPgKVtDj4TUpJTQaEpCYDQlKTASGpyYCQ1GRASGoyICQ1GRCSmgwI\nSU0GhKQmA0JSkwEhqcmAkNRkQEhqMiAkNRkQkpoMCElNBoSkprn8JuVG4BJgb1W9pxu7DTi9m/I2\n4H+qatWA9z4N/AR4HdhXVatHVLekBXDQgGC69d51wJf2D1TVH+xfTnIN8MKbvP/cqvrRsAVKGp+5\n/Gjt3UneOWhbkgAfA353tGVJmgR9n0H8NvBsVT3V2F7AnUkeSLKh57EkLbC53GK8mUuBW95k+9lV\nNZXkeGBbkse7ZsAH6AJkA8ARHNmzLEmjMPQVRJJDgN8DbmvNqaqp7nUvsInBLfr2z7X1njRh+txi\nfAB4vKr2DNqYZHmSo/YvA+czuEWfpAl10IDoWu99Czg9yZ4kl3Wb1jHr9iLJiUn2d9JaAdyT5CHg\nPuAbVXXH6EqXNN+Gbb1HVX18wNgbrfeqahdwZs/6JI2R36SU1GRASGoyICQ1GRCSmgwISU0GhKQm\nA0JSkwEhqcmAkNRkQEhqMiAkNRkQkpoMCElNBoSkJgNCUpMBIanJgJDUlKoadw0HSPJD4Huzho8D\nlmIDnqV6XrB0z20pnNepVfX2g02ayIAYJMn2pdi6b6meFyzdc1uq5zWItxiSmgwISU2LKSA+N+4C\n5slSPS9Yuue2VM/rAIvmGYSkhbeYriAkLbBFERBJLkzyRJKdSa4cdz2jkuTpJI8k2ZFk+7jr6SPJ\nxiR7kzw6Y+yYJNuSPNW9Hj3OGofROK9PJ5nq/m47klw8zhrn08QHRJJlwPXARcAZwKVJzhhvVSN1\nblWtWgIfm90EXDhr7ErgrqpaCdzVrS82N3HgeQFc2/3dVlXVlgHbl4SJDwimO4LvrKpdVfUacCuw\ndsw1aZaquht4ftbwWuDmbvlm4MMLWtQINM7rLWMxBMRJwO4Z63u6saWggDuTPJBkw7iLmQcrquqZ\nbvkHTDd0XiquSPJwdwuy6G6d5moxBMRSdnZVvY/p26dPJvmdcRc0X2r647Kl8pHZDcC7gFXAM8A1\n4y1n/iyGgJgCTpmxfnI3tuhV1VT3uhfYxPTt1FLybJITALrXvWOuZySq6tmqer2qfgF8nqX3d3vD\nYgiI+4GVSU5LchiwDtg85pp6S7I8yVH7l4HzgUff/F2LzmZgfbe8Hvj6GGsZmf2h1/kIS+/v9oZD\nxl3AwVTVviRXAFuBZcDGqnpszGWNwgpgUxKY/jt8paruGG9Jw0tyC3AOcFySPcDVwGeBrya5jOn/\nnfux8VU4nMZ5nZNkFdO3TE8Dl4+twHnmNyklNS2GWwxJY2JASGoyICQ1GRCSmgwISU0GhKQmA0JS\nkwEhqel/ARiL50/jSCiIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x112e85250>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(input_image)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "conv_filter1 = np.zeros((4, 4))\n",
    "conv_filter1[1:3, :] = 1\n",
    "conv_filter2 = np.zeros((4, 4))\n",
    "conv_filter2[:, 1:3] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.  0.  0.  0.]\n",
      " [ 1.  1.  1.  1.]\n",
      " [ 1.  1.  1.  1.]\n",
      " [ 0.  0.  0.  0.]]\n"
     ]
    }
   ],
   "source": [
    "print(conv_filter1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.  1.  1.  0.]\n",
      " [ 0.  1.  1.  0.]\n",
      " [ 0.  1.  1.  0.]\n",
      " [ 0.  1.  1.  0.]]\n"
     ]
    }
   ],
   "source": [
    "print(conv_filter2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAADYJJREFUeJzt3W2MXOV5h/Hrjm1w7TjYDpSAbQpE\nCIlGTbEsQ9LIjeKWGBfhRMoHo6aFEAmhlhYqKuQUqYn6KWmaNH2JErlAQ1sLUAkkVgQB10mEKtVL\nwLUBYxIMpQZjMA0UKAi/hLsf5jgaLzv2eM6Ld/1cP2m1Z+Y8M+feZ+a/58yZ0dyRmUgqz7uOdQGS\njg3DLxXK8EuFMvxSoQy/VCjDLxXK8EuFMvxSoQy/VKjpXW7shDgxZzK7y00el96ee/RzuH9OC4U0\nbMbro93uXf/7RrOFTGFv8Qb7cm8MM7bT8M9kNhfE8i43eVx682MXHPVtnl821PPhmDr9gdE+aj7r\n7rGGK5m6xnLj0GM97JcKVSv8EbEiIn4SETsiYk1TRUlq38jhj4hpwNeBi4HzgMsi4rymCpPUrjp7\n/qXAjsx8OjP3AbcDq5opS1Lb6oR/AfBs3+XnquskTQGtn+2PiKuAqwBmMqvtzUkaUp09/y5gUd/l\nhdV1h8jMtZm5JDOXzODEGpuT1KQ64f8xcE5EnBURJwCrgfXNlCWpbSMf9mfmgYi4BrgPmAbckpnb\nGqtMUqtqvebPzHuAexqqRVKH/ISfVCjDLxXK8EuFMvxSoQy/VCjDLxXK8EuFMvxSoQy/VCjDLxXK\n8EuFMvxSoQy/VCjDLxXK8EuFMvxSoQy/VCjDLxXK8EuFqtOua1FE/DAiHo+IbRFxbZOFSWpXnS/w\nPABcn5mbI2IO8HBEbMjMxxuqTVKLRt7zZ+buzNxcLb8ObMd2XdKU0Ui7rog4EzgfGJtgne26pEmo\n9gm/iHg38G3gusx8bfx623VJk1Ot8EfEDHrBX5eZdzVTkqQu1DnbH8DNwPbM/GpzJUnqQp09/28A\nvwd8LCK2VD8rG6pLUsvqNOr8dyAarEVSh/yEn1Qowy8VyvBLhTL8UqEMv1Qowy8VyvBLhTL8UqEM\nv1Qowy8VyvBLhTL8UqEMv1Qowy8VyvBLhTL8UqEMv1Qowy8Vqomv7p4WEf8ZEd9roiBJ3Whiz38t\nvW49kqaQut/bvxD4HeCmZsqR1JW6e/6vATcAbzdQi6QO1WnacQmwJzMfPsK4qyLioYh4aD97R92c\npIbVbdpxaUQ8A9xOr3nHv4wfZK8+aXKq06L7c5m5MDPPBFYDP8jMTzdWmaRW+T6/VKiR23X1y8wf\nAT9q4r4kdcM9v1Qowy8VyvBLhTL8UqEMv1Qowy8VyvBLhTL8UqEMv1Qowy8VyvBLhTL8UqEMv1Qo\nwy8VyvBLhTL8UqEMv1Qowy8Vqm7TjrkRcWdEPBER2yPiQ00VJqlddb/D72+A72fmpyLiBGBWAzVJ\n6sDI4Y+Ik4BlwBUAmbkP2NdMWZLaVuew/yzgJeAfqy69N0XE7IbqktSyOuGfDiwGvpGZ5wNvAGvG\nD7JdlzQ51Qn/c8BzmTlWXb6T3j+DQ9iuS5qc6rTregF4NiLOra5aDjzeSFWSWlf3bP8fAeuqM/1P\nA5+pX5KkLtQKf2ZuAZY0VIukDvkJP6lQjTTqHNbeRbPZcf2FXW7yuDTzjNeP+jbLFuxsoZJmPXjG\nGSPd7vllPqcO2vuVTUOPdc8vFcrwS4Uy/FKhDL9UKMMvFcrwS4Uy/FKhDL9UKMMvFcrwS4Uy/FKh\nDL9UKMMvFSoys7ONLfngzHzwvkWdbU8qzdKPP8tDW9+KYca655cKZfilQtVt1/UnEbEtIh6LiNsi\nYmZThUlq18jhj4gFwB8DSzLzA8A0YHVThUlqV93D/unAL0XEdHp9+p6vX5KkLtT53v5dwF8BO4Hd\nwKuZeX9ThUlqV53D/nnAKno9+04HZkfEpycY94t2XS/97OejVyqpUXUO+38L+K/MfCkz9wN3AR8e\nP6i/Xdcp751WY3OSmlQn/DuBCyNiVkQEvXZd25spS1Lb6rzmH6PXnHMz8Gh1X2sbqktSy+q26/o8\n8PmGapHUIT/hJxXK8EuF6rRX36OvnML777i6y00el0bp1bd0KvTq2zVar763ds5puJKp6/lXvjb0\nWPf8UqEMv1Qowy8VyvBLhTL8UqEMv1Qowy8VyvBLhTL8UqEMv1Qowy8VyvBLheq0Xdd7Yn5eEMs7\n297x6s1PXnDUt3l+2VAdnI6p0x8Y7bk46+6xhiuZusZyI6/ly7brkjSY4ZcKdcTwR8QtEbEnIh7r\nu25+RGyIiCer3/PaLVNS04bZ838LWDHuujXAxsw8B9hYXZY0hRwx/Jn5APDyuKtXAbdWy7cCn2i4\nLkktG/U1/6mZubtafgE4taF6JHWk9gm/7L1XOPA9mv52XfvZW3dzkhoyavhfjIjTAKrfewYN7G/X\nNYMTR9ycpKaNGv71wOXV8uXAd5spR1JXhnmr7zbgP4BzI+K5iPgs8EXgtyPiSXoNO7/YbpmSmnbE\n7+3PzMsGrPJzutIU5if8pEIZfqlQhl8qlOGXCmX4pUIZfqlQhl8qlOGXCmX4pUIZfqlQhl8qlOGX\nCmX4pUIZfqlQhl8qlOGXCmX4pUIZfqlQhl8q1Ki9+r4cEU9ExCMRcXdEzG23TElNG7VX3wbgA5n5\na8BPgc81XJeklo3Uqy8z78/MA9XFTcDCFmqT1KImXvNfCdw7aKXtuqTJqVb4I+JG4ACwbtAY23VJ\nk9MRm3YMEhFXAJcAy6tmnZKmkJHCHxErgBuA38zMN5stSVIXRu3V9/fAHGBDRGyJiG+2XKekho3a\nq+/mFmqR1CE/4ScVyvBLhTL8UqEMv1Qowy8VyvBLhTL8UqEMv1Qowy8VyvBLhTL8UqEMv1Qowy8V\nyvBLhTL8UqEMv1Qowy8VyvBLhRqpXVffuusjIiPi5HbKk9SWUdt1ERGLgIuAnQ3XJKkDI7Xrqvw1\nva/v9jv7pSlopNf8EbEK2JWZW4cYa7suaRI66qYdETEL+DN6h/xHlJlrgbUA74n5HiVIk8Qoe/73\nA2cBWyPiGXodejdHxPuaLExSu456z5+ZjwK/fPBy9Q9gSWb+T4N1SWrZqO26JE1xo7br6l9/ZmPV\nSOqMn/CTCmX4pUIZfqlQhl8qlOGXCmX4pUIZfqlQhl8qlOGXCmX4pUIZfqlQhl8qlOGXCmX4pUIZ\nfqlQkdnd1+pFxEvAfw9YfTIwGb4NyDoOZR2Hmux1/EpmnjLMHXQa/sOJiIcyc4l1WId1dFOHh/1S\noQy/VKjJFP61x7qAinUcyjoOddzUMWle80vq1mTa80vqUKfhj4gVEfGTiNgREWsmWH9iRNxRrR+L\niDNbqGFRRPwwIh6PiG0Rce0EYz4aEa9GxJbq58+brqNvW89ExKPVdh6aYH1ExN9Wc/JIRCxuePvn\n9v2dWyLitYi4btyY1uZjohbwETE/IjZExJPV73kDbnt5NebJiLi8hTq+HBFPVPN+d0TMHXDbwz6G\nDdTxhYjY1Tf/Kwfc9rD5eofM7OQHmAY8BZwNnABsBc4bN+YPgG9Wy6uBO1qo4zRgcbU8B/jpBHV8\nFPheR/PyDHDyYdavBO4FArgQGGv5MXqB3nvFncwHsAxYDDzWd91fAmuq5TXAlya43Xzg6er3vGp5\nXsN1XARMr5a/NFEdwzyGDdTxBeBPh3jsDpuv8T9d7vmXAjsy8+nM3AfcDqwaN2YVcGu1fCewPCKi\nySIyc3dmbq6WXwe2Awua3EbDVgH/lD2bgLkRcVpL21oOPJWZgz6I1bicuAV8//PgVuATE9z048CG\nzHw5M18BNgArmqwjM+/PzAPVxU30+lK2asB8DGOYfB2iy/AvAJ7tu/wc7wzdL8ZUk/4q8N62Cqpe\nVpwPjE2w+kMRsTUi7o2IX22rBiCB+yPi4Yi4aoL1w8xbU1YDtw1Y19V8AJyambur5ReAUycY0+W8\nAFxJ7whsIkd6DJtwTfXy45YBL4OOej6KPeEXEe8Gvg1cl5mvjVu9md6h7weBvwO+02IpH8nMxcDF\nwB9GxLIWtzVQRJwAXAr86wSru5yPQ2TvmPaYviUVETcCB4B1A4a0/Rh+g1537F8HdgNfaeJOuwz/\nLmBR3+WF1XUTjomI6cBJwM+aLiQiZtAL/rrMvGv8+sx8LTP/r1q+B5gRESc3XUd1/7uq33uAu+kd\nvvUbZt6acDGwOTNfnKDGzuaj8uLBlzbV7z0TjOlkXiLiCuAS4Herf0TvMMRjWEtmvpiZP8/Mt4F/\nGHD/Rz0fXYb/x8A5EXFWtZdZDawfN2Y9cPCs7aeAHwya8FFV5xBuBrZn5lcHjHnfwXMNEbGU3jy1\n8U9odkTMObhM7wTTY+OGrQd+vzrrfyHwat8hcZMuY8Ahf1fz0af/eXA58N0JxtwHXBQR86rD4Iuq\n6xoTESuAG4BLM/PNAWOGeQzr1tF/jueTA+5/mHwdqokzlEdxJnMlvbPrTwE3Vtf9Bb3JBZhJ77Bz\nB/AgcHYLNXyE3mHkI8CW6mclcDVwdTXmGmAbvTOmm4APtzQfZ1fb2Fpt7+Cc9NcSwNerOXsUWNJC\nHbPphfmkvus6mQ96/3B2A/vpvU79LL3zPBuBJ4F/A+ZXY5cAN/Xd9srqubID+EwLdeyg9zr64PPk\n4DtRpwP3HO4xbLiOf64e+0foBfq08XUMytfhfvyEn1SoYk/4SaUz/FKhDL9UKMMvFcrwS4Uy/FKh\nDL9UKMMvFer/ARsr3RRcboFTAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x112f89610>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "output1 = convolution(input_image, conv_filter1, 1)\n",
    "plt.imshow(output1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAADYZJREFUeJzt3WusXNV5h/Hnxdf6UjAlJYBNgRQh\nkfQCsohJIzeKW2pchBMpUo2a1oRICLW0EFEhp0hN1E9J06bXKMgFUtpagEqgsSJocCERqlSbi2tj\njElsKDUYc2mTgjGKL+Hth9kmcw5n7DmzLxx3PT/paPbMXnvW6zXnf/ZlxrMiM5FUnhPe7QIkvTsM\nv1Qowy8VyvBLhTL8UqEMv1Qowy8VyvBLhTL8UqGmd9nZzJiVs5nbZZeqHFg02rj/3IJXR9pu2w/e\nM+ltZj2/f6S+9GM/ZD8H80AM07bT8M9mLh+MZV12qcquG5aMtN0jv3HzSNu9765rJr3Nz35m40h9\n6cc25YNDt/WwXypUrfBHxPKI+G5E7IqINU0VJal9I4c/IqYBXwEuBc4HroiI85sqTFK76uz5LwJ2\nZeazmXkQuBNY2UxZktpWJ/xnAM/33X+hekzScaD1q/0RcTVwNcBs5rTdnaQh1dnz7wEW9d1fWD02\nRmauzczFmbl4BrNqdCepSXXC/yhwbkScHREzgVXA+mbKktS2kQ/7M/NwRFwLfAuYBtyWmdsbq0xS\nq2qd82fmfcB9DdUiqUN+wk8qlOGXCmX4pUIZfqlQhl8qlOGXCmX4pUIZfqlQhl8qlOGXCmX4pUIZ\nfqlQhl8qlOGXCmX4pUIZfqlQhl8qlOGXCmX4pULVma5rUUR8OyKeiojtEXFdk4VJaledL/A8DNyQ\nmZsjYj7weERsyMynGqpNUotG3vNn5t7M3Fwt7wN24HRd0nGjkem6IuIs4AJg0wTrnK5LmoJqX/CL\niHnA14HrM/P18eudrkuammqFPyJm0Av+usy8p5mSJHWhztX+AG4FdmTml5srSVIX6uz5fwn4LeCj\nEbGl+lnRUF2SWlZnos5/A6LBWiR1yE/4SYUy/FKhDL9UKMMvFcrwS4Uy/FKhDL9UKMMvFcrwS4Uy\n/FKhDL9UKMMvFcrwS4Uy/FKhDL9UKMMvFcrwS4Uy/FKhmvjq7mkR8R8R8c0mCpLUjSb2/NfRm61H\n0nGk7vf2LwR+HbilmXIkdaXunv8vgBuBtxqoRVKH6kzacRnwSmY+fox2V0fEYxHx2CEOjNqdpIbV\nnbTj8oh4DriT3uQd/zi+kXP1SVNTnSm6P5uZCzPzLGAV8FBmfrKxyiS1yvf5pUKNPF1Xv8z8DvCd\nJp5LUjfc80uFMvxSoQy/VCjDLxXK8EuFMvxSoQy/VCjDLxXK8EuFMvxSoQy/VCjDLxXK8EuFMvxS\noQy/VKhG/j//sN46aS5vfvSDXXapyuwz9035/t78uL8bdb310Mah27rnlwpl+KVC1Z2046SIuDsi\nno6IHRFxcVOFSWpX3XP+vwT+JTM/EREzgTkN1CSpAyOHPyJOBJYCVwJk5kHgYDNlSWpbncP+s4FX\nga9Vs/TeEhFzG6pLUsvqhH86cCHw1cy8ANgPrBnfaMx0XQfeqNGdpCbVCf8LwAuZuam6fze9PwZj\njJmua9a8Gt1JalKd6bpeAp6PiPOqh5YBTzVSlaTW1b3a/3vAuupK/7PAp+qXJKkLtcKfmVuAxQ3V\nIqlDfsJPKlSn/7Hn0Hx4cWl02aUqS8/Y3Wl/F43Q38NL399CJWU59Ojwbd3zS4Uy/FKhDL9UKMMv\nFcrwS4Uy/FKhDL9UKMMvFcrwS4Uy/FKhDL9UKMMvFcrwS4Uy/FKhDL9UKMMvFarudF2fiYjtEfFk\nRNwREbObKkxSu0YOf0ScAfw+sDgzPwBMA1Y1VZikdtU97J8O/ERETKc3T9+L9UuS1IU639u/B/hT\nYDewF3gtMx9oqjBJ7apz2L8AWElvzr7TgbkR8ckJ2r09XdeP3tg/eqWSGlXnsP9XgP/MzFcz8xBw\nD/Ch8Y36p+uaNs95PKWpok74dwNLImJORAS96bp2NFOWpLbVOeffRG9yzs3Atuq51jZUl6SW1Z2u\n63PA5xqqRVKH/ISfVCjDLxXK8EuFMvxSoQy/VCjDLxXK8EuFMvxSoQy/VCjDLxXK8EuFMvxSoQy/\nVCjDLxXK8EuFqvX/+Sdrxj44/eHssktVHjnzzNE2HHGzR/ZMfkN/N+p7dd/wbd3zS4Uy/FKhjhn+\niLgtIl6JiCf7Hjs5IjZExM7qdkG7ZUpq2jB7/r8Dlo97bA3wYGaeCzxY3Zd0HDlm+DPzYeD74x5e\nCdxeLd8OfKzhuiS1bNRz/lMzc2+1/BJwakP1SOpI7Qt+mZnAwPdo+qfrOnTgjbrdSWrIqOF/OSJO\nA6huXxnUsH+6rhmz5o3YnaSmjRr+9cDqank18I1mypHUlWHe6rsD+HfgvIh4ISI+DXwB+NWI2Elv\nws4vtFumpKYd8+O9mXnFgFXLGq5FUof8hJ9UKMMvFarT/9V3wv/uZ869m7rsUpUXly4ZbcOLR9vs\nh7vnT3qbOfduHK0zve2E3D982xbrkDSFGX6pUIZfKpThlwpl+KVCGX6pUIZfKpThlwpl+KVCGX6p\nUIZfKpThlwpl+KVCGX6pUIZfKpThlwo16lx9X4qIpyPiiYi4NyJOardMSU0bda6+DcAHMvPnge8B\nn224LkktG2muvsx8IDMPV3c3AgtbqE1Si5o4578KuH/QyjHTdXGgge4kNaFW+CPiJuAwsG5QmzHT\ndTGrTneSGjTyt/dGxJXAZcCyarJOSceRkcIfEcuBG4Ffzsw3my1JUhdGnavvb4D5wIaI2BIRN7dc\np6SGjTpX360t1CKpQ37CTyqU4ZcKZfilQhl+qVCGXyqU4ZcKZfilQhl+qVCGXyqU4ZcKZfilQhl+\nqVCGXyqU4ZcKZfilQhl+qVCGXyqU4ZcKNdJ0XX3rboiIjIhT2ilPUltGna6LiFgEXALsbrgmSR0Y\nabquyp/T+/puv7NfOg6NdM4fESuBPZm5dYi2TtclTUGTnrQjIuYAf0jvkP+YMnMtsBbgJ+NkjxKk\nKWKUPf/7gLOBrRHxHL0ZejdHxHubLExSuya958/MbcBPH7lf/QFYnJn/3WBdklo26nRdko5zo07X\n1b/+rMaqkdQZP+EnFcrwS4Uy/FKhDL9UKMMvFcrwS4Uy/FKhDL9UKMMvFcrwS4Uy/FKhDL9UKMMv\nFcrwS4Uy/FKhIrO7r9WLiFeB/xqw+hRgKnwbkHWMZR1jTfU6fiYz3zPME3Qa/qOJiMcyc7F1WId1\ndFOHh/1SoQy/VKipFP6173YBFesYyzrG+n9Tx5Q555fUram055fUoU7DHxHLI+K7EbErItZMsH5W\nRNxVrd8UEWe1UMOiiPh2RDwVEdsj4roJ2nwkIl6LiC3Vzx81XUdfX89FxLaqn8cmWB8R8VfVmDwR\nERc23P95ff/OLRHxekRcP65Na+Mx0RTwEXFyRGyIiJ3V7YIB266u2uyMiNUt1PGliHi6Gvd7I+Kk\nAdse9TVsoI7PR8SevvFfMWDbo+brHTKzkx9gGvAMcA4wE9gKnD+uze8AN1fLq4C7WqjjNODCank+\n8L0J6vgI8M2OxuU54JSjrF8B3A8EsATY1PJr9BK994o7GQ9gKXAh8GTfY38CrKmW1wBfnGC7k4Fn\nq9sF1fKChuu4BJheLX9xojqGeQ0bqOPzwB8M8dodNV/jf7rc818E7MrMZzPzIHAnsHJcm5XA7dXy\n3cCyiIgmi8jMvZm5uVreB+wAzmiyj4atBP4+ezYCJ0XEaS31tQx4JjMHfRCrcTnxFPD9vwe3Ax+b\nYNNfAzZk5vcz8wfABmB5k3Vk5gOZebi6u5HevJStGjAewxgmX2N0Gf4zgOf77r/AO0P3dptq0F8D\nfqqtgqrTiguATROsvjgitkbE/RHx/rZqABJ4ICIej4irJ1g/zLg1ZRVwx4B1XY0HwKmZubdafgk4\ndYI2XY4LwFX0jsAmcqzXsAnXVqcftw04DZr0eBR7wS8i5gFfB67PzNfHrd5M79D3F4C/Bv65xVI+\nnJkXApcCvxsRS1vsa6CImAlcDvzTBKu7HI8xsndM+66+JRURNwGHgXUDmrT9Gn6V3uzYvwjsBf6s\niSftMvx7gEV99xdWj03YJiKmAycC/9N0IRExg17w12XmPePXZ+brmflGtXwfMCMiTmm6jur591S3\nrwD30jt86zfMuDXhUmBzZr48QY2djUfl5SOnNtXtKxO06WRcIuJK4DLgN6s/RO8wxGtYS2a+nJk/\nysy3gL8d8PyTHo8uw/8ocG5EnF3tZVYB68e1WQ8cuWr7CeChQQM+quoawq3Ajsz88oA27z1yrSEi\nLqI3Tm38EZobEfOPLNO7wPTkuGbrgd+urvovAV7rOyRu0hUMOOTvajz69P8erAa+MUGbbwGXRMSC\n6jD4kuqxxkTEcuBG4PLMfHNAm2Few7p19F/j+fiA5x8mX2M1cYVyElcyV9C7uv4McFP12B/TG1yA\n2fQOO3cBjwDntFDDh+kdRj4BbKl+VgDXANdUba4FttO7YroR+FBL43FO1cfWqr8jY9JfSwBfqcZs\nG7C4hTrm0gvziX2PdTIe9P7g7AUO0TtP/TS96zwPAjuBfwVOrtouBm7p2/aq6ndlF/CpFurYRe88\n+sjvyZF3ok4H7jvaa9hwHf9QvfZP0Av0aePrGJSvo/34CT+pUMVe8JNKZ/ilQhl+qVCGXyqU4ZcK\nZfilQhl+qVCGXyrU/wEw19UFg9FdtwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1160b55d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "output2 = convolution(input_image, conv_filter2, 1)\n",
    "plt.imshow(output2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As images have multiple channels, convolution is actually three dimensional."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![Filter](https://raw.githubusercontent.com/UCIDataScienceInitiative/LearningWithPython/master/images/cnn.jpeg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convolution is translation invariant, which means the same features can de detected regardless of their locations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "input_image = np.zeros((50, 50))\n",
    "for i in range(5):\n",
    "    x = int(np.random.uniform(5, 45, 1))\n",
    "    y = int(np.random.uniform(5, 45, 1))\n",
    "    input_image[x, y] = 1\n",
    "    input_image[(x - 2):(x + 3), y] = 1\n",
    "    input_image[x, (y - 2):(y + 3)] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAACzVJREFUeJzt3V/I3Qd9x/H3Z/nrFGnrSsiSsna0\nTHqhKTx0le5C2pVlVWwvyrDIyEUgNw4qOly6wUDYhb2xerGbYIu5EFutQkMRJMsiMhhpo63aNmhj\nQUyXNhs2qIPFRL+7eH6Vx5D0OXnOn+ecfN8veHjO73fO6e9LOW9+5/c7v/MkVYWkXv5gvQeQNHuG\nLzVk+FJDhi81ZPhSQ4YvNWT4UkOGLzU0VvhJdif5UZKTSfZPaihJ05W1XrmXZAPwY+Ae4BTwHPBg\nVb18uedszpbayjvXtD1Jq/s//pdf17ms9riNY2zjduBkVb0KkOQJ4D7gsuFv5Z38ee4eY5OS3s6x\nOjLS48Z5q78D+NmK5VPDOklzbpw9/kiS7AP2AWzlD6e9OUkjGGeP/xpww4rlncO631NVB6pqqaqW\nNrFljM1JmpRxwn8OuCXJTUk2Ax8FDk1mLEnTtOa3+lV1IcnfAd8CNgCPV9VLE5tM0tSMdYxfVd8E\nvjmhWSTNiFfuSQ0ZvtSQ4UsNGb7UkOFLDRm+1JDhSw0ZvtSQ4UsNGb7UkOFLDRm+1JDhSw0ZvtSQ\n4UsNGb7UkOFLDRm+1JDhSw0ZvtSQ4UsNGb7UkOFLDRm+1JDhSw0ZvtSQ4UsNGb7UkOFLDRm+1JDh\nSw0ZvtSQ4UsNGb7U0KrhJ3k8yZkkL65Yd12Sw0leGX5fO90xJU3SKHv8LwG7L1q3HzhSVbcAR4Zl\nSQti1fCr6jvAzy9afR9wcLh9ELh/wnNJmqK1HuNvq6rTw+3XgW0TmkfSDIx9cq+qCqjL3Z9kX5Lj\nSY6f59y4m5M0AWsN/40k2wGG32cu98CqOlBVS1W1tIkta9ycpElaa/iHgD3D7T3A05MZR9IsjPJx\n3leA/wT+LMmpJHuBzwL3JHkF+MthWdKC2LjaA6rqwcvcdfeEZ5E0I165JzVk+FJDhi81ZPhSQ4Yv\nNWT4UkOGLzVk+FJDhi81ZPhSQ4YvNWT4UkOGLzVk+FJDhi81ZPhSQ4YvNWT4UkOGLzVk+FJDhi81\nZPhSQ4YvNWT4UkOGLzVk+FJDhi81ZPhSQ4YvNWT4UkOGLzVk+FJDhi81ZPhSQ4YvNbRq+EluSHI0\nyctJXkry0LD+uiSHk7wy/L52+uNKmoRR9vgXgE9V1a3AHcDHk9wK7AeOVNUtwJFhWdICWDX8qjpd\nVd8bbv8SOAHsAO4DDg4POwjcP60hJU3WFR3jJ7kRuA04BmyrqtPDXa8D2yY6maSpGTn8JO8Cvg58\noqp+sfK+qiqgLvO8fUmOJzl+nnNjDStpMkYKP8kmlqP/clV9Y1j9RpLtw/3bgTOXem5VHaiqpapa\n2sSWScwsaUwbV3tAkgCPASeq6nMr7joE7AE+O/x+eioTal19679e+L3lv/rjXes0iSZp1fCBO4G/\nBX6Y5K1XwT+yHPxXk+wFfgr8zXRGlDRpq4ZfVf8B5DJ33z3ZcSTNglfuSQ2N8la/LY9vdbVyjy81\nZPhSQ4YvNeQxfmMXn8OY1HM8FzL/3ONLDRm+1JDhSw0ZvtRQ25N7ntgabVYvYro6uceXGjJ8qSHD\nlxpqe4zv8a06c48vNWT4UkOGLzXU9hhfo/G8xtXJPb7UkOFLDRm+1JDhSw15cu9teGJLVyv3+FJD\nhi81ZPhSQ4YvNWT4UkOGLzVk+FJDhi81ZPhSQ4YvNbRq+Em2Jnk2yfeTvJTkM8P6m5IcS3IyyZNJ\nNk9/XEmTMMoe/xxwV1W9H9gF7E5yB/AI8GhV3Qy8Ceyd3piSJmnV8GvZr4bFTcNPAXcBTw3rDwL3\nT2VCSRM30jF+kg1JXgDOAIeBnwBnq+rC8JBTwI7pjChp0kYKv6p+U1W7gJ3A7cB7R91Akn1Jjic5\nfp5zaxxT0iRd0Vn9qjoLHAU+AFyT5K3v8+8EXrvMcw5U1VJVLW1iy1jDSpqMVf8QR5LrgfNVdTbJ\nO4B7WD6xdxR4AHgC2AM8Pc1BF5X/Go/m0Sh/gWc7cDDJBpbfIXy1qp5J8jLwRJJ/AZ4HHpvinJIm\naNXwq+oHwG2XWP8qy8f7khaMV+5JDRm+1JDhSw0ZvtSQ4UsNGb7UkP+SzhguvjhnUs/xIh9Nm3t8\nqSHDlxoyfKkhj/HHMMqxuF/S0Txyjy81ZPhSQ4YvNWT4UkOGLzVk+FJDhi81ZPhSQ17AMwa/pKNF\n5R5fasjwpYYMX2rIY/wx+CUdLSr3+FJDhi81ZPhSQ4YvNWT4UkOGLzVk+FJDhi815AU8UzbPF+xc\n6gtD8zyvJsc9vtSQ4UsNjRx+kg1Jnk/yzLB8U5JjSU4meTLJ5umNKWmSrmSP/xBwYsXyI8CjVXUz\n8Cawd5KDSZqekcJPshP4EPDFYTnAXcBTw0MOAvdPY0BJkzfqHv/zwKeB3w7L7wHOVtWFYfkUsONS\nT0yyL8nxJMfPc26sYSVNxqrhJ/kwcKaqvruWDVTVgapaqqqlTWxZy39C0oSN8jn+ncBHktwLbAXe\nDXwBuCbJxmGvvxN4bXpjSpqkVcOvqoeBhwGSfBD4+6r6WJKvAQ8ATwB7gKenOKfWYBp/BdgLfK4O\n43yO/w/AJ5OcZPmY/7HJjCRp2q7okt2q+jbw7eH2q8Dtkx9J0rR55Z7UkF/SuYqtdjzul3T6co8v\nNWT4UkOGLzVk+FJDhi81ZPhSQ4YvNWT4UkNewNOYF+v05R5fasjwpYYMX2rI8KWGDF9qyPClhgxf\nasjwpYYMX2rI8KWGDF9qyPClhgxfasjwpYYMX2rI8KWGDF9qyPClhgxfasjwpYYMX2rI8KWGDF9q\nyPClhgxfaihVNbuNJf8N/BT4I+B/Zrbh8SzSrLBY8y7SrLAY8/5JVV2/2oNmGv7vNpocr6qlmW94\nDRZpVliseRdpVli8ed+Ob/Wlhgxfami9wj+wTttdi0WaFRZr3kWaFRZv3stal2N8SevLt/pSQzMN\nP8nuJD9KcjLJ/lluexRJHk9yJsmLK9Zdl+RwkleG39eu54xvSXJDkqNJXk7yUpKHhvXzOu/WJM8m\n+f4w72eG9TclOTa8Jp5Msnm9Z31Lkg1Jnk/yzLA8t7NeqZmFn2QD8K/AXwO3Ag8muXVW2x/Rl4Dd\nF63bDxypqluAI8PyPLgAfKqqbgXuAD4+/P+c13nPAXdV1fuBXcDuJHcAjwCPVtXNwJvA3nWc8WIP\nASdWLM/zrFdklnv824GTVfVqVf0aeAK4b4bbX1VVfQf4+UWr7wMODrcPAvfPdKjLqKrTVfW94fYv\nWX6B7mB+562q+tWwuGn4KeAu4Klh/dzMm2Qn8CHgi8NymNNZ12KW4e8AfrZi+dSwbt5tq6rTw+3X\ngW3rOcylJLkRuA04xhzPO7x1fgE4AxwGfgKcraoLw0Pm6TXxeeDTwG+H5fcwv7NeMU/uXYFa/ghk\nrj4GSfIu4OvAJ6rqFyvvm7d5q+o3VbUL2MnyO8D3rvNIl5Tkw8CZqvrues8yLRtnuK3XgBtWLO8c\n1s27N5Jsr6rTSbazvLeaC0k2sRz9l6vqG8PquZ33LVV1NslR4APANUk2DnvSeXlN3Al8JMm9wFbg\n3cAXmM9Z12SWe/zngFuGM6ObgY8Ch2a4/bU6BOwZbu8Bnl7HWX5nOOZ8DDhRVZ9bcde8znt9kmuG\n2+8A7mH5vMRR4IHhYXMxb1U9XFU7q+pGll+n/15VH2MOZ12zqprZD3Av8GOWj+3+aZbbHnG+rwCn\ngfMsH8PtZfnY7gjwCvBvwHXrPecw61+w/Db+B8ALw8+9czzv+4Dnh3lfBP55WP+nwLPASeBrwJb1\nnvWiuT8IPLMIs17Jj1fuSQ15ck9qyPClhgxfasjwpYYMX2rI8KWGDF9qyPClhv4f9mpJmTC8NgEA\nAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x112f6f810>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(input_image)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "conv_filter = np.zeros((5, 5))\n",
    "conv_filter[2, :] = 1\n",
    "conv_filter[:, 2] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.  0.  1.  0.  0.]\n",
      " [ 0.  0.  1.  0.  0.]\n",
      " [ 1.  1.  1.  1.  1.]\n",
      " [ 0.  0.  1.  0.  0.]\n",
      " [ 0.  0.  1.  0.  0.]]\n"
     ]
    }
   ],
   "source": [
    "print(conv_filter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "output = convolution(input_image, conv_filter, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAADslJREFUeJzt3W2MHeV5xvH/VWNj4sQiTqhr2aQ4\nQItQVRyJECzygZrSum4aiIQiUFS5kotTpUhEjRpMK7VESiSQmjj5UCKRQPGHNCYliYyQm9YxjqJI\nyLw61LwEG4ckXi04UbAMqDW2ufvhPK62e2Z3Z8+ZOS++r590tGdm5+y5x9rLc+bZmftRRGBm+fzG\nsAsws+Fw+M2ScvjNknL4zZJy+M2ScvjNknL4zZJy+M2S6iv8ktZL+omkg5K2NFWUmbVPvV7hJ2kB\n8CJwLXAYeBy4KSKem+k1i3R2LGZJT+9nZnP7H97krTiuOtue1cf7XAEcjIhDAJK2A9cBM4Z/MUv4\nkK7p4y3NbDZ7Y3ftbfv52L8S+MWU5cNlnZmNgX6O/LVI2gxsBljMO9p+OzOrqZ8j/wRw/pTlVWXd\n/xMR90TE5RFx+ULO7uPtzKxJ/YT/ceBiSaslLQJuBB5qpiwza1vPH/sj4qSkW4D/ABYA90XEs41V\nZmat6uucPyJ2AjsbqsXMBshX+Jkl5fCbJeXwmyXl8Jsl5fCbJeXwmyXl8Jsl5fCbJeXwmyXl8Jsl\n5fCbJeXwmyXl8Jsl5fCbJeXwmyXV1/38kl4GXgdOAScj4vImijKz9jXRwPMPIuJXDfwcMxsgf+w3\nS6rf8Afwn5KeLC26zWxM9Pux/8MRMSHpN4Fdkl6IiB9O3cB9+81GU19H/oiYKF+PAN+lM4XX9G3c\nt99sBPUcfklLJL3r9HPgj4D9TRVmZu3q52P/cuC7kk7/nH+NiO81UpWZta6fSTsOAZc1WIuZDZD/\n1GeWlMNvlpTDb5aUw2+WlMNvlpTDb5aUw2+WlMNvlpTDb5aUw2+WlMNvlpTDb5aUw2+WlMNvlpTD\nb5bUnOGXdJ+kI5L2T1m3TNIuSQfK13e3W6aZNa3Okf9+YP20dVuA3RFxMbC7LJvZGJkz/KUb76+n\nrb4O2FaebwOub7guM2tZr+f8yyNisjx/hU4/PzMbI30P+EVE0Jm8o5KkzZKekPTECY73+3Zm1pBe\nw/+qpBUA5euRmTZ0336z0dRr+B8CNpbnG4EdzZRjZoNS50993wQeBX5X0mFJm4A7gWslHQD+sCyb\n2RiZs29/RNw0w7euabgWMxsgX+FnlpTDb5aUw2+WlMNvlpTDb5aUw2+WlMNvlpTDb5aUw2+WlMNv\nlpTDb5aUw2+WlMNvlpTDb5aUw2+WlMNvllSvk3bcIWlC0r7y2NBumWbWtF4n7QDYGhFrymNns2WZ\nWdt6nbTDzMZcP+f8t0h6ppwWzDhXn/v2m42mXsP/VeBCYA0wCXxxpg3dt99sNPUU/oh4NSJORcTb\nwNeAK5oty8za1lP4T8/WU3wM2D/TtmY2mubs218m7bgaeK+kw8A/AldLWkNnjr6XgU+2WKOZtaDX\nSTvubaEWMxsgX+FnltScR34zgAW/c2HXulMvvjSESqwpPvKbJeXwmyXl8Jsl5fCbJeUBv8IDWrP7\n7/cv61q3yP8+Y81HfrOkHH6zpBx+s6QcfrOkPOBXZB7Qemv9B+fc5tj7un9VltZ4HcCi7z0+75qs\nfT7ymyXl8Jsl5fCbJVWnb//5kvZIek7Ss5JuLeuXSdol6UD5OmMTTzMbPXUG/E4Cn4mIpyS9C3hS\n0i7gL4DdEXGnpC3AFuC29krtTZ3BLOh9QCvLYNabK9W1bunPh1CINaZO3/7JiHiqPH8deB5YCVwH\nbCubbQOub6tIM2vevP7UJ+kC4APAXmB5REyWb70CLJ/hNZuBzQCLeUevdZpZw2oP+El6J/Bt4NMR\ncWzq9yIi6DTz7OK+/WajqVb4JS2kE/xvRMR3yupXT7fwLl+PtFOimbWhTutu0enW+3xEfGnKtx4C\nNgJ3lq87WqlwQM7EAa2q25SrrmSsGuyc/u/xws13d21zCZ/qWrdkovsDYNXA6TmHuqd/9C3Ug1Xn\nnP8q4M+B/5K0r6z7Ozqh/5akTcDPgI+3U6KZtaFO3/4fAd2HxY5rmi3HzAbFV/iZJeXwmyU1trf0\nNjmYBfUGtMZtMKvqPatuU67ah+mDnVWDeysePVGrjqqrIE/VeqW1yUd+s6QcfrOkHH6zpMb2nL/J\n81no/Zw2y/ls1XiHjTcf+c2ScvjNknL4zZJy+M2SGtsBv6ZlHtCq04qsauA0SwuzM5WP/GZJOfxm\nSTn8Zkn107f/DkkTkvaVx4b2yzWzpvTTtx9ga0T8U3vl9a/uoJQHtGZXeafiEOqw5tTp5DMJTJbn\nr0s63bffzMbYvM75p/XtB7hF0jOS7vN0XWbjpZ++/V8FLgTW0Plk8MUZXrdZ0hOSnjjB8QZKNrMm\n9Ny3PyJejYhTEfE28DXgiqrXetIOs9HUc99+SSumTNf1MWB/OyUOhge0Zuee+meefvr23yRpDZ1p\nul4GPtlKhWbWin769u9svhwzGxRf4WeWlMNvlpRv6S2aHNCqmlPAA2Y2anzkN0vK4TdLyuE3S8rh\nN0vKA34tqJowtGpCEbNh8pHfLCmH3ywph98sKZ/zz8NbFa2+qhx7X/c/a1WbsCpuHWaD4iO/WVIO\nv1lSDr9ZUnX69i+W9JikH5e+/Z8r61dL2ivpoKQHJC1qv1wza0qdI/9xYF1EXEanWed6SVcCd9Hp\n238R8Bqwqb0yx8ubK9X1MBs1c4Y/Ot4oiwvLI4B1wINl/Tbg+lYqNLNW1O3eu6D07zsC7AJeAo5G\nxMmyyWE8kYfZWKkV/tKiew2wik6L7kvqvoH79puNpnmN9kfEUWAPsBY4V9Lpq1lWARMzvMZ9+81G\nUJ2+/ecBJyLiqKRzgGvpDPbtAW4AtgMbgR1tFtq2qtZb0+/Oq7pyr2ow74Wb7+5adwmf6lq3ZCK6\n1lVdCTh9TgG3BLMm1Lm8dwWwTdICOp8UvhURD0t6Dtgu6fPA03Qm9jCzMVGnb/8zdCbnnL7+EDNM\n0WVmo89X+Jkl5fCbJeVbeouqQbSu1lub13ZtUzVot3rnX3atW1axXZWqW3o9Yai1wUd+s6QcfrOk\nHH6zpBx+s6QcfrOkHH6zpBx+s6QcfrOkHH6zpHyFX1Hnlt4qVbf0/nTD17vWXTJR75beqolBfEuv\ntcFHfrOkHH6zpPrp23+/pJ9K2lcea9ov18yaUuec/3Tf/jckLQR+JOnfy/f+NiIenOW1Zjai6nTy\nCaCqb/8Zpc4tvVX99Zb+vPtnVfXrW/HoiVp1DOOW3qrBTg8qnvl66tsfEXvLt74g6RlJWyW5Na/Z\nGOmpb7+k3wNup9O//4PAMuC2qte6b7/ZaOq1b//6iJgsU3kdB/6FGZp5um+/2WjquW+/pBURMSlJ\ndObp299yrWOj6uKdUVZ1MVNXCzM74/TTt/+R8h+DgH3AX7VYp5k1rJ++/etaqcjMBsJX+Jkl5fCb\nJeW7+uah6gKcKlUXA9V9bZOq7hCsUjUBadU+TDeMfbLm+MhvlpTDb5aUw2+WlMNvlpQH/Fowve0W\njPZkm1WtyKruVrQzi4/8Zkk5/GZJOfxmSTn8Zkl5wK8FbbfAqjvHQNWVe1WDey/cfHfXuumtyKpu\nU666CrBysNO3B48kH/nNknL4zZKqHf7SxPNpSQ+X5dWS9ko6KOkBSYvaK9PMmjafI/+twPNTlu8C\ntkbERcBrwKYmCzOzdqnTln+OjaRVwDbgC8DfAH8G/BL4rYg4KWktcEdE/PFsP2eplsWHdE3/VVst\ndW/pnVy7sGtdnXkGfEvv6NkbuzkWv+4e1a1Q98j/ZeCzwNtl+T3A0Yg4WZYPAyvnVaWZDVWdufo+\nAhyJiCd7eQP37TcbTXX+zn8V8FFJG4DFwFLgK8C5ks4qR/9VwETViyPiHuAe6Hzsb6RqM+vbnEf+\niLg9IlZFxAXAjcAjEfEJOpN33FA22wjsaK1KM2tcP1f43QZsl/R54Gng3mZKskEbt0lGrBnzCn9E\n/AD4QXl+iBmm6DKz0ecr/MyScvjNkvJdfWewcZtnwAbLR36zpBx+s6QcfrOkHH6zpDzgZ2M3z4A1\nw0d+s6QcfrOkHH6zpBx+s6Q84Gfuq5+Uj/xmSTn8Zkk5/GZJOfxmSdXq29/Ym0m/BH4GvBf41cDe\nuB3jvg+uf/ja2Iffjojz6mw40PD/35tKT0TE5QN/4waN+z64/uEb9j74Y79ZUg6/WVLDCv89Q3rf\nJo37Prj+4RvqPgzlnN/Mhs8f+82SGnj4Ja2X9BNJByVtGfT7z5ek+yQdkbR/yrplknZJOlC+vnuY\nNc5G0vmS9kh6TtKzkm4t68dpHxZLekzSj8s+fK6sXy1pb/ldekDSomHXOhtJCyQ9LenhsjzU+gca\nfkkLgH8G/gS4FLhJ0qWDrKEH9wPrp63bAuyOiIuB3WV5VJ0EPhMRlwJXAn9d/s3HaR+OA+si4jJg\nDbBe0pXAXcDWiLgIeA3YNMQa67gVeH7K8lDrH/SR/wrgYEQcioi3gO3AdQOuYV4i4ofA9D5X1wHb\nyvNtwPUDLWoeImIyIp4qz1+n88u3kvHah4iIN8riwvIIYB3wYFk/0vsgaRXwp8DXy7IYcv2DDv9K\n4BdTlg+XdeNmeURMluevAMuHWUxdki4APgDsZcz2oXxk3gccAXYBLwFHyxTxMPq/S18GPgu8XZbf\nw5Dr94Bfn6Lz55KR/5OJpHcC3wY+HRHHpn5vHPYhIk5FxBpgFZ1PkJcMuaTaJH0EOBIRTw67lqkG\n3cxjAjh/yvKqsm7cvCppRURMSlpB52g0siQtpBP8b0TEd8rqsdqH0yLiqKQ9wFrgXElnlaPnKP8u\nXQV8VNIGYDGwFPgKQ65/0Ef+x4GLyyjnIuBG4KEB19CEh4CN5flGYMcQa5lVObe8F3g+Ir405Vvj\ntA/nSTq3PD8HuJbO2MUe4Iay2cjuQ0TcHhGrIuICOr/zj0TEJxh2/REx0AewAXiRzjnb3w/6/Xuo\n95vAJHCCznnZJjrna7uBA8D3gWXDrnOW+j9M5yP9M8C+8tgwZvvw+8DTZR/2A/9Q1r8feAw4CPwb\ncPawa62xL1cDD49C/b7CzywpD/iZJeXwmyXl8Jsl5fCbJeXwmyXl8Jsl5fCbJeXwmyX1v2SwKqJh\niSycAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x116088490>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(output)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Large convolution filters have substantially more parameters and are more difficult to learn. Therefore, to learn features at different scales, we would downsample the images instead."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![Maxpool](https://raw.githubusercontent.com/UCIDataScienceInitiative/LearningWithPython/master/images/maxpool.jpeg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def pooling(feature_map, size, stride):\n",
    "    \"\"\"\n",
    "    Perform max pooling on feature_map.\n",
    "    \n",
    "    Args\n",
    "        feature_map: (d x d) numpy array\n",
    "        size: pooling window size\n",
    "        stride: sliding width / height\n",
    "    \"\"\"\n",
    "    d = feature_map.shape[0]\n",
    "    n = (d - size) / stride\n",
    "    reduced_map = np.zeros((n, n))\n",
    "    for i in range(n):\n",
    "        for j in range(n):\n",
    "            reduced_map[i, j] = 0\n",
    "    return reduced_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAAD8CAYAAABaQGkdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAACjFJREFUeJzt3V+InXedx/H3Z/PXxF0Vdlk0KW1Y\nXCUIS3XW7R+Q2nhRq9gL96JChVVowF21iiB1b9R7Eb1wC9mqNxZ7kfZCpESX1V4sKyGTtKBJqtSq\naWKl8WJVCps/+t2LmYVYzJwnmefpM/Pl/YJC5vT01w/DvPucc+bMNFWFpJ7+bO4BkqZj4FJjBi41\nZuBSYwYuNWbgUmMGLjVm4FJjBi41tnWKQ7dnR+1k9xRHSwL+l5e4WBey6H6TBL6T3fxDDkxxtCTg\naP3noPv5EF1qzMClxgxcaszApcYMXGrMwKXGBgWe5K4kP07ybJIHpx4laRwLA0+yBfgK8G5gP/CB\nJPunHiZp/YZcwd8OPFtVz1XVReBR4J5pZ0kaw5DA9wDPX/Hx2dXb/kiSg0mWkyxf4sJY+yStw2gv\nslXVoapaqqqlbewY61hJ6zAk8HPADVd8vHf1Nkkb3JDAjwFvTLIvyXbgXuBb086SNIaFP01WVZeT\nfBT4DrAF+FpVnZx8maR1G/TjolX1BPDExFskjcx3skmNGbjUmIFLjRm41JiBS40ZuNSYgUuNGbjU\nmIFLjRm41JiBS40ZuNSYgUuNGbjUmIFLjRm41JiBS40ZuNSYgUuNGbjUmIFLjRm41JiBS40ZuNSY\ngUuNGbjUmIFLjRm41JiBS40ZuNSYgUuNGbjUmIFLjRm41NjCwJPckOT7SU4lOZnkgVdimKT12zrg\nPpeBT1XViSR/DhxP8h9VdWribZLWaeEVvKpeqKoTq3/+HXAa2DP1MEnrd03PwZPcBNwMHJ1ijKRx\nDXmIDkCSVwOPAZ+oqt/+ib9/EDgIsJNdow2UdP0GXcGTbGMl7keq6vE/dZ+qOlRVS1W1tI0dY26U\ndJ2GvIoe4KvA6ar64vSTJI1lyBX8duCDwJ1Jnl796+6Jd0kawcLn4FX1X0BegS2SRuY72aTGDFxq\nzMClxgxcaszApcYMXGrMwKXGDFxqzMClxgxcaszApcYMXGrMwKXGDFxqzMClxgxcaszApcYMXGrM\nwKXGDFxqzMClxgxcaszApcYMXGrMwKXGDFxqzMClxgb//8E7u3jX349+5vYjx0Y/c0q/PnjrJOf+\nxZnLk5y72T6/c/EKLjVm4FJjBi41ZuBSYwYuNWbgUmMGLjU2OPAkW5I8leTbUw6SNJ5ruYI/AJye\naoik8Q0KPMle4D3Aw9POkTSmoVfwLwGfBv5wtTskOZhkOcnyJS6MMk7S+iwMPMl7gRer6vha96uq\nQ1W1VFVL29gx2kBJ12/IFfx24H1Jfg48CtyZ5BuTrpI0ioWBV9VnqmpvVd0E3At8r6rum3yZpHXz\n++BSY9f08+BV9STw5CRLJI3OK7jUmIFLjRm41JiBS40ZuNTYpvqtqr/4/G2TnPvM/f82+pl33/H+\n0c+c0vHPPTTJuW/+93+e5Nwbj0xybDtewaXGDFxqzMClxgxcaszApcYMXGrMwKXGDFxqzMClxgxc\naszApcYMXGrMwKXGDFxqzMClxgxcaszApcYMXGrMwKXGDFxqzMClxjbVb1V9/Q8uTXLu2859ZPQz\n/5oXRz8T4Pc/+ekk577zw/dPcu6NR/57knM1jFdwqTEDlxozcKkxA5caM3CpMQOXGhsUeJLXJjmc\n5Jkkp5PcOvUwSes39PvgXwaOVNU/JtkO7Jpwk6SRLAw8yWuAdwD/BFBVF4GL086SNIYhD9H3AeeB\nryd5KsnDSXZPvEvSCIYEvhV4K/BQVd0MvAQ8+PI7JTmYZDnJ8iUujDxT0vUYEvhZ4GxVHV39+DAr\nwf+RqjpUVUtVtbSNHWNulHSdFgZeVb8Cnk/yptWbDgCnJl0laRRDX0X/GPDI6ivozwEfmm6SpLEM\nCryqngaWJt4iaWS+k01qzMClxgxcaszApcYMXGrMwKXGNtVvVd1+5Ngk5770+dtGP/OJJx8b/UyA\nu+94/yTnTvW51by8gkuNGbjUmIFLjRm41JiBS40ZuNSYgUuNGbjUmIFLjRm41JiBS40ZuNSYgUuN\nGbjUmIFLjRm41JiBS40ZuNSYgUuNGbjU2Kb6pYuazpa//ZtJzv39T346ybkaxiu41JiBS40ZuNSY\ngUuNGbjUmIFLjRm41NigwJN8MsnJJD9K8s0kO6ceJmn9FgaeZA/wcWCpqt4CbAHunXqYpPUb+hB9\nK/CqJFuBXcAvp5skaSwLA6+qc8AXgDPAC8Bvquq7L79fkoNJlpMsX+LC+EslXbMhD9FfB9wD7APe\nAOxOct/L71dVh6pqqaqWtrFj/KWSrtmQh+jvAn5WVeer6hLwOHDbtLMkjWFI4GeAW5LsShLgAHB6\n2lmSxjDkOfhR4DBwAvjh6j9zaOJdkkYw6OfBq+qzwGcn3iJpZL6TTWrMwKXGDFxqzMClxgxcaszf\nqgq8/geXRj/zbec+MvqZANwxzbHHP/fQJOe+88P3T3Lu9iPHJjm3G6/gUmMGLjVm4FJjBi41ZuBS\nYwYuNWbgUmMGLjVm4FJjBi41ZuBSYwYuNWbgUmMGLjVm4FJjBi41ZuBSYwYuNWbgUmMGLjVm4FJj\nqarxD03OA78YcNe/BH49+oDpbKa9m2krbK69G2HrjVX1V4vuNEngQyVZrqql2QZco820dzNthc21\ndzNt9SG61JiBS43NHfihmf/912oz7d1MW2Fz7d00W2d9Di5pWnNfwSVNaLbAk9yV5MdJnk3y4Fw7\nFklyQ5LvJzmV5GSSB+beNESSLUmeSvLtubesJclrkxxO8kyS00lunXvTWpJ8cvXr4EdJvplk59yb\n1jJL4Em2AF8B3g3sBz6QZP8cWwa4DHyqqvYDtwD/soG3XukB4PTcIwb4MnCkqt4M/B0beHOSPcDH\ngaWqeguwBbh33lVrm+sK/nbg2ap6rqouAo8C98y0ZU1V9UJVnVj98+9Y+QLcM++qtSXZC7wHeHju\nLWtJ8hrgHcBXAarqYlX9z7yrFtoKvCrJVmAX8MuZ96xprsD3AM9f8fFZNng0AEluAm4Gjs67ZKEv\nAZ8G/jD3kAX2AeeBr68+nXg4ye65R11NVZ0DvgCcAV4AflNV35131dp8kW2gJK8GHgM+UVW/nXvP\n1SR5L/BiVR2fe8sAW4G3Ag9V1c3AS8BGfj3mdaw80twHvAHYneS+eVetba7AzwE3XPHx3tXbNqQk\n21iJ+5GqenzuPQvcDrwvyc9ZeepzZ5JvzDvpqs4CZ6vq/x8RHWYl+I3qXcDPqup8VV0CHgdum3nT\nmuYK/BjwxiT7kmxn5YWKb820ZU1JwspzxNNV9cW59yxSVZ+pqr1VdRMrn9fvVdWGvMpU1a+A55O8\nafWmA8CpGSctcga4Jcmu1a+LA2zgFwVh5SHSK66qLif5KPAdVl6J/FpVnZxjywC3Ax8Efpjk6dXb\n/rWqnphxUycfAx5Z/Q/9c8CHZt5zVVV1NMlh4AQr3115ig3+rjbfySY15otsUmMGLjVm4FJjBi41\nZuBSYwYuNWbgUmMGLjX2f3zmJ0fhTw6SAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x116145290>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(pooling(output, 4, 4))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hierarchical representation\n",
    "\n",
    "https://distill.pub/2017/feature-visualization/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![Filter](https://raw.githubusercontent.com/UCIDataScienceInitiative/LearningWithPython/master/figures/filter.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import keras.datasets.mnist as mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://s3.amazonaws.com/img-datasets/mnist.npz\n",
      "11493376/11490434 [==============================] - 7s 1us/step\n",
      "11501568/11490434 [==============================] - 7s 1us/step\n"
     ]
    }
   ],
   "source": [
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This makes sure the image has the correct order in the axis for Tensorflow, it would be different for Theano backend\n",
    "x_train = x_train.reshape(x_train.shape[0], 28, 28, 1)\n",
    "x_test = x_test.reshape(x_test.shape[0], 28, 28, 1)\n",
    "\n",
    "# Convert values to floats, originally they are integers\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "\n",
    "# Convert values of labels from 0 to 9 to categorical (one_hot encoding)\n",
    "y_train = keras.utils.to_categorical(y_train, 10)\n",
    "y_test = keras.utils.to_categorical(y_test, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Flatten, Dropout\n",
    "from keras.layers import Conv2D, MaxPooling2D"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Keras documentation: https://keras.io/layers/convolutional/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Conv2D(32, (3, 3), padding='same', input_shape=(28, 28, 1), activation='relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Flatten())\n",
    "model.add(Dropout(0.1))\n",
    "model.add(Dense(10, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_3 (Conv2D)            (None, 28, 28, 32)        320       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2 (None, 14, 14, 32)        0         \n",
      "_________________________________________________________________\n",
      "flatten_3 (Flatten)          (None, 6272)              0         \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 6272)              0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 10)                62730     \n",
      "=================================================================\n",
      "Total params: 63,050\n",
      "Trainable params: 63,050\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Common pre-processing steps include rotation and zoom, which will make the neural network learn features that are angle and scale invariant. These steps can be performed within a data generator on-the-fly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "\n",
    "data_generator = ImageDataGenerator(rescale=1./255,\n",
    "                                    #featurewise_center=True,\n",
    "                                    #featurewise_std_normalization=True,\n",
    "                                    rotation_range=10,\n",
    "                                    width_shift_range=0.1,\n",
    "                                    height_shift_range=0.1,\n",
    "                                    #horizontal_flip=True,\n",
    "                                    zoom_range=0.1\n",
    "                                    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 361,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      " - 26s - loss: 0.8140 - acc: 0.7562 - val_loss: 5.2912 - val_acc: 0.6683\n",
      "Epoch 2/10\n",
      " - 25s - loss: 0.3456 - acc: 0.9010 - val_loss: 6.3462 - val_acc: 0.6038\n",
      "Epoch 3/10\n",
      " - 25s - loss: 0.2771 - acc: 0.9173 - val_loss: 5.8151 - val_acc: 0.6369\n",
      "Epoch 4/10\n",
      " - 29s - loss: 0.2475 - acc: 0.9269 - val_loss: 6.2446 - val_acc: 0.6103\n",
      "Epoch 5/10\n",
      " - 23s - loss: 0.2387 - acc: 0.9291 - val_loss: 6.2826 - val_acc: 0.6079\n",
      "Epoch 6/10\n",
      " - 24s - loss: 0.2260 - acc: 0.9309 - val_loss: 5.8294 - val_acc: 0.6362\n",
      "Epoch 7/10\n",
      " - 24s - loss: 0.2183 - acc: 0.9341 - val_loss: 6.2320 - val_acc: 0.6112\n",
      "Epoch 8/10\n",
      " - 24s - loss: 0.2077 - acc: 0.9376 - val_loss: 6.1975 - val_acc: 0.6131\n",
      "Epoch 9/10\n",
      " - 25s - loss: 0.2009 - acc: 0.9392 - val_loss: 6.2829 - val_acc: 0.6071\n",
      "Epoch 10/10\n",
      " - 28s - loss: 0.1958 - acc: 0.9424 - val_loss: 7.7532 - val_acc: 0.5164\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x12735fdd0>"
      ]
     },
     "execution_count": 361,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit_generator(data_generator.flow(x_train[0:40000], y_train[0:40000], batch_size=100), steps_per_epoch=40000/100,\n",
    "                    validation_data=(x_train[40000:], y_train[40000:]), \n",
    "                    epochs=10, verbose=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Further readings\n",
    "\n",
    "https://qz.com/1034972/the-data-that-changed-the-direction-of-ai-research-and-possibly-the-world/\n",
    "\n",
    "https://www.youtube.com/watch?v=KfV8CJh7hE0&list=PLkDaE6sCZn6Gl29AoE31iwdVwSG-KnDzF&index=18\n",
    "\n",
    "https://blog.openai.com/adversarial-example-research/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "![Adverserial](https://blog.openai.com/content/images/2017/02/adversarial_img_1.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:tensorflow]",
   "language": "python",
   "name": "conda-env-tensorflow-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}