GAN_on_NotMNIST.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4embtkV0pNxM"
   },
   "source": [
    "Generative Adversarial Net experiment on NotMNIST\n",
    "------------\n",
    "\n",
    "A basic GAN in Tensorflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "tm2CQN_Cpwj0"
   },
   "outputs": [],
   "source": [
    "# These are all the modules we'll be using later. Make sure you can import them\n",
    "# before proceeding further.\n",
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from six.moves import cPickle as pickle\n",
    "from six.moves import range\n",
    "import matplotlib.pyplot as plt\n",
    "from mpl_toolkits.axes_grid1 import AxesGrid\n",
    "import random\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 11948,
     "status": "ok",
     "timestamp": 1446658914837,
     "user": {
      "color": "",
      "displayName": "",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "",
      "photoUrl": "",
      "sessionId": "0",
      "userId": ""
     },
     "user_tz": 480
    },
    "id": "y3-cj1bpmuxc",
    "outputId": "016b1a51-0290-4b08-efdb-8c95ffc3cd01",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set (200000, 28, 28) (200000,)\n",
      "Validation set (10000, 28, 28) (10000,)\n",
      "Test set (10000, 28, 28) (10000,)\n"
     ]
    }
   ],
   "source": [
    "# load the notmnist dataset as is\n",
    "pickle_file = '../udacity_deep_learning_course/notMNIST.pickle'\n",
    "with open(pickle_file, 'rb') as f:\n",
    "    save = pickle.load(f)\n",
    "    train_dataset = save['train_dataset']\n",
    "    train_labels = save['train_labels']\n",
    "    valid_dataset = save['valid_dataset']\n",
    "    valid_labels = save['valid_labels']\n",
    "    test_dataset = save['test_dataset']\n",
    "    test_labels = save['test_labels']\n",
    "    del save  # hint to help gc free up memory\n",
    "\n",
    "    print('Training set', train_dataset.shape, train_labels.shape)\n",
    "    print('Validation set', valid_dataset.shape, valid_labels.shape)\n",
    "    print('Test set', test_dataset.shape, test_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def show_imagelist_as_grid(img_list, nrow, ncol):\n",
    "    fig = plt.figure(figsize=(5,5))\n",
    "    grid = AxesGrid(fig, 111, nrows_ncols=(nrow, ncol), axes_pad=0.05, label_mode=\"1\")\n",
    "    for i in range(nrow*ncol):\n",
    "        im = grid[i].imshow(img_list[i], interpolation=\"none\", cmap='gray', vmin=-0.5, vmax=0.5)\n",
    "    # grid.axes_llc.set_xticks([-1, 0, 1])\n",
    "    # grid.axes_llc.set_yticks([-1, 0, 1])\n",
    "\n",
    "    plt.draw()\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 11952,
     "status": "ok",
     "timestamp": 1446658914857,
     "user": {
      "color": "",
      "displayName": "",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "",
      "photoUrl": "",
      "sessionId": "0",
      "userId": ""
     },
     "user_tz": 480
    },
    "id": "IRSyYiIIGIzS",
    "outputId": "650a208c-8359-4852-f4f5-8bf10e80ef6c",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "image_size = 28\n",
    "num_labels = 10\n",
    "num_channels = 1 # grayscale\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "def reformat(dataset, labels):\n",
    "    dataset = dataset.reshape(\n",
    "        (-1, image_size*image_size)).astype(np.float32)\n",
    "    labels = (np.arange(num_labels) == labels[:,None]).astype(np.float32)\n",
    "    return dataset, labels\n",
    "\n",
    "train_dataset, train_labels = reformat(train_dataset, train_labels)\n",
    "valid_dataset, valid_labels = reformat(valid_dataset, valid_labels)\n",
    "test_dataset, test_labels = reformat(test_dataset, test_labels)\n",
    "print('Training set', train_dataset.shape, train_labels.shape)\n",
    "print('Validation set', valid_dataset.shape, valid_labels.shape)\n",
    "print('Test set', test_dataset.shape, test_labels.shape)\n",
    "\n",
    "dataset_mean = np.mean(train_dataset)\n",
    "dataset_std = np.std(train_dataset)\n",
    "print(\"mean and std: \", dataset_mean, dataset_std)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "IZYv70SvvOan"
   },
   "outputs": [],
   "source": [
    "batch_size = 16\n",
    "stddev=0.05\n",
    "dropout_prob = 0.7\n",
    "uni_weight = 0.0\n",
    "dummy_size = 1024\n",
    "\n",
    "# discriminator: 2 hidden layers\n",
    "num_discr_layer1 = dummy_size\n",
    "num_discr_layer2 = dummy_size\n",
    "num_discr_layer3 = dummy_size\n",
    "\n",
    "\n",
    "\n",
    "# generator: 3 hidden layers\n",
    "num_gen_input_size = 16\n",
    "num_gen_layer1 = dummy_size\n",
    "num_gen_layer2 = dummy_size\n",
    "num_gen_layer3 = dummy_size\n",
    "\n",
    "graph = tf.Graph()\n",
    "\n",
    "with graph.as_default():\n",
    "\n",
    "    # number of steps taken in training.\n",
    "    global_step = tf.Variable(0)  \n",
    "    \n",
    "    # Input data.\n",
    "    tf_train_dataset = tf.placeholder(\n",
    "        tf.float32, shape=(batch_size, image_size*image_size))\n",
    "\n",
    "    tf_noise = tf.placeholder(\n",
    "        tf.float32, shape=(batch_size, num_gen_input_size))  \n",
    "\n",
    "    # Variables for discriminator network.\n",
    "    discr_w1 = tf.Variable(uni_weight/(image_size*image_size) + tf.truncated_normal(\n",
    "        [image_size*image_size, num_discr_layer1], stddev=stddev), name='discr_w1')\n",
    "    discr_b1 = tf.Variable(tf.zeros([num_discr_layer1]), name='discr_b1')\n",
    "    \n",
    "    discr_w2 = tf.Variable(uni_weight/num_discr_layer1 + tf.truncated_normal(\n",
    "        [num_discr_layer1, num_discr_layer2], stddev=stddev), name='discr_w2')\n",
    "    discr_b2 = tf.Variable(tf.zeros([num_discr_layer2]), name='discr_b2')\n",
    "\n",
    "    discr_w3 = tf.Variable(uni_weight/num_discr_layer2 + tf.truncated_normal(\n",
    "        [num_discr_layer2, num_discr_layer3], stddev=stddev), name='discr_w3')\n",
    "    discr_b3 = tf.Variable(tf.zeros([num_discr_layer2]), name='discr_b3')\n",
    "    \n",
    "    discr_w4 = tf.Variable(uni_weight/num_discr_layer3 + tf.truncated_normal(\n",
    "        [num_discr_layer3, 1], stddev=stddev), name='discr_w4')\n",
    "    #discr_b3 = tf.Variable(tf.zeros([1]))\n",
    "\n",
    "    # Variables for the generator network.\n",
    "    gen_w1 = tf.Variable(uni_weight/num_gen_input_size + tf.truncated_normal(\n",
    "        [num_gen_input_size, num_gen_layer1], stddev=stddev), name='gen_w1')\n",
    "    gen_b1 = tf.Variable(tf.zeros([num_gen_layer1]), name='gen_b1')\n",
    "    \n",
    "    gen_w2 = tf.Variable(uni_weight/num_gen_layer1 + tf.truncated_normal(\n",
    "        [num_gen_layer1, num_gen_layer2], stddev=stddev), name='gen_w2')\n",
    "    gen_b2 = tf.Variable(tf.zeros([num_gen_layer2]), name='gen_b2')\n",
    "\n",
    "    gen_w3 = tf.Variable(uni_weight/num_gen_layer2 + tf.truncated_normal(\n",
    "        [num_gen_layer2, num_gen_layer3], stddev=stddev), name='gen_w3')\n",
    "    gen_b3 = tf.Variable(tf.zeros([num_gen_layer3]), name='gen_b3')\n",
    "\n",
    "    gen_w4 = tf.Variable(uni_weight/num_gen_layer3 + tf.truncated_normal(\n",
    "        [num_gen_layer3, image_size*image_size], stddev=stddev), name='gen_w4')\n",
    "    #gen_b3 = tf.Variable(tf.zeros([image_size*image_size]))\n",
    "\n",
    "    # Model.\n",
    "    def discr_model_dropout(data):\n",
    "        discr_w1_do = tf.nn.dropout(discr_w1, dropout_prob)\n",
    "        discr_w2_do = tf.nn.dropout(discr_w2, dropout_prob)\n",
    "        discr_w3_do = tf.nn.dropout(discr_w3, dropout_prob)\n",
    "        discr_w4_do = tf.nn.dropout(discr_w4, dropout_prob)\n",
    "        discr_o1 = tf.nn.relu(tf.matmul(data, discr_w1_do) + discr_b1)\n",
    "        discr_o2 = tf.nn.relu(tf.matmul(discr_o1, discr_w2_do) + discr_b2)\n",
    "        discr_o3 = tf.nn.relu(tf.matmul(discr_o2, discr_w3_do) + discr_b2)\n",
    "        discr_o4 = tf.nn.sigmoid(tf.matmul(discr_o3, discr_w4_do))\n",
    "        return discr_o4\n",
    "\n",
    "    def discr_model(data):\n",
    "        discr_w1_do = discr_w1\n",
    "        discr_w2_do = discr_w2\n",
    "        discr_w3_do = discr_w3\n",
    "        discr_o1 = tf.nn.relu(tf.matmul(data, discr_w1_do) + discr_b1)\n",
    "        discr_o2 = tf.nn.relu(tf.matmul(discr_o1, discr_w2_do) + discr_b2)\n",
    "        discr_o3 = tf.nn.sigmoid(tf.matmul(discr_o2, discr_w3_do))/num_discr_layer2\n",
    "        return discr_o3\n",
    "    \n",
    "    # generator model, data will be noise in this case\n",
    "    def gen_model(data):\n",
    "        gen_o1 = tf.nn.relu(tf.matmul(data, gen_w1) + gen_b1)\n",
    "        gen_o2 = tf.nn.relu(tf.matmul(gen_o1, gen_w2) + gen_b2)\n",
    "        gen_o3 = tf.nn.relu(tf.matmul(gen_o2, gen_w3) + gen_b3)\n",
    "        gen_o4 = tf.nn.tanh(tf.matmul(gen_o3, gen_w4))\n",
    "        return gen_o4\n",
    "        \n",
    "    # computation\n",
    "    discr_out_on_real = discr_model_dropout(tf_train_dataset)\n",
    "    gen_out = gen_model(tf_noise)\n",
    "    discr_out_on_gen = discr_model_dropout(gen_out)\n",
    "    gen_loss = tf.reduce_mean(tf.log(1 - discr_out_on_gen))\n",
    "    discr_loss = (-0.2)*(tf.reduce_mean(tf.log(discr_out_on_real)) + gen_loss)\n",
    "    \n",
    "    trainable_vars = tf.trainable_variables()\n",
    "    discr_vars = [ x for x in trainable_vars if 'discr_' in x.name]\n",
    "    gen_vars = [ x for x in trainable_vars if 'gen_' in x.name]\n",
    "    \n",
    "    learn_rate = 0.01\n",
    "    gen_learn_rate = tf.train.exponential_decay(learn_rate, global_step, num_steps, 0.99)\n",
    "    discr_learn_rate = tf.train.exponential_decay(learn_rate, global_step, num_steps, 0.99)\n",
    "    \n",
    "    # Optimizers\n",
    "    discr_optimizer = tf.train.GradientDescentOptimizer(discr_learn_rate).minimize(discr_loss, var_list=discr_vars)\n",
    "    gen_optimizer = tf.train.GradientDescentOptimizer(gen_learn_rate).minimize(gen_loss, var_list=gen_vars)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 37
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 63292,
     "status": "ok",
     "timestamp": 1446658966251,
     "user": {
      "color": "",
      "displayName": "",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "",
      "photoUrl": "",
      "sessionId": "0",
      "userId": ""
     },
     "user_tz": 480
    },
    "id": "noKFb2UovVFR",
    "outputId": "28941338-2ef9-4088-8bd1-44295661e628",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "num_steps = 2500\n",
    "print_step = 100\n",
    "\n",
    "gen_loss_total = 0.0\n",
    "gen_trained = 0\n",
    "discr_loss_total = 0.0\n",
    "discr_trained = 0\n",
    "with tf.Session(graph=graph) as session:\n",
    "    tf.initialize_all_variables().run()\n",
    "    print('Initialized')\n",
    "    for step in range(num_steps):\n",
    "        offset = (round(random.uniform(0, 100000)) + step * batch_size) % (train_labels.shape[0] - batch_size)\n",
    "        batch_data = train_dataset[offset:(offset + batch_size), :]\n",
    "\n",
    "        for rep in range(1):\n",
    "            batch_noise = np.random.normal(loc=dataset_mean, scale=dataset_std, size=(batch_size, num_gen_input_size))\n",
    "            feed_dict = {tf_noise : batch_noise}\n",
    "            _, lg = session.run(\n",
    "              [gen_optimizer, gen_loss], feed_dict=feed_dict)\n",
    "            gen_loss_total += lg\n",
    "            gen_trained += 1\n",
    "\n",
    "        #if random.random()< abs(lg):\n",
    "        #if (abs(lg)>0.5):\n",
    "            #print(lg, \"training the discriminator\")\n",
    "        \n",
    "        batch_noise = np.random.normal(loc=dataset_mean, scale=dataset_std, size=(batch_size, num_gen_input_size))\n",
    "        feed_dict = {tf_train_dataset : batch_data, tf_noise : batch_noise}\n",
    "        _, ld = session.run(\n",
    "          [discr_optimizer, discr_loss], feed_dict=feed_dict)\n",
    "        discr_loss_total += ld\n",
    "        discr_trained += 1\n",
    "        \n",
    "        if (step % print_step == print_step-1):\n",
    "            if discr_trained == 0:\n",
    "                discr_trained = 1\n",
    "            print('Minibatch loss before step %d: discriminator %f, generator: %f' % (step+1, discr_loss_total/discr_trained, gen_loss_total/gen_trained))\n",
    "            gen_loss_total = 0.0\n",
    "            discr_loss_total = 0.0\n",
    "            gen_trained = 0\n",
    "            discr_trained = 0\n",
    "            \n",
    "\n",
    "            \n",
    "    batch_noise = np.random.normal(loc=dataset_mean, scale=dataset_std, size=(batch_size, num_gen_input_size))\n",
    "    feed_dict = {tf_noise : batch_noise}\n",
    "    example_outs = gen_out.eval(feed_dict=feed_dict)\n",
    "    img_list = example_outs.reshape(batch_size, image_size, image_size)\n",
    "    show_imagelist_as_grid(img_list, 4, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def unblockshaped(arr, h, w):\n",
    "    \"\"\"\n",
    "    Return an array of shape (h, w) where\n",
    "    h * w = arr.size\n",
    "\n",
    "    If arr is of shape (n, nrows, ncols), n sublocks of shape (nrows, ncols),\n",
    "    then the returned array preserves the \"physical\" layout of the sublocks.\n",
    "    \"\"\"\n",
    "    n, nrows, ncols = arr.shape\n",
    "    return (arr.reshape(h//nrows, -1, nrows, ncols)\n",
    "               .swapaxes(1,2)\n",
    "               .reshape(h, w))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "default_view": {},
   "name": "4_convolutions.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}