Keras_GAN_on_MNIST_better_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4embtkV0pNxM"
   },
   "source": [
    "Generative Adversarial Net experiment on MNIST data\n",
    "------------\n",
    "\n",
    "A basic GAN in Keras\n",
    "\n",
    "maxout layers and then convolutional"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "tm2CQN_Cpwj0"
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "from six.moves import cPickle as pickle\n",
    "from six.moves import range\n",
    "import matplotlib.pyplot as plt\n",
    "from PIL import Image\n",
    "from mpl_toolkits.axes_grid1 import AxesGrid\n",
    "\n",
    "import keras\n",
    "from six.moves import cPickle as pickle\n",
    "\n",
    "import random\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers import Convolution2D, MaxPooling2D\n",
    "from keras.optimizers import SGD\n",
    "\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.layers.advanced_activations import LeakyReLU\n",
    "from keras.layers.core import MaxoutDense\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 11948,
     "status": "ok",
     "timestamp": 1446658914837,
     "user": {
      "color": "",
      "displayName": "",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "",
      "photoUrl": "",
      "sessionId": "0",
      "userId": ""
     },
     "user_tz": 480
    },
    "id": "y3-cj1bpmuxc",
    "outputId": "016b1a51-0290-4b08-efdb-8c95ffc3cd01",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets('MNIST_data', one_hot=True)\n",
    "\n",
    "train_dataset = mnist.train.images - 0.5\n",
    "train_labels = mnist.train.labels\n",
    "valid_dataset = mnist.validation.images -0.5\n",
    "valid_labels = mnist.validation.labels\n",
    "test_dataset = mnist.test.images - 0.5\n",
    "test_labels = mnist.test.labels\n",
    "\n",
    "print('Training set', train_dataset.shape, train_labels.shape)\n",
    "print('Validation set', valid_dataset.shape, valid_labels.shape)\n",
    "print('Test set', test_dataset.shape, test_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# tools for showing and saving images\n",
    "\n",
    "def show_imagelist_as_grid(img_list, nrow, ncol):\n",
    "    fig = plt.figure(figsize=(5,5))\n",
    "    grid = AxesGrid(fig, 111, nrows_ncols=(nrow, ncol), axes_pad=0.05, label_mode=\"1\")\n",
    "    for i in range(nrow*ncol):\n",
    "        im = grid[i].imshow(img_list[i], interpolation=\"none\", cmap='gray', vmin=-0.5, vmax=0.5)\n",
    "    # grid.axes_llc.set_xticks([-1, 0, 1])\n",
    "    # grid.axes_llc.set_yticks([-1, 0, 1])\n",
    "\n",
    "    plt.draw()\n",
    "    plt.show()\n",
    "        \n",
    "def picture_grid(img_list,nrow,ncol):\n",
    "    #print(img_list.shape)\n",
    "    imsx, imsy = img_list.shape[1], img_list.shape[2]\n",
    "    mosarr=np.zeros([ncol*(imsx+1),nrow*(imsy+1)])+0.25\n",
    "    for i in range(ncol):\n",
    "        for j in range(nrow):\n",
    "            if img_list.shape[0]>j*ncol+i:\n",
    "                mosarr[i*(imsx+1):(i+1)*(imsx+1)-1, j*(imsy+1) : (j+1)*(imsy+1)-1] = img_list[j*ncol+i,:,:]\n",
    "    return mosarr\n",
    "\n",
    "def save_images(img_list,nrow,ncol,label):\n",
    "    imgtosave = picture_grid(img_list,nrow,ncol)*127.5+127.5\n",
    "    Image.fromarray(imgtosave.astype(np.uint8)).save('outputimages/' + label + '.jpeg', quality=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 11952,
     "status": "ok",
     "timestamp": 1446658914857,
     "user": {
      "color": "",
      "displayName": "",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "",
      "photoUrl": "",
      "sessionId": "0",
      "userId": ""
     },
     "user_tz": 480
    },
    "id": "IRSyYiIIGIzS",
    "outputId": "650a208c-8359-4852-f4f5-8bf10e80ef6c",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "image_size = 28\n",
    "num_labels = 10\n",
    "num_channels = 1 # grayscale\n",
    "\n",
    "dataset_mean = np.mean(train_dataset)\n",
    "dataset_std = np.std(train_dataset)\n",
    "print(\"mean and std: \", dataset_mean, dataset_std)\n",
    "show_imagelist_as_grid(train_dataset[:16].reshape(-1,image_size,image_size),4,4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# parameters\n",
    "batch_size = 24\n",
    "num_steps = 10000\n",
    "dummy_size = 768\n",
    "stddev=0.05\n",
    "dropout_prob_gen = 0.05\n",
    "dropout_prob_discr = 0.3\n",
    "lrelu_alpha = 0.1\n",
    "\n",
    "num_discr_layer1 = dummy_size//2\n",
    "num_discr_layer2 = dummy_size//2\n",
    "num_discr_layer3 = dummy_size//2\n",
    "\n",
    "# generator: 3 hidden layers\n",
    "num_gen_input_size = dummy_size//4\n",
    "num_gen_layer1 = dummy_size*2\n",
    "num_gen_layer2 = dummy_size*2\n",
    "num_gen_layer3 = dummy_size*2\n",
    "\n",
    "def gen_model():\n",
    "    model = Sequential()\n",
    "    model.add(Dense(num_gen_layer1, input_shape=(num_gen_input_size,), init='lecun_uniform'))\n",
    "    model.add(Dropout(dropout_prob_gen))\n",
    "    model.add(LeakyReLU(alpha=lrelu_alpha))\n",
    "    model.add(Dense(num_gen_layer2, init='lecun_uniform'))\n",
    "    model.add(Dropout(dropout_prob_gen))\n",
    "    model.add(LeakyReLU(alpha=lrelu_alpha))\n",
    "#     model.add(Dense(num_gen_layer3, init='normal'))\n",
    "#     model.add(Dropout(dropout_prob_gen))\n",
    "#     model.add(LeakyReLU(alpha=lrelu_alpha))\n",
    "    model.add(Dense(image_size*image_size, init='lecun_uniform'))\n",
    "    model.add(Activation('tanh'))\n",
    "    return model\n",
    "\n",
    "def discr_model():\n",
    "    model = Sequential()\n",
    "    model.add(MaxoutDense(num_discr_layer1, nb_feature=5, input_shape=(image_size*image_size,), init='lecun_uniform'))\n",
    "    model.add(Dropout(dropout_prob_discr))\n",
    "    #model.add(LeakyReLU(alpha=lrelu_alpha))\n",
    "    model.add(MaxoutDense(num_discr_layer2, nb_feature=5, init='lecun_uniform'))\n",
    "    model.add(Dropout(dropout_prob_discr))\n",
    "    #model.add(LeakyReLU(alpha=lrelu_alpha))\n",
    "    #model.add(MaxoutDense(num_discr_layer3, init='he_normal'))\n",
    "    #model.add(Dropout(dropout_prob_discr))\n",
    "    #model.add(LeakyReLU(alpha=lrelu_alpha))\n",
    "    model.add(Dense(1, init='lecun_uniform'))\n",
    "    model.add(Activation('sigmoid'))\n",
    "    return model\n",
    "\n",
    "def discr_on_gen_model(gen, discr):\n",
    "    model = Sequential()\n",
    "    model.add(gen)\n",
    "    discr.trainable = False\n",
    "    model.add(discr)\n",
    "    return model\n",
    "    \n",
    "discriminator = discr_model();\n",
    "generator = gen_model();\n",
    "discriminator_on_generator = discr_on_gen_model(generator, discriminator);\n",
    "\n",
    "discr_optimizer = SGD(lr=0.01)\n",
    "gen_optimizer = SGD(lr=0.01)\n",
    "\n",
    "generator.compile(loss='binary_crossentropy', optimizer=\"SGD\")\n",
    "discriminator_on_generator.compile(loss='binary_crossentropy', optimizer=gen_optimizer)\n",
    "discriminator.trainable = True # we had made it untrainable when we added to the discriminator_on_generator\n",
    "discriminator.compile(loss='binary_crossentropy', optimizer=discr_optimizer)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# training\n",
    "print_step = 30\n",
    "\n",
    "gen_loss_total = 0.0\n",
    "gen_trained = 0\n",
    "discr_loss_total = 0.0\n",
    "discr_trained = 0\n",
    "\n",
    "batch_1s = np.zeros(batch_size)+1\n",
    "batch_0s = np.zeros(batch_size)\n",
    "\n",
    "\n",
    "print(\"Starting training.\")\n",
    "for step in range(60000,60000+num_steps):\n",
    "\n",
    "    for i in range(3):    \n",
    "        #train discriminator\n",
    "        offset = (round(random.uniform(0, 100000)) + step * batch_size) % (train_labels.shape[0] - batch_size)\n",
    "        batch_data = train_dataset[offset:(offset + batch_size), :]\n",
    "        batch_noise = np.random.normal(loc=dataset_mean, scale=dataset_std, size=(batch_size, num_gen_input_size))\n",
    "        #batch_noise = np.random.uniform(-0.5, 0.5, size=(batch_size, num_gen_input_size))\n",
    "        generated_images = generator.predict(batch_noise, verbose=0)\n",
    "        # batch_X = np.concatenate((batch_data, generated_images))\n",
    "        # batch_Y = np.concatenate((np.zeros(batch_size)+1, np.zeros(batch_size)))\n",
    "        discriminator.trainable = True # just to make sure it's still trainable\n",
    "        ld1 = discriminator.train_on_batch(batch_data, batch_1s)\n",
    "        ld2 = discriminator.train_on_batch(generated_images, batch_0s)\n",
    "        discr_loss_total += (ld1+ld2)/2\n",
    "        discr_trained += 1\n",
    "    \n",
    "    for i in range(1):\n",
    "        #train generator\n",
    "        batch_noise = np.random.normal(loc=dataset_mean, scale=dataset_std, size=(batch_size, num_gen_input_size))    \n",
    "        #batch_noise = np.random.uniform(-0.5, 0.5, size=(batch_size, num_gen_input_size))    \n",
    "        discriminator.trainable = False # make sure we train only the generator\n",
    "        lg = discriminator_on_generator.train_on_batch(batch_noise, np.zeros(batch_size) + 1)\n",
    "        discriminator.trainable = True # just to make sure it's still trainable\n",
    "        gen_loss_total += lg\n",
    "        gen_trained += 1\n",
    "        \n",
    "    if (step % print_step == print_step-1):\n",
    "        if discr_trained == 0:\n",
    "            discr_trained = 1\n",
    "        print('Minibatch loss before step %d: discriminator %f, generator: %f' % (step+1, discr_loss_total/discr_trained, gen_loss_total/gen_trained))\n",
    "        gen_loss_total = 0.0\n",
    "        discr_loss_total = 0.0\n",
    "        gen_trained = 0\n",
    "        discr_trained = 0\n",
    "        save_images(generated_images.reshape(-1,image_size,image_size),4,4,\"gen_\"+ str(step))\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# cherry-picking nicely generated images\n",
    "cherry_picked = np.zeros([batch_size, image_size*image_size])\n",
    "num_picked = 0\n",
    "while num_picked < batch_size:\n",
    "    batch_noise = np.random.uniform(-0.5, 0.5, size=(batch_size, num_gen_input_size))    \n",
    "    generated_images = generator.predict(batch_noise, verbose=0)\n",
    "    preds = discriminator.predict(generated_images)\n",
    "    for i in range(batch_size):\n",
    "        if preds[i]>0.85:\n",
    "            cherry_picked[num_picked,:] = generated_images[i,:]\n",
    "            num_picked+=1\n",
    "            if(num_picked>=batch_size):\n",
    "                break\n",
    "#cher_no = 0\n",
    "#show_imagelist_as_grid(cherry_picked.reshape(-1,image_size,image_size), 4,4)\n",
    "save_images(cherry_picked.reshape(-1,image_size,image_size), 4,4, \"cherry_no\" + str(cher_no))\n",
    "cher_no+=1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# serialize model to JSON\n",
    "generator_json = generator.to_json()\n",
    "with open(\"generator.json\", \"w\") as json_file:\n",
    "    json_file.write(generator_json)\n",
    "# serialize weights to HDF5\n",
    "generator.save_weights(\"generator.h5\")\n",
    "print(\"Saved model to disk\")\n",
    "\n",
    "# serialize model to JSON\n",
    "discr_json = discriminator.to_json()\n",
    "with open(\"discriminator.json\", \"w\") as json_file:\n",
    "    json_file.write(discr_json)\n",
    "# serialize weights to HDF5\n",
    "discriminator.save_weights(\"discriminator.h5\")\n",
    "print(\"Saved model to disk\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# load the models from files:\n",
    "from keras.models import model_from_json\n",
    "\n",
    "# load and create generator\n",
    "json_file = open('generator.json', 'r')\n",
    "loaded_model_json = json_file.read()\n",
    "json_file.close()\n",
    "generator = model_from_json(loaded_model_json)\n",
    "generator.load_weights(\"generator.h5\")\n",
    "print(\"Loaded generator\")\n",
    "\n",
    "json_file = open('discriminator.json', 'r')\n",
    "loaded_model_json = json_file.read()\n",
    "json_file.close()\n",
    "discriminator = model_from_json(loaded_model_json)\n",
    "discriminator.load_weights(\"discriminator.h5\")\n",
    "print(\"Loaded discriminator\")\n",
    "\n",
    "discriminator_on_generator = discr_on_gen_model(generator, discriminator);\n",
    "\n",
    "discr_optimizer = SGD(lr=0.001, decay=1e-6, momentum=0.7)\n",
    "gen_optimizer = SGD(lr=0.001, decay=1e-6, momentum=0.7)\n",
    "\n",
    "generator.compile(loss='binary_crossentropy', optimizer=\"SGD\")\n",
    "discriminator_on_generator.compile(loss='binary_crossentropy', optimizer=gen_optimizer)\n",
    "discriminator.trainable = True # this will have to be switched on and off\n",
    "discriminator.compile(loss='binary_crossentropy', optimizer=discr_optimizer)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "generator.optimizer.lr=0.01\n",
    "discriminator_on_generator.optimizer.lr = 0.01"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "default_view": {},
   "name": "4_convolutions.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}