06_vgg-neural-network-image-similarity.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This notebook uses the VGG16 neural network to create features for\n",
    "# finding similar images. It requires Tensorflow and the Keras application.\n",
    "# VGG16 was trained in ImageNet and can provide object detection, which can\n",
    "# be useful with these images and the network's representation of scaled images\n",
    "# as a vector of 1,000 values provides useful features for distance measurements,\n",
    "# classifications, and other image tasks. \n",
    "#\n",
    "# James E. Dobson\n",
    "# James.E.Dobson@Dartmouth.EDU\n",
    "# https://jeddobson.github.io/\n",
    "\n",
    "import numpy as np\n",
    "from glob import glob\n",
    "import cv2\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy, os, re\n",
    "import tensorflow as tf\n",
    "import sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# download and extract data\n",
    "!wget 'https://www.dropbox.com/s/7iq7kg623z5f0sd/opencv_data.tgz?dl=0' -O opencv_data.tgz\n",
    "!tar -zxf opencv_data.tgz > /dev/null 2>&1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.preprocessing import image\n",
    "from tensorflow.keras.applications.vgg16 import preprocess_input, decode_predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.applications.vgg16 import VGG16\n",
    "model = VGG16(weights='imagenet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_prediction(image_file,display_flag=False):\n",
    "    img = image.load_img(image_file,color_mode='rgb', target_size=(224, 224))\n",
    "    img_array = image.img_to_array(img)\n",
    "    img_array = np.expand_dims(img_array, axis=0)\n",
    "    img_array = preprocess_input(img_array)\n",
    "    features = model.predict(img_array)\n",
    "    \n",
    "    if display_flag:\n",
    "        display(img)\n",
    "    decoded = [x[1:] for x in decode_predictions(features)[0]]\n",
    "    return decoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_prediction('train/14719691_657210427789980_5511428924740993024_n.jpg',display_flag=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_prediction('train/14717662_219665258446439_2538176967283310592_n.jpg',display_flag=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_prediction('train/14482663_1739187826342894_146245723782905856_n.jpg',display_flag=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_prediction('train/14566625_1760095214239415_7171580328929132544_n.jpg',display_flag=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dartmouth_instagram = list()\n",
    "files = glob(\"data/*.jpg\")\n",
    "for file in files:\n",
    "    img = image.load_img(file,color_mode='rgb', target_size=(224, 224))\n",
    "    img_array = image.img_to_array(img)\n",
    "    img_array = np.expand_dims(img_array, axis=0)\n",
    "    img_array = preprocess_input(img_array)\n",
    "    dartmouth_instagram.append(model.predict(img_array).flatten())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KDTree\n",
    "kdt = KDTree(dartmouth_instagram, leaf_size=30, metric='euclidean')\n",
    "neighbors = kdt.query(dartmouth_instagram, k=10, return_distance=False)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_neighbors(neighboring_images):\n",
    "    plt.figure(figsize=(15,7), facecolor='white')\n",
    "    i = 1\n",
    "    for image in neighboring_images:\n",
    "        ax = plt.subplot(2, 5, i)\n",
    "        \n",
    "        # read and resize image to 200 x 200\n",
    "        img = cv2.imread(files[image])\n",
    "        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "        img = cv2.resize(img, (200, 200)) \n",
    "        ax.axis('off')\n",
    "        ax.imshow(img)\n",
    "        i += 1\n",
    "    plt.tight_layout()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display_neighbors(neighbors[16])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculate similarities with Euclidean distance metric\n",
    "from sklearn.metrics import euclidean_distances\n",
    "euclidean_dist_matrix = euclidean_distances(dartmouth_instagram)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in np.argsort(euclidean_dist_matrix[16]):\n",
    "    print(files[i],euclidean_dist_matrix[16][i])\n",
    "    img = cv2.imread(files[i])\n",
    "    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "    plt.imshow(img)\n",
    "    plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}