01_18-07-2018_srm_iwildcam-analyse-dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['info', 'images', 'annotations', 'categories'])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta_train = json.load(open('data/train_annotations.json'))\n",
    "meta_train.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "images_train = pd.DataFrame(meta_train['images'])\n",
    "annotations_train = pd.DataFrame(meta_train['annotations'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>height</th>\n",
       "      <th>id</th>\n",
       "      <th>location</th>\n",
       "      <th>rights_holder</th>\n",
       "      <th>width</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>train_val/5968c0f9-23d2-11e8-a6a3-ec086b02610b...</td>\n",
       "      <td>1494</td>\n",
       "      <td>5968c0f9-23d2-11e8-a6a3-ec086b02610b</td>\n",
       "      <td>26</td>\n",
       "      <td>Erin Boydston</td>\n",
       "      <td>2048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>train_val/5a0b016f-23d2-11e8-a6a3-ec086b02610b...</td>\n",
       "      <td>1494</td>\n",
       "      <td>5a0b016f-23d2-11e8-a6a3-ec086b02610b</td>\n",
       "      <td>26</td>\n",
       "      <td>Erin Boydston</td>\n",
       "      <td>2048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>train_val/59b93afb-23d2-11e8-a6a3-ec086b02610b...</td>\n",
       "      <td>1494</td>\n",
       "      <td>59b93afb-23d2-11e8-a6a3-ec086b02610b</td>\n",
       "      <td>38</td>\n",
       "      <td>Justin Brown</td>\n",
       "      <td>2048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>train_val/59641f56-23d2-11e8-a6a3-ec086b02610b...</td>\n",
       "      <td>1494</td>\n",
       "      <td>59641f56-23d2-11e8-a6a3-ec086b02610b</td>\n",
       "      <td>38</td>\n",
       "      <td>Justin Brown</td>\n",
       "      <td>2048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>train_val/5a1e5306-23d2-11e8-a6a3-ec086b02610b...</td>\n",
       "      <td>1494</td>\n",
       "      <td>5a1e5306-23d2-11e8-a6a3-ec086b02610b</td>\n",
       "      <td>33</td>\n",
       "      <td>Justin Brown</td>\n",
       "      <td>2048</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           file_name  height  \\\n",
       "0  train_val/5968c0f9-23d2-11e8-a6a3-ec086b02610b...    1494   \n",
       "1  train_val/5a0b016f-23d2-11e8-a6a3-ec086b02610b...    1494   \n",
       "2  train_val/59b93afb-23d2-11e8-a6a3-ec086b02610b...    1494   \n",
       "3  train_val/59641f56-23d2-11e8-a6a3-ec086b02610b...    1494   \n",
       "4  train_val/5a1e5306-23d2-11e8-a6a3-ec086b02610b...    1494   \n",
       "\n",
       "                                     id  location  rights_holder  width  \n",
       "0  5968c0f9-23d2-11e8-a6a3-ec086b02610b        26  Erin Boydston   2048  \n",
       "1  5a0b016f-23d2-11e8-a6a3-ec086b02610b        26  Erin Boydston   2048  \n",
       "2  59b93afb-23d2-11e8-a6a3-ec086b02610b        38   Justin Brown   2048  \n",
       "3  59641f56-23d2-11e8-a6a3-ec086b02610b        38   Justin Brown   2048  \n",
       "4  5a1e5306-23d2-11e8-a6a3-ec086b02610b        33   Justin Brown   2048  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "images_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category_id</th>\n",
       "      <th>id</th>\n",
       "      <th>image_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>825024ef-23d2-11e8-a6a3-ec086b02610b</td>\n",
       "      <td>5968c0f9-23d2-11e8-a6a3-ec086b02610b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>82937b1d-23d2-11e8-a6a3-ec086b02610b</td>\n",
       "      <td>5a0b016f-23d2-11e8-a6a3-ec086b02610b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>825c44a2-23d2-11e8-a6a3-ec086b02610b</td>\n",
       "      <td>59b93afb-23d2-11e8-a6a3-ec086b02610b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>821f3f55-23d2-11e8-a6a3-ec086b02610b</td>\n",
       "      <td>59641f56-23d2-11e8-a6a3-ec086b02610b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>823c54b7-23d2-11e8-a6a3-ec086b02610b</td>\n",
       "      <td>5a1e5306-23d2-11e8-a6a3-ec086b02610b</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   category_id                                    id  \\\n",
       "0            0  825024ef-23d2-11e8-a6a3-ec086b02610b   \n",
       "1            1  82937b1d-23d2-11e8-a6a3-ec086b02610b   \n",
       "2            1  825c44a2-23d2-11e8-a6a3-ec086b02610b   \n",
       "3            1  821f3f55-23d2-11e8-a6a3-ec086b02610b   \n",
       "4            1  823c54b7-23d2-11e8-a6a3-ec086b02610b   \n",
       "\n",
       "                               image_id  \n",
       "0  5968c0f9-23d2-11e8-a6a3-ec086b02610b  \n",
       "1  5a0b016f-23d2-11e8-a6a3-ec086b02610b  \n",
       "2  59b93afb-23d2-11e8-a6a3-ec086b02610b  \n",
       "3  59641f56-23d2-11e8-a6a3-ec086b02610b  \n",
       "4  5a1e5306-23d2-11e8-a6a3-ec086b02610b  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "annotations_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_train = pd.merge(images_train, annotations_train[['category_id', 'image_id']], left_on='id', right_on='image_id').drop('image_id', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>height</th>\n",
       "      <th>id</th>\n",
       "      <th>location</th>\n",
       "      <th>rights_holder</th>\n",
       "      <th>width</th>\n",
       "      <th>category_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>train_val/5968c0f9-23d2-11e8-a6a3-ec086b02610b...</td>\n",
       "      <td>1494</td>\n",
       "      <td>5968c0f9-23d2-11e8-a6a3-ec086b02610b</td>\n",
       "      <td>26</td>\n",
       "      <td>Erin Boydston</td>\n",
       "      <td>2048</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>train_val/5a0b016f-23d2-11e8-a6a3-ec086b02610b...</td>\n",
       "      <td>1494</td>\n",
       "      <td>5a0b016f-23d2-11e8-a6a3-ec086b02610b</td>\n",
       "      <td>26</td>\n",
       "      <td>Erin Boydston</td>\n",
       "      <td>2048</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>train_val/59b93afb-23d2-11e8-a6a3-ec086b02610b...</td>\n",
       "      <td>1494</td>\n",
       "      <td>59b93afb-23d2-11e8-a6a3-ec086b02610b</td>\n",
       "      <td>38</td>\n",
       "      <td>Justin Brown</td>\n",
       "      <td>2048</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>train_val/59641f56-23d2-11e8-a6a3-ec086b02610b...</td>\n",
       "      <td>1494</td>\n",
       "      <td>59641f56-23d2-11e8-a6a3-ec086b02610b</td>\n",
       "      <td>38</td>\n",
       "      <td>Justin Brown</td>\n",
       "      <td>2048</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>train_val/5a1e5306-23d2-11e8-a6a3-ec086b02610b...</td>\n",
       "      <td>1494</td>\n",
       "      <td>5a1e5306-23d2-11e8-a6a3-ec086b02610b</td>\n",
       "      <td>33</td>\n",
       "      <td>Justin Brown</td>\n",
       "      <td>2048</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           file_name  height  \\\n",
       "0  train_val/5968c0f9-23d2-11e8-a6a3-ec086b02610b...    1494   \n",
       "1  train_val/5a0b016f-23d2-11e8-a6a3-ec086b02610b...    1494   \n",
       "2  train_val/59b93afb-23d2-11e8-a6a3-ec086b02610b...    1494   \n",
       "3  train_val/59641f56-23d2-11e8-a6a3-ec086b02610b...    1494   \n",
       "4  train_val/5a1e5306-23d2-11e8-a6a3-ec086b02610b...    1494   \n",
       "\n",
       "                                     id  location  rights_holder  width  \\\n",
       "0  5968c0f9-23d2-11e8-a6a3-ec086b02610b        26  Erin Boydston   2048   \n",
       "1  5a0b016f-23d2-11e8-a6a3-ec086b02610b        26  Erin Boydston   2048   \n",
       "2  59b93afb-23d2-11e8-a6a3-ec086b02610b        38   Justin Brown   2048   \n",
       "3  59641f56-23d2-11e8-a6a3-ec086b02610b        38   Justin Brown   2048   \n",
       "4  5a1e5306-23d2-11e8-a6a3-ec086b02610b        33   Justin Brown   2048   \n",
       "\n",
       "   category_id  \n",
       "0            0  \n",
       "1            1  \n",
       "2            1  \n",
       "3            1  \n",
       "4            1  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26    28670\n",
       "57    12805\n",
       "23    12375\n",
       "38     9751\n",
       "46     6056\n",
       "43     4082\n",
       "33     3928\n",
       "61     2268\n",
       "7      1837\n",
       "0      1784\n",
       "51     1489\n",
       "27     1435\n",
       "41     1375\n",
       "34     1342\n",
       "28     1279\n",
       "36     1216\n",
       "40     1060\n",
       "64      963\n",
       "6       948\n",
       "18      783\n",
       "58      767\n",
       "10      722\n",
       "14      624\n",
       "24      616\n",
       "44      590\n",
       "56      556\n",
       "13      493\n",
       "53      428\n",
       "42      407\n",
       "60      405\n",
       "      ...  \n",
       "55      263\n",
       "22      259\n",
       "1       248\n",
       "39      246\n",
       "37      244\n",
       "2       235\n",
       "63      220\n",
       "4       211\n",
       "20      198\n",
       "29      196\n",
       "25      155\n",
       "35      152\n",
       "12      143\n",
       "48      143\n",
       "17      138\n",
       "31      117\n",
       "62      116\n",
       "45      106\n",
       "30      105\n",
       "16       67\n",
       "9        66\n",
       "21       52\n",
       "50       40\n",
       "8        24\n",
       "52       23\n",
       "11       21\n",
       "54       19\n",
       "49       12\n",
       "15        9\n",
       "47        8\n",
       "Name: location, Length: 65, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.location.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1494    101523\n",
       "1536      2755\n",
       "584       1447\n",
       "1920       703\n",
       "Name: height, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.height.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2048    104278\n",
       "800       1447\n",
       "2560       703\n",
       "Name: width, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.width.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "450.0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "2048, 1494\n",
    "0.75 * 600"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    55902\n",
       "0    50526\n",
       "Name: category_id, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.category_id.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_map_files(src, dest):\n",
    "    data = json.load(open(src))\n",
    "    imgs = pd.DataFrame(data['images'])\n",
    "    if dest is not 'test':\n",
    "        print('In if: {}'.format(dest))\n",
    "        annotations = pd.DataFrame(data['annotations'])\n",
    "        annotations = annotations.replace({'category_id': {0: 'no_animal', 1: 'animal'}})\n",
    "        df = pd.merge(imgs, annotations[['category_id', 'image_id']], left_on='id', right_on='image_id').drop('image_id', axis=1)\n",
    "        df = df[['file_name', 'location', 'category_id']]\n",
    "    else:\n",
    "        print('In else: {}'.format(dest))\n",
    "        df = imgs[['file_name', 'id', 'location']]\n",
    "    df.to_csv('data/{}.csv'.format(dest), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "In if: train\n",
      "In if: val\n"
     ]
    }
   ],
   "source": [
    "map_files = [\n",
    "    ('data/train_annotations.json', 'train'),\n",
    "    ('data/val_annotations.json', 'val'),\n",
    "#     ('data/test_information.json', 'test')\n",
    "]\n",
    "\n",
    "for src, dest in map_files:\n",
    "    create_map_files(src, dest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('data/train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>location</th>\n",
       "      <th>category_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>train_val/5968c0f9-23d2-11e8-a6a3-ec086b02610b...</td>\n",
       "      <td>26</td>\n",
       "      <td>no_animal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>train_val/5a0b016f-23d2-11e8-a6a3-ec086b02610b...</td>\n",
       "      <td>26</td>\n",
       "      <td>animal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>train_val/59b93afb-23d2-11e8-a6a3-ec086b02610b...</td>\n",
       "      <td>38</td>\n",
       "      <td>animal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>train_val/59641f56-23d2-11e8-a6a3-ec086b02610b...</td>\n",
       "      <td>38</td>\n",
       "      <td>animal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>train_val/5a1e5306-23d2-11e8-a6a3-ec086b02610b...</td>\n",
       "      <td>33</td>\n",
       "      <td>animal</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           file_name  location category_id\n",
       "0  train_val/5968c0f9-23d2-11e8-a6a3-ec086b02610b...        26   no_animal\n",
       "1  train_val/5a0b016f-23d2-11e8-a6a3-ec086b02610b...        26      animal\n",
       "2  train_val/59b93afb-23d2-11e8-a6a3-ec086b02610b...        38      animal\n",
       "3  train_val/59641f56-23d2-11e8-a6a3-ec086b02610b...        38      animal\n",
       "4  train_val/5a1e5306-23d2-11e8-a6a3-ec086b02610b...        33      animal"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_information.json\ttrain.csv  val_annotations.json\r\n",
      "train_annotations.json\ttrain_val  val.csv\r\n"
     ]
    }
   ],
   "source": [
    "!ls data/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}