sk_resnet-50.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import division,print_function\n",
    "\n",
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import sys\n",
    "sys.path.append('../')\n",
    "from tqdm.notebook import tqdm\n",
    "import numpy as np\n",
    "import os\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torchvision import transforms, models, datasets\n",
    "import utils.calculate_log as callog\n",
    "from my_models import resnet_50\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setting the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "torch_model = resnet_50.Net(models.resnet50(pretrained=False), 8)\n",
    "ckpt = torch.load(\"checkpoints/resnet-50_checkpoint.pth\")\n",
    "torch_model.load_state_dict(ckpt['model_state_dict'])\n",
    "torch_model.eval()\n",
    "torch_model.cuda()\n",
    "print(\"Done!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setting the hook register"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feat_maps = list()\n",
    "def _hook_fn(self, input, output):\n",
    "    feat_maps.append(output)\n",
    "    \n",
    "\n",
    "def hook_layers(model):\n",
    "    hooked_layers = list()\n",
    "    for layer in torch_model.modules():\n",
    "        if isinstance(layer, models.resnet.Bottleneck):\n",
    "#         if isinstance(layer, nn.ReLU) or isinstance(layer, nn.Conv2d):\n",
    "            hooked_layers.append(layer)\n",
    "    return hooked_layers\n",
    "\n",
    "\n",
    "def register_layers(layers):\n",
    "    regs_layers = list()\n",
    "    for lay in layers:\n",
    "        regs_layers.append(lay.register_forward_hook(_hook_fn))\n",
    "    return regs_layers\n",
    "\n",
    "\n",
    "def unregister_layers(reg_layers):\n",
    "    for lay in reg_layers:\n",
    "        lay.remove()\n",
    "                    \n",
    "\n",
    "def get_feat_maps(model, batch_img):\n",
    "    batch_img = batch_img.cuda()\n",
    "    with torch.no_grad():\n",
    "        preds = model(batch_img)\n",
    "\n",
    "    preds = F.softmax(preds, dim=1)\n",
    "    maps = feat_maps.copy()\n",
    "    feat_maps.clear()\n",
    "    return preds, maps\n",
    "\n",
    "## Setting the hook\n",
    "hl = hook_layers (torch_model)\n",
    "rgl = register_layers (hl)\n",
    "print (\"Total number of registered hooked layers:\", len(rgl))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading the data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## In distributions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 30\n",
    "trans = transforms.Compose([\n",
    "#             transforms.Resize((224,224)),\n",
    "            transforms.ToTensor(),\n",
    "            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n",
    "        ])\n",
    "\n",
    "\n",
    "sk_train = torch.utils.data.DataLoader(\n",
    "                datasets.ImageFolder(\"data/skin_cancer/train/\",transform=trans),\n",
    "                batch_size=batch_size,\n",
    "                shuffle=False)\n",
    "\n",
    "sk_test = torch.utils.data.DataLoader(\n",
    "                datasets.ImageFolder(\"data/skin_cancer/test/\",transform=trans),\n",
    "                batch_size=batch_size,\n",
    "                shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Out-of-distributions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "skin_cli = torch.utils.data.DataLoader(\n",
    "                datasets.ImageFolder(\"data/skins/clinical/\",transform=trans),\n",
    "                batch_size=batch_size,\n",
    "                shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "skin_derm = torch.utils.data.DataLoader(\n",
    "                datasets.ImageFolder(\"data/skins/dermoscopy/\",transform=trans),\n",
    "                batch_size=batch_size,\n",
    "                shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "imgnet = torch.utils.data.DataLoader(\n",
    "                datasets.ImageFolder(\"data/imagenet/\",transform=trans),\n",
    "                batch_size=batch_size,\n",
    "                shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corrupted = torch.utils.data.DataLoader(\n",
    "                datasets.ImageFolder(\"data/corrupted/bbox/\",transform=trans),\n",
    "                batch_size=batch_size,\n",
    "                shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corrupted_70 = torch.utils.data.DataLoader(\n",
    "                datasets.ImageFolder(\"data/corrupted/bbox_70/\",transform=trans),\n",
    "                batch_size=batch_size,\n",
    "                shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nct = torch.utils.data.DataLoader(\n",
    "                datasets.ImageFolder(\"data/nct/\",transform=trans),     \n",
    "                batch_size=batch_size,\n",
    "                shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gram-Matrix operations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gram matrix operator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def norm_min_max(x):\n",
    "    ma = torch.max(x,dim=1)[0].unsqueeze(1)\n",
    "    mi = torch.min(x,dim=1)[0].unsqueeze(1)\n",
    "    x = (x-mi)/(ma-mi)\n",
    "    return x\n",
    "\n",
    "def get_sims_gram_matrix (maps, power):\n",
    "    maps = F.relu(maps)    \n",
    "    maps = maps ** power    \n",
    "    maps = maps.reshape(maps.shape[0],maps.shape[1],-1)\n",
    "    gram = ((torch.matmul(maps,maps.transpose(dim0=2,dim1=1)))).sum(2)\n",
    "    gram = (gram.sign()*torch.abs(gram)**(1/power)).reshape(gram.shape[0],-1)  \n",
    "    gram = norm_min_max(gram)\n",
    "    return gram\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Considering samples per label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _get_sim_per_labels(data_loader, power, use_preds=True):\n",
    "    \n",
    "    sims_per_label = None\n",
    "    if not isinstance(power, list) and not isinstance(power, range):\n",
    "        power = [power]\n",
    "    \n",
    "    for data in tqdm(data_loader):\n",
    "        img_batch, labels = data \n",
    "        preds, maps_list = get_feat_maps(torch_model, img_batch)\n",
    "      \n",
    "        if use_preds:\n",
    "            labels = preds.argmax(dim=1)  \n",
    "                \n",
    "        if sims_per_label is None:\n",
    "            sims_per_label = [[[] for _ in range(len(maps_list))] for _ in range(preds.shape[1])]  \n",
    "           \n",
    "        for layer, maps in enumerate(maps_list): \n",
    "            for p in power:\n",
    "                sims = get_sims_gram_matrix (maps, p)\n",
    "\n",
    "                for sim, lab in zip(sims, labels):              \n",
    "                    sims_per_label[lab.item()][layer].append(sim.cpu()) \n",
    "                \n",
    "    return sims_per_label\n",
    "\n",
    "\n",
    "def get_min_max_per_label(data_loader, power):\n",
    "    \n",
    "    sims_per_label = _get_sim_per_labels(data_loader, power)\n",
    "    sims_per_label_min = [[[] for _ in range(len(sims_per_label[0]))] for _ in range(len(sims_per_label))] \n",
    "    sims_per_label_max = [[[] for _ in range(len(sims_per_label[0]))] for _ in range(len(sims_per_label))] \n",
    "    \n",
    "    \n",
    "    print (\"-- Computing the values...\")\n",
    "    for lab_idx in range(len(sims_per_label)):\n",
    "        for layer_idx in range(len(sims_per_label[lab_idx])):\n",
    "            temp = torch.stack(sims_per_label[lab_idx][layer_idx])\n",
    "            sims_per_label_min[lab_idx][layer_idx] = temp.min(dim=0)[0] \n",
    "            sims_per_label_max[lab_idx][layer_idx] = temp.max(dim=0)[0]\n",
    "    \n",
    "    del sims_per_label\n",
    "    \n",
    "    return sims_per_label_min, sims_per_label_max\n",
    "\n",
    "\n",
    "def get_layer_gaps(mins, maxs):  \n",
    "    num_lab, num_lay = len(mins), len(mins[0])    \n",
    "    gaps = torch.zeros(num_lab, num_lay)\n",
    "    gaps = gaps.cuda()\n",
    "    \n",
    "    for lab in range(num_lab):      \n",
    "        for layer in range(num_lay):\n",
    "            gaps[lab][layer] = (maxs[lab][layer]-mins[lab][layer]).sum()\n",
    "            \n",
    "    return gaps.cpu().numpy()\n",
    "\n",
    "\n",
    "def get_dev_scores_per_label(data_loader, power, sims_min, sims_max, ep=10e-6):\n",
    "    \n",
    "    if not isinstance(power, list) and not isinstance(power, range):\n",
    "        power = [power]\n",
    "    \n",
    "    dev_scores = list()    \n",
    "    for data in tqdm(data_loader):\n",
    "        img_batch, _ = data \n",
    "        preds_batch, maps_list = get_feat_maps(torch_model, img_batch)                \n",
    "        labels = preds_batch.argmax(dim=1)\n",
    "        batch_scores = list()\n",
    "       \n",
    "        for layer, maps in enumerate(maps_list):\n",
    "                \n",
    "            score_layer = 0\n",
    "            for p in power:\n",
    "                sims = get_sims_gram_matrix (maps, p)  \n",
    "                _sim_min = torch.zeros(sims.shape[0], sims.shape[1]).cuda()\n",
    "                _sim_max = torch.zeros(sims.shape[0], sims.shape[1]).cuda()\n",
    "            \n",
    "                for k, lab in enumerate(labels):\n",
    "                    _sim_min[k] = sims_min[lab.item()][layer]\n",
    "                    _sim_max[k] = sims_max[lab.item()][layer]            \n",
    "            \n",
    "                score_layer += (F.relu(_sim_min-sims)/torch.abs(_sim_min+ep)).sum(dim=1, keepdim=True)\n",
    "                score_layer += (F.relu(sims-_sim_max)/torch.abs(_sim_max+ep)).sum(dim=1, keepdim=True)\n",
    "           \n",
    "            batch_scores.append(score_layer)            \n",
    "            \n",
    "        dev_scores.append(torch.cat(batch_scores, dim=1)) \n",
    "\n",
    "    return torch.cat(dev_scores).cpu().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def detect_mean(all_test_std, all_ood_std, gaps=None): \n",
    "    \n",
    "    avg_results = dict()\n",
    "    indices = list(range(len(all_test_std)))\n",
    "    split = int(np.floor(0.1 * len(all_test_std))) \n",
    "    for i in range(1,11):\n",
    "        np.random.seed(i)\n",
    "        np.random.shuffle(indices)\n",
    "        \n",
    "        val_std = all_test_std[indices[:split]]\n",
    "        test_std = all_test_std[indices[split:]]\n",
    "        \n",
    "        if gaps is not None:\n",
    "            t95 = (val_std.sum(axis=0) + gaps.mean(0))\n",
    "        else:\n",
    "            t95 = val_std.mean(axis=0) + 10**-7\n",
    "        \n",
    "        test_std = ((test_std)/t95[np.newaxis,:]).sum(axis=1)\n",
    "        ood_std = ((all_ood_std)/t95[np.newaxis,:]).sum(axis=1)\n",
    "\n",
    "        results = callog.compute_metric(-test_std,-ood_std)  \n",
    "\n",
    "        for m in results:\n",
    "            avg_results[m] = avg_results.get(m,0)+results[m]\n",
    "    \n",
    "    for m in avg_results:\n",
    "        avg_results[m] /= i\n",
    "        \n",
    "        \n",
    "    callog.print_results(avg_results)\n",
    "    \n",
    "    return avg_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# OOD detection per label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "power = 1\n",
    "\n",
    "print (\"- Getting mins/maxs\")\n",
    "mins, maxs = get_min_max_per_label(sk_train, power)\n",
    "\n",
    "print(\"- Getting the gaps\")\n",
    "gaps = get_layer_gaps(mins, maxs) \n",
    "\n",
    "print (\"- Getting test stdevs\")\n",
    "sk_test_stdev = get_dev_scores_per_label(sk_test, power, mins, maxs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Releasing the GPU cache memory\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "print(\"Skins dermoscopy\")\n",
    "skin_derm_stdev = get_dev_scores_per_label(skin_derm, power, mins, maxs)\n",
    "skin_derm_results = detect_mean(sk_test_stdev, skin_derm_stdev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Skins clinical\")\n",
    "skin_cli_stdev = get_dev_scores_per_label(skin_cli, power, mins, maxs)\n",
    "skin_cli_results = detect_mean(sk_test_stdev, skin_cli_stdev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"ImageNet\")\n",
    "imgnet_stdev = get_dev_scores_per_label(imgnet, power, mins, maxs)\n",
    "imgent_results = detect_mean(sk_test_stdev, imgnet_stdev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Corrupted images\")\n",
    "corrupted_stdev = get_dev_scores_per_label(corrupted, power, mins, maxs)\n",
    "corrupted_results = detect_mean(sk_test_stdev, corrupted_stdev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Corrupted 70 images\")\n",
    "corrupted_70_stdev = get_dev_scores_per_label(corrupted_70, power, mins, maxs)\n",
    "corrupted_70_results = detect_mean(sk_test_stdev, corrupted_70_stdev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"NCT\")\n",
    "nct_stdev = get_dev_scores_per_label(nct, power, mins, maxs)\n",
    "nct_results = detect_mean(sk_test_stdev, nct_stdev)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(round(skin_derm_results['TNR']*100,3))\n",
    "print(round(skin_cli_results['TNR']*100,3))\n",
    "print(round(imgent_results['TNR']*100,3))\n",
    "print(round(corrupted_results['TNR']*100,3))\n",
    "print(round(corrupted_70_results['TNR']*100,3))\n",
    "print(round(nct_results['TNR']*100,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}