Dataset_preparation_disease_classification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Dataset preparation disease classification.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Dataset preparation for healthy-unhealthy classification & disease classification"
      ],
      "metadata": {
        "id": "-nlxnl8EOhUU"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p6_BzGj5NRpB",
        "outputId": "c7bc9e11-2543-4bf9-c05e-4760fe98b35e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import csv\n",
        "import pandas as pd\n",
        "import scipy.io\n",
        "import numpy as np\n",
        "import h5py"
      ],
      "metadata": {
        "id": "b73M7GqbNczc"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "healthy = np.loadtxt('/content/drive/MyDrive/FYP/A1_A2_A3_B_data/Data_csv/bal_healthy.csv', delimiter=',')\n",
        "ins = np.loadtxt('/content/drive/MyDrive/FYP/A1_A2_A3_B_data/Data_csv/bal_ins.csv', delimiter=',')\n",
        "nfle = np.loadtxt('/content/drive/MyDrive/FYP/A1_A2_A3_B_data/Data_csv/bal_nfle.csv', delimiter=',')\n",
        "narco = np.loadtxt('/content/drive/MyDrive/FYP/A1_A2_A3_B_data/Data_csv/bal_narco.csv', delimiter=',')\n",
        "rbd = np.loadtxt('/content/drive/MyDrive/FYP/A1_A2_A3_B_data/Data_csv/bal_rbd.csv', delimiter=',')\n",
        "plm = np.loadtxt('/content/drive/MyDrive/FYP/A1_A2_A3_B_data/Data_csv/bal_plm.csv', delimiter=',')"
      ],
      "metadata": {
        "id": "BarJ71GrOMQx"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Healthy-unhealthy dataset"
      ],
      "metadata": {
        "id": "cjJk78_KPzbp"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Both phases"
      ],
      "metadata": {
        "id": "xLJJcERuPzbt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "n1=np.shape(healthy)[0]\n",
        "n = (n1//10)*10   # 2 phases of 5 disorders each = 10\n",
        "n11=int(n/2)\n",
        "n12=int(n11/3)\n",
        "print('samples of healthy subjects:', n)\n",
        "n2=int(n/5)\n",
        "n3=int(n2/2)\n",
        "n4=int(n3/3)\n",
        "print('samples of each disorders:', n2)\n",
        "print('samples of unhealthy subjects:', n)\n",
        "total=int(2*n)\n",
        "print('Total samples:', total)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EWNsYPqkP28Q",
        "outputId": "0ad613e6-fae2-4776-dbb1-a07dc3cb2a8c"
      },
      "execution_count": 83,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "samples of healthy subjects: 9300\n",
            "samples of each disorders: 1860\n",
            "samples of unhealthy subjects: 9300\n",
            "Total samples: 18600\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data=np.zeros([total, 1026])\n",
        "\n",
        "# healthy\n",
        "i=0\n",
        "rows=np.where(healthy[:,1024]==0)\n",
        "data[i:i+n11,0:1025]=healthy[rows][0:n11,:]\n",
        "i+=n11\n",
        "rows=np.where(healthy[:,1024]==1)\n",
        "data[i:i+n12,0:1025]=healthy[rows][0:n12,:]\n",
        "i+=n12\n",
        "rows=np.where(healthy[:,1024]==2)\n",
        "data[i:i+n12,0:1025]=healthy[rows][0:n12,:]\n",
        "i+=n12\n",
        "rows=np.where(healthy[:,1024]==3)\n",
        "data[i:i+n12,0:1025]=healthy[rows][0:n12,:]\n",
        "i+=n12\n",
        "\n",
        "# insomnia\n",
        "rows=np.where(ins[:,1024]==0)\n",
        "data[i:i+n3,0:1025]=ins[rows][0:n3,:]\n",
        "i+=n3\n",
        "rows=np.where(ins[:,1024]==1)\n",
        "data[i:i+n4,0:1025]=ins[rows][0:n4,:]\n",
        "i+=n4\n",
        "rows=np.where(ins[:,1024]==2)\n",
        "data[i:i+n4,0:1025]=ins[rows][0:n4,:]\n",
        "i+=n4\n",
        "rows=np.where(ins[:,1024]==3)\n",
        "data[i:i+n4,0:1025]=ins[rows][0:n4,:]\n",
        "i+=n4\n",
        "\n",
        "# nfle\n",
        "rows=np.where(nfle[:,1024]==0)\n",
        "data[i:i+n3,0:1025]=nfle[rows][0:n3,:]\n",
        "i+=n3\n",
        "rows=np.where(nfle[:,1024]==1)\n",
        "data[i:i+n4,0:1025]=nfle[rows][0:n4,:]\n",
        "i+=n4\n",
        "rows=np.where(nfle[:,1024]==2)\n",
        "data[i:i+n4,0:1025]=nfle[rows][0:n4,:]\n",
        "i+=n4\n",
        "rows=np.where(nfle[:,1024]==3)\n",
        "data[i:i+n4,0:1025]=nfle[rows][0:n4,:]\n",
        "i+=n4\n",
        "\n",
        "# narcolepsy\n",
        "rows=np.where(narco[:,1024]==0)\n",
        "data[i:i+n3,0:1025]=narco[rows][0:n3,:]\n",
        "i+=n3\n",
        "rows=np.where(narco[:,1024]==1)\n",
        "data[i:i+n4,0:1025]=narco[rows][0:n4,:]\n",
        "i+=n4\n",
        "rows=np.where(narco[:,1024]==2)\n",
        "data[i:i+n4,0:1025]=narco[rows][0:n4,:]\n",
        "i+=n4\n",
        "rows=np.where(narco[:,1024]==3)\n",
        "data[i:i+n4,0:1025]=narco[rows][0:n4,:]\n",
        "i+=n4\n",
        "\n",
        "# rbd\n",
        "rows=np.where(rbd[:,1024]==0)\n",
        "data[i:i+n3,0:1025]=rbd[rows][0:n3,:]\n",
        "i+=n3\n",
        "rows=np.where(rbd[:,1024]==1)\n",
        "data[i:i+n4,0:1025]=rbd[rows][0:n4,:]\n",
        "i+=n4\n",
        "rows=np.where(rbd[:,1024]==2)\n",
        "data[i:i+n4,0:1025]=rbd[rows][0:n4,:]\n",
        "i+=n4\n",
        "rows=np.where(rbd[:,1024]==3)\n",
        "data[i:i+n4,0:1025]=rbd[rows][0:n4,:]\n",
        "i+=n4\n",
        "\n",
        "# plm\n",
        "rows=np.where(plm[:,1024]==0)\n",
        "data[i:i+n3,0:1025]=plm[rows][0:n3,:]\n",
        "i+=n3\n",
        "rows=np.where(plm[:,1024]==1)\n",
        "data[i:i+n4,0:1025]=plm[rows][0:n4,:]\n",
        "i+=n4\n",
        "rows=np.where(plm[:,1024]==2)\n",
        "data[i:i+n4,0:1025]=plm[rows][0:n4,:]\n",
        "i+=n4\n",
        "rows=np.where(plm[:,1024]==3)\n",
        "data[i:i+n4,0:1025]=plm[rows][0:n4,:]\n",
        "i+=n4\n",
        "\n",
        "# labelling unhealthy\n",
        "data[n:total,1025]=1\n",
        "np.savetxt('/content/drive/MyDrive/FYP/healthy_unhealthy_classification/dataset/healthy_unhealthy1.csv', data, delimiter=',')"
      ],
      "metadata": {
        "id": "4DU2UQhPQJcd"
      },
      "execution_count": 86,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "rows=np.where(data[:,1024]==0)\n",
        "print(np.shape(rows))\n",
        "rows=np.where(data[:,1024]==1)\n",
        "print(np.shape(rows))\n",
        "rows=np.where(data[:,1024]==2)\n",
        "print(np.shape(rows))\n",
        "rows=np.where(data[:,1024]==3)\n",
        "print(np.shape(rows))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3vEZ1XNraWtr",
        "outputId": "eb81b864-9597-4a96-cef6-e79a4ec37852"
      },
      "execution_count": 85,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(1, 9300)\n",
            "(1, 3100)\n",
            "(1, 3100)\n",
            "(1, 3100)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### B phase"
      ],
      "metadata": {
        "id": "2TiU1mYgUnSr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "data = np.loadtxt('/content/drive/MyDrive/FYP/healthy_unhealthy_classification/dataset/healthy_unhealthy1.csv', delimiter=',')\n",
        "rows = np.where(data[:,1024]==0)\n",
        "dataB=np.empty([int(np.shape(data)[0]/2), np.shape(data)[1]])\n",
        "dataB=data[rows]\n",
        "np.savetxt('/content/drive/MyDrive/FYP/healthy_unhealthy_classification/dataset/healthy_unhealthyB.csv', dataB, delimiter=',')"
      ],
      "metadata": {
        "id": "MJPVCtO2Upu_"
      },
      "execution_count": 92,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### A phase"
      ],
      "metadata": {
        "id": "jiC2bzLjV85U"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "data = np.loadtxt('/content/drive/MyDrive/FYP/healthy_unhealthy_classification/dataset/healthy_unhealthy1.csv', delimiter=',')\n",
        "n1=int(np.shape(data)[0]/2)\n",
        "n2=int(n1/3)\n",
        "dataA=np.empty([n1, np.shape(data)[1]])\n",
        "rows = np.where(data[:,1024]==1)\n",
        "dataA[0:n2]=data[rows]\n",
        "rows = np.where(data[:,1024]==2)\n",
        "dataA[n2:2*n2]=data[rows]\n",
        "rows = np.where(data[:,1024]==3)\n",
        "dataA[2*n2:3*n2]=data[rows]\n",
        "np.savetxt('/content/drive/MyDrive/FYP/healthy_unhealthy_classification/dataset/healthy_unhealthyA.csv', dataA, delimiter=',')"
      ],
      "metadata": {
        "id": "0A9ja4u3V85W"
      },
      "execution_count": 97,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Disease classification dataset"
      ],
      "metadata": {
        "id": "n8uAEV6hWWFj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "n1=np.shape(ins)[0]\n",
        "n2=np.shape(nfle)[0]\n",
        "n3=np.shape(narco)[0]\n",
        "n4=np.shape(rbd)[0]\n",
        "n5=np.shape(plm)[0]\n",
        "n_min=min(n1, n2, n3, n4, n5)\n",
        "print('Available Samples of ins:', n1)\n",
        "print('Available Samples of nfle:', n2)\n",
        "print('Available Samples of narco:', n3)\n",
        "print('Available Samples of rbd:', n4)\n",
        "print('Available Samples of plm:', n5)\n",
        "print('Samples considered of each disease:', n_min)\n",
        "n3=int(n_min/2)\n",
        "n4=int(n3/3)\n",
        "total=int(n_min*5)\n",
        "print('Total samples:', total)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QlgJem_JWZ6e",
        "outputId": "1056986c-a649-4251-b668-2daa23df53d6"
      },
      "execution_count": 109,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Available Samples of ins: 9960\n",
            "Available Samples of nfle: 75576\n",
            "Available Samples of narco: 9558\n",
            "Available Samples of rbd: 44100\n",
            "Available Samples of plm: 17934\n",
            "Samples considered of each disease: 9558\n",
            "Total samples: 47790\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data=np.zeros([total, 1026])\n",
        "\n",
        "i=0\n",
        "i1=0\n",
        "# insomnia\n",
        "rows=np.where(ins[:,1024]==0)\n",
        "data[i:i+n3,0:1025]=ins[rows][0:n3,:]\n",
        "i+=n3\n",
        "rows=np.where(ins[:,1024]==1)\n",
        "data[i:i+n4,0:1025]=ins[rows][0:n4,:]\n",
        "i+=n4\n",
        "rows=np.where(ins[:,1024]==2)\n",
        "data[i:i+n4,0:1025]=ins[rows][0:n4,:]\n",
        "i+=n4\n",
        "rows=np.where(ins[:,1024]==3)\n",
        "data[i:i+n4,0:1025]=ins[rows][0:n4,:]\n",
        "i+=n4\n",
        "data[i1:i,-1]=0\n",
        "i1=i\n",
        "\n",
        "# nfle\n",
        "rows=np.where(nfle[:,1024]==0)\n",
        "data[i:i+n3,0:1025]=nfle[rows][0:n3,:]\n",
        "i+=n3\n",
        "rows=np.where(nfle[:,1024]==1)\n",
        "data[i:i+n4,0:1025]=nfle[rows][0:n4,:]\n",
        "i+=n4\n",
        "rows=np.where(nfle[:,1024]==2)\n",
        "data[i:i+n4,0:1025]=nfle[rows][0:n4,:]\n",
        "i+=n4\n",
        "rows=np.where(nfle[:,1024]==3)\n",
        "data[i:i+n4,0:1025]=nfle[rows][0:n4,:]\n",
        "i+=n4\n",
        "data[i1:i,-1]=1\n",
        "i1=i\n",
        "\n",
        "# narcolepsy\n",
        "rows=np.where(narco[:,1024]==0)\n",
        "data[i:i+n3,0:1025]=narco[rows][0:n3,:]\n",
        "i+=n3\n",
        "rows=np.where(narco[:,1024]==1)\n",
        "data[i:i+n4,0:1025]=narco[rows][0:n4,:]\n",
        "i+=n4\n",
        "rows=np.where(narco[:,1024]==2)\n",
        "data[i:i+n4,0:1025]=narco[rows][0:n4,:]\n",
        "i+=n4\n",
        "rows=np.where(narco[:,1024]==3)\n",
        "data[i:i+n4,0:1025]=narco[rows][0:n4,:]\n",
        "i+=n4\n",
        "data[i1:i,-1]=2\n",
        "i1=i\n",
        "\n",
        "# rbd\n",
        "rows=np.where(rbd[:,1024]==0)\n",
        "data[i:i+n3,0:1025]=rbd[rows][0:n3,:]\n",
        "i+=n3\n",
        "rows=np.where(rbd[:,1024]==1)\n",
        "data[i:i+n4,0:1025]=rbd[rows][0:n4,:]\n",
        "i+=n4\n",
        "rows=np.where(rbd[:,1024]==2)\n",
        "data[i:i+n4,0:1025]=rbd[rows][0:n4,:]\n",
        "i+=n4\n",
        "rows=np.where(rbd[:,1024]==3)\n",
        "data[i:i+n4,0:1025]=rbd[rows][0:n4,:]\n",
        "i+=n4\n",
        "data[i1:i,-1]=3\n",
        "i1=i\n",
        "\n",
        "# plm\n",
        "rows=np.where(plm[:,1024]==0)\n",
        "data[i:i+n3,0:1025]=plm[rows][0:n3,:]\n",
        "i+=n3\n",
        "rows=np.where(plm[:,1024]==1)\n",
        "data[i:i+n4,0:1025]=plm[rows][0:n4,:]\n",
        "i+=n4\n",
        "rows=np.where(plm[:,1024]==2)\n",
        "data[i:i+n4,0:1025]=plm[rows][0:n4,:]\n",
        "i+=n4\n",
        "rows=np.where(plm[:,1024]==3)\n",
        "data[i:i+n4,0:1025]=plm[rows][0:n4,:]\n",
        "i+=n4\n",
        "data[i1:i,-1]=4\n",
        "i1=i\n",
        "\n",
        "np.savetxt('/content/drive/MyDrive/ML/Disease_classification/Dataset/both_phases.csv', data, delimiter=',')"
      ],
      "metadata": {
        "id": "xV5QxBmyW_sp"
      },
      "execution_count": 112,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### B phase"
      ],
      "metadata": {
        "id": "u_PYgyTIj6R0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "data = np.loadtxt('/content/drive/MyDrive/ML/Disease_classification/Dataset/both_phases.csv', delimiter=',')\n",
        "rows = np.where(data[:,1024]==0)\n",
        "dataB=np.empty([int(np.shape(data)[0]/2), np.shape(data)[1]])\n",
        "dataB=data[rows]\n",
        "np.savetxt('/content/drive/MyDrive/ML/Disease_classification/Dataset/B_phases.csv', dataB, delimiter=',')"
      ],
      "metadata": {
        "id": "IxV51_upj6R1"
      },
      "execution_count": 118,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### A phase"
      ],
      "metadata": {
        "id": "qaaHiJRxj6R8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "data = np.loadtxt'/content/drive/MyDrive/ML/Disease_classification/Dataset/both_phases.csv', delimiter=',')\n",
        "n1=int(np.shape(data)[0]/2)\n",
        "n2=int(n1/3)\n",
        "dataA=np.empty([n1, np.shape(data)[1]])\n",
        "rows = np.where(data[:,1024]==1)\n",
        "dataA[0:n2]=data[rows]\n",
        "rows = np.where(data[:,1024]==2)\n",
        "dataA[n2:2*n2]=data[rows]\n",
        "rows = np.where(data[:,1024]==3)\n",
        "dataA[2*n2:3*n2]=data[rows]\n",
        "np.savetxt('/content/drive/MyDrive/ML/Disease_classification/Dataset/A_phases.csv', dataA, delimiter=',')"
      ],
      "metadata": {
        "id": "XC9D5iaWj6R9"
      },
      "execution_count": 125,
      "outputs": []
    }
  ]
}