AdultEngineer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from lib.data.wrangler import PreProcess, FeatureSelector\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# turns marital-status into boolean with-spouse and w/o spouse\n",
    "def reducemarriage(coldict, marriage):\n",
    "    spouses = {'Married-AF-spouse','Married-civ-spouse'}\n",
    "    withspouse = {v for k,v in coldict['marital-status'].iteritems() if k.strip() in spouses}\n",
    "    return marriage.isin(withspouse).astype(\"int\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# turns race into boolean white-nonwhite\n",
    "def reducerace(coldict, race):\n",
    "    white = None\n",
    "    for k,v in coldict.get('race').iteritems():\n",
    "        if k.strip() == 'White':\n",
    "            white = v\n",
    "    return (race==white).astype(\"int\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# turns native country into boolean developed vs under-developed\n",
    "def reducenativecountry(coldict, native):\n",
    "    rdev = set()\n",
    "    devlist = [\"Canada\", \"England\", \"France\", \"Germany\", \"Italy\",\n",
    "               \"Ireland\", \"Japan\", \"Portugal\", \"Taiwan\", \"India\",\n",
    "               \"Holand-Netherlands\", \"China\", \"United-States\"]\n",
    "    rdev = {v for k,v in coldict.get('native-country').iteritems() if k.strip() in devlist}\n",
    "    return native.isin(rdev).astype(\"int\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# reduces edu levels to 4 categories Nohighschool, HighSchool, Associate/Vocational, Graduate\n",
    "def reducedu(coldict, education):\n",
    "    noHS  = {\"10th\", \"11th\", \"12th\", \"1st-4th\", \"5th-6th\", \"7th-8th\", \"9th\", \"Preschool\"}\n",
    "    HS = {\"HS-grad\", \"Some-college\"}\n",
    "    AS = {\"Prof-school\", \"Assoc-acdm\", \"Assoc-voc\"}\n",
    "    nhs = set()\n",
    "    hs = set()\n",
    "    ass = set()\n",
    "    for k,v in coldict['education'].iteritems():\n",
    "        if k.strip() in noHS:\n",
    "            nhs.add(v)\n",
    "        elif k.strip() in HS:\n",
    "            hs.add(v)\n",
    "        elif k.strip() in AS:\n",
    "            ass.add(v)\n",
    "    edulevel = [1 if x in nhs else 2 \n",
    "                      if x in hs else 3\n",
    "                      if x in ass else 4\n",
    "                      for x in education.values]\n",
    "    return edulevel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Reduces Workclass to 3 categories gov-employee, self-employed, private-employee\n",
    "def reduceworkclass(coldict,workclass):\n",
    "    govemp = {\"Federal-gov\", \"Local-gov\", \"State-gov\"}\n",
    "    selfemp = {\"Self-emp-inc\", \"Self-emp-not-inc\"}\n",
    "    govnum = set()\n",
    "    selfnum = set()\n",
    "    privnum = set()\n",
    "\n",
    "    for k,v in coldict[\"workclass\"].iteritems():\n",
    "        if k.strip() in govemp:\n",
    "            govnum.add(v)\n",
    "        elif k.strip in selfemp:\n",
    "            selfnum.add(v)\n",
    "        else:\n",
    "            privnum.add(v)\n",
    "    worktype = [1 if x in govnum else 2\n",
    "                if x in selfnum else 3\n",
    "                for x in workclass.values]\n",
    "    return worktype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# reduces Occupation to boolean highly paid and lower-pay'\n",
    "def reduceoccup(coldict,occupation):\n",
    "    whitecols = {\"Exec-managerial\", \"Prof-specialty\", \"Tech-support\", \"Sales\"}\n",
    "    highsals = {v for k,v in coldict[\"occupation\"].iteritems() if k.strip() in whitecols}\n",
    "    return occupation.isin(highsals).astype(\"int\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# transforms the pre-preocessed data to get derived features\n",
    "def transformer(prepdata,coldict):\n",
    "    dervcols = [\"age\",\"hours-per-week\",\"sex\",\"income\"]\n",
    "    dervdata = prepdata[dervcols]\n",
    "    dervdata[\"marital-status\"] = reducemarriage(coldict, prepdata[\"marital-status\"])\n",
    "    dervdata[\"netcapita\"] = prepdata[\"capital-gain\"] - prepdata[\"capital-loss\"]\n",
    "    dervdata[\"race\"] = reducerace(coldict, prepdata[\"race\"])\n",
    "    dervdata[\"native-country\"] = reducenativecountry(coldict, prepdata[\"native-country\"])\n",
    "    dervdata[\"education-num\"] = reducedu(coldict, prepdata[\"education-num\"])\n",
    "    dervdata[\"workclass\"] = reduceworkclass(coldict,prepdata[\"workclass\"])\n",
    "    dervdata[\"occupation\"] = reduceoccup(coldict, prepdata[\"occupation\"])\n",
    "    dervdata = dervdata[[\"age\",\"netcapita\",\"education-num\",\n",
    "                         \"hours-per-week\",\"marital-status\",\"native-country\",\n",
    "                         \"occupation\",\"race\",\"sex\",\"workclass\",\"income\"]]\n",
    "    return dervdata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#trains an baseline classsifier and returns the score (Accuracy)\n",
    "def trainclf(clf, *args):\n",
    "    Xtrain,ytrain,Xtest,ytest = args\n",
    "    clf.fit(Xtrain,ytrain)\n",
    "    return clf.score(Xtest,ytest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Gets train and CV data for the classifier\n",
    "def getdata(selector):\n",
    "    Xtrain, ytrain = prepselector.features, prepselector.labels\n",
    "    Xtest, ytest = prepselector.testfeatures, prepselector.testlabels\n",
    "    return Xtrain, ytrain, Xtest, ytest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "# gets a performance dictionary for the types of data\n",
    "def comparedata(prepselector, dervselector, dumdervselector):\n",
    "    clfs = [DecisionTreeClassifier(), RandomForestClassifier(),\n",
    "            ExtraTreesClassifier(), LogisticRegression(),\n",
    "            GaussianNB(), SVC()]\n",
    "    prepdata = getdata(prepselector)\n",
    "    dervdata = getdata(dervselector)\n",
    "    dumdervdata = getdata(dumdervselector)\n",
    "    perf_dict = {}\n",
    "    for clf in clfs:\n",
    "        key = clf.__class__.__name__\n",
    "        perf_dict[key] = {\"prep\":trainclf(clf,*prepdata),\n",
    "                          \"derv\":trainclf(clf,*dervdata),\n",
    "                          \"dumderv\": trainclf(clf,*dumdervdata) }\n",
    "    return perf_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Creating a pre-process object.\n",
    "data = PreProcess()\n",
    "# this is the pre-processed data with encoded categories instead of strings\n",
    "prepdata = data.__getprep__()\n",
    "# a dictionay mapping the strings in raw to the encoded categories in prepdata\n",
    "coldict = data.col_map\n",
    "# this is the raw data\n",
    "rawdata = data.__getraw__()\n",
    "# add education mapping to coldict since there is no mapping for that column\n",
    "edudict = pd.Series(rawdata[\"education-num\"].values,index=rawdata[\"education\"]).to_dict()\n",
    "coldict['education'] = edudict\n",
    "# This is the derived data after doing some compression on the data\n",
    "dervdata = transformer(prepdata,coldict)\n",
    "\n",
    "# A temporary df containing only to categorical features for one-hot-encoding\n",
    "catFrame = dervdata[[\"education-num\",\"marital-status\",\"native-country\",\n",
    "                     \"occupation\",\"race\",\"sex\",\"workclass\"]]\n",
    "frames = [pd.get_dummies(catFrame[col], prefix=col) for col in catFrame.columns]\n",
    "\n",
    "# The one-hot-encoded features\n",
    "dumdervdata = pd.concat(frames, axis=1)\n",
    "dumdervdata = dumdervdata.merge(dervdata[[\"age\",\"netcapita\",\"hours-per-week\",\"income\"]], left_index=True, right_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Defining the featureselector objects for the 3 types of features\n",
    "prepselector = FeatureSelector(prepdata)\n",
    "dervselector = FeatureSelector(dervdata)\n",
    "dumdervselector = FeatureSelector(dumdervdata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'DecisionTreeClassifier': {'derv': 0.8150271266250384,\n",
       "  'dumderv': 0.81205855256423376,\n",
       "  'prep': 0.8112396355819429},\n",
       " 'ExtraTreesClassifier': {'derv': 0.83672842665574776,\n",
       "  'dumderv': 0.83539768655952507,\n",
       "  'prep': 0.83335039410379774},\n",
       " 'GaussianNB': {'derv': 0.79752277612856992,\n",
       "  'dumderv': 0.79752277612856992,\n",
       "  'prep': 0.79752277612856992},\n",
       " 'LogisticRegression': {'derv': 0.81963353465042477,\n",
       "  'dumderv': 0.81963353465042477,\n",
       "  'prep': 0.81963353465042477},\n",
       " 'RandomForestClassifier': {'derv': 0.84389395025079328,\n",
       "  'dumderv': 0.84399631487357973,\n",
       "  'prep': 0.84266557477735693},\n",
       " 'SVC': {'derv': 0.81052308322243838,\n",
       "  'dumderv': 0.81052308322243838,\n",
       "  'prep': 0.81052308322243838}}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# final comparision - prep - pre-processed data, derv - Compressed data, dumderv - compressed and one-hot encoded data\n",
    "comparedata(prepselector, dervselector, dumdervselector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}