20190903_Partition_with_NAN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-09-03T08:15:44-0400 - INFO - Great Expectations logging enabled at INFO level by JupyterUX module.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'0.7.8__develop'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import great_expectations as ge\n",
    "from great_expectations import jupyter_ux\n",
    "\n",
    "ge.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Build a sample dataset\n",
    "\n",
    "arr = np.random.normal(0, 1, size=(1000, 2))\n",
    "df = ge.dataset.PandasDataset(arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-09-03T08:15:44-0400 - INFO - \t0 expectation(s) included in expectation_suite.\n"
     ]
    }
   ],
   "source": [
    "## Add nan to one of the columns\n",
    "df.iloc[df.sample(n=200, random_state=42).index, 0] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.010992152593699036\n",
      "0.02775779505956932\n",
      "1000\n",
      "1000\n"
     ]
    }
   ],
   "source": [
    "# Sanity check\n",
    "\n",
    "print(df[0].mean())\n",
    "print(df[1].mean())\n",
    "print(len(df[0]))\n",
    "print(len(df[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'success': True,\n",
       " 'result': {'element_count': 1000,\n",
       "  'missing_count': 200,\n",
       "  'missing_percent': 0.2,\n",
       "  'unexpected_count': 35,\n",
       "  'unexpected_percent': 0.035,\n",
       "  'unexpected_percent_nonmissing': 0.04375,\n",
       "  'partial_unexpected_list': [-2.4033208540028412,\n",
       "   2.1502800125318458,\n",
       "   2.073732781983799,\n",
       "   -2.249650417687381,\n",
       "   2.04795047563228,\n",
       "   -2.1726062732984786,\n",
       "   -3.0591044634104514,\n",
       "   -2.2027013079982862,\n",
       "   -3.4556200297278057,\n",
       "   2.2348324470898677,\n",
       "   2.4558580336475506,\n",
       "   -2.2315575942268113,\n",
       "   2.8059498930506224,\n",
       "   2.4093065996175094,\n",
       "   -2.2105236145371396,\n",
       "   2.613132377841119,\n",
       "   2.165261195568698,\n",
       "   3.5796465541410876,\n",
       "   2.181309684837725,\n",
       "   -2.4369409996835043]}}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Sanity check with expectation; notice missing count == 200 as expected\n",
    "df.expect_column_values_to_be_between(0, -2, 2, mostly=0.95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'success': True,\n",
       " 'result': {'element_count': 1000,\n",
       "  'missing_count': 0,\n",
       "  'missing_percent': 0.0,\n",
       "  'unexpected_count': 47,\n",
       "  'unexpected_percent': 0.047,\n",
       "  'unexpected_percent_nonmissing': 0.047,\n",
       "  'partial_unexpected_list': [-3.125848356837646,\n",
       "   -2.020496744927424,\n",
       "   3.361073270662882,\n",
       "   2.002711626637642,\n",
       "   -2.202524145192944,\n",
       "   -2.057317226179811,\n",
       "   -2.0523993802762965,\n",
       "   2.151414645665574,\n",
       "   2.399518487894312,\n",
       "   -2.6512801651040903,\n",
       "   2.2140311059608813,\n",
       "   -2.0331049035979034,\n",
       "   -2.513228680180754,\n",
       "   -2.0609690728228163,\n",
       "   2.8796064746097585,\n",
       "   -2.4002678271683173,\n",
       "   -2.534450679847885,\n",
       "   2.692123055869081,\n",
       "   -2.0694862842788857,\n",
       "   2.289306339883892]}}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Sanity check with expectation; notice missing count == 0 as expected\n",
    "df.expect_column_values_to_be_between(1, -2, 2, mostly=0.95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "autodetected range of [nan, nan] is not finite",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-54a5e47580ff>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m## Observe failure: we cannot use continuous partition without\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mpart_0\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mge\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontinuous_partition_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/dev/great_expectations/great_expectations/dataset/util.py\u001b[0m in \u001b[0;36mcontinuous_partition_data\u001b[0;34m(data, bins, n_bins)\u001b[0m\n\u001b[1;32m    158\u001b[0m         \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Invalid parameter for bins argument\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    159\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 160\u001b[0;31m     \u001b[0mhist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbin_edges\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistogram\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdensity\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    161\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    162\u001b[0m     return {\n",
      "\u001b[0;32m~/anaconda3/envs/dsci/lib/python3.6/site-packages/numpy/lib/histograms.py\u001b[0m in \u001b[0;36mhistogram\u001b[0;34m(a, bins, range, normed, weights, density)\u001b[0m\n\u001b[1;32m    778\u001b[0m     \u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweights\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_ravel_and_check_weights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    779\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 780\u001b[0;31m     \u001b[0mbin_edges\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muniform_bins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_get_bin_edges\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    781\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    782\u001b[0m     \u001b[0;31m# Histogram is an integer or a float array depending on the weights.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/dsci/lib/python3.6/site-packages/numpy/lib/histograms.py\u001b[0m in \u001b[0;36m_get_bin_edges\u001b[0;34m(a, bins, range, weights)\u001b[0m\n\u001b[1;32m    385\u001b[0m                             \"bins is not supported for weighted data\")\n\u001b[1;32m    386\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 387\u001b[0;31m         \u001b[0mfirst_edge\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlast_edge\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_get_outer_edges\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    388\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    389\u001b[0m         \u001b[0;31m# truncate the range if needed\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/dsci/lib/python3.6/site-packages/numpy/lib/histograms.py\u001b[0m in \u001b[0;36m_get_outer_edges\u001b[0;34m(a, range)\u001b[0m\n\u001b[1;32m    313\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misfinite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfirst_edge\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misfinite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlast_edge\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    314\u001b[0m             raise ValueError(\n\u001b[0;32m--> 315\u001b[0;31m                 \"autodetected range of [{}, {}] is not finite\".format(first_edge, last_edge))\n\u001b[0m\u001b[1;32m    316\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    317\u001b[0m     \u001b[0;31m# expand empty range to avoid divide by zero\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: autodetected range of [nan, nan] is not finite"
     ]
    }
   ],
   "source": [
    "## Observe failure: we cannot use continuous partition without \n",
    "part_0 = ge.dataset.util.continuous_partition_data(df[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Observe success: without NAN everything is as expected\n",
    "part_1 = ge.dataset.util.continuous_partition_data(df[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28\n",
      "11\n"
     ]
    }
   ],
   "source": [
    "## Simple workaround: we can ask for 'uniform' bins instead of auto.\n",
    "part_0_uniform = ge.dataset.util.continuous_partition_data(df[0], bins='uniform')\n",
    "\n",
    "# DRAWBACK: we lose the auto guessing about the *number* of bins\n",
    "print(len(part_1[\"bins\"]))\n",
    "print(len(part_0_uniform[\"bins\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28\n",
      "11\n"
     ]
    }
   ],
   "source": [
    "## Sligthly better workaround: use a heuristic like numpy does to guess the right n_bins\n",
    "# See: https://docs.scipy.org/doc/numpy/reference/generated/numpy.histogram_bin_edges.html#numpy.histogram_bin_edges\n",
    "# Manually use max of sturges and fd\n",
    "\n",
    "n_bins = np.max(\n",
    "  np.log2(len(df[0])),\n",
    "  # cannot use sturges since np.percentile(df[0], [0.25, 0.75]) returns nan\n",
    ")\n",
    "part_0_uniform_n_bins = ge.dataset.util.continuous_partition_data(df[0], bins='uniform', n_bins=np.log2(len(df[0])))\n",
    "\n",
    "# This works, and will eventually scale with bigger data, but still suggests fewer bins than fd\n",
    "# However, fd requires quartiles, which np also fails to provide when the data contains NAN\n",
    "print(len(part_1[\"bins\"]))\n",
    "print(len(part_0_uniform[\"bins\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# np.histogram also gives another workaround: specifying range. \n",
    "## Recommendation: We should accept kwargs to pass to np.histogram\n",
    "b, e = np.histogram(df[0], range=(df[0].min(), df[0].max()))\n",
    "\n",
    "# Note that this still doesn't give the same nice behavior as auto\n",
    "len(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}