Prevent crossfilter to fail when data is missing. Replaced missing value with __missing__. Allow to pass missingValue in query

Author: christophe-g

src/column.js

@@ -120,7 +120,7 @@ export default function(service) {
           column.type = getType(sample)
         }
 
-        return dimension.make(column.key, column.type, column.complex)
+        return dimension.make(column.key, column.type, column.complex, column.missingValue)
       })
       .then(function(dim) {
         column.dimension = dim
@@ -136,7 +136,7 @@ export default function(service) {
             return Promise.resolve()
           }
 
-          var accessor = dimension.makeAccessor(column.key, column.complex)
+          var accessor = dimension.makeAccessor(column.key, column.complex, column.missingValue)
           column.values = column.values || []
 
           return new Promise(function(resolve, reject) {

src/dimension.js

@@ -6,19 +6,20 @@ export default function(service) {
     makeAccessor: makeAccessor,
   }
 
-  function make(key, type, complex) {
-    var accessor = makeAccessor(key, complex)
+  function make(key, type, complex, missingValue = '__missing__') {
+    var accessor = makeAccessor(key, complex, missingValue)
     // Promise.resolve will handle promises or non promises, so
     // this crossfilter async is supported if present
     return Promise.resolve(service.cf.dimension(accessor, type === 'array'))
   }
 
-  function makeAccessor(key, complex) {
+  function makeAccessor(key, complex, missingValue) {
     var accessorFunction
 
     if (complex === 'string') {
       accessorFunction = function(d) {
-        return _.get(d, key)
+        const value = _.get(d, key)
+        return value === undefined || isNaN(value) ? missingValue : value
       }
     } else if (complex === 'function') {
       accessorFunction = key
@@ -32,7 +33,10 @@ export default function(service) {
         // Index Dimension
         key === true ? (d, i) => i :
         // Value Accessor Dimension
-        (d) => d[key]
+        (d) => {
+          const value = d[key]
+          return value === undefined ? missingValue : value
+        }
     }
     return accessorFunction
   }

src/query.js

@@ -62,6 +62,7 @@ export default function(service) {
           key: query.original.groupBy,
           type: _.isUndefined(query.type) ? null : query.type,
           array: Boolean(query.array),
+          missingValue: query.original.missingValue
         })
         .then(function() {
           // Attach the column to the query

test/column.spec.js

@@ -2,6 +2,7 @@ import test from 'ava'
 
 import universe from '../src/universe'
 import data from './fixtures/data.json'
+import dataMissing from './fixtures/dataMissing.json'
 
 test('has the columns properties', async t => {
   const u = await universe(data)
@@ -45,6 +46,18 @@ test('can add a column with a complex key', async t => {
   t.is(typeof res.columns[0].dimension, 'object')
 })
 
+test('can add a column with a complex key and missing value', async t => {
+  const u = await universe(dataMissing)
+
+  const res = await u.column({
+    key: ['type', 'total', 'quantity', 'tip']
+  })
+
+  t.deepEqual(res.columns[0].key, ['type', 'total', 'quantity', 'tip'])
+  t.is(res.columns[0].type, 'complex')
+  t.is(typeof res.columns[0].dimension, 'object')
+})
+
 test('can add a column with nested string format', async t => {
   const u = await universe(data)
 

test/fixtures/dataMissing.json

@@ -0,0 +1,95 @@
+[
+{
+  "date": "2011-11-14T16:17:54Z",
+  "quantity": 2,
+  "total": 190,
+  "tip": 100,
+  "type": "tab",
+  "productIDs": ["001"]
+},
+{
+  "date": "2011-11-14T16:20:19Z",
+  "quantity": 2,
+  "total": 190,
+  "tip": 100,
+  "productIDs": ["001", "005"]
+},
+{
+  "date": "2011-11-14T16:28:54Z",
+  "quantity": 1,
+  "total": 300,
+  "tip": 200,
+  "type": "visa",
+  "productIDs": ["004", "005"]
+},
+{
+  "date": "2011-11-14T16:30:43Z",
+  "quantity": 2,
+  "total": 90,
+  "tip": 0,
+  "productIDs": ["001", "002"]
+},
+{
+  "date": "2011-11-14T16:48:46Z",
+  "quantity": 2,
+  "total": 90,
+  "tip": 0,
+  "type": "tab",
+  "productIDs": ["005"]
+},
+{
+  "date": "2011-11-14T16:53:41Z",
+  "quantity": 2,
+  "total": 90,
+  "tip": 0,
+  "type": "tab",
+  "productIDs": ["001", "004", "005"]
+},
+{
+  "date": "2011-11-14T16:54:06Z",
+  "quantity": 1,
+  "total": 100,
+  "tip": 0,
+  "type": "cash",
+  "productIDs": ["001", "002", "003", "004", "005"]
+},
+{
+  "date": "2011-11-14T16:58:03Z",
+  "quantity": 2,
+  "total": 90,
+  "tip": 0,
+  "type": "tab",
+  "productIDs": ["001"]
+},
+{
+  "date": "2011-11-14T17:07:21Z",
+  "quantity": 2,
+  "total": 90,
+  "tip": 0,
+  "type": "tab",
+  "productIDs": ["004", "005"]
+},
+{
+  "date": "2011-11-14T17:22:59Z",
+  "quantity": 2,
+  "total": 90,
+  "tip": 0,
+  "type": "tab",
+  "productIDs": ["001", "002", "004", "005"]
+},
+{
+  "date": "2011-11-14T17:25:45Z",
+  "quantity": 2,
+  "total": 200,
+  "tip": 0,
+  "type": "cash",
+  "productIDs": ["002"]
+},
+{
+  "date": "2011-11-14T17:29:52Z",
+  "quantity": 1,
+  "total": 200,
+  "tip": 100,
+  "type": "visa",
+  "productIDs": ["004"]
+}]

test/query.spec.js

@@ -2,6 +2,7 @@ import test from 'ava'
 
 import universe from '../src/universe'
 import data from './fixtures/data.json'
+import dataMissing from './fixtures/dataMissing.json'
 
 test('has the query method', async t => {
   const u = await universe(data)
@@ -198,6 +199,21 @@ test('supports groupBy', async t => {
   ])
 })
 
+test('supports groupBy - event with missing value', async t => {
+  const u = await universe(dataMissing)
+
+  const q = await u.query({
+    groupBy: 'type'
+  })
+
+  t.deepEqual(q.data, [
+    {key: '__missing__', value: {count: 2}},
+    {key: 'cash', value: {count: 2}},
+    {key: 'tab', value: {count: 6}},
+    {key: 'visa', value: {count: 2}}
+  ])
+})
+
 test('can query using the valueList aggregation', async t => {
   const u = await universe(data)