fix bug with processing service catalog (#134)

quest/plugins/base/service_base.py

@@ -187,10 +187,7 @@ def fn(row):
  # merge extra data columns/fields into metadata as a dictionary
  extra_fields = list(set(catalog_entries.columns.tolist()) - set(reserved_catalog_entry_fields))
  # change NaN to None so it can be JSON serialized properly
- catalog_entries['metadata'] = [
- {k: None if v != v else v for k, v in record.items()}
- for record in catalog_entries[extra_fields].to_dict(orient='records')
- ]
+ catalog_entries['metadata'] = json.loads(catalog_entries[extra_fields].to_json(orient='records'))
  catalog_entries.drop(extra_fields, axis=1, inplace=True)
  columns = list(set(catalog_entries.columns.tolist()).intersection(reserved_geometry_fields))
  catalog_entries.drop(columns, axis=1, inplace=True)

quest_provider_plugins/usgs_nwis.py

@@ -223,7 +223,7 @@ def _states():
 
 
 def _parse_rdb(url, index=None):
- df = pd.read_table(url, comment='#')
+ df = pd.read_csv(url, sep='\t', comment='#')
  if index is not None:
  df.index = df[index]
 

test/data.py

@@ -292,7 +292,7 @@
 SERVICES_CATALOG_COUNT = [
  # ('svc://nasa:srtm-3-arc-second', 14297, 1000),
  # ('svc://nasa:srtm-30-arc-second', 27, 10),
- ('svc://noaa-ncdc:ghcn-daily', 104126, 5000),
+ ('svc://noaa-ncdc:ghcn-daily', 113948, 5000),
  ('svc://noaa-ncdc:gsod', 28754, 1500),
  ('svc://noaa-coast:coops-meteorological', 375, 50),
  ('svc://noaa-coast:coops-water', 234, 50),

test/test_catalog.py

@@ -48,7 +48,7 @@ def test_search_catalog_from_service(api, service, expected, tolerance):
 @pytest.mark.slow
 def test_search_catalog_with_search_term(api):
  catalog_entries = api.search_catalog('svc://noaa-ncdc:ghcn-daily', filters={'search_terms': ['ZI']})
- expected = 270
+ expected = 342
  tolerance = 10
  assert abs(len(catalog_entries) - expected) < tolerance