Merge pull request #138 from Sage-Bionetworks/update-table

[GEN-1424] Make sure 'None' gets read in as a string

Author: thomasyu888

scripts/table_updates/update_data_table.py

@@ -289,15 +289,18 @@ def main():
     with open(project_config) as config_file:
         cohort_info = json.load(config_file)
         logger.info("Read cohort information successful.")
-        config_file.close()
 
     # get master table
+    # This is the internal tables with non redacted
     table_id, condition = list(TABLE_INFO[table_type])
     master_table = download_synapse_table(syn, table_id, condition)
+    # This contains external tables with redacted
     TABLE_INFO["redacted"] = ('syn21446696',"table_type='data' and double_curated is false")
-    
-   # download data files 
-   # TODO: find the cohort that has new data
+
+    # download data files
+    # TODO: find the cohort that has new data
+    # This is a mapping to all the intake data. e.g: ProstateBPCIntake_data
+    # found here: https://www.synapse.org/Synapse:syn23286928
     cohort_info_selected = cohort_info[table_type]
     cohort_data_list = []
     for cohort in cohort_info_selected:

scripts/table_updates/utilities.py

@@ -69,7 +69,8 @@ def get_data(syn, label_data_id, cohort):
     Returns:
         Dataframe: label data
     """
-    label_data = pandas.read_csv(syn.get(label_data_id).path,low_memory=False)
+    na_values = [" ", "#N/A", "#N/A N/A", "#NA", "-1.#IND", "-1.#QNAN", "-NaN", "-nan", "1.#IND", "1.#QNAN", "<NA>", "N/A", "NA", "NULL", "NaN", "n/a", "nan", "null"]
+    label_data = pandas.read_csv(syn.get(label_data_id).path, low_memory=False, na_values=na_values, keep_default_na=False)
     label_data['cohort'] = cohort
     return(label_data)