From 426d996882a050e39bf85b44fab1994241466a40 Mon Sep 17 00:00:00 2001 From: Charles Tapley Hoyt Date: Wed, 13 Jun 2018 17:24:38 +0200 Subject: [PATCH] Add additional logging and tqdm Still need to add tqdm to requirements.txt and setup.py --- src/pyctd/manager/database.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/pyctd/manager/database.py b/src/pyctd/manager/database.py index 8feebec..78283fd 100755 --- a/src/pyctd/manager/database.py +++ b/src/pyctd/manager/database.py @@ -17,12 +17,11 @@ from requests.compat import urlparse from sqlalchemy import create_engine, inspect from sqlalchemy.engine import reflection -from sqlalchemy.orm import sessionmaker, scoped_session +from sqlalchemy.orm import scoped_session, sessionmaker from sqlalchemy.sql import sqltypes +from tqdm import tqdm -from . import defaults -from . import models -from . import table_conf +from . import defaults, models, table_conf from .table import get_table_configurations from ..constants import PYCTD_DATA_DIR, PYCTD_DIR, bcolors @@ -225,10 +224,13 @@ def import_tables(self, only_tables=None, exclude_tables=None): """ for table in self.tables: if only_tables is not None and table.name not in only_tables: + log.debug('skipping table: %s', table) continue if exclude_tables is not None and table.name in exclude_tables: + log.debug('skipping table: %s', table) continue + self.import_table(table) @classmethod @@ -368,7 +370,7 @@ def import_table_in_db(self, file_path, use_columns_with_index, column_names_in_ dtype=self.get_dtypes(table.model) ) - for chunk in chunks: + for chunk in tqdm(chunks): # this is an evil hack because CTD is not using the MESH prefix in this table if table.name == 'exposure_event': chunk.disease_id = 'MESH:' + chunk.disease_id @@ -417,6 +419,7 @@ def download_urls(cls, urls, force_download=False): :param iter[str] urls: iterable of URL of CTD :param bool force_download: force method to download """ + log.info('downloading CTD data') for url in urls: file_path = cls.get_path_to_file_from_url(url)