Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add additional logging and tqdm #23

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add additional logging and tqdm #23

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 8 additions & 5 deletions src/pyctd/manager/database.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,11 @@
from requests.compat import urlparse
from sqlalchemy import create_engine, inspect
from sqlalchemy.engine import reflection
from sqlalchemy.orm import sessionmaker, scoped_session
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.sql import sqltypes
from tqdm import tqdm

from . import defaults
from . import models
from . import table_conf
from . import defaults, models, table_conf
from .table import get_table_configurations
from ..constants import PYCTD_DATA_DIR, PYCTD_DIR, bcolors

Expand Down Expand Up @@ -225,10 +224,13 @@ def import_tables(self, only_tables=None, exclude_tables=None):
"""
for table in self.tables:
if only_tables is not None and table.name not in only_tables:
log.debug('skipping table: %s', table)
continue

if exclude_tables is not None and table.name in exclude_tables:
log.debug('skipping table: %s', table)
continue

self.import_table(table)

@classmethod
Expand Down Expand Up @@ -368,7 +370,7 @@ def import_table_in_db(self, file_path, use_columns_with_index, column_names_in_
dtype=self.get_dtypes(table.model)
)

for chunk in chunks:
for chunk in tqdm(chunks):
# this is an evil hack because CTD is not using the MESH prefix in this table
if table.name == 'exposure_event':
chunk.disease_id = 'MESH:' + chunk.disease_id
Expand Down Expand Up @@ -417,6 +419,7 @@ def download_urls(cls, urls, force_download=False):
:param iter[str] urls: iterable of URL of CTD
:param bool force_download: force method to download
"""
log.info('downloading CTD data')
for url in urls:
file_path = cls.get_path_to_file_from_url(url)

Expand Down