From 52853b0a8751fd2fb79d0ba2cb3d98b7305c1740 Mon Sep 17 00:00:00 2001 From: bobby-didcoding Date: Fri, 15 Mar 2024 15:56:13 +0000 Subject: [PATCH 1/6] Added import_world_economic_outlook_data to IMF pipeline --- .../commands/import_market_guides_data.py | 4 ++++ .../import_world_economic_outlook_data.py | 19 ++++++++++++------- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/dataservices/management/commands/import_market_guides_data.py b/dataservices/management/commands/import_market_guides_data.py index 878488da..c596d00f 100644 --- a/dataservices/management/commands/import_market_guides_data.py +++ b/dataservices/management/commands/import_market_guides_data.py @@ -24,6 +24,10 @@ class Command(BaseCommand): 'table_name': 'trade__uk_services_nsa', 'view_name': 'TopFiveServicesExportsByCountryView', }, + 'import_world_economic_outlook_data': { + 'table_name': 'world_economic_outlook__by_country', + 'view_name': 'EconomicHighlightsView', + }, } def add_arguments(self, parser): diff --git a/dataservices/management/commands/import_world_economic_outlook_data.py b/dataservices/management/commands/import_world_economic_outlook_data.py index e100f8f5..d123cfba 100644 --- a/dataservices/management/commands/import_world_economic_outlook_data.py +++ b/dataservices/management/commands/import_world_economic_outlook_data.py @@ -5,11 +5,12 @@ from dataservices.models import Country, WorldEconomicOutlookByCountry +from .helpers import MarketGuidesDataIngestionCommand -class Command(BaseCommand): + +class Command(MarketGuidesDataIngestionCommand): help = 'Import IMF world economic outlook data by country from Data Workspace' - engine = sa.create_engine(settings.DATA_WORKSPACE_DATASETS_URL, execution_options={'stream_results': True}) sql = ''' SELECT iso AS ons_iso_alpha_3_code, @@ -38,7 +39,14 @@ class Command(BaseCommand): AND NULLIF(TRIM(x.value), '') IS NOT NULL; ''' - def handle(self, *args, **options): + def add_arguments(self, parser): + parser.add_argument( + '--write', + action='store_true', + help='Store dataset records', + ) + + def load_data(self): data = [] chunks = pd.read_sql(sa.text(self.sql), self.engine, chunksize=10000) @@ -64,7 +72,4 @@ def handle(self, *args, **options): ) ) - WorldEconomicOutlookByCountry.objects.all().delete() - WorldEconomicOutlookByCountry.objects.bulk_create(data) - - self.stdout.write(self.style.SUCCESS('All done, bye!')) + return data From bad330db54d120ced3b385025b777b9029481824 Mon Sep 17 00:00:00 2001 From: bobby-didcoding Date: Fri, 15 Mar 2024 16:01:35 +0000 Subject: [PATCH 2/6] Run checks --- .../management/commands/import_world_economic_outlook_data.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/dataservices/management/commands/import_world_economic_outlook_data.py b/dataservices/management/commands/import_world_economic_outlook_data.py index d123cfba..df1a7102 100644 --- a/dataservices/management/commands/import_world_economic_outlook_data.py +++ b/dataservices/management/commands/import_world_economic_outlook_data.py @@ -1,7 +1,5 @@ import pandas as pd import sqlalchemy as sa -from django.conf import settings -from django.core.management import BaseCommand from dataservices.models import Country, WorldEconomicOutlookByCountry From ce3d9d035c300615a5c9b07c402aa15b7aff360d Mon Sep 17 00:00:00 2001 From: bobby-didcoding Date: Mon, 18 Mar 2024 14:30:43 +0000 Subject: [PATCH 3/6] fixed broken IMF ingest tests --- .../commands/import_market_guides_data.py | 1 + .../commands/tests/test_import_data.py | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/dataservices/management/commands/import_market_guides_data.py b/dataservices/management/commands/import_market_guides_data.py index c596d00f..2f42d471 100644 --- a/dataservices/management/commands/import_market_guides_data.py +++ b/dataservices/management/commands/import_market_guides_data.py @@ -56,6 +56,7 @@ def handle(self, *args, **options): self.stderr.write(self.style.ERROR(e)) send_ingest_error_notify_email(table_view_names['view_name'], e) else: + print("test") self.stdout.write(self.style.NOTICE(f'{table_view_names["view_name"]} does not need updating')) self.stdout.write(self.style.SUCCESS('Finished Market Guides import!')) diff --git a/dataservices/management/commands/tests/test_import_data.py b/dataservices/management/commands/tests/test_import_data.py index 9485f98c..ef122429 100644 --- a/dataservices/management/commands/tests/test_import_data.py +++ b/dataservices/management/commands/tests/test_import_data.py @@ -388,8 +388,15 @@ def test_import_world_economic_outlook_data(read_sql_mock, world_economic_outloo assert len(models.WorldEconomicOutlookByCountry.objects.all()) == 0 management.call_command('import_countries') + + # Dry run management.call_command('import_world_economic_outlook_data') + assert len(models.WorldEconomicOutlookByCountry.objects.all()) == 0 + + # Write option + management.call_command('import_world_economic_outlook_data', '--write') + assert len(models.WorldEconomicOutlookByCountry.objects.all()) == 10 @@ -426,7 +433,7 @@ def test_import_metadata_source_data_filter_tables(): @pytest.mark.django_db @pytest.mark.parametrize( 'env, review_requested_x_times', - [('dev', 0), ('staging', 3), ('uat', 0), ('production', 0)], + [('dev', 0), ('staging', 4), ('uat', 0), ('production', 0)], ) @mock.patch('dataservices.management.commands.import_market_guides_data.call_command') @mock.patch('dataservices.management.commands.helpers.MarketGuidesDataIngestionCommand.should_ingestion_run') @@ -439,6 +446,7 @@ def test_import_market_guides_data( 'import_uk_total_trade_data', 'import_uk_trade_in_goods_data', 'import_uk_trade_in_services_data', + 'import_world_economic_outlook_data', ] mock_should_run.return_value = False management.call_command('import_market_guides_data', '--write') @@ -446,7 +454,7 @@ def test_import_market_guides_data( mock_should_run.return_value = True management.call_command('import_market_guides_data', '--write') - assert mock_call_command.call_count == 6 + assert mock_call_command.call_count == 8 assert mock_send_review_request.call_count == review_requested_x_times for command in command_list: @@ -462,12 +470,13 @@ def test_import_market_guides_data_dry_run(mock_call_command, mock_should_run): 'import_uk_total_trade_data', 'import_uk_trade_in_goods_data', 'import_uk_trade_in_services_data', + 'import_world_economic_outlook_data', ] mock_should_run.return_value = True management.call_command('import_market_guides_data') - assert mock_call_command.call_count == 6 + assert mock_call_command.call_count == 8 for command in command_list: assert command in str(mock_call_command.call_args_list) @@ -486,9 +495,9 @@ def test_import_market_guides_data_error(mock_call_command, mock_should_run, moc management.call_command('import_market_guides_data') mock_email_string = str(mock_error_email.mock_calls) - assert mock_call_command.call_count == 3 + assert mock_call_command.call_count == 4 assert 'oops' in str(mock_call_command.side_effect) - assert mock_error_email.call_count == 3 + assert mock_error_email.call_count == 4 assert 'area_of_error' in mock_email_string assert 'error_type' in mock_email_string assert 'error_details' in mock_email_string From 8a36fa3643d67b73e9d3ebd4022609fccb9d426b Mon Sep 17 00:00:00 2001 From: bobby-didcoding Date: Mon, 18 Mar 2024 14:53:32 +0000 Subject: [PATCH 4/6] removed rouge debugging print statement --- dataservices/management/commands/import_market_guides_data.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dataservices/management/commands/import_market_guides_data.py b/dataservices/management/commands/import_market_guides_data.py index 2f42d471..c596d00f 100644 --- a/dataservices/management/commands/import_market_guides_data.py +++ b/dataservices/management/commands/import_market_guides_data.py @@ -56,7 +56,6 @@ def handle(self, *args, **options): self.stderr.write(self.style.ERROR(e)) send_ingest_error_notify_email(table_view_names['view_name'], e) else: - print("test") self.stdout.write(self.style.NOTICE(f'{table_view_names["view_name"]} does not need updating')) self.stdout.write(self.style.SUCCESS('Finished Market Guides import!')) From 5a84042ecce6359344de015180c84b1ca60c35d6 Mon Sep 17 00:00:00 2001 From: Halidu Abubakar Date: Thu, 28 Mar 2024 08:45:09 +0000 Subject: [PATCH 5/6] GREATUK-440 Fix Black vulnerable to Regular Expression Denial of Service --- .pre-commit-config.yaml | 2 +- requirements.txt | 2 -- requirements_test.in | 2 +- requirements_test.txt | 4 +--- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8beb5d53..215bf768 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/psf/black - rev: 23.1.0 + rev: 24.3.0 hooks: - id: black # Config for black lives in pyproject.toml diff --git a/requirements.txt b/requirements.txt index e79411a4..9d2e99c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -170,8 +170,6 @@ faker==22.6.0 # via factory-boy future==0.18.3 # via django-json-widget -greenlet==3.0.3 - # via sqlalchemy html5lib==1.1 # via xhtml2pdf idna==3.6 diff --git a/requirements_test.in b/requirements_test.in index 352145aa..cd6f3ebf 100644 --- a/requirements_test.in +++ b/requirements_test.in @@ -10,7 +10,7 @@ pytest-codecov GitPython==3.1.41 # Pinned because pre-commit features specific versions -black==24.1.1 +black==24.3.0 # Pinned pin dataclasses as 0.08 specified by black doesn't exist dataclasses==0.06 blacken-docs==1.13.0 diff --git a/requirements_test.txt b/requirements_test.txt index a8c18ca2..80b0e4f0 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -25,7 +25,7 @@ beautifulsoup4==4.12.3 # via directory-components billiard==4.2.0 # via celery -black==24.1.1 +black==24.3.0 # via # -r requirements_test.in # blacken-docs @@ -193,8 +193,6 @@ gitdb==4.0.11 # via gitpython gitpython==3.1.41 # via -r requirements_test.in -greenlet==3.0.3 - # via sqlalchemy html5lib==1.1 # via xhtml2pdf idna==3.6 From f6a9788fc920945cca2e1b8883c235d6387b1c5e Mon Sep 17 00:00:00 2001 From: bobby-didcoding Date: Thu, 28 Mar 2024 09:42:55 +0000 Subject: [PATCH 6/6] Bump django to 4.2.11 --- requirements.in | 2 +- requirements.txt | 2 +- requirements_test.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.in b/requirements.in index 6e4ebb40..834ab867 100644 --- a/requirements.in +++ b/requirements.in @@ -1,4 +1,4 @@ -django==4.2.10 +django==4.2.11 django-celery-beat==2.5.0 django-cleanup==8.1.0 django-cte==1.3.2 diff --git a/requirements.txt b/requirements.txt index e79411a4..d6f1343c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -92,7 +92,7 @@ directory-validators==9.3.3 # via -r requirements.in dj-database-url==2.1.0 # via -r requirements.in -django==4.2.10 +django==4.2.11 # via # -r requirements.in # directory-client-core diff --git a/requirements_test.txt b/requirements_test.txt index a8c18ca2..853244e2 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -107,7 +107,7 @@ directory-validators==9.3.3 # via -r requirements.in dj-database-url==2.1.0 # via -r requirements.in -django==4.2.10 +django==4.2.11 # via # -r requirements.in # directory-client-core