Merge pull request #1192 from uktrade/KLS-1067-market-guides-update-notification

KLS 1067 market guides update notification

Author: timothyPatterson

conf/settings.py

@@ -207,6 +207,8 @@
     'VERSION': os.environ.get('GIT_TAG', 'dev'),
 }
 
+APP_ENVIRONMENT = env.str('APP_ENVIRONMENT', 'dev')
+SENTRY_ENVIRONMENT = env.str('SENTRY_ENVIRONMENT', APP_ENVIRONMENT)
 # Sentry
 if env.str('SENTRY_DSN', ''):
     sentry_sdk.init(
@@ -288,6 +290,8 @@
     'COLLABORATOR_INVITE_SUBJECT', 'Confirm you’ve been added to {company_name}’s Find a buyer profile'
 )
 GREAT_MARKETGUIDES_TEAMS_CHANNEL_EMAIL = env.str('GREAT_MARKETGUIDES_TEAMS_CHANNEL_EMAIL')
+GREAT_MARKETGUIDES_REVIEW_PERIOD_DAYS = env.int('GREAT_MARKETGUIDES_REVIEW_PERIOD_DAYS', 10)
+
 
 # Public storage for company profile logo
 STORAGE_CLASSES = {
@@ -396,6 +400,10 @@
     'GOVNOTIFY_ACCOUNT_OWNERSHIP_TRANSFER_TEMPLATE_ID', 'eecd214c-8072-4d16-87ea-4283d5925f16'
 )
 
+GOVNOTIFY_GREAT_MARKETGUIDES_REVIEW_REQUEST_TEMPLATE_ID = env.str(
+    'GOVNOTIFY_GREAT_MARKETGUIDES_REVIEW_REQUEST_TEMPLATE_ID', '2c49305b-6632-44fa-8d7d-830086363258'
+)
+
 # Duplicate companies notification
 
 GOVNOTIFY_DUPLICATE_COMPANIES = env.str('GOVNOTIFY_DUPLICATE_COMPANIES', '9d93b6c9-ff75-4797-b841-2f7a6c78a277')

dataservices/management/commands/helpers.py

@@ -1,5 +1,6 @@
 import io
-from datetime import datetime
+from datetime import datetime, timedelta
+from sys import stdout
 from zipfile import ZipFile
 
 import pandas as pd
@@ -45,6 +46,35 @@ def send_ingest_error_notify_email(view_name, error_details):
     )
 
 
+def send_review_request_message(view_name):
+    instance, _created = Metadata.objects.get_or_create(view_name=view_name)
+    last_release = datetime.strptime(instance.data['source']['last_release'], '%Y-%m-%dT%H:%M:%S')
+
+    try:
+        last_notification_sent = datetime.strptime(
+            instance.data['review_process']['notification_sent'], '%Y-%m-%dT%H:%M:%S'
+        )
+    except KeyError:
+        instance.data['review_process'] = {'notification_sent': None}
+        last_notification_sent = None
+
+    if last_notification_sent is None or (((last_notification_sent.timestamp() - last_release.timestamp())) < 0):
+        notifications_client().send_email_notification(
+            email_address=settings.GREAT_MARKETGUIDES_TEAMS_CHANNEL_EMAIL,
+            template_id=settings.GOVNOTIFY_GREAT_MARKETGUIDES_REVIEW_REQUEST_TEMPLATE_ID,
+            personalisation={
+                'view_name': view_name,
+                'review_url': 'https://great.staging.uktrade.digital/markets/',
+                'release_date': (
+                    last_release + timedelta(days=settings.GREAT_MARKETGUIDES_REVIEW_PERIOD_DAYS)
+                ).strftime('%d/%m/%Y'),
+            },
+        )
+        stdout.write(f"Sent review request notification for {view_name}")
+        instance.data['review_process']['notification_sent'] = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
+        instance.save()
+
+
 class MarketGuidesDataIngestionCommand(BaseCommand):
     engine = sa.create_engine(settings.DATA_WORKSPACE_DATASETS_URL, execution_options={'stream_results': True})
 
@@ -82,7 +112,15 @@ def should_ingestion_run(self, view_name, table_name):
         if great_metadata is not None:
             great_metadata_date = datetime.strptime(great_metadata, '%Y-%m-%dT%H:%M:%S').date()
             if swapped_date > great_metadata_date:
-                return True
+                if settings.APP_ENVIRONMENT != 'production' or (
+                    settings.APP_ENVIRONMENT == 'production'
+                    and datetime.now().date()
+                    > (swapped_date + timedelta(days=settings.GREAT_MARKETGUIDES_REVIEW_PERIOD_DAYS))
+                ):
+                    self.stdout.write(
+                        self.style.SUCCESS(f'Importing {view_name} data into {settings.APP_ENVIRONMENT} env.')
+                    )
+                    return True
 
         return False
 

dataservices/management/commands/import_market_guides_data.py

@@ -1,6 +1,11 @@
+from django.conf import settings
 from django.core.management import BaseCommand, call_command
 
-from dataservices.management.commands.helpers import MarketGuidesDataIngestionCommand, send_ingest_error_notify_email
+from dataservices.management.commands.helpers import (
+    MarketGuidesDataIngestionCommand,
+    send_ingest_error_notify_email,
+    send_review_request_message,
+)
 
 
 class Command(BaseCommand):
@@ -37,6 +42,10 @@ def handle(self, *args, **options):
                 try:
                     call_command(command_name, **options)
                     call_command('import_metadata_source_data', table=table_view_names['table_name'])
+
+                    if settings.APP_ENVIRONMENT == 'staging':
+                        send_review_request_message(table_view_names['view_name'])
+
                     self.stdout.write(self.style.SUCCESS(f'Finished import for {table_view_names["view_name"]}'))
                 except Exception as e:
                     self.stderr.write(self.style.ERROR(f'Failed import for {table_view_names["view_name"]}'))

dataservices/management/commands/tests/test_import_data.py

@@ -1,6 +1,6 @@
-import datetime
 import json
 import re
+from datetime import date, datetime
 from itertools import cycle, islice
 from unittest import mock
 
@@ -9,6 +9,7 @@
 import sqlalchemy
 from django.core import management
 from django.test import override_settings
+from freezegun import freeze_time
 from import_export import results
 from sqlalchemy import TIMESTAMP, Column, MetaData, String, Table
 
@@ -301,10 +302,10 @@ def metadata_last_release_raw_data():
     return {
         'table_name': 'trade__uk_goods_nsa trade__uk_services_nsa trade__uk_totals_sa xxx'.split(),
         'last_release': [
-            datetime.date(2018, 12, 30),
-            datetime.date(2019, 12, 30),
-            datetime.date(2020, 12, 30),
-            datetime.date(2021, 12, 30),
+            date(2018, 12, 30),
+            date(2019, 12, 30),
+            date(2020, 12, 30),
+            date(2021, 12, 30),
         ],
     }
 
@@ -423,25 +424,34 @@ def test_import_metadata_source_data_filter_tables():
 
 
 @pytest.mark.django_db
-@mock.patch('dataservices.management.commands.helpers.MarketGuidesDataIngestionCommand.should_ingestion_run')
+@pytest.mark.parametrize(
+    'env, review_requested_x_times',
+    [('dev', 0), ('staging', 3), ('uat', 0), ('production', 0)],
+)
 @mock.patch('dataservices.management.commands.import_market_guides_data.call_command')
-def test_import_market_guides_data(mock_call_command, mock_should_run):
-    command_list = [
-        'import_uk_total_trade_data',
-        'import_uk_trade_in_goods_data',
-        'import_uk_trade_in_services_data',
-    ]
-    mock_should_run.return_value = False
-    management.call_command('import_market_guides_data', '--write')
-    assert mock_call_command.call_count == 0
-
-    mock_should_run.return_value = True
-    management.call_command('import_market_guides_data', '--write')
-    assert mock_call_command.call_count == 6
-
-    for command in command_list:
-        assert command in str(mock_call_command.call_args_list)
-        assert 'write=True' in str(mock_call_command.call_args_list)
+@mock.patch('dataservices.management.commands.helpers.MarketGuidesDataIngestionCommand.should_ingestion_run')
+@mock.patch('dataservices.management.commands.import_market_guides_data.send_review_request_message')
+def test_import_market_guides_data(
+    mock_send_review_request, mock_should_run, mock_call_command, env, review_requested_x_times
+):
+    with override_settings(APP_ENVIRONMENT=env):
+        command_list = [
+            'import_uk_total_trade_data',
+            'import_uk_trade_in_goods_data',
+            'import_uk_trade_in_services_data',
+        ]
+        mock_should_run.return_value = False
+        management.call_command('import_market_guides_data', '--write')
+        assert mock_call_command.call_count == 0
+
+        mock_should_run.return_value = True
+        management.call_command('import_market_guides_data', '--write')
+        assert mock_call_command.call_count == 6
+        assert mock_send_review_request.call_count == review_requested_x_times
+
+        for command in command_list:
+            assert command in str(mock_call_command.call_args_list)
+            assert 'write=True' in str(mock_call_command.call_args_list)
 
 
 @pytest.mark.django_db
@@ -511,24 +521,32 @@ def workspace_data():
             'trade__uk_goods_nsa',
         ],
         'source_data_modified_utc': [
-            datetime.datetime(2023, 6, 10),
+            datetime(2023, 9, 3),
         ],
-        'dataflow_swapped_tables_utc': [datetime.datetime(2023, 6, 10)],
+        'dataflow_swapped_tables_utc': [datetime(2023, 9, 3)],
     }
 
 
 @pytest.mark.parametrize(
-    'view_date, expected',
-    [('2023-04-27T00:00:00', True), ('2023-06-10T00:00:00', False), ('2023-07-01T00:00:00', False)],
+    'env, view_date, swap_date, expected',
+    [
+        ('staging', datetime(2023, 9, 12), datetime(2023, 9, 13), True),
+        ('staging', datetime(2023, 9, 14), datetime(2023, 9, 13), False),
+        ('production', datetime(2023, 9, 6), datetime(2023, 9, 6), False),
+        ('production', datetime(2023, 9, 1), datetime(2023, 9, 2), True),
+    ],
 )
 @mock.patch('dataservices.management.commands.helpers.MarketGuidesDataIngestionCommand.get_view_metadata')
 @mock.patch('dataservices.management.commands.helpers.MarketGuidesDataIngestionCommand.get_dataflow_metadata')
-def test_helper_should_ingest_run(dataflow_mock, view_mock, view_date, expected, workspace_data):
-    m = MarketGuidesDataIngestionCommand()
-    dataflow_mock.return_value = pd.DataFrame(workspace_data)
-    view_mock.return_value = view_date
-    actual = m.should_ingestion_run('UKMarketTrendsView', 'trade__uk_goods_nsa')
-    assert actual == expected
+@freeze_time('2023-09-14T15:21:10')
+def test_helper_should_ingest_run(dataflow_mock, view_mock, env, view_date, swap_date, expected, workspace_data):
+    with override_settings(APP_ENVIRONMENT=env):
+        m = MarketGuidesDataIngestionCommand()
+        workspace_data['dataflow_swapped_tables_utc'] = swap_date
+        dataflow_mock.return_value = pd.DataFrame(workspace_data)
+        view_mock.return_value = view_date.strftime('%Y-%m-%dT%H:%M:%S')
+        actual = m.should_ingestion_run('UKMarketTrendsView', 'trade__uk_goods_nsa')
+        assert actual == expected
 
 
 @pytest.mark.django_db
@@ -576,22 +594,22 @@ def test_helper_get_dataflow_metadata():
     m.engine.execute(
         tbl.insert().values(
             table_name='trade__uk_goods_nsa',
-            source_data_modified_utc=datetime.date(2023, 1, 1),
-            dataflow_swapped_tables_utc=datetime.date(2023, 1, 1),
+            source_data_modified_utc=date(2023, 1, 1),
+            dataflow_swapped_tables_utc=date(2023, 1, 1),
         )
     )
     m.engine.execute(
         tbl.insert().values(
             table_name='trade__uk_goods_nsa',
-            source_data_modified_utc=datetime.date(2023, 4, 1),
-            dataflow_swapped_tables_utc=datetime.date(2023, 4, 1),
+            source_data_modified_utc=date(2023, 4, 1),
+            dataflow_swapped_tables_utc=date(2023, 4, 1),
         )
     )
     m.engine.execute(
         tbl.insert().values(
             table_name='trade__uk_services_nsa',
-            source_data_modified_utc=datetime.date(2023, 6, 1),
-            dataflow_swapped_tables_utc=datetime.date(2023, 6, 1),
+            source_data_modified_utc=date(2023, 6, 1),
+            dataflow_swapped_tables_utc=date(2023, 6, 1),
         )
     )
     result = m.get_dataflow_metadata('trade__uk_goods_nsa')

dataservices/tests/test_helpers.py

@@ -1,8 +1,10 @@
 import re
+from datetime import datetime, timedelta
 from unittest import mock
 
 import pytest
 from django.conf import settings
+from freezegun import freeze_time
 
 from dataservices import helpers, models
 from dataservices.management.commands import helpers as dmch
@@ -196,3 +198,39 @@ def test_notify_error_message(mock_notify):
         },
     )
     assert mock_notify.call_count == 1
+
+
+@freeze_time('2023-09-13T15:21:10')
+@pytest.mark.django_db
+@mock.patch('dataservices.management.commands.helpers.notifications_client')
+@pytest.mark.parametrize(
+    "last_release, notification_sent, result",
+    [
+        (datetime.now().strftime('%Y-%m-%dT%H:%M:%S'), None, 1),
+        (
+            datetime.now().strftime('%Y-%m-%dT%H:%M:%S'),
+            (datetime.now() - timedelta(days=10)).strftime('%Y-%m-%dT%H:%M:%S'),
+            1,
+        ),
+        (
+            datetime.now().strftime('%Y-%m-%dT%H:%M:%S'),
+            (datetime.now() + timedelta(minutes=1)).strftime('%Y-%m-%dT%H:%M:%S'),
+            0,
+        ),
+    ],
+)
+def test_send_review_request_message(mock_notify, last_release, notification_sent, result):
+    data = {'source': {'last_release': last_release}}
+    data['review_process'] = {} if notification_sent is None else {'notification_sent': notification_sent}
+    factories.MetadataFactory(view_name='TestView', data=data)
+    dmch.send_review_request_message('TestView')
+    mock_notify.call_args = mock.call(
+        email_address='[email protected]',
+        template_id=settings.GOVNOTIFY_GREAT_MARKETGUIDES_REVIEW_REQUEST_TEMPLATE_ID,
+        personalisation={
+            'view_name': 'view_name',
+            'review_url': 'https://great.staging.uktrade.digital/markets/',
+            'release_date': 'dd/mm/YYYY',
+        },
+    )
+    assert mock_notify.call_count == result