Merge pull request #2797 from uktrade/feature/catalogue-page-pipeline…

…-details

add pipeline last success period to context

dataworkspace/dataworkspace/apps/datasets/models.py

@@ -85,6 +85,7 @@
     get_latest_row_count_for_query,
     get_latest_row_count_for_table,
     get_pipeline_id_for_source_table,
+    get_pipeline_last_success_date,
 )
 
 
@@ -890,6 +891,12 @@ def get_pipeline_last_run_state(self):
             return None
         return get_last_run_state_for_pipeline(pipeline_name)
 
+    def get_pipeline_last_success_date(self):
+        pipeline_name = self.get_metadata_pipeline_name()
+        if pipeline_name is None:
+            return None
+        return get_pipeline_last_success_date(pipeline_name)
+
 
 class SourceTableFieldDefinition(models.Model):
     field = models.CharField(

dataworkspace/dataworkspace/apps/datasets/views.py

@@ -1,5 +1,6 @@
 import json
 import logging
+from datetime import datetime
 import uuid
 from collections import defaultdict, namedtuple
 from itertools import chain
@@ -124,6 +125,7 @@
 from dataworkspace.apps.eventlog.models import EventLog
 from dataworkspace.apps.eventlog.utils import log_event, log_permission_change
 from dataworkspace.apps.explorer.utils import invalidate_data_explorer_user_cached_credentials
+from dataworkspace.datasets_db import get_pipeline_last_success_date
 
 logger = logging.getLogger("app")
 
@@ -429,11 +431,18 @@ def _get_user_tools_access(self) -> bool:
 
         return user_has_tools_access
 
+    def _get_pipeline_info(self, source_table):
+        last_success_date = get_pipeline_last_success_date(source_table)
+        if last_success_date:
+            return abs((last_success_date - datetime.now()).days)
+        return 0
+
     def _get_context_data_for_master_dataset(self, ctx, **kwargs):
         source_tables = sorted(self.object.sourcetable_set.all(), key=lambda x: x.name)
 
         MasterDatasetInfo = namedtuple(
-            "MasterDatasetInfo", ("source_table", "code_snippets", "columns", "tools_links")
+            "MasterDatasetInfo",
+            ("source_table", "code_snippets", "columns", "tools_links", "pipeline_info"),
         )
         master_datasets_info = [
             MasterDatasetInfo(
@@ -446,6 +455,7 @@ def _get_context_data_for_master_dataset(self, ctx, **kwargs):
                     include_types=True,
                 ),
                 tools_links=get_tools_links_for_user(self.request.user, self.request.scheme),
+                pipeline_info=self._get_pipeline_info(source_table),
             )
             for source_table in sorted(source_tables, key=lambda x: x.name)
         ]

dataworkspace/dataworkspace/datasets_db.py

@@ -303,6 +303,39 @@ def get_changelog_from_metadata_rows(rows):
     return list(reversed(changelog))
 
 
+def get_pipeline_last_success_date(pipeline_name):
+    with connections[list(settings.DATABASES_DATA.items())[0][0]].cursor() as cursor:
+        try:
+            cursor.execute(
+                SQL(
+                    """
+                    WITH dag_runs AS (
+                        SELECT *
+                        FROM dataflow.pipeline_dag_runs_v2
+                        WHERE pipeline_name = {}
+                        AND pipeline_active = 'active'
+                    )
+                    SELECT last_success_of_day
+                    FROM dag_runs
+                    WHERE dag_runs.run_end_date = (
+                        SELECT MAX(run_end_date)
+                        FROM dag_runs
+                    );
+                    """
+                ).format(
+                    Literal(DataSetType.MASTER),
+                    Literal(pipeline_name),
+                )
+            )
+        except Exception:  # pylint: disable=broad-except
+            logger.error(
+                "Failed to get last success date for pipeline %s", pipeline_name, exc_info=True
+            )
+            return None
+        result = cursor.fetchone()
+        return result[0] if result else None
+
+
 def get_latest_row_count_for_table(table):
     with connections[table.database.memorable_name].cursor() as cursor:
         try:

dataworkspace/dataworkspace/templates/datasets/master_dataset.html

@@ -181,7 +181,7 @@ <h2 class="govuk-heading-l">Data</h2>
         </thead>
         <tbody>
 
-        {% for source_table, code_snippets, columns, tools_links in master_datasets_info %}
+        {% for source_table, code_snippets, columns, tools_links, pipeline_info in master_datasets_info %}
           <tr class="govuk-table__row">
             <td class="govuk-table__cell app-table_cell--no-border">{{ source_table.name }}</td>
             <td class="govuk-table__cell app-table_cell--no-border">