Merge pull request #721 from hmcts/feature-databricks-active-listing-ariadm-1473-listing-state

Feature databricks active listing ariadm 1473 listing state

Author: tchow8

.gitignore

@@ -0,0 +1 @@
+__pycache__/

ARIA_DABs/resources/pipelines/PL_ARIADM_ACTIVE_LISTING_GOLD.yml

@@ -0,0 +1,19 @@
+resources:
+  pipelines:
+    pipeline_pl_listing_gold:
+      name: PL_listing_gold
+      configuration:
+        pipelines.tableManagedByMultiplePipelinesCheck.enabled: "false"
+      clusters:
+        - label: default
+          node_type_id: Standard_D4ads_v5
+          driver_node_type_id: Standard_D4ads_v5
+          autoscale:
+            min_workers: 1
+            max_workers: 8
+            mode: ENHANCED
+      libraries:
+        - notebook:
+            path: "/Workspace/live/ACTIVE/APPEALS/GOLD_LISTING_JSON"
+      target: listing_gold
+      development: true

ARIA_DABs/resources/workflows/listing_payload_databricks_job.yml

@@ -0,0 +1,38 @@
+resources:
+  jobs:
+    listing_payload_databricks_job:
+      name: listing_payload_databricks_job
+      job_clusters:
+        - job_cluster_key: listing_payload_databricks_job
+          new_cluster:
+            spark_version: 14.3.x-scala2.12
+            node_type_id: Standard_D8ds_v5
+            num_workers: 8
+      tasks:
+        - task_key: run-my-wheel
+          job_cluster_key: listing_payload_databricks_job
+          python_wheel_task:
+            entry_point: main
+            package_name: shared_functions
+          libraries: 
+            - whl: "dbfs:/FileStore/packages/shared_functions-0.6.6-py3-none-any.whl"
+
+        - task_key: DLT_listing
+          depends_on:
+            - task_key: run-my-wheel
+          pipeline_task:
+            pipeline_id: ${resources.pipelines.pipeline_pl_listing_gold.id}
+            full_refresh: true
+                
+        - task_key: active_publish_EH_HTML_JSON
+          depends_on: 
+            - task_key: DLT_listing
+          job_cluster_key: listing_payload_databricks_job
+          notebook_task:
+            notebook_path: "/Workspace/live/ACTIVE/MVP/Active_Publish_EH_HTML_JSON"
+            source: WORKSPACE
+            base_parameters:
+              state: listing
+
+      queue:
+        enabled: true

Databricks/ACTIVE/APPEALS/Active_Functions/__pycache__/paymentPending.cpython-312.pyc

@@ -2263,6 +2263,72 @@
     "display(bronze_appealtype)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "fda46dc4-e7af-4243-926b-f6efaad6af03",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "source": [
+    "### Transformation bronze_interpreter_languages"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 0,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "d77d7399-47a1-44c0-89c8-2cf3b39944b4",
+     "showTitle": false,
+     "tableResultSettingsMap": {},
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "@dlt.table(\n",
+    "    name=\"bronze_interpreter_languages\",\n",
+    "    comment=\"DLT table for interpreter languages, associated to the case by m1 appealCase LanguageId\",\n",
+    "    path=f\"{bronze_path}/bronze_interpreter_language\"\n",
+    ")\n",
+    "def bronze_interpreter_languages():\n",
+    "    schema = StructType([\n",
+    "        StructField(\"LanguageId\", IntegerType(), True),\n",
+    "        StructField(\"Description\", StringType(), True),\n",
+    "        StructField(\"appellantinterpreterLanguageCategory\", StringType(), True),\n",
+    "        StructField(\"languageCode\", StringType(), True),\n",
+    "        StructField(\"languageLabel\", StringType(), True),\n",
+    "        StructField(\"manualEntry\", StringType(), True),\n",
+    "        StructField(\"manualEntryDescription\", StringType(), True)\n",
+    "    ])\n",
+    "    \n",
+    "    return (\n",
+    "        spark.read.format(\"csv\")\n",
+    "            .option(\"header\", \"true\")\n",
+    "            .option(\"inferSchema\", \"false\")\n",
+    "            .option(\"encoding\", \"UTF-8\")\n",
+    "            .option(\"multiline\", \"true\")\n",
+    "            .option(\"escape\", \"\\\"\")\n",
+    "            .schema(schema)\n",
+    "            .load(f\"abfss://external-csv@{external_storage}.dfs.core.windows.net/ReferenceData/interpreter_language.csv\")\n",
+    "            .select(\"*\")\n",
+    "    )"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {