fix(airflow): fix spark_history_ui_link by removing -driver suffix (#68)

hussein-awala · May 13, 2024 · 03bc979 · 03bc979
1 parent 376b083
commit 03bc979
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/spark_on_k8s/airflow/operators.py b/spark_on_k8s/airflow/operators.py
@@ -229,7 +229,7 @@ def _persist_spark_history_ui_link(self, context: Context):
                 context,
                 self,
                 spark_on_k8s_service_url=self.spark_on_k8s_service_url,
-                spark_app_id=self._driver_pod_name,
+                spark_app_id=self._driver_pod_name[: -len("-driver")],
             )
 
     def _try_to_adopt_job(self, context: Context, spark_app_manager: SparkAppManager) -> bool: