fix(airflow): load namespace and pod_name from trigger event (#89)

hussein-awala · Jul 29, 2024 · 9fad5e8 · 9fad5e8
1 parent 1a1ac96
commit 9fad5e8
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/spark_on_k8s/airflow/operators.py b/spark_on_k8s/airflow/operators.py
@@ -407,6 +407,8 @@ def execute(self, context: Context):
         raise AirflowException(f"The job finished with status: {app_status}")
 
     def execute_complete(self, context: Context, event: dict, **kwargs):
+        self.namespace = event["namespace"]
+        self._driver_pod_name = event["pod_name"]
         if self.app_waiter == "log":
             from spark_on_k8s.utils.app_manager import SparkAppManager