fix(airflow): wait for termination status when there is no more log (#40

)
hussein-awala · May 3, 2024 · 6ffe952 · 6ffe952
1 parent ba6812b
commit 6ffe952
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/spark_on_k8s/airflow/operators.py b/spark_on_k8s/airflow/operators.py
@@ -287,6 +287,12 @@ def execute(self, context):
                 namespace=self.namespace,
                 pod_name=self._driver_pod_name,
             )
+            # wait for termination status
+            spark_app_manager.wait_for_app(
+                namespace=self.namespace,
+                pod_name=self._driver_pod_name,
+                poll_interval=1,
+            )
         app_status = spark_app_manager.app_status(
             namespace=self.namespace,
             pod_name=self._driver_pod_name,