From 912278012a91684ef4f6874ff5c641def76d3ae0 Mon Sep 17 00:00:00 2001 From: Tomek Urbaszek Date: Tue, 28 Jul 2020 12:25:14 +0200 Subject: [PATCH] Add try clause to DataFusionHook.wait_for_pipeline_state Sometimes it may happen that the pipeline is not visible instatntly in DataFusion so retriving it will result in 404 closes: #10030 --- .../providers/google/cloud/hooks/datafusion.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/airflow/providers/google/cloud/hooks/datafusion.py b/airflow/providers/google/cloud/hooks/datafusion.py index 50d552c8f7122..8e19acaa03c66 100644 --- a/airflow/providers/google/cloud/hooks/datafusion.py +++ b/airflow/providers/google/cloud/hooks/datafusion.py @@ -112,13 +112,15 @@ def wait_for_pipeline_state( start_time = monotonic() current_state = None while monotonic() - start_time < timeout: - current_state = self._get_workflow_state( - pipeline_name=pipeline_name, - pipeline_id=pipeline_id, - instance_url=instance_url, - namespace=namespace, - ) - + try: + current_state = self._get_workflow_state( + pipeline_name=pipeline_name, + pipeline_id=pipeline_id, + instance_url=instance_url, + namespace=namespace, + ) + except AirflowException: + pass # Because the pipeline may not be visible in system yet if current_state in success_states: return if current_state in failure_states: