From e0d1c59a186431298ca3383010e969a00378b973 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 22 May 2025 13:03:36 +0200 Subject: [PATCH] Isolate celery tests to separate container The celery tests hang intermittently from time to time, and it's rather difficult to pin-point the root cause. This PR attempts to isolate the tests and make them fail faster in case the problem happens. Currently, after some recent refactoring - none of the tests usually run longer that 18-19 minutes, so we can set much lower timeouts for the test job - 30 minutes "soft" timeout (SIGTERM sent to stop the container and dump logs) and 35 minutes for "hard" failure of GitHub Action. If we see that we are still hanging despite the isolation, we can later introduce more debug logging for **just** the celery container run. --- .github/workflows/run-unit-tests.yml | 2 +- dev/breeze/doc/images/output_testing_core-tests.txt | 2 +- dev/breeze/doc/images/output_testing_providers-tests.txt | 2 +- dev/breeze/src/airflow_breeze/commands/testing_commands.py | 5 +++-- dev/breeze/src/airflow_breeze/utils/selective_checks.py | 6 +++++- dev/breeze/tests/test_selective_checks.py | 6 +++--- 6 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 435b49897da14..8bd5816345c97 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -124,7 +124,7 @@ permissions: contents: read jobs: tests: - timeout-minutes: 120 + timeout-minutes: 35 name: "\ ${{ inputs.test-scope }}-${{ inputs.test-group }}:\ ${{ inputs.test-name }}${{ inputs.test-name-separator }}${{ matrix.backend-version }}:\ diff --git a/dev/breeze/doc/images/output_testing_core-tests.txt b/dev/breeze/doc/images/output_testing_core-tests.txt index 3566b8e39961c..db0767e5fdf88 100644 --- a/dev/breeze/doc/images/output_testing_core-tests.txt +++ b/dev/breeze/doc/images/output_testing_core-tests.txt @@ -1 +1 @@ -e17e8722da2c099cb96f37dc51c279e0 +de2875b9ee563468daf60d78527ff40c diff --git a/dev/breeze/doc/images/output_testing_providers-tests.txt b/dev/breeze/doc/images/output_testing_providers-tests.txt index e3969eae7d715..901b4ab678a4d 100644 --- a/dev/breeze/doc/images/output_testing_providers-tests.txt +++ b/dev/breeze/doc/images/output_testing_providers-tests.txt @@ -1 +1 @@ -e38e20ca833d2fc03b5f6a8fb746f47c +71fa5f575e7628b431cb71a6ef749e00 diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index cdaaa3c11c34f..617486f80080c 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -102,8 +102,8 @@ GRACE_CONTAINER_STOP_TIMEOUT = 10 # Timeout in seconds to wait for containers to get killed -LOW_MEMORY_CONDITION = 8 * 1024 * 1024 * 1024 -DEFAULT_TOTAL_TEST_TIMEOUT = 6500 # 6500 seconds = 1h 48 minutes +LOW_MEMORY_CONDITION = 8 * 1024 * 1024 * 1024 # 8 GB +DEFAULT_TOTAL_TEST_TIMEOUT = 30 * 60 # 30 minutes logs_already_dumped = False @@ -327,6 +327,7 @@ def _run_tests_in_pool( "CLI", "Serialization", "Always", + "Providers[celery]", ] sort_key = {item: i for i, item in enumerate(sorting_order)} # Put the test types in the order we want them to run diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 3c186a5da4ef2..cc76ee7769fa0 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -965,8 +965,12 @@ def _extract_long_provider_tests(current_test_types: set[str]): In case of Providers, we need to replace it with Providers[-], but in case of Providers[list_of_tests] we need to remove the long tests from the list. + In case of celery tests we want to isolate them from the rest, because they seem to be hanging + infrequently when running together with other tests + + :param current_test_types: The set of test types to run """ - long_tests = ["amazon", "google", "standard"] + long_tests = ["amazon", "celery", "google", "standard"] for original_test_type in tuple(current_test_types): if original_test_type == "Providers": current_test_types.remove(original_test_type) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index d4b307be97545..5f20da3c753d7 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -80,9 +80,9 @@ ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON = json.dumps( [ { - "description": "-amazon,googl...standard", - "test_types": "Providers[-amazon,google,standard] " - "Providers[amazon] Providers[google] Providers[standard]", + "description": "-amazon,celer...standard", + "test_types": "Providers[-amazon,celery,google,standard] " + "Providers[amazon] Providers[celery] Providers[google] Providers[standard]", } ] )