From dfb14caabf4a0bfd0d7feab7e693210274e28105 Mon Sep 17 00:00:00 2001 From: rjgoyln Date: Sun, 16 Aug 2026 20:07:19 +0800 Subject: [PATCH] Drop provider dependency from airflow-core secrets backend tests These tests cover how core loads and configures a custom secrets backend, not how any particular backend behaves. Pinning them to the amazon and hashicorp providers made a core-only environment unable to run them, and kept them skipped whenever the full provider set is not installed. --- .../tests/unit/always/test_secrets.py | 98 +++++------------ .../tests/unit/core/test_configuration.py | 102 +++++------------- .../test_utils/fake_secrets_backend.py | 59 ++++++++++ 3 files changed, 114 insertions(+), 145 deletions(-) create mode 100644 devel-common/src/tests_common/test_utils/fake_secrets_backend.py diff --git a/airflow-core/tests/unit/always/test_secrets.py b/airflow-core/tests/unit/always/test_secrets.py index 9372cb08a8354..4d7371303b643 100644 --- a/airflow-core/tests/unit/always/test_secrets.py +++ b/airflow-core/tests/unit/always/test_secrets.py @@ -29,10 +29,9 @@ from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_variables -from tests_common.test_utils.markers import skip_if_force_lowest_dependencies_marker +from tests_common.test_utils.fake_secrets_backend import FAKE_BACKEND_PATH, FakeSecretsBackend -@skip_if_force_lowest_dependencies_marker class TestConnectionsFromSecrets: def setup_method(self) -> None: SecretCache.reset() @@ -55,11 +54,8 @@ def test_get_connection_first_try(self, mock_env_get, mock_meta_get): @conf_vars( { - ( - "secrets", - "backend", - ): "airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend", - ("secrets", "backend_kwargs"): '{"connections_prefix": "/airflow", "profile_name": null}', + ("secrets", "backend"): FAKE_BACKEND_PATH, + ("secrets", "backend_kwargs"): '{"connections_prefix": "/airflow"}', } ) def test_initialize_secrets_backends(self): @@ -67,34 +63,26 @@ def test_initialize_secrets_backends(self): backend_classes = [backend.__class__.__name__ for backend in backends] assert len(backends) == 3 - assert "SystemsManagerParameterStoreBackend" in backend_classes + assert "FakeSecretsBackend" in backend_classes @conf_vars( { - ( - "secrets", - "backend", - ): "airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend", + ("secrets", "backend"): FAKE_BACKEND_PATH, ("secrets", "backend_kwargs"): '{"use_ssl": false}', } ) def test_backends_kwargs(self): backends = initialize_secrets_backends() - systems_manager = next( - backend - for backend in backends - if backend.__class__.__name__ == "SystemsManagerParameterStoreBackend" + fake_backend = next( + backend for backend in backends if backend.__class__.__name__ == "FakeSecretsBackend" ) - assert systems_manager.kwargs == {} - assert systems_manager.use_ssl is False + assert fake_backend.kwargs == {} + assert fake_backend.use_ssl is False @conf_vars( { - ( - "secrets", - "backend", - ): "airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend", - ("secrets", "backend_kwargs"): '{"connections_prefix": "/airflow", "profile_name": null}', + ("secrets", "backend"): FAKE_BACKEND_PATH, + ("secrets", "backend_kwargs"): '{"connections_prefix": "/airflow"}', } ) @mock.patch.dict( @@ -103,20 +91,16 @@ def test_backends_kwargs(self): "AIRFLOW_CONN_TEST_MYSQL": "mysql://airflow:airflow@host:5432/airflow", }, ) - @mock.patch( - "airflow.providers.amazon.aws.secrets.systems_manager." - "SystemsManagerParameterStoreBackend.get_connection" - ) + @mock.patch.object(FakeSecretsBackend, "get_connection") def test_backend_fallback_to_env_var(self, mock_get_connection): mock_get_connection.return_value = None backends = ensure_secrets_loaded() backend_classes = [backend.__class__.__name__ for backend in backends] - assert "SystemsManagerParameterStoreBackend" in backend_classes + assert "FakeSecretsBackend" in backend_classes conn = Connection.get_connection_from_secrets(conn_id="test_mysql") - # Assert that SystemsManagerParameterStoreBackend.get_conn_uri was called mock_get_connection.assert_called_once_with(conn_id="test_mysql", team_name=None) assert conn.get_uri() == "mysql://airflow:airflow@host:5432/airflow" @@ -134,7 +118,6 @@ def test_connection_env_var_do_not_access_team_specific(self): Connection.get_connection_from_secrets(conn_id="_team___test_mysql") -@skip_if_force_lowest_dependencies_marker @pytest.mark.db_test class TestVariableFromSecrets: def setup_method(self) -> None: @@ -181,11 +164,8 @@ def test_backend_fallback_to_default_var(self): @conf_vars( { - ( - "secrets", - "backend", - ): "airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend", - ("secrets", "backend_kwargs"): '{"variables_prefix": "/airflow", "profile_name": null}', + ("secrets", "backend"): FAKE_BACKEND_PATH, + ("secrets", "backend_kwargs"): '{"variables_prefix": "/airflow"}', } ) @mock.patch.dict( @@ -195,14 +175,11 @@ def test_backend_fallback_to_default_var(self): }, ) @mock.patch("airflow.secrets.metastore.MetastoreBackend.get_variable") - @mock.patch( - "airflow.providers.amazon.aws.secrets.systems_manager." - "SystemsManagerParameterStoreBackend.get_variable" - ) + @mock.patch.object(FakeSecretsBackend, "get_variable") def test_backend_variable_order(self, mock_secret_get, mock_meta_get): backends = ensure_secrets_loaded() backend_classes = [backend.__class__.__name__ for backend in backends] - assert "SystemsManagerParameterStoreBackend" in backend_classes + assert "FakeSecretsBackend" in backend_classes mock_secret_get.return_value = None mock_meta_get.return_value = None @@ -250,7 +227,6 @@ def get_variable(self, key): return self._vars.get(key) -@skip_if_force_lowest_dependencies_marker class TestTeamUnawareBackendSignatureCompat: """Backends whose overrides predate the ``team_name`` keyword must keep working (issue #1333).""" @@ -279,21 +255,13 @@ def test_get_variable_with_team_unaware_override(self, team_name): assert value == "secret_value" -@skip_if_force_lowest_dependencies_marker class TestSecretBackendKwargEnvVars: """Test per-key env var overrides for secrets backend kwargs.""" def setup_method(self) -> None: SecretCache.reset() - @conf_vars( - { - ( - "secrets", - "backend", - ): "airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend", - } - ) + @conf_vars({("secrets", "backend"): FAKE_BACKEND_PATH}) @mock.patch.dict( "os.environ", {"AIRFLOW__SECRETS__BACKEND_KWARG__CONNECTIONS_PREFIX": "/airflow/connections"}, @@ -301,17 +269,12 @@ def setup_method(self) -> None: def test_backend_kwarg_env_vars_basic(self): """Per-key env var is picked up when no JSON blob is set.""" backends = initialize_secrets_backends() - systems_manager = next( - b for b in backends if b.__class__.__name__ == "SystemsManagerParameterStoreBackend" - ) - assert systems_manager.connections_prefix == "/airflow/connections" + fake_backend = next(b for b in backends if b.__class__.__name__ == "FakeSecretsBackend") + assert fake_backend.connections_prefix == "/airflow/connections" @conf_vars( { - ( - "secrets", - "backend", - ): "airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend", + ("secrets", "backend"): FAKE_BACKEND_PATH, ("secrets", "backend_kwargs"): '{"connections_prefix": "/old"}', } ) @@ -322,17 +285,12 @@ def test_backend_kwarg_env_vars_basic(self): def test_backend_kwarg_env_vars_override_json(self): """Per-key env var overrides the same key in the JSON blob.""" backends = initialize_secrets_backends() - systems_manager = next( - b for b in backends if b.__class__.__name__ == "SystemsManagerParameterStoreBackend" - ) - assert systems_manager.connections_prefix == "/new" + fake_backend = next(b for b in backends if b.__class__.__name__ == "FakeSecretsBackend") + assert fake_backend.connections_prefix == "/new" @conf_vars( { - ( - "secrets", - "backend", - ): "airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend", + ("secrets", "backend"): FAKE_BACKEND_PATH, ("secrets", "backend_kwargs"): '{"connections_prefix": "/airflow"}', } ) @@ -343,8 +301,6 @@ def test_backend_kwarg_env_vars_override_json(self): def test_backend_kwarg_env_vars_merge_with_json(self): """Per-key env var is merged with (not replacing) the JSON blob.""" backends = initialize_secrets_backends() - systems_manager = next( - b for b in backends if b.__class__.__name__ == "SystemsManagerParameterStoreBackend" - ) - assert systems_manager.connections_prefix == "/airflow" - assert systems_manager.variables_prefix == "/airflow/variables" + fake_backend = next(b for b in backends if b.__class__.__name__ == "FakeSecretsBackend") + assert fake_backend.connections_prefix == "/airflow" + assert fake_backend.variables_prefix == "/airflow/variables" diff --git a/airflow-core/tests/unit/core/test_configuration.py b/airflow-core/tests/unit/core/test_configuration.py index 815574a30e137..e173215234ac4 100644 --- a/airflow-core/tests/unit/core/test_configuration.py +++ b/airflow-core/tests/unit/core/test_configuration.py @@ -52,6 +52,11 @@ conf_vars, create_fresh_airflow_config, ) +from tests_common.test_utils.fake_secrets_backend import ( + FAKE_BACKEND_PATH, + FAKE_CONFIG_BACKEND_PATH, + FAKE_UNREACHABLE_BACKEND_PATH, +) from tests_common.test_utils.markers import skip_if_force_lowest_dependencies_marker from tests_common.test_utils.reset_warning_registry import reset_warning_registry from unit.utils.test_config import ( @@ -421,37 +426,16 @@ def test_can_read_dot_section(self): ): assert test_conf.getboolean(section, key) is False - @skip_if_force_lowest_dependencies_marker - @mock.patch("airflow.providers.hashicorp._internal_client.vault_client.hvac") @conf_vars( { - ("secrets", "backend"): "airflow.providers.hashicorp.secrets.vault.VaultBackend", - ("secrets", "backend_kwargs"): '{"url": "http://127.0.0.1:8200", "token": "token"}', + ("secrets", "backend"): FAKE_CONFIG_BACKEND_PATH, + ("secrets", "backend_kwargs"): ( + '{"config_values": {"sql_alchemy_conn": "sqlite:////Users/airflow/airflow/airflow.db"}}' + ), } ) - def test_config_from_secret_backend(self, mock_hvac): + def test_config_from_secret_backend(self): """Get Config Value from a Secret Backend""" - mock_client = mock.MagicMock() - mock_hvac.Client.return_value = mock_client - mock_client.secrets.kv.v2.read_secret_version.return_value = { - "request_id": "2d48a2ad-6bcb-e5b6-429d-da35fdf31f56", - "lease_id": "", - "renewable": False, - "lease_duration": 0, - "data": { - "data": {"value": "sqlite:////Users/airflow/airflow/airflow.db"}, - "metadata": { - "created_time": "2020-03-28T02:10:54.301784Z", - "deletion_time": "", - "destroyed": False, - "version": 1, - }, - }, - "wrap_info": None, - "warnings": None, - "auth": None, - } - test_config = textwrap.dedent( """ [test] @@ -507,21 +491,9 @@ def test_hidding_of_sensitive_config_values(self): # is True assert test_conf.as_dict(display_sensitive=True, include_cmds=False) - @skip_if_force_lowest_dependencies_marker - @mock.patch("airflow.providers.hashicorp._internal_client.vault_client.hvac") - @conf_vars( - { - ("secrets", "backend"): "airflow.providers.hashicorp.secrets.vault.VaultBackend", - ("secrets", "backend_kwargs"): '{"url": "http://127.0.0.1:8200", "token": "token"}', - } - ) - def test_config_raise_exception_from_secret_backend_connection_error(self, mock_hvac): + @conf_vars({("secrets", "backend"): FAKE_UNREACHABLE_BACKEND_PATH}) + def test_config_raise_exception_from_secret_backend_connection_error(self): """Get Config Value from a Secret Backend""" - mock_client = mock.MagicMock() - # mock_client.side_effect = AirflowConfigException - mock_hvac.Client.return_value = mock_client - mock_client.secrets.kv.v2.read_secret_version.return_value = Exception - test_config = textwrap.dedent( """ [test] @@ -815,14 +787,10 @@ def test_as_dict_respects_sensitive_cmds_from_env(self): assert conf_materialize_cmds["database"]["sql_alchemy_conn"] == "postgresql+psycopg2://" - @skip_if_force_lowest_dependencies_marker @conf_vars( { - ( - "workers", - "secrets_backend", - ): "airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend", - ("workers", "secrets_backend_kwargs"): '{"connections_prefix": "/airflow", "profile_name": null}', + ("workers", "secrets_backend"): FAKE_BACKEND_PATH, + ("workers", "secrets_backend_kwargs"): '{"connections_prefix": "/airflow"}', } ) def test_initialize_secrets_backends_on_workers(self): @@ -831,32 +799,25 @@ def test_initialize_secrets_backends_on_workers(self): backend_classes = [backend.__class__.__name__ for backend in backends] assert len(backends) == 3 - assert "SystemsManagerParameterStoreBackend" in backend_classes + assert "FakeSecretsBackend" in backend_classes assert "EnvironmentVariablesBackend" in backend_classes assert "ExecutionAPISecretsBackend" in backend_classes - @skip_if_force_lowest_dependencies_marker @conf_vars( { - ( - "workers", - "secrets_backend", - ): "airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend", + ("workers", "secrets_backend"): FAKE_BACKEND_PATH, ("workers", "secrets_backend_kwargs"): '{"use_ssl": false}', } ) def test_secrets_backends_kwargs_on_workers(self): """Tests if secrets backend kwargs are loaded correctly for workers.""" backends = initialize_secrets_backends(DEFAULT_SECRETS_SEARCH_PATH_WORKERS) - systems_manager = next( - backend - for backend in backends - if backend.__class__.__name__ == "SystemsManagerParameterStoreBackend" + fake_backend = next( + backend for backend in backends if backend.__class__.__name__ == "FakeSecretsBackend" ) - assert systems_manager.kwargs == {} - assert systems_manager.use_ssl is False + assert fake_backend.kwargs == {} + assert fake_backend.use_ssl is False - @skip_if_force_lowest_dependencies_marker @pytest.mark.parametrize( ( "secrets_backend", @@ -869,21 +830,21 @@ def test_secrets_backends_kwargs_on_workers(self): [ # pick right backend and kwargs pytest.param( - "airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend", - '{"connections_prefix": "/airflow", "profile_name": null}', + FAKE_BACKEND_PATH, + '{"connections_prefix": "/airflow", "variables_prefix": null}', "airflow.secrets.local_filesystem.LocalFilesystemBackend", '{"connections_file_path": "/files/conn.json", "variables_file_path": "/files/var.json"}', - "airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend", - {"connections_prefix": "/airflow", "profile_name": None}, + FAKE_BACKEND_PATH, + {"connections_prefix": "/airflow", "variables_prefix": None}, id="both-defined", ), # do not pick kwargs of secrets backend when not defined for worker pytest.param( - "airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend", + FAKE_BACKEND_PATH, "", "airflow.secrets.local_filesystem.LocalFilesystemBackend", '{"connections_file_path": "/files/conn.json", "variables_file_path": "/files/var.json"}', - "airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend", + FAKE_BACKEND_PATH, {}, id="worker-backend-defined-not-kwargs", ), @@ -960,14 +921,7 @@ def test_collect_kwarg_env_vars(self): result = _collect_kwarg_env_vars("AIRFLOW__SECRETS__BACKEND_KWARG__") assert result == {"role_id": "abc"} - @conf_vars( - { - ( - "workers", - "secrets_backend", - ): "airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend", - } - ) + @conf_vars({("workers", "secrets_backend"): FAKE_BACKEND_PATH}) @mock.patch.dict( "os.environ", {"AIRFLOW__WORKERS__SECRETS_BACKEND_KWARG__CONNECTIONS_PREFIX": "/worker/connections"}, @@ -976,7 +930,7 @@ def test_worker_backend_kwarg_env_vars(self): """Per-key env var is picked up for the workers secrets backend.""" backends = ensure_secrets_loaded(DEFAULT_SECRETS_SEARCH_PATH_WORKERS) secrets_backend = backends[0] - assert secrets_backend.__class__.__name__ == "SystemsManagerParameterStoreBackend" + assert secrets_backend.__class__.__name__ == "FakeSecretsBackend" assert secrets_backend.connections_prefix == "/worker/connections" @mock.patch("airflow._shared.secrets_masker.mask_secret") diff --git a/devel-common/src/tests_common/test_utils/fake_secrets_backend.py b/devel-common/src/tests_common/test_utils/fake_secrets_backend.py new file mode 100644 index 0000000000000..2980b3faf6fc5 --- /dev/null +++ b/devel-common/src/tests_common/test_utils/fake_secrets_backend.py @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.secrets import BaseSecretsBackend + +FAKE_BACKEND_PATH = f"{__name__}.FakeSecretsBackend" +FAKE_CONFIG_BACKEND_PATH = f"{__name__}.FakeConfigSecretsBackend" +FAKE_UNREACHABLE_BACKEND_PATH = f"{__name__}.FakeUnreachableSecretsBackend" + + +class FakeSecretsBackend(BaseSecretsBackend): + """Secrets backend for tests that exercise backend configuration rather than lookups.""" + + # Defaults deliberately differ from the values the tests pass in, so an assertion on an + # attribute fails unless the configured kwarg — including an explicit JSON null — reached here. + def __init__( + self, + connections_prefix: str | None = "/connections", + variables_prefix: str | None = "/variables", + use_ssl: bool = True, + **kwargs, + ) -> None: + self.connections_prefix = connections_prefix + self.variables_prefix = variables_prefix + self.use_ssl = use_ssl + self.kwargs = kwargs + + +class FakeConfigSecretsBackend(BaseSecretsBackend): + """Secrets backend serving config values from a mapping passed as a backend kwarg.""" + + def __init__(self, config_values: dict[str, str] | None = None, **kwargs) -> None: + self.config_values = config_values or {} + self.kwargs = kwargs + + def get_config(self, key: str) -> str | None: + return self.config_values.get(key) + + +class FakeUnreachableSecretsBackend(BaseSecretsBackend): + """Secrets backend whose config lookups fail, standing in for an unreachable backend.""" + + def get_config(self, key: str) -> str | None: + raise ConnectionError(f"Cannot reach the secrets backend to look up {key!r}")