Skip to content

Add fallback region_name value to AWS Executors #38704

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
potiuk merged 2 commits into from
Apr 3, 2024
Merged

Add fallback region_name value to AWS Executors #38704

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
43d616e
Add fallback `region_name` value to AWS Executors
Taragolis Apr 3, 2024
e57e937
Get rid of os.environ in test_ecs_executor.py
Taragolis Apr 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion airflow/providers/amazon/aws/executors/batch/batch_executor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ def load_batch_connection(self, check_connection: bool = True):
AllBatchConfigKeys.AWS_CONN_ID,
fallback=CONFIG_DEFAULTS[AllBatchConfigKeys.AWS_CONN_ID],
)
region_name = conf.get(CONFIG_GROUP_NAME, AllBatchConfigKeys.REGION_NAME)
region_name = conf.get(CONFIG_GROUP_NAME, AllBatchConfigKeys.REGION_NAME, fallback=None)
self.batch = BatchClientHook(aws_conn_id=aws_conn_id, region_name=region_name).conn
self.attempts_since_last_successful_connection += 1
self.last_connection_reload = timezone.utcnow()
Expand Down
2 changes: 1 addition & 1 deletion airflow/providers/amazon/aws/executors/ecs/ecs_executor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ def load_ecs_connection(self, check_connection: bool = True):
AllEcsConfigKeys.AWS_CONN_ID,
fallback=CONFIG_DEFAULTS[AllEcsConfigKeys.AWS_CONN_ID],
)
region_name = conf.get(CONFIG_GROUP_NAME, AllEcsConfigKeys.REGION_NAME)
region_name = conf.get(CONFIG_GROUP_NAME, AllEcsConfigKeys.REGION_NAME, fallback=None)
self.ecs = EcsHook(aws_conn_id=aws_conn_id, region_name=region_name).conn
self.attempts_since_last_successful_connection += 1
self.last_connection_reload = timezone.utcnow()
Expand Down
17 changes: 11 additions & 6 deletions tests/providers/amazon/aws/executors/batch/test_batch_executor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
)
from airflow.utils.helpers import convert_camel_to_snake
from airflow.utils.state import State
from tests.test_utils.config import conf_vars

ARN1 = "arn1"

Expand All @@ -49,11 +50,15 @@

@pytest.fixture
def set_env_vars():
os.environ[f"AIRFLOW__{CONFIG_GROUP_NAME}__{AllBatchConfigKeys.REGION_NAME}".upper()] = "us-west-1"
os.environ[f"AIRFLOW__{CONFIG_GROUP_NAME}__{AllBatchConfigKeys.JOB_NAME}".upper()] = "some-job-name"
os.environ[f"AIRFLOW__{CONFIG_GROUP_NAME}__{AllBatchConfigKeys.JOB_QUEUE}".upper()] = "some-job-queue"
os.environ[f"AIRFLOW__{CONFIG_GROUP_NAME}__{AllBatchConfigKeys.JOB_DEFINITION}".upper()] = "some-job-def"
os.environ[f"AIRFLOW__{CONFIG_GROUP_NAME}__{AllBatchConfigKeys.MAX_SUBMIT_JOB_ATTEMPTS}".upper()] = "3"
overrides: dict[tuple[str, str], str] = {
(CONFIG_GROUP_NAME, AllBatchConfigKeys.REGION_NAME): "us-east-1",
(CONFIG_GROUP_NAME, AllBatchConfigKeys.JOB_NAME): "some-job-name",
(CONFIG_GROUP_NAME, AllBatchConfigKeys.JOB_QUEUE): "some-job-queue",
(CONFIG_GROUP_NAME, AllBatchConfigKeys.JOB_DEFINITION): "some-job-def",
(CONFIG_GROUP_NAME, AllBatchConfigKeys.MAX_SUBMIT_JOB_ATTEMPTS): "3",
}
with conf_vars(overrides):
yield


@pytest.fixture
Expand Down Expand Up @@ -514,7 +519,7 @@ def test_start_success(self, set_env_vars, caplog):
@mock.patch(
"airflow.providers.amazon.aws.executors.batch.boto_schema.BatchDescribeJobsResponseSchema.load"
)
def test_health_check_failure(self, mock_executor):
def test_health_check_failure(self, mock_executor, set_env_vars):
mock_executor.batch.describe_jobs.side_effect = Exception("Test_failure")
executor = AwsBatchExecutor()
batch_mock = mock.Mock(spec=executor.batch)
Expand Down
Loading