diff --git a/scripts/in_container/provider_suspension.py b/scripts/in_container/provider_suspension.py new file mode 100644 index 0000000000000..947d1d2f916aa --- /dev/null +++ b/scripts/in_container/provider_suspension.py @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Filters that drop what a suspended provider leaves behind in the provider.yaml checks.""" + +from __future__ import annotations + +from collections.abc import Collection + + +def remove_suspended_doc_urls(doc_urls: Collection[str], suspended_packages: Collection[str]) -> set[str]: + """Drop documentation urls that belong to one of the suspended providers.""" + return {url for url in doc_urls if not any(package in url for package in suspended_packages)} + + +def remove_suspended_import_errors(errors: Collection[str], suspended_packages: Collection[str]) -> list[str]: + """Drop import errors raised for the modules of one of the suspended providers.""" + suspended_modules = { + package.replace("apache-", "", 1).replace("-", ".") for package in suspended_packages + } + return [ + error + for error in errors + if not any(f"No module named '{module}'" in error for module in suspended_modules) + ] diff --git a/scripts/in_container/run_provider_yaml_files_check.py b/scripts/in_container/run_provider_yaml_files_check.py index 12e082864f23f..cddac1c80cd2e 100755 --- a/scripts/in_container/run_provider_yaml_files_check.py +++ b/scripts/in_container/run_provider_yaml_files_check.py @@ -43,6 +43,7 @@ AIRFLOW_ROOT_PATH, ) from jsonpath_ng.ext import parse +from provider_suspension import remove_suspended_doc_urls, remove_suspended_import_errors from rich.console import Console from tabulate import tabulate @@ -1046,13 +1047,7 @@ def check_doc_files(yaml_files: dict[str, dict]) -> tuple[int, int]: if f.name != "index.rst" and "_partials" not in f.parts and f.parts[2] == "docs" } - if suspended_providers: - expected_doc_urls = { - doc_url - for doc_url in expected_doc_urls - for suspend_provider in suspended_providers - if suspend_provider not in doc_url - } + expected_doc_urls = remove_suspended_doc_urls(expected_doc_urls, suspended_providers) if suspended_logos: console.print("[yellow]Suspended logos:[/]") @@ -1216,15 +1211,8 @@ def check_providers_have_all_documentation_files(yaml_files: dict[str, dict]): check_invalid_integration(all_parsed_yaml_files) check_providers_are_mentioned_in_issue_template(all_parsed_yaml_files) - # remove errors related to suspended module imports. print("suspended_providers ", suspended_providers) - if suspended_providers and errors: - errors = [ - error - for error in errors - for module in suspended_providers - if f"No module named '{module.replace('apache-', '', 1).replace('-', '.')}'" not in error - ] + errors = remove_suspended_import_errors(errors, suspended_providers) if errors: error_num = len(errors) diff --git a/scripts/tests/in_container/test_provider_suspension.py b/scripts/tests/in_container/test_provider_suspension.py new file mode 100644 index 0000000000000..52bb2b7674f14 --- /dev/null +++ b/scripts/tests/in_container/test_provider_suspension.py @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest +from provider_suspension import remove_suspended_doc_urls, remove_suspended_import_errors + +BEAM_URL = "/docs/apache-airflow-providers-apache-beam/operators.rst" +YANDEX_URL = "/docs/apache-airflow-providers-yandex/operators/dataproc.rst" +GOOGLE_URL = "/docs/apache-airflow-providers-google/operators/bigquery.rst" +ALL_URLS = [BEAM_URL, YANDEX_URL, GOOGLE_URL] + +BEAM_ERROR = "No module named 'airflow.providers.apache.beam'" +YANDEX_ERROR = "No module named 'airflow.providers.yandex'" +OTHER_ERROR = "Provider google has an invalid integration" +ALL_ERRORS = [BEAM_ERROR, YANDEX_ERROR, OTHER_ERROR] + + +@pytest.mark.parametrize( + "suspended_packages, expected", + [ + pytest.param(set(), ALL_URLS, id="none-suspended"), + pytest.param({"apache-airflow-providers-apache-beam"}, [YANDEX_URL, GOOGLE_URL], id="one-suspended"), + pytest.param( + {"apache-airflow-providers-apache-beam", "apache-airflow-providers-yandex"}, + [GOOGLE_URL], + id="several-suspended", + ), + ], +) +def test_remove_suspended_doc_urls(suspended_packages, expected): + assert remove_suspended_doc_urls(ALL_URLS, suspended_packages) == set(expected) + + +@pytest.mark.parametrize( + "suspended_packages, expected", + [ + pytest.param(set(), ALL_ERRORS, id="none-suspended"), + pytest.param( + {"apache-airflow-providers-apache-beam"}, [YANDEX_ERROR, OTHER_ERROR], id="one-suspended" + ), + pytest.param( + {"apache-airflow-providers-apache-beam", "apache-airflow-providers-yandex"}, + [OTHER_ERROR], + id="several-suspended", + ), + ], +) +def test_remove_suspended_import_errors(suspended_packages, expected): + assert remove_suspended_import_errors(ALL_ERRORS, suspended_packages) == expected