Fix provider.yaml doc URL check with several suspended providers

scripts/in_container/provider_suspension.py

@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Filters that drop what a suspended provider leaves behind in the provider.yaml checks."""
+
+from __future__ import annotations
+
+from collections.abc import Collection
+
+
+def remove_suspended_doc_urls(doc_urls: Collection[str], suspended_packages: Collection[str]) -> set[str]:
+    """Drop documentation urls that belong to one of the suspended providers."""
+    return {url for url in doc_urls if not any(package in url for package in suspended_packages)}
+
+
+def remove_suspended_import_errors(errors: Collection[str], suspended_packages: Collection[str]) -> list[str]:
+    """Drop import errors raised for the modules of one of the suspended providers."""
+    suspended_modules = {
+        package.replace("apache-", "", 1).replace("-", ".") for package in suspended_packages
+    }
+    return [
+        error
+        for error in errors
+        if not any(f"No module named '{module}'" in error for module in suspended_modules)
+    ]

scripts/in_container/run_provider_yaml_files_check.py

@@ -43,6 +43,7 @@
     AIRFLOW_ROOT_PATH,
 )
 from jsonpath_ng.ext import parse
+from provider_suspension import remove_suspended_doc_urls, remove_suspended_import_errors
 from rich.console import Console
 from tabulate import tabulate
 
@@ -1046,13 +1047,7 @@ def check_doc_files(yaml_files: dict[str, dict]) -> tuple[int, int]:
         if f.name != "index.rst" and "_partials" not in f.parts and f.parts[2] == "docs"
     }
 
-    if suspended_providers:
-        expected_doc_urls = {
-            doc_url
-            for doc_url in expected_doc_urls
-            for suspend_provider in suspended_providers
-            if suspend_provider not in doc_url
-        }
+    expected_doc_urls = remove_suspended_doc_urls(expected_doc_urls, suspended_providers)
 
     if suspended_logos:
         console.print("[yellow]Suspended logos:[/]")
@@ -1216,15 +1211,8 @@ def check_providers_have_all_documentation_files(yaml_files: dict[str, dict]):
         check_invalid_integration(all_parsed_yaml_files)
         check_providers_are_mentioned_in_issue_template(all_parsed_yaml_files)
 
-    # remove errors related to suspended module imports.
     print("suspended_providers ", suspended_providers)
-    if suspended_providers and errors:
-        errors = [
-            error
-            for error in errors
-            for module in suspended_providers
-            if f"No module named '{module.replace('apache-', '', 1).replace('-', '.')}'" not in error
-        ]
+    errors = remove_suspended_import_errors(errors, suspended_providers)
 
     if errors:
         error_num = len(errors)

scripts/tests/in_container/test_provider_suspension.py

@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import pytest
+from provider_suspension import remove_suspended_doc_urls, remove_suspended_import_errors
+
+BEAM_URL = "/docs/apache-airflow-providers-apache-beam/operators.rst"
+YANDEX_URL = "/docs/apache-airflow-providers-yandex/operators/dataproc.rst"
+GOOGLE_URL = "/docs/apache-airflow-providers-google/operators/bigquery.rst"
+ALL_URLS = [BEAM_URL, YANDEX_URL, GOOGLE_URL]
+
+BEAM_ERROR = "No module named 'airflow.providers.apache.beam'"
+YANDEX_ERROR = "No module named 'airflow.providers.yandex'"
+OTHER_ERROR = "Provider google has an invalid integration"
+ALL_ERRORS = [BEAM_ERROR, YANDEX_ERROR, OTHER_ERROR]
+
+
+@pytest.mark.parametrize(
+    "suspended_packages, expected",
+    [
+        pytest.param(set(), ALL_URLS, id="none-suspended"),
+        pytest.param({"apache-airflow-providers-apache-beam"}, [YANDEX_URL, GOOGLE_URL], id="one-suspended"),
+        pytest.param(
+            {"apache-airflow-providers-apache-beam", "apache-airflow-providers-yandex"},
+            [GOOGLE_URL],
+            id="several-suspended",
+        ),
+    ],
+)
+def test_remove_suspended_doc_urls(suspended_packages, expected):
+    assert remove_suspended_doc_urls(ALL_URLS, suspended_packages) == set(expected)
+
+
+@pytest.mark.parametrize(
+    "suspended_packages, expected",
+    [
+        pytest.param(set(), ALL_ERRORS, id="none-suspended"),
+        pytest.param(
+            {"apache-airflow-providers-apache-beam"}, [YANDEX_ERROR, OTHER_ERROR], id="one-suspended"
+        ),
+        pytest.param(
+            {"apache-airflow-providers-apache-beam", "apache-airflow-providers-yandex"},
+            [OTHER_ERROR],
+            id="several-suspended",
+        ),
+    ],
+)
+def test_remove_suspended_import_errors(suspended_packages, expected):
+    assert remove_suspended_import_errors(ALL_ERRORS, suspended_packages) == expected