Skip to content

Commit 23c6e77

Browse files
authored
Fix provider.yaml doc URL check with several suspended providers (#73782)
Both suspended-provider filters in the check cross-multiply their inputs, so an entry survives whenever it fails to match any single suspended provider. The check has only ever run with zero or one suspended provider, where that reading is accidentally correct, so suspending a second one would fail the check on the suspended providers' own documentation and report unrelated errors once per suspended provider.
1 parent 3107af0 commit 23c6e77

3 files changed

Lines changed: 105 additions & 15 deletions

File tree

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
"""Filters that drop what a suspended provider leaves behind in the provider.yaml checks."""
18+
19+
from __future__ import annotations
20+
21+
from collections.abc import Collection
22+
23+
24+
def remove_suspended_doc_urls(doc_urls: Collection[str], suspended_packages: Collection[str]) -> set[str]:
25+
"""Drop documentation urls that belong to one of the suspended providers."""
26+
return {url for url in doc_urls if not any(package in url for package in suspended_packages)}
27+
28+
29+
def remove_suspended_import_errors(errors: Collection[str], suspended_packages: Collection[str]) -> list[str]:
30+
"""Drop import errors raised for the modules of one of the suspended providers."""
31+
suspended_modules = {
32+
package.replace("apache-", "", 1).replace("-", ".") for package in suspended_packages
33+
}
34+
return [
35+
error
36+
for error in errors
37+
if not any(f"No module named '{module}'" in error for module in suspended_modules)
38+
]

‎scripts/in_container/run_provider_yaml_files_check.py‎

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
AIRFLOW_ROOT_PATH,
4444
)
4545
from jsonpath_ng.ext import parse
46+
from provider_suspension import remove_suspended_doc_urls, remove_suspended_import_errors
4647
from rich.console import Console
4748
from tabulate import tabulate
4849

@@ -1048,13 +1049,7 @@ def check_doc_files(yaml_files: dict[str, dict]) -> tuple[int, int]:
10481049
if f.name != "index.rst" and "_partials" not in f.parts and f.parts[2] == "docs"
10491050
}
10501051

1051-
if suspended_providers:
1052-
expected_doc_urls = {
1053-
doc_url
1054-
for doc_url in expected_doc_urls
1055-
for suspend_provider in suspended_providers
1056-
if suspend_provider not in doc_url
1057-
}
1052+
expected_doc_urls = remove_suspended_doc_urls(expected_doc_urls, suspended_providers)
10581053

10591054
if suspended_logos:
10601055
console.print("[yellow]Suspended logos:[/]")
@@ -1218,15 +1213,8 @@ def check_providers_have_all_documentation_files(yaml_files: dict[str, dict]):
12181213
check_invalid_integration(all_parsed_yaml_files)
12191214
check_providers_are_mentioned_in_issue_template(all_parsed_yaml_files)
12201215

1221-
# remove errors related to suspended module imports.
12221216
print("suspended_providers ", suspended_providers)
1223-
if suspended_providers and errors:
1224-
errors = [
1225-
error
1226-
for error in errors
1227-
for module in suspended_providers
1228-
if f"No module named '{module.replace('apache-', '', 1).replace('-', '.')}'" not in error
1229-
]
1217+
errors = remove_suspended_import_errors(errors, suspended_providers)
12301218

12311219
if errors:
12321220
error_num = len(errors)
Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
from __future__ import annotations
18+
19+
import pytest
20+
from provider_suspension import remove_suspended_doc_urls, remove_suspended_import_errors
21+
22+
BEAM_URL = "/docs/apache-airflow-providers-apache-beam/operators.rst"
23+
YANDEX_URL = "/docs/apache-airflow-providers-yandex/operators/dataproc.rst"
24+
GOOGLE_URL = "/docs/apache-airflow-providers-google/operators/bigquery.rst"
25+
ALL_URLS = [BEAM_URL, YANDEX_URL, GOOGLE_URL]
26+
27+
BEAM_ERROR = "No module named 'airflow.providers.apache.beam'"
28+
YANDEX_ERROR = "No module named 'airflow.providers.yandex'"
29+
OTHER_ERROR = "Provider google has an invalid integration"
30+
ALL_ERRORS = [BEAM_ERROR, YANDEX_ERROR, OTHER_ERROR]
31+
32+
33+
@pytest.mark.parametrize(
34+
"suspended_packages, expected",
35+
[
36+
pytest.param(set(), ALL_URLS, id="none-suspended"),
37+
pytest.param({"apache-airflow-providers-apache-beam"}, [YANDEX_URL, GOOGLE_URL], id="one-suspended"),
38+
pytest.param(
39+
{"apache-airflow-providers-apache-beam", "apache-airflow-providers-yandex"},
40+
[GOOGLE_URL],
41+
id="several-suspended",
42+
),
43+
],
44+
)
45+
def test_remove_suspended_doc_urls(suspended_packages, expected):
46+
assert remove_suspended_doc_urls(ALL_URLS, suspended_packages) == set(expected)
47+
48+
49+
@pytest.mark.parametrize(
50+
"suspended_packages, expected",
51+
[
52+
pytest.param(set(), ALL_ERRORS, id="none-suspended"),
53+
pytest.param(
54+
{"apache-airflow-providers-apache-beam"}, [YANDEX_ERROR, OTHER_ERROR], id="one-suspended"
55+
),
56+
pytest.param(
57+
{"apache-airflow-providers-apache-beam", "apache-airflow-providers-yandex"},
58+
[OTHER_ERROR],
59+
id="several-suspended",
60+
),
61+
],
62+
)
63+
def test_remove_suspended_import_errors(suspended_packages, expected):
64+
assert remove_suspended_import_errors(ALL_ERRORS, suspended_packages) == expected

0 commit comments

Comments
 (0)