Skip to content

Commit 53de1c1

Browse files
dablapotiuk
andauthored
Added static check for conn-fields defined in provider.yaml (#69473)
* refactor: Added static check for conn-fields defined in provider.yaml to make sure those correspond with the ones defined in the get_connection_form_widgets method of the corresponding hook * Update scripts/in_container/run_provider_yaml_files_check.py Co-authored-by: Jarek Potiuk <[email protected]> * refactor: Updated comment in _get_widget_keys method --------- Co-authored-by: Jarek Potiuk <[email protected]>
1 parent d9bfef0 commit 53de1c1

3 files changed

Lines changed: 301 additions & 0 deletions

File tree

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
#!/usr/bin/env python
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
"""
20+
Validation helpers for the conn-fields ↔ get_connection_form_widgets() check.
21+
22+
These functions have no third-party dependencies so they can be unit-tested
23+
outside of the Breeze container without any stubbing.
24+
25+
Used by ``scripts/in_container/run_provider_yaml_files_check.py``.
26+
"""
27+
28+
from __future__ import annotations
29+
30+
from collections.abc import Callable
31+
32+
33+
def check_conn_fields_for_entry(
34+
conn_type_entry: dict,
35+
yaml_file_path: str,
36+
get_widget_keys: Callable[[str], set[str] | None],
37+
) -> list[str]:
38+
"""
39+
Validate a single connection-type entry. Returns a (possibly empty) list of error strings.
40+
41+
*get_widget_keys(hook_class_name)* is a callable supplied by the caller that:
42+
43+
- returns the set of field keys from ``get_connection_form_widgets()`` on success,
44+
- returns ``None`` to signal that the hook could not be imported, its UI
45+
dependencies are absent, or it does not implement ``get_connection_form_widgets()``
46+
at all — the entry is then skipped entirely (no ``conn-fields`` check), or
47+
- raises any other ``Exception`` to signal an unexpected failure (converted
48+
here into an error string so callers never need to catch it).
49+
"""
50+
hook_class_name: str = conn_type_entry["hook-class-name"]
51+
connection_type: str = conn_type_entry.get("connection-type", "?")
52+
53+
try:
54+
widget_keys = get_widget_keys(hook_class_name)
55+
except Exception as exc:
56+
return [
57+
f"Failed to call `{hook_class_name}.get_connection_form_widgets()` "
58+
f"while checking {yaml_file_path}: {exc}"
59+
]
60+
61+
if widget_keys is None:
62+
return []
63+
64+
conn_fields = conn_type_entry.get("conn-fields")
65+
if conn_fields is None:
66+
# No conn-fields declared: the new UI simply exposes no custom fields for this
67+
# connection type, which is intentional. Nothing to validate.
68+
return []
69+
70+
error = build_mismatch_error(
71+
set(conn_fields.keys()), widget_keys, connection_type, yaml_file_path, hook_class_name
72+
)
73+
return [error] if error else []
74+
75+
76+
def build_mismatch_error(
77+
yaml_keys: set[str],
78+
hook_keys: set[str],
79+
connection_type: str,
80+
yaml_file_path: str,
81+
hook_class_name: str,
82+
) -> str | None:
83+
"""
84+
Check that every key declared in ``conn-fields`` exists in
85+
``get_connection_form_widgets()``.
86+
87+
``conn-fields`` is the new React UI's view of a connection type and is
88+
intentionally a *subset* of the Flask form widgets — fields can be omitted
89+
from ``conn-fields`` on purpose. We therefore only flag keys that appear in
90+
``conn-fields`` but are absent from the hook's form (invalid / stale
91+
declarations). The reverse direction (hook fields not listed in
92+
``conn-fields``) is not an error.
93+
94+
Return an error string when stale keys are found, or ``None`` when the
95+
declared keys are all valid.
96+
"""
97+
only_in_yaml = yaml_keys - hook_keys
98+
99+
if not only_in_yaml:
100+
return None
101+
102+
lines = [
103+
f"Mismatch between `conn-fields` in {yaml_file_path} and "
104+
f"`{hook_class_name}.get_connection_form_widgets()` "
105+
f"for connection-type '{connection_type}':"
106+
]
107+
lines.append(
108+
" Fields in provider.yaml conn-fields but NOT in get_connection_form_widgets(): "
109+
+ ", ".join(sorted(only_in_yaml))
110+
)
111+
lines.append("[yellow]How to fix it[/]: Remove the stale key(s) from conn-fields in provider.yaml.")
112+
return "\n".join(lines)

‎scripts/in_container/run_provider_yaml_files_check.py‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,11 @@
5050
from airflow.exceptions import AirflowOptionalProviderFeatureException, AirflowProviderDeprecationWarning
5151
from airflow.providers_manager import ProvidersManager
5252

53+
# check_provider_conn_fields lives in scripts/ci/prek/ which is not on sys.path when
54+
# this script runs inside Breeze; resolve it relative to this file.
55+
sys.path.insert(0, str(pathlib.Path(__file__).parent.parent / "ci" / "prek"))
56+
from check_provider_conn_fields import check_conn_fields_for_entry
57+
5358
# Those are deprecated modules that contain removed Hooks/Sensors/Operators that we left in the code
5459
# so that users can get a very specific error message when they try to use them.
5560

@@ -473,6 +478,65 @@ def check_hook_class_name_entries_in_connection_types(yaml_files: dict[str, dict
473478
return num_connection_types, num_errors
474479

475480

481+
@run_check("Checking that conn-fields in provider.yaml match get_connection_form_widgets() of the hook class")
482+
def check_conn_fields_match_form_widgets(yaml_files: dict[str, dict]) -> tuple[int, int]:
483+
"""
484+
For every connection-type entry whose hook declares ``conn-fields``,
485+
verify that every key in ``conn-fields`` also exists in the hook's
486+
``get_connection_form_widgets()``.
487+
488+
``conn-fields`` is optional and is intentionally allowed to be a *subset*
489+
of the hook's form widgets (the new React UI may expose fewer fields than
490+
the legacy Flask form), so extra hook widgets are not flagged — only
491+
``conn-fields`` keys absent from the hook are reported as stale.
492+
"""
493+
num_checks = 0
494+
num_errors = 0
495+
496+
for yaml_file_path, provider_data in yaml_files.items():
497+
for conn_type_entry in provider_data.get("connection-types", []):
498+
num_checks += 1
499+
for error in check_conn_fields_for_entry(conn_type_entry, yaml_file_path, _get_widget_keys):
500+
errors.append(error)
501+
num_errors += 1
502+
503+
return num_checks, num_errors
504+
505+
506+
def _get_widget_keys(hook_class_name: str) -> set[str] | None:
507+
"""
508+
Import *hook_class_name* and return the keys of ``get_connection_form_widgets()``.
509+
510+
Returns ``None`` when the hook or its UI dependencies cannot be imported,
511+
or when the hook does not override ``get_connection_form_widgets()`` (meaning it
512+
has no custom connection fields and the conn-fields check should be skipped).
513+
Raises for unexpected errors so ``check_conn_fields_for_entry`` can convert them
514+
to an error string.
515+
"""
516+
try:
517+
module_name, class_name = hook_class_name.rsplit(".", maxsplit=1)
518+
with warnings.catch_warnings(record=True):
519+
hook_class = getattr(importlib.import_module(module_name), class_name)
520+
except (ImportError, AirflowOptionalProviderFeatureException, AttributeError):
521+
return None
522+
523+
# Only validate hooks that override get_connection_form_widgets() in their own __dict__,
524+
# because that method is the source-of-truth for what conn-fields should be declared.
525+
# Hooks that inherit it without overriding have no provider-specific widget definition
526+
# to diff against, so the check is intentionally skipped for them. As of writing this
527+
# includes HttpHook, the common/ai hooks, and AzureComputeHook — any provider whose hook
528+
# falls into this category will NOT be validated here, even if it declares conn-fields.
529+
if "get_connection_form_widgets" not in hook_class.__dict__:
530+
return None
531+
532+
with warnings.catch_warnings(record=True):
533+
try:
534+
form_widgets: dict[str, Any] = hook_class.get_connection_form_widgets()
535+
return set(form_widgets.keys())
536+
except (ImportError, AirflowOptionalProviderFeatureException, AttributeError):
537+
return None
538+
539+
476540
@run_check("Checking that hook classes defining conn_type are registered in connection-types")
477541
def check_hook_classes_with_conn_type_are_registered(yaml_files: dict[str, dict]) -> tuple[int, int]:
478542
"""Find Hook subclasses that define conn_type but are not listed in connection-types."""
@@ -1046,6 +1110,7 @@ def check_providers_have_all_documentation_files(yaml_files: dict[str, dict]):
10461110

10471111
check_completeness_of_list_of_transfers(all_parsed_yaml_files)
10481112
check_hook_class_name_entries_in_connection_types(all_parsed_yaml_files)
1113+
check_conn_fields_match_form_widgets(all_parsed_yaml_files)
10491114
check_hook_classes_with_conn_type_are_registered(all_parsed_yaml_files)
10501115
check_executor_classes(all_parsed_yaml_files)
10511116
check_queue_classes(all_parsed_yaml_files)
Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
from __future__ import annotations
18+
19+
import pytest
20+
from check_provider_conn_fields import (
21+
build_mismatch_error,
22+
check_conn_fields_for_entry,
23+
)
24+
25+
YAML_PATH = "providers/my_provider/provider.yaml"
26+
HOOK_CLASS = "my_provider.hooks.my_hook.MyHook"
27+
CONN_TYPE = "my_conn_type"
28+
29+
30+
def _entry(conn_fields: list[str] | None) -> dict:
31+
entry: dict = {"hook-class-name": HOOK_CLASS, "connection-type": CONN_TYPE}
32+
if conn_fields is not None:
33+
entry["conn-fields"] = {k: {} for k in conn_fields}
34+
return entry
35+
36+
37+
def _get_keys(*keys: str):
38+
"""Return a get_widget_keys callable that always returns the given keys."""
39+
return lambda _hook_class_name: set(keys)
40+
41+
42+
def _skip(_hook_class_name: str) -> None:
43+
"""get_widget_keys callable that signals 'skip silently'."""
44+
return None
45+
46+
47+
def _raise(_hook_class_name: str) -> None:
48+
raise RuntimeError("boom")
49+
50+
51+
class TestBuildMismatchError:
52+
@pytest.mark.parametrize(
53+
"yaml_keys, hook_keys",
54+
[
55+
pytest.param({"a", "b"}, {"a", "b"}, id="matching-keys"),
56+
pytest.param(set(), set(), id="empty-sets"),
57+
# Hook may have more keys than conn-fields — that is intentional (subset allowed).
58+
pytest.param({"a"}, {"a", "extra_hook"}, id="hook-has-extra-keys-no-error"),
59+
],
60+
)
61+
def test_no_mismatch_returns_none(self, yaml_keys, hook_keys):
62+
assert build_mismatch_error(yaml_keys, hook_keys, CONN_TYPE, YAML_PATH, HOOK_CLASS) is None
63+
64+
@pytest.mark.parametrize(
65+
"yaml_keys, hook_keys, expected_in_error, not_expected_in_error",
66+
[
67+
pytest.param(
68+
{"a", "extra_yaml"},
69+
{"a"},
70+
["extra_yaml", "NOT in get_connection_form_widgets"],
71+
["NOT in provider.yaml conn-fields"],
72+
id="extra-in-yaml",
73+
),
74+
# only_in_hook no longer triggers an error — only only_in_yaml does
75+
pytest.param(
76+
{"a", "only_yaml"},
77+
{"a", "only_hook"},
78+
[
79+
"only_yaml",
80+
"NOT in get_connection_form_widgets",
81+
],
82+
["only_hook", "NOT in provider.yaml conn-fields"],
83+
id="both-sides-only-yaml-reported",
84+
),
85+
],
86+
)
87+
def test_mismatch_error_content(self, yaml_keys, hook_keys, expected_in_error, not_expected_in_error):
88+
error = build_mismatch_error(yaml_keys, hook_keys, CONN_TYPE, YAML_PATH, HOOK_CLASS)
89+
assert error is not None
90+
for expected in expected_in_error:
91+
assert expected in error
92+
for not_expected in not_expected_in_error:
93+
assert not_expected not in error
94+
95+
96+
class TestCheckConnFieldsForEntry:
97+
@pytest.mark.parametrize(
98+
"conn_fields, get_keys",
99+
[
100+
pytest.param(["a", "b"], _get_keys("a", "b"), id="matching-keys"),
101+
pytest.param([], _get_keys(), id="empty-on-both-sides"),
102+
pytest.param(["a"], _skip, id="skip-hook-without-get-connection-form-widgets"),
103+
pytest.param(None, _skip, id="skip-missing-conn-fields-when-hook-has-no-widgets"),
104+
# Hook with widgets but no conn-fields is allowed: new UI intentionally omits custom fields.
105+
pytest.param(None, _get_keys("field_a"), id="no-conn-fields-with-hook-widgets-is-ok"),
106+
# Hook has extra keys not in conn-fields — allowed (conn-fields is a valid subset).
107+
pytest.param(["a"], _get_keys("a", "extra_hook"), id="hook-extra-keys-no-error"),
108+
],
109+
)
110+
def test_no_errors(self, conn_fields, get_keys):
111+
assert check_conn_fields_for_entry(_entry(conn_fields), YAML_PATH, get_keys) == []
112+
113+
@pytest.mark.parametrize(
114+
"conn_fields, get_keys, expected_in_error",
115+
[
116+
pytest.param(["a", "extra"], _get_keys("a"), "extra", id="extra-key-in-yaml"),
117+
pytest.param(["a"], _raise, "boom", id="unexpected-exception-message"),
118+
pytest.param(["a"], _raise, HOOK_CLASS, id="unexpected-exception-mentions-hook-class"),
119+
],
120+
)
121+
def test_one_error_containing(self, conn_fields, get_keys, expected_in_error):
122+
errors = check_conn_fields_for_entry(_entry(conn_fields), YAML_PATH, get_keys)
123+
assert len(errors) == 1
124+
assert expected_in_error in errors[0]

0 commit comments

Comments
 (0)