Skip to content

Commit 58b5a80

Browse files
dablaBKD7702
andauthored
Flag conn-fields in hook but absent from provider.yaml in static checks (#69655)
* Flag conn-fields in hook but absent from provider.yaml * Added missing conn-fields in provider.yaml of Hive, Google, Azure and Snowflake providers so those are in sync with defined fields in get_connection_form_widgets method of corresponding hook * Added cloud_environment in provider.yaml for AzureBaseHook * Fixed addition of cloud_environment in get_provider_info --------- Co-authored-by: BKD7702 <[email protected]>
1 parent b6f140e commit 58b5a80

11 files changed

Lines changed: 93 additions & 35 deletions

File tree

‎providers/apache/hive/provider.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,20 @@ connection-types:
195195
- boolean
196196
- 'null'
197197
default: false
198+
ssl:
199+
label: Ssl
200+
schema:
201+
type:
202+
- boolean
203+
- 'null'
204+
default: true
205+
zoo_keeper_namespace:
206+
label: Zoo Keeper Namespace
207+
schema:
208+
type:
209+
- string
210+
- 'null'
211+
default: 'hiveserver2'
198212
- hook-class-name: airflow.providers.apache.hive.hooks.hive.HiveServer2Hook
199213
hook-name: "Hive Server 2 Thrift"
200214
connection-type: hiveserver2

‎providers/apache/hive/src/airflow/providers/apache/hive/get_provider_info.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,11 @@ def get_provider_info():
131131
"label": "High Availability mode",
132132
"schema": {"type": ["boolean", "null"], "default": False},
133133
},
134+
"ssl": {"label": "Ssl", "schema": {"type": ["boolean", "null"], "default": True}},
135+
"zoo_keeper_namespace": {
136+
"label": "Zoo Keeper Namespace",
137+
"schema": {"type": ["string", "null"], "default": "hiveserver2"},
138+
},
134139
},
135140
},
136141
{

‎providers/google/provider.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1202,6 +1202,10 @@ connection-types:
12021202
schema:
12031203
type: ["boolean", "null"]
12041204
default: false
1205+
quota_project_id:
1206+
label: "Quota Project ID"
1207+
schema:
1208+
type: ["string", "null"]
12051209
- hook-class-name: airflow.providers.google.cloud.hooks.spanner.SpannerHook
12061210
connection-type: gcpspanner
12071211
- hook-class-name: airflow.providers.google.cloud.hooks.dataprep.GoogleDataprepHook
@@ -1281,6 +1285,10 @@ connection-types:
12811285
schema:
12821286
type: ["boolean", "null"]
12831287
default: false
1288+
quota_project_id:
1289+
label: "Quota Project ID"
1290+
schema:
1291+
type: ["string", "null"]
12841292
use_legacy_sql:
12851293
label: "Use Legacy SQL"
12861294
schema:

‎providers/google/src/airflow/providers/google/get_provider_info.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1434,6 +1434,7 @@ def get_provider_info():
14341434
"label": "Anonymous credentials (ignores all other settings)",
14351435
"schema": {"type": ["boolean", "null"], "default": False},
14361436
},
1437+
"quota_project_id": {"label": "Quota Project ID", "schema": {"type": ["string", "null"]}},
14371438
},
14381439
},
14391440
{
@@ -1512,6 +1513,7 @@ def get_provider_info():
15121513
"label": "Anonymous credentials (ignores all other settings)",
15131514
"schema": {"type": ["boolean", "null"], "default": False},
15141515
},
1516+
"quota_project_id": {"label": "Quota Project ID", "schema": {"type": ["string", "null"]}},
15151517
"use_legacy_sql": {"label": "Use Legacy SQL", "schema": {"type": ["boolean", "null"]}},
15161518
"location": {"label": "Location", "schema": {"type": ["string", "null"]}},
15171519
"priority": {

‎providers/microsoft/azure/provider.yaml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -384,7 +384,6 @@ transfers:
384384
how-to-guide: /docs/apache-airflow-providers-microsoft-azure/transfer/gcs_to_wasb.rst
385385
python-module: airflow.providers.microsoft.azure.transfers.gcs_to_wasb
386386

387-
388387
connection-types:
389388
- hook-class-name: airflow.providers.microsoft.azure.hooks.base_azure.AzureBaseHook
390389
hook-name: "Azure"
@@ -417,6 +416,10 @@ connection-types:
417416
label: Workload Identity Tenant ID
418417
schema:
419418
type: ["string", "null"]
419+
cloud_environment:
420+
label: Azure Cloud Environment
421+
schema:
422+
type: ["string", "null"]
420423
- hook-class-name: airflow.providers.microsoft.azure.hooks.ai_agents.AzureAIAgentsHook
421424
hook-name: "Azure AI Foundry Hosted Agents"
422425
connection-type: azure_ai_agents

‎providers/microsoft/azure/src/airflow/providers/microsoft/azure/get_provider_info.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -408,6 +408,10 @@ def get_provider_info():
408408
"label": "Workload Identity Tenant ID",
409409
"schema": {"type": ["string", "null"]},
410410
},
411+
"cloud_environment": {
412+
"label": "Azure Cloud Environment",
413+
"schema": {"type": ["string", "null"]},
414+
},
411415
},
412416
},
413417
{

‎providers/snowflake/provider.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -252,6 +252,12 @@ connection-types:
252252
- string
253253
- 'null'
254254
format: password
255+
workload_identity_provider:
256+
label: Workload Identity Provider
257+
schema:
258+
type:
259+
- string
260+
- 'null'
255261
insecure_mode:
256262
label: Insecure Mode
257263
schema:

‎providers/snowflake/src/airflow/providers/snowflake/get_provider_info.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,10 @@ def get_provider_info():
142142
"label": "Private key (Text)",
143143
"schema": {"type": ["string", "null"], "format": "password"},
144144
},
145+
"workload_identity_provider": {
146+
"label": "Workload Identity Provider",
147+
"schema": {"type": ["string", "null"]},
148+
},
145149
"insecure_mode": {
146150
"label": "Insecure Mode",
147151
"schema": {"type": ["boolean", "null"]},

‎scripts/ci/prek/check_provider_conn_fields.py‎

Lines changed: 26 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -81,32 +81,40 @@ def build_mismatch_error(
8181
hook_class_name: str,
8282
) -> str | None:
8383
"""
84-
Check that every key declared in ``conn-fields`` exists in
85-
``get_connection_form_widgets()``.
86-
87-
``conn-fields`` is the new React UI's view of a connection type and is
88-
intentionally a *subset* of the Flask form widgets — fields can be omitted
89-
from ``conn-fields`` on purpose. We therefore only flag keys that appear in
90-
``conn-fields`` but are absent from the hook's form (invalid / stale
91-
declarations). The reverse direction (hook fields not listed in
92-
``conn-fields``) is not an error.
93-
94-
Return an error string when stale keys are found, or ``None`` when the
95-
declared keys are all valid.
84+
Check that ``conn-fields`` and ``get_connection_form_widgets()`` are in sync.
85+
86+
Once a provider declares ``conn-fields``, those declarations must exactly
87+
match the hook's form widgets — we flag both directions:
88+
89+
- Keys present in ``conn-fields`` but absent from the hook → stale / invalid
90+
declarations that should be removed.
91+
- Keys present in the hook but absent from ``conn-fields`` → fields that will
92+
be invisible in the new React connection UI, almost certainly unintentional.
93+
94+
Return an error string when any mismatch is found, or ``None`` when the sets
95+
are identical.
9696
"""
9797
only_in_yaml = yaml_keys - hook_keys
98+
only_in_hook = hook_keys - yaml_keys
9899

99-
if not only_in_yaml:
100+
if not only_in_yaml and not only_in_hook:
100101
return None
101102

102103
lines = [
103104
f"Mismatch between `conn-fields` in {yaml_file_path} and "
104105
f"`{hook_class_name}.get_connection_form_widgets()` "
105106
f"for connection-type '{connection_type}':"
106107
]
107-
lines.append(
108-
" Fields in provider.yaml conn-fields but NOT in get_connection_form_widgets(): "
109-
+ ", ".join(sorted(only_in_yaml))
110-
)
111-
lines.append("[yellow]How to fix it[/]: Remove the stale key(s) from conn-fields in provider.yaml.")
108+
if only_in_yaml:
109+
lines.append(
110+
" Fields in provider.yaml conn-fields but NOT in get_connection_form_widgets(): "
111+
+ ", ".join(sorted(only_in_yaml))
112+
)
113+
lines.append("[yellow]How to fix it[/]: Remove the stale key(s) from conn-fields in provider.yaml.")
114+
if only_in_hook:
115+
lines.append(
116+
" Fields in get_connection_form_widgets() but NOT in provider.yaml conn-fields: "
117+
+ ", ".join(sorted(only_in_hook))
118+
)
119+
lines.append("[yellow]How to fix it[/]: Add the missing key(s) to conn-fields in provider.yaml.")
112120
return "\n".join(lines)

‎scripts/in_container/run_provider_yaml_files_check.py‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -481,14 +481,11 @@ def check_hook_class_name_entries_in_connection_types(yaml_files: dict[str, dict
481481
@run_check("Checking that conn-fields in provider.yaml match get_connection_form_widgets() of the hook class")
482482
def check_conn_fields_match_form_widgets(yaml_files: dict[str, dict]) -> tuple[int, int]:
483483
"""
484-
For every connection-type entry whose hook declares ``conn-fields``,
485-
verify that every key in ``conn-fields`` also exists in the hook's
486-
``get_connection_form_widgets()``.
487-
488-
``conn-fields`` is optional and is intentionally allowed to be a *subset*
489-
of the hook's form widgets (the new React UI may expose fewer fields than
490-
the legacy Flask form), so extra hook widgets are not flagged — only
491-
``conn-fields`` keys absent from the hook are reported as stale.
484+
For every connection-type entry that both declares ``conn-fields`` and whose
485+
hook overrides ``get_connection_form_widgets()``, verify the two sets are
486+
identical — stale YAML keys (in ``conn-fields`` but not in the hook) and
487+
missing YAML keys (in the hook but absent from ``conn-fields``) are both
488+
reported as errors.
492489
"""
493490
num_checks = 0
494491
num_errors = 0

0 commit comments

Comments
 (0)