Skip to content

Commit df9a3e7

Browse files
authored
Move AzureFileShareToGCSOperator directory_name alias out of __init__ (#70740)
1 parent 3680493 commit df9a3e7

3 files changed

Lines changed: 111 additions & 5 deletions

File tree

‎providers/google/src/airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -69,8 +69,8 @@ class AzureFileShareToGCSOperator(BaseOperator):
6969
:param return_gcs_uris: If True, return a list of GCS URIs. If False (default), return the legacy
7070
list of Azure FileShare filenames and emit a deprecation warning.
7171
72-
Note that ``share_name``, ``directory_path``, ``prefix``, and ``dest_gcs`` are
73-
templated, so you can use variables in them if you wish.
72+
Note that ``share_name``, ``directory_name``, ``directory_path``, ``prefix``, and
73+
``dest_gcs`` are templated, so you can use variables in them if you wish.
7474
"""
7575

7676
template_fields: Sequence[str] = (
@@ -102,8 +102,11 @@ def __init__(
102102
self.share_name = share_name
103103
self.directory_path = directory_path
104104
self.directory_name = directory_name
105-
if self.directory_path is None and self.directory_name is not None:
106-
self.directory_path = self.directory_name
105+
# The deprecated directory_name->directory_path alias is decided here on the un-rendered
106+
# values (native rendering can turn a supplied directory_path into None) and applied in
107+
# execute(), which runs for mapped tasks too — render_template_fields overrides do not.
108+
self._use_directory_name = directory_path is None and directory_name is not None
109+
if self._use_directory_name:
107110
warnings.warn(
108111
"Use 'directory_path' instead of 'directory_name'. Planned removal date: October 5, 2026.",
109112
AirflowProviderDeprecationWarning,
@@ -139,6 +142,8 @@ def _check_inputs(self) -> None:
139142
)
140143

141144
def execute(self, context: Context) -> list[str]:
145+
if self._use_directory_name:
146+
self.directory_path = self.directory_name
142147
self._check_inputs()
143148
azure_fileshare_hook = AzureFileShareHook(
144149
share_name=self.share_name,

‎providers/google/tests/unit/google/cloud/transfers/test_azure_fileshare_to_gcs.py‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,18 @@
1616
# under the License.
1717
from __future__ import annotations
1818

19+
import datetime
1920
from unittest import mock
2021

2122
import pytest
2223

24+
from airflow import DAG
25+
from airflow.exceptions import AirflowProviderDeprecationWarning
2326
from airflow.providers.google.cloud.transfers.azure_fileshare_to_gcs import AzureFileShareToGCSOperator
2427

2528
pytestmark = pytest.mark.filterwarnings("ignore::FutureWarning")
2629

30+
DEFAULT_DATE = datetime.datetime(2024, 1, 1)
2731
TASK_ID = "test-azure-fileshare-to-gcs"
2832
AZURE_FILESHARE_SHARE = "test-share"
2933
AZURE_FILESHARE_DIRECTORY_PATH = "/path/to/dir"
@@ -56,6 +60,104 @@ def test_init(self):
5660
assert operator.dest_gcs == GCS_PATH_PREFIX
5761
assert operator.google_impersonation_chain == IMPERSONATION_CHAIN
5862

63+
@mock.patch("airflow.providers.google.cloud.transfers.azure_fileshare_to_gcs.AzureFileShareHook")
64+
@mock.patch("airflow.providers.google.cloud.transfers.azure_fileshare_to_gcs.GCSHook")
65+
def test_directory_name_alias_uses_rendered_value(self, gcs_mock_hook, azure_fileshare_mock_hook):
66+
"""A templated directory_name is aliased to directory_path using its rendered value, not the Jinja expression."""
67+
dag = DAG("test_azure_fileshare_alias", schedule=None, start_date=DEFAULT_DATE)
68+
with pytest.warns(AirflowProviderDeprecationWarning, match="Use 'directory_path' instead"):
69+
operator = AzureFileShareToGCSOperator(
70+
task_id=TASK_ID,
71+
share_name=AZURE_FILESHARE_SHARE,
72+
directory_name="{{ params.legacy_dir }}",
73+
params={"legacy_dir": "rendered/dir"},
74+
azure_fileshare_conn_id=AZURE_FILESHARE_CONN_ID,
75+
gcp_conn_id=GCS_CONN_ID,
76+
dest_gcs=GCS_PATH_PREFIX,
77+
return_gcs_uris=True,
78+
dag=dag,
79+
)
80+
assert operator.directory_path is None
81+
82+
operator.render_template_fields({"params": {"legacy_dir": "rendered/dir"}})
83+
assert operator.directory_path is None
84+
85+
azure_fileshare_mock_hook.return_value.list_files.return_value = MOCK_FILES
86+
operator.execute(None)
87+
azure_fileshare_mock_hook.assert_any_call(
88+
share_name=AZURE_FILESHARE_SHARE,
89+
azure_fileshare_conn_id=AZURE_FILESHARE_CONN_ID,
90+
directory_path="rendered/dir",
91+
)
92+
93+
@mock.patch("airflow.providers.google.cloud.transfers.azure_fileshare_to_gcs.AzureFileShareHook")
94+
@mock.patch("airflow.providers.google.cloud.transfers.azure_fileshare_to_gcs.GCSHook")
95+
def test_native_directory_path_rendering_to_none_is_not_aliased(
96+
self, gcs_mock_hook, azure_fileshare_mock_hook
97+
):
98+
"""
99+
Behaviour-preservation guard: the alias decision must come from the un-rendered values.
100+
101+
With render_template_as_native_obj an explicitly supplied directory_path can render to None,
102+
and re-deciding the alias after rendering would wrongly fall back to the deprecated
103+
directory_name. Deciding in __init__ (pre-render) keeps this case an explicit error.
104+
"""
105+
dag = DAG(
106+
"test_azure_fileshare_native",
107+
schedule=None,
108+
start_date=DEFAULT_DATE,
109+
render_template_as_native_obj=True,
110+
)
111+
operator = AzureFileShareToGCSOperator(
112+
task_id=TASK_ID,
113+
share_name=AZURE_FILESHARE_SHARE,
114+
directory_path="{{ params.p }}",
115+
directory_name="legacy",
116+
params={"p": None},
117+
azure_fileshare_conn_id=AZURE_FILESHARE_CONN_ID,
118+
gcp_conn_id=GCS_CONN_ID,
119+
dest_gcs=GCS_PATH_PREFIX,
120+
return_gcs_uris=True,
121+
dag=dag,
122+
)
123+
124+
operator.render_template_fields({"params": {"p": None}})
125+
assert operator.directory_path is None
126+
127+
# The deprecated alias must not silently substitute directory_name; a genuinely unset
128+
# directory surfaces as the operator's own error instead of listing the wrong directory.
129+
azure_fileshare_mock_hook.return_value.list_files.return_value = MOCK_FILES
130+
with pytest.raises(RuntimeError, match="directory_name must be set"):
131+
operator.execute(None)
132+
133+
@mock.patch("airflow.providers.google.cloud.transfers.azure_fileshare_to_gcs.AzureFileShareHook")
134+
@mock.patch("airflow.providers.google.cloud.transfers.azure_fileshare_to_gcs.GCSHook")
135+
def test_mapped_task_aliases_directory_name(self, gcs_mock_hook, azure_fileshare_mock_hook):
136+
"""
137+
Mapped tasks reach execute() through MappedOperator.unmap(), never through
138+
render_template_fields overrides, so the alias must not depend on one.
139+
"""
140+
mapped = AzureFileShareToGCSOperator.partial(
141+
task_id=TASK_ID,
142+
share_name=AZURE_FILESHARE_SHARE,
143+
directory_name=AZURE_FILESHARE_DIRECTORY_PATH,
144+
azure_fileshare_conn_id=AZURE_FILESHARE_CONN_ID,
145+
gcp_conn_id=GCS_CONN_ID,
146+
dest_gcs=GCS_PATH_PREFIX,
147+
return_gcs_uris=True,
148+
).expand(prefix=["sub/a/", "sub/b/"])
149+
150+
with pytest.warns(AirflowProviderDeprecationWarning, match="Use 'directory_path' instead"):
151+
operator = mapped.unmap({"prefix": "sub/a/"})
152+
153+
azure_fileshare_mock_hook.return_value.list_files.return_value = MOCK_FILES
154+
operator.execute(None)
155+
azure_fileshare_mock_hook.assert_any_call(
156+
share_name=AZURE_FILESHARE_SHARE,
157+
azure_fileshare_conn_id=AZURE_FILESHARE_CONN_ID,
158+
directory_path=AZURE_FILESHARE_DIRECTORY_PATH,
159+
)
160+
59161
@pytest.mark.parametrize("return_gcs_uris", [True, False])
60162
@mock.patch("airflow.providers.google.cloud.transfers.azure_fileshare_to_gcs.AzureFileShareHook")
61163
@mock.patch("airflow.providers.google.cloud.transfers.azure_fileshare_to_gcs.GCSHook")

‎scripts/ci/prek/validate_operators_init_exemptions.txt‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,4 @@ providers/google/src/airflow/providers/google/cloud/operators/functions.py::Clou
1717
providers/google/src/airflow/providers/google/cloud/operators/gcs.py::GCSFileTransformOperator
1818
providers/google/src/airflow/providers/google/cloud/sensors/bigquery_dts.py::BigQueryDataTransferServiceTransferRunSensor
1919
providers/google/src/airflow/providers/google/cloud/sensors/cloud_composer.py::CloudComposerExternalTaskSensor
20-
providers/google/src/airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py::AzureFileShareToGCSOperator
2120
providers/microsoft/psrp/src/airflow/providers/microsoft/psrp/operators/psrp.py::PsrpOperator

0 commit comments

Comments
 (0)