Skip to content

Commit bac7b30

Browse files
authored
Fix SFTP operator's template fields processing (#29068)
The SFTP operator had logic in `__init__` that was running checks on passed arguments, but that means that the operator woudl fail when using TaskFlow. Fixes: #27328
1 parent 10f0f8b commit bac7b30

2 files changed

Lines changed: 50 additions & 27 deletions

File tree

‎airflow/providers/sftp/operators/sftp.py‎

Lines changed: 22 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -100,23 +100,26 @@ def __init__(
100100
self.operation = operation
101101
self.confirm = confirm
102102
self.create_intermediate_dirs = create_intermediate_dirs
103+
self.local_filepath = local_filepath
104+
self.remote_filepath = remote_filepath
103105

104-
self.local_filepath_was_str = False
105-
if isinstance(local_filepath, str):
106-
self.local_filepath = [local_filepath]
107-
self.local_filepath_was_str = True
106+
def execute(self, context: Any) -> str | list[str] | None:
107+
local_filepath_was_str = False
108+
if isinstance(self.local_filepath, str):
109+
local_filepath_array = [self.local_filepath]
110+
local_filepath_was_str = True
108111
else:
109-
self.local_filepath = local_filepath
112+
local_filepath_array = self.local_filepath
110113

111-
if isinstance(remote_filepath, str):
112-
self.remote_filepath = [remote_filepath]
114+
if isinstance(self.remote_filepath, str):
115+
remote_filepath_array = [self.remote_filepath]
113116
else:
114-
self.remote_filepath = remote_filepath
117+
remote_filepath_array = self.remote_filepath
115118

116-
if len(self.local_filepath) != len(self.remote_filepath):
119+
if len(local_filepath_array) != len(remote_filepath_array):
117120
raise ValueError(
118-
f"{len(self.local_filepath)} paths in local_filepath "
119-
f"!= {len(self.remote_filepath)} paths in remote_filepath"
121+
f"{len(local_filepath_array)} paths in local_filepath "
122+
f"!= {len(remote_filepath_array)} paths in remote_filepath"
120123
)
121124

122125
if not (self.operation.lower() == SFTPOperation.GET or self.operation.lower() == SFTPOperation.PUT):
@@ -145,7 +148,6 @@ def __init__(
145148
)
146149
self.sftp_hook = SFTPHook(ssh_hook=self.ssh_hook)
147150

148-
def execute(self, context: Any) -> str | list[str] | None:
149151
file_msg = None
150152
try:
151153
if self.ssh_conn_id:
@@ -168,23 +170,23 @@ def execute(self, context: Any) -> str | list[str] | None:
168170
)
169171
self.sftp_hook.remote_host = self.remote_host
170172

171-
for local_filepath, remote_filepath in zip(self.local_filepath, self.remote_filepath):
173+
for _local_filepath, _remote_filepath in zip(local_filepath_array, remote_filepath_array):
172174
if self.operation.lower() == SFTPOperation.GET:
173-
local_folder = os.path.dirname(local_filepath)
175+
local_folder = os.path.dirname(_local_filepath)
174176
if self.create_intermediate_dirs:
175177
Path(local_folder).mkdir(parents=True, exist_ok=True)
176-
file_msg = f"from {remote_filepath} to {local_filepath}"
178+
file_msg = f"from {_remote_filepath} to {_local_filepath}"
177179
self.log.info("Starting to transfer %s", file_msg)
178-
self.sftp_hook.retrieve_file(remote_filepath, local_filepath)
180+
self.sftp_hook.retrieve_file(_remote_filepath, _local_filepath)
179181
else:
180-
remote_folder = os.path.dirname(remote_filepath)
182+
remote_folder = os.path.dirname(_remote_filepath)
181183
if self.create_intermediate_dirs:
182184
self.sftp_hook.create_directory(remote_folder)
183-
file_msg = f"from {local_filepath} to {remote_filepath}"
185+
file_msg = f"from {_local_filepath} to {_remote_filepath}"
184186
self.log.info("Starting to transfer file %s", file_msg)
185-
self.sftp_hook.store_file(remote_filepath, local_filepath, confirm=self.confirm)
187+
self.sftp_hook.store_file(_remote_filepath, _local_filepath, confirm=self.confirm)
186188

187189
except Exception as e:
188190
raise AirflowException(f"Error while transferring {file_msg}, error: {str(e)}")
189191

190-
return self.local_filepath[0] if self.local_filepath_was_str else self.local_filepath
192+
return local_filepath_array[0] if local_filepath_was_str else local_filepath_array

‎tests/providers/sftp/operators/test_sftp.py‎

Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -396,16 +396,22 @@ def test_unequal_local_remote_file_paths(self):
396396
task_id="test_sftp_unequal_paths",
397397
local_filepath="/tmp/test",
398398
remote_filepath=["/tmp/test1", "/tmp/test2"],
399-
)
399+
).execute(None)
400400

401-
def test_str_filepaths_converted_to_lists(self):
401+
@mock.patch("airflow.providers.sftp.operators.sftp.SFTPHook.retrieve_file")
402+
def test_str_filepaths_get(self, mock_get):
402403
local_filepath = "/tmp/test"
403404
remote_filepath = "/tmp/remotetest"
404-
sftp_op = SFTPOperator(
405-
task_id="test_str_to_list", local_filepath=local_filepath, remote_filepath=remote_filepath
406-
)
407-
assert sftp_op.local_filepath == [local_filepath]
408-
assert sftp_op.remote_filepath == [remote_filepath]
405+
SFTPOperator(
406+
task_id="test_str_to_list",
407+
sftp_hook=self.sftp_hook,
408+
local_filepath=local_filepath,
409+
remote_filepath=remote_filepath,
410+
operation=SFTPOperation.GET,
411+
).execute(None)
412+
assert mock_get.call_count == 1
413+
args, _ = mock_get.call_args_list[0]
414+
assert args == (remote_filepath, local_filepath)
409415

410416
@mock.patch("airflow.providers.sftp.operators.sftp.SFTPHook.retrieve_file")
411417
def test_multiple_paths_get(self, mock_get):
@@ -425,6 +431,21 @@ def test_multiple_paths_get(self, mock_get):
425431
assert args0 == (remote_filepath[0], local_filepath[0])
426432
assert args1 == (remote_filepath[1], local_filepath[1])
427433

434+
@mock.patch("airflow.providers.sftp.operators.sftp.SFTPHook.store_file")
435+
def test_str_filepaths_put(self, mock_get):
436+
local_filepath = "/tmp/test"
437+
remote_filepath = "/tmp/remotetest"
438+
SFTPOperator(
439+
task_id="test_str_to_list",
440+
sftp_hook=self.sftp_hook,
441+
local_filepath=local_filepath,
442+
remote_filepath=remote_filepath,
443+
operation=SFTPOperation.PUT,
444+
).execute(None)
445+
assert mock_get.call_count == 1
446+
args, _ = mock_get.call_args_list[0]
447+
assert args == (remote_filepath, local_filepath)
448+
428449
@mock.patch("airflow.providers.sftp.operators.sftp.SFTPHook.store_file")
429450
def test_multiple_paths_put(self, mock_put):
430451
local_filepath = ["/tmp/ltest1", "/tmp/ltest2"]

0 commit comments

Comments
 (0)