Skip to content

Commit 5f073e3

Browse files
SFTPOperator - add support for list of file paths (#26666)
1 parent 2f86e6d commit 5f073e3

2 files changed

Lines changed: 110 additions & 22 deletions

File tree

‎airflow/providers/sftp/operators/sftp.py‎

Lines changed: 39 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -52,8 +52,8 @@ class SFTPOperator(BaseOperator):
5252
:param remote_host: remote host to connect (templated)
5353
Nullable. If provided, it will replace the `remote_host` which was
5454
defined in `sftp_hook`/`ssh_hook` or predefined in the connection of `ssh_conn_id`.
55-
:param local_filepath: local file path to get or put. (templated)
56-
:param remote_filepath: remote file path to get or put. (templated)
55+
:param local_filepath: local file path or list of local file paths to get or put. (templated)
56+
:param remote_filepath: remote file path or list of remote file paths to get or put. (templated)
5757
:param operation: specify operation 'get' or 'put', defaults to put
5858
:param confirm: specify if the SFTP operation should be confirmed, defaults to True
5959
:param create_intermediate_dirs: create missing intermediate directories when
@@ -85,8 +85,8 @@ def __init__(
8585
sftp_hook: SFTPHook | None = None,
8686
ssh_conn_id: str | None = None,
8787
remote_host: str | None = None,
88-
local_filepath: str,
89-
remote_filepath: str,
88+
local_filepath: str | list[str],
89+
remote_filepath: str | list[str],
9090
operation: str = SFTPOperation.PUT,
9191
confirm: bool = True,
9292
create_intermediate_dirs: bool = False,
@@ -97,12 +97,28 @@ def __init__(
9797
self.sftp_hook = sftp_hook
9898
self.ssh_conn_id = ssh_conn_id
9999
self.remote_host = remote_host
100-
self.local_filepath = local_filepath
101-
self.remote_filepath = remote_filepath
102100
self.operation = operation
103101
self.confirm = confirm
104102
self.create_intermediate_dirs = create_intermediate_dirs
105103

104+
self.local_filepath_was_str = False
105+
if isinstance(local_filepath, str):
106+
self.local_filepath = [local_filepath]
107+
self.local_filepath_was_str = True
108+
else:
109+
self.local_filepath = local_filepath
110+
111+
if isinstance(remote_filepath, str):
112+
self.remote_filepath = [remote_filepath]
113+
else:
114+
self.remote_filepath = remote_filepath
115+
116+
if len(self.local_filepath) != len(self.remote_filepath):
117+
raise ValueError(
118+
f'{len(self.local_filepath)} paths in local_filepath '
119+
f'!= {len(self.remote_filepath)} paths in remote_filepath'
120+
)
121+
106122
if not (self.operation.lower() == SFTPOperation.GET or self.operation.lower() == SFTPOperation.PUT):
107123
raise TypeError(
108124
f"Unsupported operation value {self.operation}, "
@@ -129,7 +145,7 @@ def __init__(
129145
)
130146
self.sftp_hook = SFTPHook(ssh_hook=self.ssh_hook)
131147

132-
def execute(self, context: Any) -> str | None:
148+
def execute(self, context: Any) -> str | list[str] | None:
133149
file_msg = None
134150
try:
135151
if self.ssh_conn_id:
@@ -152,22 +168,23 @@ def execute(self, context: Any) -> str | None:
152168
)
153169
self.sftp_hook.remote_host = self.remote_host
154170

155-
if self.operation.lower() == SFTPOperation.GET:
156-
local_folder = os.path.dirname(self.local_filepath)
157-
if self.create_intermediate_dirs:
158-
Path(local_folder).mkdir(parents=True, exist_ok=True)
159-
file_msg = f"from {self.remote_filepath} to {self.local_filepath}"
160-
self.log.info("Starting to transfer %s", file_msg)
161-
self.sftp_hook.retrieve_file(self.remote_filepath, self.local_filepath)
162-
else:
163-
remote_folder = os.path.dirname(self.remote_filepath)
164-
if self.create_intermediate_dirs:
165-
self.sftp_hook.create_directory(remote_folder)
166-
file_msg = f"from {self.local_filepath} to {self.remote_filepath}"
167-
self.log.info("Starting to transfer file %s", file_msg)
168-
self.sftp_hook.store_file(self.remote_filepath, self.local_filepath, confirm=self.confirm)
171+
for local_filepath, remote_filepath in zip(self.local_filepath, self.remote_filepath):
172+
if self.operation.lower() == SFTPOperation.GET:
173+
local_folder = os.path.dirname(local_filepath)
174+
if self.create_intermediate_dirs:
175+
Path(local_folder).mkdir(parents=True, exist_ok=True)
176+
file_msg = f"from {remote_filepath} to {local_filepath}"
177+
self.log.info("Starting to transfer %s", file_msg)
178+
self.sftp_hook.retrieve_file(remote_filepath, local_filepath)
179+
else:
180+
remote_folder = os.path.dirname(remote_filepath)
181+
if self.create_intermediate_dirs:
182+
self.sftp_hook.create_directory(remote_folder)
183+
file_msg = f"from {local_filepath} to {remote_filepath}"
184+
self.log.info("Starting to transfer file %s", file_msg)
185+
self.sftp_hook.store_file(remote_filepath, local_filepath, confirm=self.confirm)
169186

170187
except Exception as e:
171188
raise AirflowException(f"Error while transferring {file_msg}, error: {str(e)}")
172189

173-
return self.local_filepath
190+
return self.local_filepath[0] if self.local_filepath_was_str else self.local_filepath

‎tests/providers/sftp/operators/test_sftp.py‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,9 @@ def setup_method(self):
4343
hook = SSHHook(ssh_conn_id='ssh_default')
4444
hook.no_host_key_check = True
4545
self.hook = hook
46+
sftp_hook = SFTPHook(ssh_conn_id='ssh_default')
47+
sftp_hook.no_host_key_check = True
48+
self.sftp_hook = sftp_hook
4649
self.test_dir = "/tmp"
4750
self.test_local_dir = "/tmp/tmp2"
4851
self.test_remote_dir = "/tmp/tmp1"
@@ -386,3 +389,71 @@ def test_arg_checking(self):
386389
except Exception:
387390
pass
388391
assert task_6.sftp_hook.remote_host == 'remotehost'
392+
393+
def test_unequal_local_remote_file_paths(self):
394+
with pytest.raises(ValueError):
395+
SFTPOperator(
396+
task_id='test_sftp_unequal_paths',
397+
local_filepath='/tmp/test',
398+
remote_filepath=['/tmp/test1', '/tmp/test2'],
399+
)
400+
401+
def test_str_filepaths_converted_to_lists(self):
402+
local_filepath = '/tmp/test'
403+
remote_filepath = '/tmp/remotetest'
404+
sftp_op = SFTPOperator(
405+
task_id='test_str_to_list', local_filepath=local_filepath, remote_filepath=remote_filepath
406+
)
407+
assert sftp_op.local_filepath == [local_filepath]
408+
assert sftp_op.remote_filepath == [remote_filepath]
409+
410+
@mock.patch('airflow.providers.sftp.operators.sftp.SFTPHook.retrieve_file')
411+
def test_multiple_paths_get(self, mock_get):
412+
local_filepath = ['/tmp/ltest1', '/tmp/ltest2']
413+
remote_filepath = ['/tmp/rtest1', '/tmp/rtest2']
414+
sftp_op = SFTPOperator(
415+
task_id='test_multiple_paths_get',
416+
sftp_hook=self.sftp_hook,
417+
local_filepath=local_filepath,
418+
remote_filepath=remote_filepath,
419+
operation=SFTPOperation.GET,
420+
)
421+
sftp_op.execute(None)
422+
assert mock_get.call_count == 2
423+
args0, _ = mock_get.call_args_list[0]
424+
args1, _ = mock_get.call_args_list[1]
425+
assert args0 == (remote_filepath[0], local_filepath[0])
426+
assert args1 == (remote_filepath[1], local_filepath[1])
427+
428+
@mock.patch('airflow.providers.sftp.operators.sftp.SFTPHook.store_file')
429+
def test_multiple_paths_put(self, mock_put):
430+
local_filepath = ['/tmp/ltest1', '/tmp/ltest2']
431+
remote_filepath = ['/tmp/rtest1', '/tmp/rtest2']
432+
sftp_op = SFTPOperator(
433+
task_id='test_multiple_paths_get',
434+
sftp_hook=self.sftp_hook,
435+
local_filepath=local_filepath,
436+
remote_filepath=remote_filepath,
437+
operation=SFTPOperation.PUT,
438+
)
439+
sftp_op.execute(None)
440+
assert mock_put.call_count == 2
441+
args0, _ = mock_put.call_args_list[0]
442+
args1, _ = mock_put.call_args_list[1]
443+
assert args0 == (remote_filepath[0], local_filepath[0])
444+
assert args1 == (remote_filepath[1], local_filepath[1])
445+
446+
@mock.patch('airflow.providers.sftp.operators.sftp.SFTPHook.retrieve_file')
447+
def test_return_str_when_local_filepath_was_str(self, mock_get):
448+
local_filepath = '/tmp/ltest1'
449+
remote_filepath = '/tmp/rtest1'
450+
sftp_op = SFTPOperator(
451+
task_id='test_returns_str',
452+
sftp_hook=self.sftp_hook,
453+
local_filepath=local_filepath,
454+
remote_filepath=remote_filepath,
455+
operation=SFTPOperation.GET,
456+
)
457+
return_value = sftp_op.execute(None)
458+
assert isinstance(return_value, str)
459+
assert return_value == local_filepath

0 commit comments

Comments
 (0)