@@ -52,8 +52,8 @@ class SFTPOperator(BaseOperator):
5252 :param remote_host: remote host to connect (templated)
5353 Nullable. If provided, it will replace the `remote_host` which was
5454 defined in `sftp_hook`/`ssh_hook` or predefined in the connection of `ssh_conn_id`.
55- :param local_filepath: local file path to get or put. (templated)
56- :param remote_filepath: remote file path to get or put. (templated)
55+ :param local_filepath: local file path or list of local file paths to get or put. (templated)
56+ :param remote_filepath: remote file path or list of remote file paths to get or put. (templated)
5757 :param operation: specify operation 'get' or 'put', defaults to put
5858 :param confirm: specify if the SFTP operation should be confirmed, defaults to True
5959 :param create_intermediate_dirs: create missing intermediate directories when
@@ -85,8 +85,8 @@ def __init__(
8585 sftp_hook : SFTPHook | None = None ,
8686 ssh_conn_id : str | None = None ,
8787 remote_host : str | None = None ,
88- local_filepath : str ,
89- remote_filepath : str ,
88+ local_filepath : str | list [ str ] ,
89+ remote_filepath : str | list [ str ] ,
9090 operation : str = SFTPOperation .PUT ,
9191 confirm : bool = True ,
9292 create_intermediate_dirs : bool = False ,
@@ -97,12 +97,28 @@ def __init__(
9797 self .sftp_hook = sftp_hook
9898 self .ssh_conn_id = ssh_conn_id
9999 self .remote_host = remote_host
100- self .local_filepath = local_filepath
101- self .remote_filepath = remote_filepath
102100 self .operation = operation
103101 self .confirm = confirm
104102 self .create_intermediate_dirs = create_intermediate_dirs
105103
104+ self .local_filepath_was_str = False
105+ if isinstance (local_filepath , str ):
106+ self .local_filepath = [local_filepath ]
107+ self .local_filepath_was_str = True
108+ else :
109+ self .local_filepath = local_filepath
110+
111+ if isinstance (remote_filepath , str ):
112+ self .remote_filepath = [remote_filepath ]
113+ else :
114+ self .remote_filepath = remote_filepath
115+
116+ if len (self .local_filepath ) != len (self .remote_filepath ):
117+ raise ValueError (
118+ f'{ len (self .local_filepath )} paths in local_filepath '
119+ f'!= { len (self .remote_filepath )} paths in remote_filepath'
120+ )
121+
106122 if not (self .operation .lower () == SFTPOperation .GET or self .operation .lower () == SFTPOperation .PUT ):
107123 raise TypeError (
108124 f"Unsupported operation value { self .operation } , "
@@ -129,7 +145,7 @@ def __init__(
129145 )
130146 self .sftp_hook = SFTPHook (ssh_hook = self .ssh_hook )
131147
132- def execute (self , context : Any ) -> str | None :
148+ def execute (self , context : Any ) -> str | list [ str ] | None :
133149 file_msg = None
134150 try :
135151 if self .ssh_conn_id :
@@ -152,22 +168,23 @@ def execute(self, context: Any) -> str | None:
152168 )
153169 self .sftp_hook .remote_host = self .remote_host
154170
155- if self .operation .lower () == SFTPOperation .GET :
156- local_folder = os .path .dirname (self .local_filepath )
157- if self .create_intermediate_dirs :
158- Path (local_folder ).mkdir (parents = True , exist_ok = True )
159- file_msg = f"from { self .remote_filepath } to { self .local_filepath } "
160- self .log .info ("Starting to transfer %s" , file_msg )
161- self .sftp_hook .retrieve_file (self .remote_filepath , self .local_filepath )
162- else :
163- remote_folder = os .path .dirname (self .remote_filepath )
164- if self .create_intermediate_dirs :
165- self .sftp_hook .create_directory (remote_folder )
166- file_msg = f"from { self .local_filepath } to { self .remote_filepath } "
167- self .log .info ("Starting to transfer file %s" , file_msg )
168- self .sftp_hook .store_file (self .remote_filepath , self .local_filepath , confirm = self .confirm )
171+ for local_filepath , remote_filepath in zip (self .local_filepath , self .remote_filepath ):
172+ if self .operation .lower () == SFTPOperation .GET :
173+ local_folder = os .path .dirname (local_filepath )
174+ if self .create_intermediate_dirs :
175+ Path (local_folder ).mkdir (parents = True , exist_ok = True )
176+ file_msg = f"from { remote_filepath } to { local_filepath } "
177+ self .log .info ("Starting to transfer %s" , file_msg )
178+ self .sftp_hook .retrieve_file (remote_filepath , local_filepath )
179+ else :
180+ remote_folder = os .path .dirname (remote_filepath )
181+ if self .create_intermediate_dirs :
182+ self .sftp_hook .create_directory (remote_folder )
183+ file_msg = f"from { local_filepath } to { remote_filepath } "
184+ self .log .info ("Starting to transfer file %s" , file_msg )
185+ self .sftp_hook .store_file (remote_filepath , local_filepath , confirm = self .confirm )
169186
170187 except Exception as e :
171188 raise AirflowException (f"Error while transferring { file_msg } , error: { str (e )} " )
172189
173- return self .local_filepath
190+ return self .local_filepath [ 0 ] if self . local_filepath_was_str else self . local_filepath
0 commit comments