11import inspect
2+ import logging
23
34from .asyn import AsyncFileSystem
45from .callbacks import _DEFAULT_CALLBACK
56from .core import filesystem , get_filesystem_class , split_protocol
67
78_generic_fs = {}
9+ logger = logging .getLogger ("fsspec.generic" )
810
911
1012def set_generic_fs (protocol , ** storage_options ):
@@ -31,6 +33,97 @@ def _resolve_fs(url, method=None, protocol=None, storage_options=None):
3133 raise ValueError (f"Unknown FS resolution method: { method } " )
3234
3335
36+ def rsync (
37+ source ,
38+ destination ,
39+ delete_missing = False ,
40+ source_field = "size" ,
41+ dest_field = "size" ,
42+ update_cond = "different" ,
43+ inst_kwargs = None ,
44+ fs = None ,
45+ ** kwargs ,
46+ ):
47+ """Sync files between two directory trees
48+
49+ (experimental)
50+
51+ Parameters
52+ ----------
53+ source: str
54+ Root of the directory tree to take files from.
55+ destination: str
56+ Root path to copy into. The contents of this location should be
57+ identical to the contents of ``source`` when done.
58+ delete_missing: bool
59+ If there are paths in the destination that don't exist in the
60+ source and this is True, delete them. Otherwise, leave them alone.
61+ source_field: str
62+ If ``update_field`` is "different", this is the key in the info
63+ of source files to consider for difference.
64+ dest_field: str
65+ If ``update_field`` is "different", this is the key in the info
66+ of destination files to consider for difference.
67+ update_cond: "different"|"always"|"never"
68+ If "always", every file is copied, regardless of whether it exists in
69+ the destination. If "never", files that exist in the destination are
70+ not copied again. If "different" (default), only copy if the info
71+ fields given by ``source_field`` and ``dest_field`` (usually "size")
72+ are different. Other comparisons may be added in the future.
73+ inst_kwargs: dict|None
74+ If ``fs`` is None, use this set of keyword arguments to make a
75+ GenericFileSystem instance
76+ fs: GenericFileSystem|None
77+ Instance to use if explicitly given. The instance defines how to
78+ to make downstream file system instances from paths.
79+ """
80+ fs = fs or GenericFileSystem (** (inst_kwargs or {}))
81+ source = fs ._strip_protocol (source )
82+ destination = fs ._strip_protocol (destination )
83+ allfiles = fs .find (source , withdirs = True , detail = True )
84+ if not fs .isdir (source ):
85+ raise ValueError ("Can only rsync on a directory" )
86+ otherfiles = fs .find (destination , withdirs = True , detail = True )
87+ dirs = [
88+ a
89+ for a , v in allfiles .items ()
90+ if v ["type" ] == "directory" and a .replace (source , destination ) not in otherfiles
91+ ]
92+ logger .debug (f"{ len (dirs )} directories to create" )
93+ for dirn in dirs :
94+ # no async
95+ fs .mkdirs (dirn .replace (source , destination ), exist_ok = True )
96+ allfiles = {a : v for a , v in allfiles .items () if v ["type" ] == "file" }
97+ logger .debug (f"{ len (allfiles )} files to consider for copy" )
98+ to_delete = [
99+ o
100+ for o , v in otherfiles .items ()
101+ if o .replace (destination , source ) not in allfiles and v ["type" ] == "file"
102+ ]
103+ for k , v in allfiles .copy ().items ():
104+ otherfile = k .replace (source , destination )
105+ if otherfile in otherfiles :
106+ if update_cond == "always" :
107+ allfiles [k ] = otherfile
108+ elif update_cond == "different" :
109+ if v [source_field ] != otherfiles [otherfile ][dest_field ]:
110+ # details mismatch, make copy
111+ allfiles [k ] = otherfile
112+ else :
113+ # details match, don't copy
114+ allfiles .pop (k )
115+ else :
116+ # file not in target yet
117+ allfiles [k ] = otherfile
118+ if allfiles :
119+ source_files , target_files = zip (* allfiles .items ())
120+ logger .debug (f"{ len (source_files )} files to copy" )
121+ fs .cp (source_files , target_files , ** kwargs )
122+ if delete_missing :
123+ logger .debug (f"{ len (to_delete )} files to delete" )
124+ fs .rm (to_delete )
125+
126+
34127class GenericFileSystem (AsyncFileSystem ):
35128 """Wrapper over all other FS types
36129
@@ -46,28 +139,49 @@ class GenericFileSystem(AsyncFileSystem):
46139
47140 protocol = "generic" # there is no real reason to ever use a protocol with this FS
48141
49- def __init__ (self , default_method = None , ** kwargs ):
142+ def __init__ (self , default_method = "default" , ** kwargs ):
50143 """
51144
52145 Parameters
53146 ----------
54147 default_method: str (optional)
55148 Defines how to configure backend FS instances. Options are:
56- - "default" (you get this with None) : instantiate like FSClass(), with no
149+ - "default": instantiate like FSClass(), with no
57150 extra arguments; this is the default instance of that FS, and can be
58151 configured via the config system
59152 - "generic": takes instances from the `_generic_fs` dict in this module,
60153 which you must populate before use. Keys are by protocol
61154 - "current": takes the most recently instantiated version of each FS
62- - "options": expect ``storage_options`` to be passed along with every call.
63155 """
64156 self .method = default_method
65157 super (GenericFileSystem , self ).__init__ (** kwargs )
66158
67- async def _info (
68- self , url , method = None , protocol = None , storage_options = None , fs = None , ** kwargs
69- ):
70- fs = fs or _resolve_fs (url , method or self .method , protocol , storage_options )
159+ def _strip_protocol (self , path ):
160+ # normalization only
161+ fs = _resolve_fs (path , self .method )
162+ return fs .unstrip_protocol (fs ._strip_protocol (path ))
163+
164+ async def _find (self , path , maxdepth = None , withdirs = False , detail = False , ** kwargs ):
165+ fs = _resolve_fs (path , self .method )
166+ if fs .async_impl :
167+ out = await fs ._find (
168+ path , maxdepth = maxdepth , withdirs = withdirs , detail = detail , ** kwargs
169+ )
170+ else :
171+ out = fs .find (
172+ path , maxdepth = maxdepth , withdirs = withdirs , detail = detail , ** kwargs
173+ )
174+ result = {}
175+ for k , v in out .items ():
176+ name = fs .unstrip_protocol (k )
177+ v ["name" ] = name
178+ result [name ] = v
179+ if detail :
180+ return result
181+ return list (result )
182+
183+ async def _info (self , url , ** kwargs ):
184+ fs = _resolve_fs (url , self .method )
71185 if fs .async_impl :
72186 out = await fs ._info (url , ** kwargs )
73187 else :
@@ -78,14 +192,10 @@ async def _info(
78192 async def _ls (
79193 self ,
80194 url ,
81- method = None ,
82- protocol = None ,
83- storage_options = None ,
84- fs = None ,
85195 detail = True ,
86196 ** kwargs ,
87197 ):
88- fs = fs or _resolve_fs (url , method or self .method , protocol , storage_options )
198+ fs = _resolve_fs (url , self .method )
89199 if fs .async_impl :
90200 out = await fs ._ls (url , detail = True , ** kwargs )
91201 else :
@@ -100,13 +210,9 @@ async def _ls(
100210 async def _cat_file (
101211 self ,
102212 url ,
103- method = None ,
104- protocol = None ,
105- storage_options = None ,
106- fs = None ,
107213 ** kwargs ,
108214 ):
109- fs = fs or _resolve_fs (url , method or self .method , protocol , storage_options )
215+ fs = _resolve_fs (url , self .method )
110216 if fs .async_impl :
111217 return await fs ._cat_file (url , ** kwargs )
112218 else :
@@ -116,47 +222,45 @@ async def _pipe_file(
116222 self ,
117223 path ,
118224 value ,
119- method = None ,
120- protocol = None ,
121- storage_options = None ,
122- fs = None ,
123225 ** kwargs ,
124226 ):
125- fs = fs or _resolve_fs (path , method or self .method , protocol , storage_options )
227+ fs = _resolve_fs (path , self .method )
126228 if fs .async_impl :
127229 return await fs ._pipe_file (path , value , ** kwargs )
128230 else :
129231 return fs .pipe_file (path , value , ** kwargs )
130232
131- async def _rm (
132- self , url , method = None , protocol = None , storage_options = None , fs = None , ** kwargs
133- ):
134- fs = fs or _resolve_fs (url , method or self .method , protocol , storage_options )
233+ async def _rm (self , url , ** kwargs ):
234+ fs = _resolve_fs (url , self .method )
135235 if fs .async_impl :
136236 await fs ._rm (url , ** kwargs )
137237 else :
138238 fs .rm (url , ** kwargs )
139239
240+ async def _makedirs (self , path , exist_ok = False ):
241+ fs = _resolve_fs (path , self .method )
242+ if fs .async_impl :
243+ await fs ._makedirs (path , exist_ok = exist_ok )
244+ else :
245+ fs .makedirs (path , exist_ok = exist_ok )
246+
247+ def rsync (self , source , destination , ** kwargs ):
248+ """Sync files between two directory trees
249+
250+ See `func:rsync` for more details.
251+ """
252+ rsync (source , destination , fs = self , ** kwargs )
253+
140254 async def _cp_file (
141255 self ,
142256 url ,
143257 url2 ,
144- method = None ,
145- protocol = None ,
146- storage_options = None ,
147- fs = None ,
148- method2 = None ,
149- protocol2 = None ,
150- storage_options2 = None ,
151- fs2 = None ,
152258 blocksize = 2 ** 20 ,
153259 callback = _DEFAULT_CALLBACK ,
154260 ** kwargs ,
155261 ):
156- fs = fs or _resolve_fs (url , method or self .method , protocol , storage_options )
157- fs2 = fs2 or _resolve_fs (
158- url2 , method2 or self .method , protocol2 , storage_options2
159- )
262+ fs = _resolve_fs (url , self .method )
263+ fs2 = _resolve_fs (url2 , self .method )
160264 if fs is fs2 :
161265 # pure remote
162266 if fs .async_impl :
0 commit comments