Skip to content

Commit 2eaa57a

Browse files
authored
rsync with genericFS (#1147)
* add docs
1 parent 51a6e05 commit 2eaa57a

7 files changed

Lines changed: 203 additions & 46 deletions

File tree

‎.github/workflows/main.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ jobs:
109109
shell: bash -l {0}
110110
run: |
111111
git clone https://github.com/dask/dask
112+
pip install -e ./dask
112113
113114
- name: Run dask tests
114115
shell: bash -l {0}

‎docs/source/api.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ User Functions
1717
fsspec.get_mapper
1818
fsspec.fuse.run
1919
fsspec.gui.FileSelector
20+
fsspec.generic.rsync
2021

2122
.. autofunction:: fsspec.open_files
2223
.. autofunction:: fsspec.open
@@ -29,6 +30,7 @@ User Functions
2930
.. autofunction:: fsspec.fuse.run
3031
.. autoclass:: fsspec.gui.FileSelector
3132
:members:
33+
.. autofunction:: fsspec.generic.rsync
3234

3335
Base Classes
3436
------------

‎fsspec/generic.py‎

Lines changed: 142 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
import inspect
2+
import logging
23

34
from .asyn import AsyncFileSystem
45
from .callbacks import _DEFAULT_CALLBACK
56
from .core import filesystem, get_filesystem_class, split_protocol
67

78
_generic_fs = {}
9+
logger = logging.getLogger("fsspec.generic")
810

911

1012
def set_generic_fs(protocol, **storage_options):
@@ -31,6 +33,97 @@ def _resolve_fs(url, method=None, protocol=None, storage_options=None):
3133
raise ValueError(f"Unknown FS resolution method: {method}")
3234

3335

36+
def rsync(
37+
source,
38+
destination,
39+
delete_missing=False,
40+
source_field="size",
41+
dest_field="size",
42+
update_cond="different",
43+
inst_kwargs=None,
44+
fs=None,
45+
**kwargs,
46+
):
47+
"""Sync files between two directory trees
48+
49+
(experimental)
50+
51+
Parameters
52+
----------
53+
source: str
54+
Root of the directory tree to take files from.
55+
destination: str
56+
Root path to copy into. The contents of this location should be
57+
identical to the contents of ``source`` when done.
58+
delete_missing: bool
59+
If there are paths in the destination that don't exist in the
60+
source and this is True, delete them. Otherwise, leave them alone.
61+
source_field: str
62+
If ``update_field`` is "different", this is the key in the info
63+
of source files to consider for difference.
64+
dest_field: str
65+
If ``update_field`` is "different", this is the key in the info
66+
of destination files to consider for difference.
67+
update_cond: "different"|"always"|"never"
68+
If "always", every file is copied, regardless of whether it exists in
69+
the destination. If "never", files that exist in the destination are
70+
not copied again. If "different" (default), only copy if the info
71+
fields given by ``source_field`` and ``dest_field`` (usually "size")
72+
are different. Other comparisons may be added in the future.
73+
inst_kwargs: dict|None
74+
If ``fs`` is None, use this set of keyword arguments to make a
75+
GenericFileSystem instance
76+
fs: GenericFileSystem|None
77+
Instance to use if explicitly given. The instance defines how to
78+
to make downstream file system instances from paths.
79+
"""
80+
fs = fs or GenericFileSystem(**(inst_kwargs or {}))
81+
source = fs._strip_protocol(source)
82+
destination = fs._strip_protocol(destination)
83+
allfiles = fs.find(source, withdirs=True, detail=True)
84+
if not fs.isdir(source):
85+
raise ValueError("Can only rsync on a directory")
86+
otherfiles = fs.find(destination, withdirs=True, detail=True)
87+
dirs = [
88+
a
89+
for a, v in allfiles.items()
90+
if v["type"] == "directory" and a.replace(source, destination) not in otherfiles
91+
]
92+
logger.debug(f"{len(dirs)} directories to create")
93+
for dirn in dirs:
94+
# no async
95+
fs.mkdirs(dirn.replace(source, destination), exist_ok=True)
96+
allfiles = {a: v for a, v in allfiles.items() if v["type"] == "file"}
97+
logger.debug(f"{len(allfiles)} files to consider for copy")
98+
to_delete = [
99+
o
100+
for o, v in otherfiles.items()
101+
if o.replace(destination, source) not in allfiles and v["type"] == "file"
102+
]
103+
for k, v in allfiles.copy().items():
104+
otherfile = k.replace(source, destination)
105+
if otherfile in otherfiles:
106+
if update_cond == "always":
107+
allfiles[k] = otherfile
108+
elif update_cond == "different":
109+
if v[source_field] != otherfiles[otherfile][dest_field]:
110+
# details mismatch, make copy
111+
allfiles[k] = otherfile
112+
else:
113+
# details match, don't copy
114+
allfiles.pop(k)
115+
else:
116+
# file not in target yet
117+
allfiles[k] = otherfile
118+
if allfiles:
119+
source_files, target_files = zip(*allfiles.items())
120+
logger.debug(f"{len(source_files)} files to copy")
121+
fs.cp(source_files, target_files, **kwargs)
122+
if delete_missing:
123+
logger.debug(f"{len(to_delete)} files to delete")
124+
fs.rm(to_delete)
125+
126+
34127
class GenericFileSystem(AsyncFileSystem):
35128
"""Wrapper over all other FS types
36129
@@ -46,28 +139,49 @@ class GenericFileSystem(AsyncFileSystem):
46139

47140
protocol = "generic" # there is no real reason to ever use a protocol with this FS
48141

49-
def __init__(self, default_method=None, **kwargs):
142+
def __init__(self, default_method="default", **kwargs):
50143
"""
51144
52145
Parameters
53146
----------
54147
default_method: str (optional)
55148
Defines how to configure backend FS instances. Options are:
56-
- "default" (you get this with None): instantiate like FSClass(), with no
149+
- "default": instantiate like FSClass(), with no
57150
extra arguments; this is the default instance of that FS, and can be
58151
configured via the config system
59152
- "generic": takes instances from the `_generic_fs` dict in this module,
60153
which you must populate before use. Keys are by protocol
61154
- "current": takes the most recently instantiated version of each FS
62-
- "options": expect ``storage_options`` to be passed along with every call.
63155
"""
64156
self.method = default_method
65157
super(GenericFileSystem, self).__init__(**kwargs)
66158

67-
async def _info(
68-
self, url, method=None, protocol=None, storage_options=None, fs=None, **kwargs
69-
):
70-
fs = fs or _resolve_fs(url, method or self.method, protocol, storage_options)
159+
def _strip_protocol(self, path):
160+
# normalization only
161+
fs = _resolve_fs(path, self.method)
162+
return fs.unstrip_protocol(fs._strip_protocol(path))
163+
164+
async def _find(self, path, maxdepth=None, withdirs=False, detail=False, **kwargs):
165+
fs = _resolve_fs(path, self.method)
166+
if fs.async_impl:
167+
out = await fs._find(
168+
path, maxdepth=maxdepth, withdirs=withdirs, detail=detail, **kwargs
169+
)
170+
else:
171+
out = fs.find(
172+
path, maxdepth=maxdepth, withdirs=withdirs, detail=detail, **kwargs
173+
)
174+
result = {}
175+
for k, v in out.items():
176+
name = fs.unstrip_protocol(k)
177+
v["name"] = name
178+
result[name] = v
179+
if detail:
180+
return result
181+
return list(result)
182+
183+
async def _info(self, url, **kwargs):
184+
fs = _resolve_fs(url, self.method)
71185
if fs.async_impl:
72186
out = await fs._info(url, **kwargs)
73187
else:
@@ -78,14 +192,10 @@ async def _info(
78192
async def _ls(
79193
self,
80194
url,
81-
method=None,
82-
protocol=None,
83-
storage_options=None,
84-
fs=None,
85195
detail=True,
86196
**kwargs,
87197
):
88-
fs = fs or _resolve_fs(url, method or self.method, protocol, storage_options)
198+
fs = _resolve_fs(url, self.method)
89199
if fs.async_impl:
90200
out = await fs._ls(url, detail=True, **kwargs)
91201
else:
@@ -100,13 +210,9 @@ async def _ls(
100210
async def _cat_file(
101211
self,
102212
url,
103-
method=None,
104-
protocol=None,
105-
storage_options=None,
106-
fs=None,
107213
**kwargs,
108214
):
109-
fs = fs or _resolve_fs(url, method or self.method, protocol, storage_options)
215+
fs = _resolve_fs(url, self.method)
110216
if fs.async_impl:
111217
return await fs._cat_file(url, **kwargs)
112218
else:
@@ -116,47 +222,45 @@ async def _pipe_file(
116222
self,
117223
path,
118224
value,
119-
method=None,
120-
protocol=None,
121-
storage_options=None,
122-
fs=None,
123225
**kwargs,
124226
):
125-
fs = fs or _resolve_fs(path, method or self.method, protocol, storage_options)
227+
fs = _resolve_fs(path, self.method)
126228
if fs.async_impl:
127229
return await fs._pipe_file(path, value, **kwargs)
128230
else:
129231
return fs.pipe_file(path, value, **kwargs)
130232

131-
async def _rm(
132-
self, url, method=None, protocol=None, storage_options=None, fs=None, **kwargs
133-
):
134-
fs = fs or _resolve_fs(url, method or self.method, protocol, storage_options)
233+
async def _rm(self, url, **kwargs):
234+
fs = _resolve_fs(url, self.method)
135235
if fs.async_impl:
136236
await fs._rm(url, **kwargs)
137237
else:
138238
fs.rm(url, **kwargs)
139239

240+
async def _makedirs(self, path, exist_ok=False):
241+
fs = _resolve_fs(path, self.method)
242+
if fs.async_impl:
243+
await fs._makedirs(path, exist_ok=exist_ok)
244+
else:
245+
fs.makedirs(path, exist_ok=exist_ok)
246+
247+
def rsync(self, source, destination, **kwargs):
248+
"""Sync files between two directory trees
249+
250+
See `func:rsync` for more details.
251+
"""
252+
rsync(source, destination, fs=self, **kwargs)
253+
140254
async def _cp_file(
141255
self,
142256
url,
143257
url2,
144-
method=None,
145-
protocol=None,
146-
storage_options=None,
147-
fs=None,
148-
method2=None,
149-
protocol2=None,
150-
storage_options2=None,
151-
fs2=None,
152258
blocksize=2**20,
153259
callback=_DEFAULT_CALLBACK,
154260
**kwargs,
155261
):
156-
fs = fs or _resolve_fs(url, method or self.method, protocol, storage_options)
157-
fs2 = fs2 or _resolve_fs(
158-
url2, method2 or self.method, protocol2, storage_options2
159-
)
262+
fs = _resolve_fs(url, self.method)
263+
fs2 = _resolve_fs(url2, self.method)
160264
if fs is fs2:
161265
# pure remote
162266
if fs.async_impl:

‎fsspec/implementations/local.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,10 @@ def rm(self, path, recursive=False, maxdepth=None):
172172
else:
173173
os.remove(p)
174174

175+
def unstrip_protocol(self, name):
176+
name = self._strip_protocol(name) # normalise for local/win/...
177+
return f"file://{name}"
178+
175179
def _open(self, path, mode="rb", block_size=None, **kwargs):
176180
path = self._strip_protocol(path)
177181
if self.auto_mkdir and "w" in mode:
@@ -247,10 +251,12 @@ def make_path_posix(path, sep=os.sep):
247251
return make_path_posix(osp.abspath(path))
248252
else:
249253
return os.getcwd() + "/" + path
254+
if path.startswith("file://"):
255+
path = path[7:]
250256
if re.match("/[A-Za-z]:", path):
251257
# for windows file URI like "file:///C:/folder/file"
252258
# or "file:///C:\\dir\\file"
253-
path = path[1:]
259+
path = path[1:].replace("\\", "/").replace("//", "/")
254260
if path.startswith("\\\\"):
255261
# special case for windows UNC/DFS-style paths, do nothing,
256262
# just flip the slashes around (case below does not work!)

‎fsspec/implementations/tests/test_local.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -465,6 +465,7 @@ def test_make_path_posix():
465465
)
466466
if WIN:
467467
assert make_path_posix("C:\\path") == "C:/path"
468+
assert make_path_posix("file://C:\\path\\file") == "C:/path/file"
468469
if WIN:
469470
assert (
470471
make_path_posix(

‎fsspec/implementations/tests/test_sftp.py‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -95,24 +95,25 @@ def test_with_url(protocol, ssh):
9595

9696

9797
@pytest.mark.parametrize("protocol", ["sftp", "ssh"])
98-
def test_get_dir(protocol, ssh, root_path):
98+
def test_get_dir(protocol, ssh, root_path, tmpdir):
99+
path = str(tmpdir)
99100
f = fsspec.filesystem(protocol, **ssh)
100101
f.mkdirs(root_path + "deeper", exist_ok=True)
101102
f.touch(root_path + "deeper/afile")
102-
f.get(root_path, ".", recursive=True)
103+
f.get(root_path, path, recursive=True)
103104

104-
assert os.path.isdir("./deeper")
105-
assert os.path.isfile("./deeper/afile")
105+
assert os.path.isdir(f"{path}/deeper")
106+
assert os.path.isfile(f"{path}/deeper/afile")
106107

107108
f.get(
108109
protocol + "://{username}:{password}@{host}:{port}"
109110
"{root_path}".format(root_path=root_path, **ssh),
110-
"./test2",
111+
f"{path}/test2",
111112
recursive=True,
112113
)
113114

114-
assert os.path.isdir("./test2/deeper")
115-
assert os.path.isfile("./test2/deeper/afile")
115+
assert os.path.isdir(f"{path}/test2/deeper")
116+
assert os.path.isfile(f"{path}/test2/deeper/afile")
116117

117118

118119
@pytest.fixture(scope="module")

0 commit comments

Comments
 (0)