Skip to content

Commit b46862a

Browse files
authored
speedup FSMap._key_to_str (#1101)
* tests: check key suffix behavior * fsspec.mapping: FSMap._key_to_str precompute _root_prefix speeds up _key_to_str by a factor of ~10
1 parent 4a67229 commit b46862a

3 files changed

Lines changed: 79 additions & 8 deletions

File tree

‎fsspec/mapping.py‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
import array
2+
import posixpath
3+
import warnings
24
from collections.abc import MutableMapping
35

46
from .core import url_to_fs
@@ -34,9 +36,8 @@ class FSMap(MutableMapping):
3436

3537
def __init__(self, root, fs, check=False, create=False, missing_exceptions=None):
3638
self.fs = fs
37-
self.root = fs._strip_protocol(root).rstrip(
38-
"/"
39-
) # we join on '/' in _key_to_str
39+
self.root = fs._strip_protocol(root).rstrip("/")
40+
self._root_key_to_str = fs._strip_protocol(posixpath.join(root, "x"))[:-1]
4041
if missing_exceptions is None:
4142
missing_exceptions = (
4243
FileNotFoundError,
@@ -120,11 +121,16 @@ def delitems(self, keys):
120121

121122
def _key_to_str(self, key):
122123
"""Generate full path for the key"""
123-
if isinstance(key, (tuple, list)):
124-
key = str(tuple(key))
125-
else:
124+
if not isinstance(key, str):
125+
# raise TypeError("key must be of type `str`, got `{type(key).__name__}`"
126+
warnings.warn(
127+
"from fsspec 2023.5 onward FSMap non-str keys will raise TypeError",
128+
DeprecationWarning,
129+
)
130+
if isinstance(key, list):
131+
key = tuple(key)
126132
key = str(key)
127-
return self.fs._strip_protocol("/".join([self.root, key]) if self.root else key)
133+
return f"{self._root_key_to_str}{key}"
128134

129135
def _str_to_key(self, s):
130136
"""Strip path of to leave key name"""

‎fsspec/tests/test_mapping.py‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import os
22
import pickle
3+
import platform
34
import sys
5+
import uuid
46

57
import pytest
68

@@ -149,3 +151,66 @@ def test_setitem_numpy():
149151
def test_empty_url():
150152
m = fsspec.get_mapper()
151153
assert isinstance(m.fs, LocalFileSystem)
154+
155+
156+
def test_fsmap_access_with_root_prefix(tmp_path):
157+
# "/a" and "a" are the same for LocalFileSystem
158+
tmp_path.joinpath("a").write_bytes(b"data")
159+
m = fsspec.get_mapper(f"file://{tmp_path}")
160+
assert m["/a"] == m["a"] == b"data"
161+
162+
# "/a" and "a" differ for MemoryFileSystem
163+
m = fsspec.get_mapper(f"memory://{uuid.uuid4()}")
164+
m["/a"] = b"data"
165+
166+
assert m["/a"] == b"data"
167+
with pytest.raises(KeyError):
168+
_ = m["a"]
169+
170+
171+
@pytest.mark.parametrize(
172+
"key",
173+
[
174+
pytest.param(b"k", id="bytes"),
175+
pytest.param(1234, id="int"),
176+
pytest.param((1,), id="tuple"),
177+
pytest.param([""], id="list"),
178+
],
179+
)
180+
def test_fsmap_non_str_keys(key):
181+
m = fsspec.get_mapper()
182+
183+
# Once the deprecation period passes
184+
# FSMap.__getitem__ should raise TypeError for non-str keys
185+
# with pytest.raises(TypeError):
186+
# _ = m[key]
187+
188+
with pytest.warns(DeprecationWarning):
189+
with pytest.raises(KeyError):
190+
_ = m[key]
191+
192+
193+
def test_fsmap_error_on_protocol_keys():
194+
root = uuid.uuid4()
195+
m = fsspec.get_mapper(f"memory://{root}", create=True)
196+
m["a"] = b"data"
197+
198+
assert m["a"] == b"data"
199+
with pytest.raises(KeyError):
200+
_ = m[f"memory://{root}/a"]
201+
202+
203+
# on Windows opening a directory will raise PermissionError
204+
# see: https://bugs.python.org/issue43095
205+
@pytest.mark.skipif(
206+
platform.system() == "Windows", reason="raises PermissionError on windows"
207+
)
208+
def test_fsmap_access_with_suffix(tmp_path):
209+
tmp_path.joinpath("b").mkdir()
210+
tmp_path.joinpath("b", "a").write_bytes(b"data")
211+
m = fsspec.get_mapper(f"file://{tmp_path}")
212+
213+
with pytest.raises(KeyError):
214+
_ = m["b/"]
215+
216+
assert m["b/a/"] == b"data"

‎tox.ini‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ conda_deps=
6565
{[core]conda_deps}
6666
httpretty
6767
aiobotocore
68-
"moto<3.0"
68+
"moto>=4"
6969
flask
7070
changedir=.tox/s3fs/tmp
7171
whitelist_externals=

0 commit comments

Comments
 (0)