Skip to content

Commit 5eec9f9

Browse files
Fix: Omit threading.get_ident() from cache token for async implementations in sync mode (#2028)
--------- Co-authored-by: Martin Durant <[email protected]>
1 parent 8643878 commit 5eec9f9

2 files changed

Lines changed: 100 additions & 3 deletions

File tree

‎fsspec/spec.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -70,9 +70,12 @@ def __call__(cls, *args, **kwargs):
7070
strip_tokenize_options = {
7171
k: kwargs.pop(k) for k in cls._strip_tokenize_options if k in kwargs
7272
}
73-
token = tokenize(
74-
cls, cls._pid, threading.get_ident(), *args, *extra_tokens, **kwargs
75-
)
73+
if getattr(cls, "async_impl", False) and not kwargs.get("asynchronous", False):
74+
token = tokenize(cls, cls._pid, *args, *extra_tokens, **kwargs)
75+
else:
76+
token = tokenize(
77+
cls, cls._pid, threading.get_ident(), *args, *extra_tokens, **kwargs
78+
)
7679
skip = kwargs.pop("skip_instance_cache", False)
7780
if os.getpid() != cls._pid:
7881
cls._cache.clear()

‎fsspec/tests/test_cache_threads.py‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
import threading
2+
3+
import pytest
4+
5+
import fsspec
6+
from fsspec.asyn import AsyncFileSystem
7+
from fsspec.registry import register_implementation
8+
from fsspec.spec import AbstractFileSystem
9+
10+
11+
class SyncDummyFS(AbstractFileSystem):
12+
cachable = True
13+
async_impl = False
14+
protocol = "syncdummy"
15+
16+
17+
class AsyncDummyFS(AsyncFileSystem):
18+
cachable = True
19+
protocol = "asyncdummy"
20+
21+
22+
@pytest.fixture(autouse=True)
23+
def dummy_fs_setup():
24+
register_implementation("syncdummy", SyncDummyFS)
25+
register_implementation("asyncdummy", AsyncDummyFS)
26+
SyncDummyFS.clear_instance_cache()
27+
AsyncDummyFS.clear_instance_cache()
28+
yield
29+
SyncDummyFS.clear_instance_cache()
30+
AsyncDummyFS.clear_instance_cache()
31+
32+
33+
def test_async_fs_sync_mode_shares_instance():
34+
results = {}
35+
lock = threading.Lock()
36+
37+
def worker(thread_id):
38+
with lock:
39+
fs = fsspec.filesystem("asyncdummy", asynchronous=False)
40+
results[thread_id] = id(fs)
41+
42+
t1 = threading.Thread(target=worker, args=(1,))
43+
t2 = threading.Thread(target=worker, args=(2,))
44+
45+
t1.start()
46+
t2.start()
47+
t1.join()
48+
t2.join()
49+
50+
assert results[1] == results[2]
51+
52+
53+
def test_async_fs_async_mode_does_not_share():
54+
results = {}
55+
lock = threading.Lock()
56+
barrier = threading.Barrier(2)
57+
58+
def worker(thread_id):
59+
barrier.wait()
60+
with lock:
61+
fs = fsspec.filesystem("asyncdummy", asynchronous=True)
62+
results[thread_id] = id(fs)
63+
64+
t1 = threading.Thread(target=worker, args=(1,))
65+
t2 = threading.Thread(target=worker, args=(2,))
66+
67+
t1.start()
68+
t2.start()
69+
t1.join()
70+
t2.join()
71+
72+
assert results[1] != results[2]
73+
74+
75+
def test_sync_fs_does_not_share():
76+
results = {}
77+
lock = threading.Lock()
78+
barrier = threading.Barrier(2)
79+
80+
def worker(thread_id):
81+
barrier.wait()
82+
with lock:
83+
fs = fsspec.filesystem("syncdummy")
84+
results[thread_id] = id(fs)
85+
86+
t1 = threading.Thread(target=worker, args=(1,))
87+
t2 = threading.Thread(target=worker, args=(2,))
88+
89+
t1.start()
90+
t2.start()
91+
t1.join()
92+
t2.join()
93+
94+
assert results[1] != results[2]

0 commit comments

Comments
 (0)