11import json
22import os
3- import pickle
43import shutil
54import tempfile
65
@@ -118,8 +117,7 @@ def test_mapper():
118117@pytest .mark .parametrize (
119118 "cache_mapper" , [BasenameCacheMapper (), BasenameCacheMapper (1 ), HashCacheMapper ()]
120119)
121- @pytest .mark .parametrize ("force_save_pickle" , [True , False ])
122- def test_metadata (tmpdir , cache_mapper , force_save_pickle ):
120+ def test_metadata (tmpdir , cache_mapper ):
123121 source = os .path .join (tmpdir , "source" )
124122 afile = os .path .join (source , "afile" )
125123 os .mkdir (source )
@@ -131,7 +129,6 @@ def test_metadata(tmpdir, cache_mapper, force_save_pickle):
131129 cache_storage = os .path .join (tmpdir , "cache" ),
132130 cache_mapper = cache_mapper ,
133131 )
134- fs ._metadata ._force_save_pickle = force_save_pickle
135132
136133 with fs .open (afile , "rb" ) as f :
137134 assert f .read (5 ) == b"test"
@@ -154,42 +151,6 @@ def test_metadata(tmpdir, cache_mapper, force_save_pickle):
154151 assert detail ["fn" ] == "source_@_afile"
155152
156153
157- def test_metadata_replace_pickle_with_json (tmpdir ):
158- # For backward compatibility will allow reading of old pickled metadata.
159- # When the metadata is next saved, it is in json format.
160- source = os .path .join (tmpdir , "source" )
161- afile = os .path .join (source , "afile" )
162- os .mkdir (source )
163- open (afile , "w" ).write ("test" )
164-
165- # Save metadata in pickle format, to simulate old metadata
166- fs = fsspec .filesystem (
167- "filecache" ,
168- target_protocol = "file" ,
169- cache_storage = os .path .join (tmpdir , "cache" ),
170- )
171- fs ._metadata ._force_save_pickle = True
172- with fs .open (afile , "rb" ) as f :
173- assert f .read (5 ) == b"test"
174-
175- # Confirm metadata is in pickle format
176- cache_fn = os .path .join (fs .storage [- 1 ], "cache" )
177- with open (cache_fn , "rb" ) as f :
178- metadata = pickle .load (f )
179- assert list (metadata .keys ()) == [make_path_posix (afile )]
180-
181- # Force rewrite of metadata, now in json format
182- fs ._metadata ._force_save_pickle = False
183- fs .pop_from_cache (afile )
184- with fs .open (afile , "rb" ) as f :
185- assert f .read (5 ) == b"test"
186-
187- # Confirm metadata is in json format
188- with open (cache_fn , "r" ) as f :
189- metadata = json .load (f )
190- assert list (metadata .keys ()) == [make_path_posix (afile )]
191-
192-
193154def test_constructor_kwargs (tmpdir ):
194155 fs = fsspec .filesystem ("filecache" , target_protocol = "file" , same_names = True )
195156 assert isinstance (fs ._mapper , BasenameCacheMapper )
@@ -212,15 +173,16 @@ def test_constructor_kwargs(tmpdir):
212173
213174
214175def test_idempotent ():
176+ import pickle
177+
215178 fs = CachingFileSystem ("file" )
216179 fs2 = CachingFileSystem ("file" )
217180 assert fs2 is fs
218181 fs3 = pickle .loads (pickle .dumps (fs ))
219182 assert fs3 .storage == fs .storage
220183
221184
222- @pytest .mark .parametrize ("force_save_pickle" , [True , False ])
223- def test_blockcache_workflow (ftp_writable , tmp_path , force_save_pickle ):
185+ def test_blockcache_workflow (ftp_writable , tmp_path ):
224186 host , port , user , pw = ftp_writable
225187 fs = FTPFileSystem (host , port , user , pw )
226188 with fs .open ("/out" , "wb" ) as f :
@@ -240,7 +202,6 @@ def test_blockcache_workflow(ftp_writable, tmp_path, force_save_pickle):
240202
241203 # Open the blockcache and read a little bit of the data
242204 fs = fsspec .filesystem ("blockcache" , ** fs_kwargs )
243- fs ._metadata ._force_save_pickle = force_save_pickle
244205 with fs .open ("/out" , "rb" , block_size = 5 ) as f :
245206 assert f .read (5 ) == b"test\n "
246207
@@ -249,18 +210,13 @@ def test_blockcache_workflow(ftp_writable, tmp_path, force_save_pickle):
249210 del fs
250211
251212 # Check that cache file only has the first two blocks
252- if force_save_pickle :
253- with open (tmp_path / "cache" , "rb" ) as f :
254- cache = pickle .load (f )
255- else :
256- with open (tmp_path / "cache" , "r" ) as f :
257- cache = json .load (f )
213+ with open (tmp_path / "cache" , "r" ) as f :
214+ cache = json .load (f )
258215 assert "/out" in cache
259216 assert cache ["/out" ]["blocks" ] == [0 , 1 ]
260217
261218 # Reopen the same cache and read some more...
262219 fs = fsspec .filesystem ("blockcache" , ** fs_kwargs )
263- fs ._metadata ._force_save_pickle = force_save_pickle
264220 with fs .open ("/out" , block_size = 5 ) as f :
265221 assert f .read (5 ) == b"test\n "
266222 f .seek (30 )
@@ -347,8 +303,7 @@ def test_clear():
347303 assert len (os .listdir (cache1 )) < 2
348304
349305
350- @pytest .mark .parametrize ("force_save_pickle" , [True , False ])
351- def test_clear_expired (tmp_path , force_save_pickle ):
306+ def test_clear_expired (tmp_path ):
352307 def __ager (cache_fn , fn , del_fn = False ):
353308 """
354309 Modify the cache file to virtually add time lag to selected files.
@@ -366,23 +321,15 @@ def __ager(cache_fn, fn, del_fn=False):
366321 import time
367322
368323 if os .path .exists (cache_fn ):
369- if force_save_pickle :
370- with open (cache_fn , "rb" ) as f :
371- cached_files = pickle .load (f )
372- else :
373- with open (cache_fn , "r" ) as f :
374- cached_files = json .load (f )
324+ with open (cache_fn , "r" ) as f :
325+ cached_files = json .load (f )
375326 fn_posix = pathlib .Path (fn ).as_posix ()
376327 cached_files [fn_posix ]["time" ] = cached_files [fn_posix ]["time" ] - 691200
377328 assert os .access (cache_fn , os .W_OK ), "Cache is not writable"
378329 if del_fn :
379330 del cached_files [fn_posix ]["fn" ]
380- if force_save_pickle :
381- with open (cache_fn , "wb" ) as f :
382- pickle .dump (cached_files , f )
383- else :
384- with open (cache_fn , "w" ) as f :
385- json .dump (cached_files , f )
331+ with open (cache_fn , "w" ) as f :
332+ json .dump (cached_files , f )
386333 time .sleep (1 )
387334
388335 origin = tmp_path .joinpath ("origin" )
@@ -414,7 +361,6 @@ def __ager(cache_fn, fn, del_fn=False):
414361 fs = fsspec .filesystem (
415362 "filecache" , target_protocol = "file" , cache_storage = str (cache1 ), cache_check = 1
416363 )
417- fs ._metadata ._force_save_pickle = force_save_pickle
418364 assert fs .cat (str (f1 )) == data
419365
420366 # populates "last" cache if file not found in first one
@@ -424,7 +370,6 @@ def __ager(cache_fn, fn, del_fn=False):
424370 cache_storage = [str (cache1 ), str (cache2 )],
425371 cache_check = 1 ,
426372 )
427- fs ._metadata ._force_save_pickle = force_save_pickle
428373 assert fs .cat (str (f2 )) == data
429374 assert fs .cat (str (f3 )) == data
430375 assert len (os .listdir (cache2 )) == 3
@@ -456,7 +401,6 @@ def __ager(cache_fn, fn, del_fn=False):
456401 same_names = True ,
457402 cache_check = 1 ,
458403 )
459- fs ._metadata ._force_save_pickle = force_save_pickle
460404 assert fs .cat (str (f4 )) == data
461405
462406 cache_fn = os .path .join (fs .storage [- 1 ], "cache" )
@@ -473,7 +417,6 @@ def __ager(cache_fn, fn, del_fn=False):
473417 same_names = True ,
474418 cache_check = 1 ,
475419 )
476- fs ._metadata ._force_save_pickle = force_save_pickle
477420 assert fs .cat (str (f1 )) == data
478421
479422 cache_fn = os .path .join (fs .storage [- 1 ], "cache" )
@@ -517,21 +460,6 @@ def test_pop():
517460 assert fs ._check_file (f1 )
518461
519462
520- def test_write_pickle_context ():
521- tmp = str (tempfile .mkdtemp ())
522- fn = tmp + "afile"
523- url = f"simplecache::file://{ fn } "
524- with fsspec .open (url , "wb" ) as f :
525- pickle .loads (pickle .dumps (f ))
526- f .write (b"hello " )
527- pickle .dumps (f )
528-
529- with pytest .raises (ValueError ):
530- pickle .dumps (f )
531-
532- assert open (fn , "rb" ).read () == b"hello "
533-
534-
535463def test_blocksize (ftp_writable ):
536464 host , port , user , pw = ftp_writable
537465 fs = FTPFileSystem (host , port , user , pw )
0 commit comments