@@ -105,7 +105,7 @@ def get_data_fileobj(dataname, cache=True):
105105 return open (get_data_filename (dataname ), 'rb' )
106106 else :
107107 url = urlparse (dataname )
108- if url . scheme != '' :
108+ if url [ 0 ] != '' : #url[0]==url.scheme, but url[0] is py 2.6-compat
109109 #it's actually a url for a net location
110110 urlres = urlopen (dataname ,timeout = REMOTE_TIMEOUT ())
111111 else :
@@ -185,7 +185,7 @@ def get_data_filename(dataname):
185185 from urlparse import urlparse
186186
187187 url = urlparse (dataname )
188- if url . scheme != '' :
188+ if url [ 0 ] != '' : #url[0]==url.scheme, but url[0] is py 2.6-compat
189189 #it's actually a url for a net location
190190 return _cache_remote (dataname )
191191 elif dataname .startswith ('hash/' ):
@@ -295,17 +295,16 @@ def _cache_remote(remoteurl):
295295 a name determined by the file's MD5 hash. If present in the cache, just
296296 returns the filename.
297297 """
298+ import hashlib
298299 from contextlib import closing
299300 from os .path import join
300301 from shutil import move
301302 from urllib2 import urlopen
302- import hashlib
303- import shelve
304303
305304 dldir ,urlmapfn = _get_data_cache_locs ()
306305 _acquire_data_cache_lock ()
307306 try :
308- with closing ( shelve . open ( urlmapfn ) ) as url2hash :
307+ with _open_shelve ( urlmapfn , True ) as url2hash :
309308 if str (remoteurl ) in url2hash :
310309 localpath = url2hash [str (remoteurl )]
311310 else :
@@ -316,7 +315,7 @@ def _cache_remote(remoteurl):
316315
317316 with open (tmpfn ,'wb' ) as f :
318317 block = remote .read (DATA_CACHE_DL_BLOCK_SIZE ())
319- while block != '' :
318+ while block :
320319 #TODO: add in something to update a progress bar
321320 #when the download occurs. Or use the log. either
322321 #requires stuff that isn't yet in master
@@ -349,11 +348,9 @@ def clear_data_cache(hashorurl=None):
349348 If the requested filename is not present in the data directory.
350349
351350 """
352- import shelve
353351 from os import unlink
354352 from os .path import join , split , exists , abspath
355353 from shutil import rmtree
356- from contextlib import closing
357354
358355 dldir ,urlmapfn = _get_data_cache_locs ()
359356 _acquire_data_cache_lock ()
@@ -365,7 +362,7 @@ def clear_data_cache(hashorurl=None):
365362 if exists (urlmapfn ):
366363 unlink (urlmapfn )
367364 else :
368- with closing ( shelve . open ( urlmapfn ) ) as url2hash :
365+ with _open_shelve ( urlmapfn , True ) as url2hash :
369366 filepath = join (dldir , hashorurl )
370367 assert abspath (filepath ).startswith (abspath (dldir )), \
371368 ("attempted to use clear_data_cache on a location" +
@@ -398,7 +395,6 @@ def _get_data_cache_locs():
398395 shelveloc : str
399396 The path to the shelve object that stores the cache info.
400397 """
401- import shelve
402398 from .paths import get_cache_dir
403399 from os .path import exists ,isdir ,join
404400 from os import mkdir
@@ -417,6 +413,28 @@ def _get_data_cache_locs():
417413
418414 return datadir ,shelveloc
419415
416+ def _open_shelve (shelffn ,withclosing = False ):
417+ """
418+ opens a shelf in a way that is py3.x and py2.x compatible. If
419+ `withclosing` is True, it will be opened with closing, allowing use like:
420+
421+ with _open_shelve('somefile',True) as s:
422+ ...
423+ """
424+ import shelve
425+ from contextlib import closing
426+ from sys import version_info
427+
428+ if version_info [0 ]> 2 :
429+ shelf = shelve .open (shelffn ,protocol = 2 )
430+ else :
431+ shelf = shelve .open (shelffn + '.db' ,protocol = 2 )
432+
433+ if withclosing :
434+ return closing (shelf )
435+ else :
436+ return shelf
437+
420438#the cache directory must be locked before any writes are performed. Same for
421439#the hash shelve, so this should be used for both. Note
422440def _acquire_data_cache_lock ():
0 commit comments