Skip to content

Commit 2f1a83e

Browse files
committed
compatibility fixes for py3.x and 2.6 in config/data
1 parent 7abe511 commit 2f1a83e

2 files changed

Lines changed: 29 additions & 11 deletions

File tree

‎astropy/config/data.py‎

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ def get_data_fileobj(dataname, cache=True):
105105
return open(get_data_filename(dataname), 'rb')
106106
else:
107107
url = urlparse(dataname)
108-
if url.scheme != '':
108+
if url[0] != '': #url[0]==url.scheme, but url[0] is py 2.6-compat
109109
#it's actually a url for a net location
110110
urlres = urlopen(dataname,timeout=REMOTE_TIMEOUT())
111111
else:
@@ -185,7 +185,7 @@ def get_data_filename(dataname):
185185
from urlparse import urlparse
186186

187187
url = urlparse(dataname)
188-
if url.scheme != '':
188+
if url[0] != '': #url[0]==url.scheme, but url[0] is py 2.6-compat
189189
#it's actually a url for a net location
190190
return _cache_remote(dataname)
191191
elif dataname.startswith('hash/'):
@@ -295,17 +295,16 @@ def _cache_remote(remoteurl):
295295
a name determined by the file's MD5 hash. If present in the cache, just
296296
returns the filename.
297297
"""
298+
import hashlib
298299
from contextlib import closing
299300
from os.path import join
300301
from shutil import move
301302
from urllib2 import urlopen
302-
import hashlib
303-
import shelve
304303

305304
dldir,urlmapfn = _get_data_cache_locs()
306305
_acquire_data_cache_lock()
307306
try:
308-
with closing(shelve.open(urlmapfn)) as url2hash:
307+
with _open_shelve(urlmapfn,True) as url2hash:
309308
if str(remoteurl) in url2hash:
310309
localpath = url2hash[str(remoteurl)]
311310
else:
@@ -316,7 +315,7 @@ def _cache_remote(remoteurl):
316315

317316
with open(tmpfn,'wb') as f:
318317
block = remote.read(DATA_CACHE_DL_BLOCK_SIZE())
319-
while block!='':
318+
while block:
320319
#TODO: add in something to update a progress bar
321320
#when the download occurs. Or use the log. either
322321
#requires stuff that isn't yet in master
@@ -349,11 +348,9 @@ def clear_data_cache(hashorurl=None):
349348
If the requested filename is not present in the data directory.
350349
351350
"""
352-
import shelve
353351
from os import unlink
354352
from os.path import join, split, exists, abspath
355353
from shutil import rmtree
356-
from contextlib import closing
357354

358355
dldir,urlmapfn = _get_data_cache_locs()
359356
_acquire_data_cache_lock()
@@ -365,7 +362,7 @@ def clear_data_cache(hashorurl=None):
365362
if exists(urlmapfn):
366363
unlink(urlmapfn)
367364
else:
368-
with closing(shelve.open(urlmapfn)) as url2hash:
365+
with _open_shelve(urlmapfn,True) as url2hash:
369366
filepath = join(dldir, hashorurl)
370367
assert abspath(filepath).startswith(abspath(dldir)), \
371368
("attempted to use clear_data_cache on a location" +
@@ -398,7 +395,6 @@ def _get_data_cache_locs():
398395
shelveloc : str
399396
The path to the shelve object that stores the cache info.
400397
"""
401-
import shelve
402398
from .paths import get_cache_dir
403399
from os.path import exists,isdir,join
404400
from os import mkdir
@@ -417,6 +413,28 @@ def _get_data_cache_locs():
417413

418414
return datadir,shelveloc
419415

416+
def _open_shelve(shelffn,withclosing=False):
417+
"""
418+
opens a shelf in a way that is py3.x and py2.x compatible. If
419+
`withclosing` is True, it will be opened with closing, allowing use like:
420+
421+
with _open_shelve('somefile',True) as s:
422+
...
423+
"""
424+
import shelve
425+
from contextlib import closing
426+
from sys import version_info
427+
428+
if version_info[0]>2:
429+
shelf = shelve.open(shelffn,protocol=2)
430+
else:
431+
shelf = shelve.open(shelffn+'.db',protocol=2)
432+
433+
if withclosing:
434+
return closing(shelf)
435+
else:
436+
return shelf
437+
420438
#the cache directory must be locked before any writes are performed. Same for
421439
#the hash shelve, so this should be used for both. Note
422440
def _acquire_data_cache_lock():

‎astropy/config/tests/test_data.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ def test_url_nocache():
1818
from ..data import get_data_fileobj
1919

2020
with get_data_fileobj(TESTURL,cache=False) as googlepage:
21-
assert googlepage.read().find('oogle</title>')>-1
21+
assert googlepage.read().decode().find('oogle</title>')>-1
2222

2323
@remote_data
2424
def test_find_by_hash():

0 commit comments

Comments
 (0)