Skip to content

Commit 92de26b

Browse files
embraydhomeier
authored andcommitted
Add optional package argument to get_pkg_data_* functions so that
package data can be retrieved from packages other than the one the function was called from. Also converted most of the examples for those functions into working doctests, which revealed some bugs in both the tests and the functions themseves. One such test is skipped due to astropy/astropy-data#4
1 parent fdba168 commit 92de26b

1 file changed

Lines changed: 100 additions & 62 deletions

File tree

‎astropy/utils/data.py‎

Lines changed: 100 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626

2727
from .. import config as _config
2828
from ..utils.exceptions import AstropyWarning
29-
from ..utils.introspection import resolve_name
29+
from ..utils.introspection import find_current_module, resolve_name
3030

3131

3232
__all__ = [
@@ -371,7 +371,7 @@ def get_file_contents(name_or_obj, encoding=None, cache=False):
371371
return f.read()
372372

373373

374-
def get_pkg_data_fileobj(data_name, encoding=None, cache=True):
374+
def get_pkg_data_fileobj(data_name, package=None, encoding=None, cache=True):
375375
"""
376376
Retrieves a data file from the standard locations for the package and
377377
provides the file as a file-like object that reads bytes.
@@ -392,10 +392,14 @@ def get_pkg_data_fileobj(data_name, encoding=None, cache=True):
392392
data server will be queried for the file.
393393
* A hash like that produced by `compute_hash` can be
394394
requested, prefixed by 'hash/'
395-
e.g. 'hash/395dd6493cc584df1e78b474fb150840'. The hash
395+
e.g. 'hash/34c33b3eb0d56eb9462003af249eff28'. The hash
396396
will first be searched for locally, and if not found,
397397
the Astropy data server will be queried.
398398
399+
package : str, optional
400+
If specified, look for a file relative to the given package, rather
401+
than the default of looking relative to the calling module's package.
402+
399403
encoding : str, optional
400404
When `None` (default), returns a file-like object with a
401405
``read`` method that on Python 2.x returns `bytes` objects and
@@ -439,32 +443,39 @@ def get_pkg_data_fileobj(data_name, encoding=None, cache=True):
439443
This will retrieve a data file and its contents for the `astropy.wcs`
440444
tests::
441445
442-
from astropy.utils.data import get_pkg_data_fileobj
443-
444-
with get_pkg_data_fileobj('data/3d_cd.hdr') as fobj:
445-
fcontents = fobj.read()
446+
>>> from astropy.utils.data import get_pkg_data_fileobj
447+
>>> with get_pkg_data_fileobj('data/3d_cd.hdr',
448+
... package='astropy.wcs.tests') as fobj:
449+
... fcontents = fobj.read()
450+
...
446451
447-
This would download a data file from the astropy data server
448-
because the ``standards/vega.fits`` file is not present in the
452+
This next example would download a data file from the astropy data server
453+
because the ``allsky/allsky_rosat.fits`` file is not present in the
449454
source distribution. It will also save the file locally so the
450455
next time it is accessed it won't need to be downloaded.::
451456
452-
from astropy.utils.data import get_pkg_data_fileobj
453-
454-
with get_pkg_data_fileobj('standards/vega.fits') as fobj:
455-
fcontents = fobj.read()
457+
>>> from astropy.utils.data import get_pkg_data_fileobj
458+
>>> with get_pkg_data_fileobj('allsky/allsky_rosat.fits',
459+
... encoding='binary') as fobj: # doctest: +REMOTE_DATA
460+
... fcontents = fobj.read()
461+
...
462+
Downloading http://data.astropy.org/allsky/allsky_rosat.fits [Done]
456463
457464
This does the same thing but does *not* cache it locally::
458465
459-
with get_pkg_data_fileobj('standards/vega.fits', cache=False) as fobj:
460-
fcontents = fobj.read()
466+
>>> with get_pkg_data_fileobj('allsky/allsky_rosat.fits',
467+
... encoding='binary', cache=False) as fobj: # doctest: +REMOTE_DATA
468+
... fcontents = fobj.read()
469+
...
470+
Downloading http://data.astropy.org/allsky/allsky_rosat.fits [Done]
461471
462472
See Also
463473
--------
464474
get_pkg_data_contents : returns the contents of a file or url as a bytes object
465475
get_pkg_data_filename : returns a local name for a file containing the data
466476
"""
467-
datafn = _find_pkg_data_path(data_name)
477+
478+
datafn = _find_pkg_data_path(data_name, package=package)
468479
if os.path.isdir(datafn):
469480
raise IOError("Tried to access a data file that's actually "
470481
"a package data directory")
@@ -475,7 +486,8 @@ def get_pkg_data_fileobj(data_name, encoding=None, cache=True):
475486
cache=cache)
476487

477488

478-
def get_pkg_data_filename(data_name, show_progress=True, remote_timeout=None):
489+
def get_pkg_data_filename(data_name, package=None, show_progress=True,
490+
remote_timeout=None):
479491
"""
480492
Retrieves a data file from the standard locations for the package and
481493
provides a local filename for the data.
@@ -501,10 +513,14 @@ def get_pkg_data_filename(data_name, show_progress=True, remote_timeout=None):
501513
data server will be queried for the file.
502514
* A hash like that produced by `compute_hash` can be
503515
requested, prefixed by 'hash/'
504-
e.g. 'hash/395dd6493cc584df1e78b474fb150840'. The hash
516+
e.g. 'hash/34c33b3eb0d56eb9462003af249eff28'. The hash
505517
will first be searched for locally, and if not found,
506518
the Astropy data server will be queried.
507519
520+
package : str, optional
521+
If specified, look for a file relative to the given package, rather
522+
than the default of looking relative to the calling module's package.
523+
508524
show_progress : bool, optional
509525
Whether to display a progress bar if the file is downloaded
510526
from a remote server. Default is `True`.
@@ -533,21 +549,21 @@ def get_pkg_data_filename(data_name, show_progress=True, remote_timeout=None):
533549
This will retrieve the contents of the data file for the `astropy.wcs`
534550
tests::
535551
536-
from astropy.utils.data import get_pkg_data_filename
537-
538-
fn = get_pkg_data_filename('data/3d_cd.hdr')
539-
with open(fn) as f:
540-
fcontents = f.read()
541-
552+
>>> from astropy.utils.data import get_pkg_data_filename
553+
>>> fn = get_pkg_data_filename('data/3d_cd.hdr',
554+
... package='astropy.wcs.tests')
555+
>>> with open(fn) as f:
556+
... fcontents = f.read()
557+
...
542558
543559
This retrieves a data file by hash either locally or from the astropy data
544560
server::
545561
546-
from astropy.utils.data import get_pkg_data_filename
547-
548-
fn = get_pkg_data_filename('hash/da34a7b07ef153eede67387bf950bb32')
549-
with open(fn) as f:
550-
fcontents = f.read()
562+
>>> from astropy.utils.data import get_pkg_data_filename
563+
>>> fn = get_pkg_data_filename('hash/34c33b3eb0d56eb9462003af249eff28') # doctest: +SKIP
564+
>>> with open(fn) as f:
565+
... fcontents = f.read()
566+
...
551567
552568
See Also
553569
--------
@@ -572,7 +588,7 @@ def get_pkg_data_filename(data_name, show_progress=True, remote_timeout=None):
572588
else:
573589
return hashfn
574590
else:
575-
datafn = _find_pkg_data_path(data_name)
591+
datafn = _find_pkg_data_path(data_name, package=package)
576592
if os.path.isdir(datafn):
577593
raise IOError("Tried to access a data file that's actually "
578594
"a package data directory")
@@ -585,7 +601,7 @@ def get_pkg_data_filename(data_name, show_progress=True, remote_timeout=None):
585601
timeout=remote_timeout)
586602

587603

588-
def get_pkg_data_contents(data_name, encoding=None, cache=True):
604+
def get_pkg_data_contents(data_name, package=None, encoding=None, cache=True):
589605
"""
590606
Retrieves a data file from the standard locations and returns its
591607
contents as a bytes object.
@@ -606,11 +622,16 @@ def get_pkg_data_contents(data_name, encoding=None, cache=True):
606622
data server will be queried for the file.
607623
* A hash like that produced by `compute_hash` can be
608624
requested, prefixed by 'hash/'
609-
e.g. 'hash/395dd6493cc584df1e78b474fb150840'. The hash
625+
e.g. 'hash/34c33b3eb0d56eb9462003af249eff28'. The hash
610626
will first be searched for locally, and if not found,
611627
the Astropy data server will be queried.
612628
* A URL to some other file.
613629
630+
package : str, optional
631+
If specified, look for a file relative to the given package, rather
632+
than the default of looking relative to the calling module's package.
633+
634+
614635
encoding : str, optional
615636
When `None` (default), returns a file-like object with a
616637
``read`` method that on Python 2.x returns `bytes` objects and
@@ -651,12 +672,14 @@ def get_pkg_data_contents(data_name, encoding=None, cache=True):
651672
get_pkg_data_fileobj : returns a file-like object with the data
652673
get_pkg_data_filename : returns a local name for a file containing the data
653674
"""
654-
with get_pkg_data_fileobj(data_name, encoding=encoding, cache=cache) as fd:
675+
676+
with get_pkg_data_fileobj(data_name, package=package, encoding=encoding,
677+
cache=cache) as fd:
655678
contents = fd.read()
656679
return contents
657680

658681

659-
def get_pkg_data_filenames(datadir, pattern='*'):
682+
def get_pkg_data_filenames(datadir, package=None, pattern='*'):
660683
"""
661684
Returns the path of all of the data files in a given directory
662685
that match a given glob pattern.
@@ -673,6 +696,10 @@ def get_pkg_data_filenames(datadir, pattern='*'):
673696
files in ``astropy/pkgname/data``.
674697
* Remote URLs are not currently supported.
675698
699+
package : str, optional
700+
If specified, look for a file relative to the given package, rather
701+
than the default of looking relative to the calling module's package.
702+
676703
pattern : str, optional
677704
A UNIX-style filename glob pattern to match files. See the
678705
`glob` module in the standard library for more information.
@@ -688,14 +715,15 @@ def get_pkg_data_filenames(datadir, pattern='*'):
688715
This will retrieve the contents of the data file for the `astropy.wcs`
689716
tests::
690717
691-
from astropy.utils.data import get_pkg_data_filenames
692-
693-
for fn in get_pkg_data_filename('maps', '*.hdr'):
694-
with open(fn) as f:
695-
fcontents = f.read()
718+
>>> from astropy.utils.data import get_pkg_data_filenames
719+
>>> for fn in get_pkg_data_filenames('maps', 'astropy.wcs.tests',
720+
... '*.hdr'):
721+
... with open(fn) as f:
722+
... fcontents = f.read()
723+
...
696724
"""
697725

698-
path = _find_pkg_data_path(datadir)
726+
path = _find_pkg_data_path(datadir, package=package)
699727
if os.path.isfile(path):
700728
raise IOError(
701729
"Tried to access a data directory that's actually "
@@ -708,7 +736,7 @@ def get_pkg_data_filenames(datadir, pattern='*'):
708736
raise IOError("Path not found")
709737

710738

711-
def get_pkg_data_fileobjs(datadir, pattern='*', encoding=None):
739+
def get_pkg_data_fileobjs(datadir, package=None, pattern='*', encoding=None):
712740
"""
713741
Returns readable file objects for all of the data files in a given
714742
directory that match a given glob pattern.
@@ -725,6 +753,10 @@ def get_pkg_data_fileobjs(datadir, pattern='*', encoding=None):
725753
files in ``astropy/pkgname/data``
726754
* Remote URLs are not currently supported
727755
756+
package : str, optional
757+
If specified, look for a file relative to the given package, rather
758+
than the default of looking relative to the calling module's package.
759+
728760
pattern : str, optional
729761
A UNIX-style filename glob pattern to match files. See the
730762
`glob` module in the standard library for more information.
@@ -756,13 +788,16 @@ def get_pkg_data_fileobjs(datadir, pattern='*', encoding=None):
756788
This will retrieve the contents of the data file for the `astropy.wcs`
757789
tests::
758790
759-
from astropy.utils.data import get_pkg_data_filenames
760-
761-
for fd in get_pkg_data_filename('maps', '*.hdr'):
762-
fcontents = fd.read()
791+
>>> from astropy.utils.data import get_pkg_data_filenames
792+
>>> for fd in get_pkg_data_fileobjs('maps', 'astropy.wcs.tests',
793+
... '*.hdr'):
794+
... fcontents = fd.read()
795+
...
763796
"""
764-
for fn in get_pkg_data_filenames(datadir, pattern):
765-
with get_pkg_data_fileobj(fn, encoding=encoding) as fd:
797+
798+
for fn in get_pkg_data_filenames(datadir, package=package,
799+
pattern=pattern):
800+
with get_readable_fileobj(fn, encoding=encoding) as fd:
766801
yield fd
767802

768803

@@ -777,7 +812,7 @@ def compute_hash(localfn):
777812
Typically, if you wish to write a test that requires a particular data
778813
file, you will want to submit that file to the astropy data servers, and
779814
use
780-
e.g. ``get_pkg_data_filename('hash/a725fa6ba642587436612c2df0451956')``,
815+
e.g. ``get_pkg_data_filename('hash/34c33b3eb0d56eb9462003af249eff28')``,
781816
but with the hash for your file in place of the hash in the example.
782817
783818
Parameters
@@ -803,30 +838,33 @@ def compute_hash(localfn):
803838
return h.hexdigest()
804839

805840

806-
def _find_pkg_data_path(data_name):
841+
def _find_pkg_data_path(data_name, package=None):
807842
"""
808843
Look for data in the source-included data directories and return the
809844
path.
810845
"""
811846

812-
from ..utils import find_current_module
847+
if package is None:
848+
module = find_current_module(1, True)
813849

814-
module = find_current_module(1, True)
815-
if module is None:
816-
# not called from inside an astropy package. So just pass name through
817-
return data_name
850+
if module is None:
851+
# not called from inside an astropy package. So just pass name
852+
# through
853+
return data_name
818854

819-
if not hasattr(module, '__package__') or not module.__package__:
820-
# The __package__ attribute may be missing or set to None; see PEP-366,
821-
# also astropy issue #1256
822-
if '.' in module.__name__:
823-
pkgname = module.__name__.rpartition('.')[0]
855+
if not hasattr(module, '__package__') or not module.__package__:
856+
# The __package__ attribute may be missing or set to None; see
857+
# PEP-366, also astropy issue #1256
858+
if '.' in module.__name__:
859+
package = module.__name__.rpartition('.')[0]
860+
else:
861+
package = module.__name__
824862
else:
825-
pkgname = module.__name__
863+
package = module.__package__
826864
else:
827-
pkgname = module.__package__
865+
module = resolve_name(package)
828866

829-
rootpkgname = pkgname.partition('.')[0]
867+
rootpkgname = package.partition('.')[0]
830868

831869
rootpkg = resolve_name(rootpkgname)
832870

0 commit comments

Comments
 (0)