Skip to content

Commit 49ace64

Browse files
committed
io.fits: allow limiting the number of read extensions
In the convenience functions only read as many extensions as necessary to satisfy the users request. This can greatly increase performance when one is only interested in an early subset of extensions. It adds the `_last_read_ext` keyword to the fitsopen function so it will also be accepted by the top level fits.open function, but this is an implementational detail and not documented. Closes astropygh-4589
1 parent f44e7b4 commit 49ace64

3 files changed

Lines changed: 87 additions & 12 deletions

File tree

‎astropy/io/fits/convenience.py‎

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,16 @@ def getdata(filename, *args, **kwargs):
191191
upper = kwargs.pop('upper', None)
192192
view = kwargs.pop('view', None)
193193

194-
hdulist, extidx = _getext(filename, mode, *args, **kwargs)
194+
# TODO this can be simplified as _getext when hdulist becomes lazy
195+
extidx = _parse_ext(args, kwargs)
196+
# return extension 1 data if extension 0 is empty
197+
# so always load at least 2 extensions
198+
if _is_int(extidx) and extidx == 0:
199+
loadext = 1
200+
else:
201+
loadext = extidx
202+
203+
hdulist = fitsopen(filename, mode=mode, _last_read_ext=loadext, **kwargs)
195204
try:
196205
hdu = hdulist[extidx]
197206
data = hdu.data
@@ -803,12 +812,13 @@ def tableload(datafile, cdfile, hfile=None):
803812
tableload.__doc__ += BinTableHDU._tdump_file_format.replace('\n', '\n ')
804813

805814

806-
def _getext(filename, mode, *args, **kwargs):
815+
def _parse_ext(args, kwargs):
807816
"""
808-
Open the input file, return the `HDUList` and the extension.
817+
parse arguments for extension selection
818+
returns integer (extname, integer) tuple
819+
modifies kwargs
809820
810-
This supports several different styles of extension selection. See the
811-
:func:`getdata()` documentation for the different possibilities.
821+
See the :func:`getdata()` documentation for the different possibilities.
812822
"""
813823

814824
ext = kwargs.pop('ext', None)
@@ -872,7 +882,18 @@ def _getext(filename, mode, *args, **kwargs):
872882
elif extver and extname is None:
873883
raise TypeError('extver alone cannot specify an extension.')
874884

875-
hdulist = fitsopen(filename, mode=mode, **kwargs)
885+
return ext
886+
887+
888+
def _getext(filename, mode, *args, **kwargs):
889+
"""
890+
Open the input file, return the `HDUList` and the extension.
891+
892+
This supports several different styles of extension selection. See the
893+
:func:`getdata()` documentation for the different possibilities.
894+
"""
895+
ext = _parse_ext(args, kwargs)
896+
hdulist = fitsopen(filename, mode=mode, _last_read_ext=ext, **kwargs)
876897

877898
return hdulist, ext
878899

‎astropy/io/fits/hdu/hdulist.py‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -525,12 +525,7 @@ def index_of(self, key):
525525
nfound = 0
526526
found = None
527527
for idx, hdu in enumerate(self):
528-
name = hdu.name
529-
if isinstance(name, string_types):
530-
name = name.strip().upper()
531-
# 'PRIMARY' should always work as a reference to the first HDU
532-
if ((name == _key or (_key == 'PRIMARY' and idx == 0)) and
533-
(_ver is None or _ver == hdu.ver)):
528+
if self._ext_match((_key, _ver), hdu, idx):
534529
found = idx
535530
nfound += 1
536531

@@ -796,6 +791,22 @@ def filename(self):
796791
return self._file.name
797792
return None
798793

794+
@classmethod
795+
def _ext_match(cls, ext, hdu, idx):
796+
if _is_int(ext):
797+
return ext == idx
798+
# it has to be a tuple now
799+
extname, extver = ext
800+
if extname is None:
801+
return False
802+
extname = extname.upper()
803+
hduname = hdu.name
804+
if isinstance(hduname, string_types):
805+
hduname = hduname.strip().upper()
806+
# 'PRIMARY' should always work as a reference to the first HDU
807+
return ((hduname == extname or (extname == 'PRIMARY' and idx == 0)) and
808+
(extver is None or extver == hdu.ver))
809+
799810
@classmethod
800811
def _readfrom(cls, fileobj=None, data=None, mode=None,
801812
memmap=None, save_backup=False, cache=True, **kwargs):
@@ -830,6 +841,9 @@ def _readfrom(cls, fileobj=None, data=None, mode=None,
830841

831842
saved_compression_enabled = compressed.COMPRESSION_ENABLED
832843

844+
# (extname, extver) tuple
845+
last_read_ext = kwargs.pop("_last_read_ext", None)
846+
idx = 0
833847
try:
834848
if ('disable_image_compression' in kwargs and
835849
kwargs['disable_image_compression']):
@@ -874,6 +888,11 @@ def _readfrom(cls, fileobj=None, data=None, mode=None,
874888
del exc
875889
break
876890

891+
if (last_read_ext is not None and
892+
HDUList._ext_match(last_read_ext, hdu, idx)):
893+
break
894+
idx += 1
895+
877896
# If we're trying to read only and no header units were found,
878897
# raise and exception
879898
if mode in ('readonly', 'denywrite') and len(hdulist) == 0:

‎astropy/io/fits/tests/test_convenience.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,41 @@ def test_resource_warning(self):
2727
header = fits.getheader(self.data('test0.fits'))
2828
assert len(w) == 0
2929

30+
def test_getdata(self):
31+
"""
32+
Basic getdata tests
33+
"""
34+
f = self.data('test0.fits')
35+
# empty primary -> getdata loads ext 1
36+
d = fits.getdata(f)
37+
d2 = fits.getdata(f, 0)
38+
np.testing.assert_array_equal(d, d2)
39+
d2 = fits.getdata(f, 1)
40+
np.testing.assert_array_equal(d, d2)
41+
d2 = fits.getdata(f, ext=0)
42+
np.testing.assert_array_equal(d, d2)
43+
d2 = fits.getdata(f, ext=1)
44+
np.testing.assert_array_equal(d, d2)
45+
d2 = fits.getdata(f, ext=('SCI', 1))
46+
np.testing.assert_array_equal(d, d2)
47+
d2 = fits.getdata(f, extname='SCI')
48+
np.testing.assert_array_equal(d, d2)
49+
d2 = fits.getdata(f, extname='SCI', extver=1)
50+
np.testing.assert_array_equal(d, d2)
51+
52+
d2 = fits.getdata(f, ext=2)
53+
assert (not np.array_equal(d, d2))
54+
55+
np.testing.assert_array_equal(fits.getdata(f, ext=('SCI', 2)),
56+
fits.getdata(f, ext=2))
57+
58+
np.testing.assert_array_equal(fits.getdata(f, ext=('SCI', 3)),
59+
fits.getdata(f, ext=3))
60+
61+
pytest.raises(IndexError, fits.getdata, f, 5)
62+
pytest.raises(IndexError, fits.getdata, f, 'PRIMARY')
63+
64+
3065
def test_fileobj_not_closed(self):
3166
"""
3267
Tests that file-like objects are not closed after being passed

0 commit comments

Comments
 (0)