Skip to content

Commit e526694

Browse files
committed
io.fits: allow limiting the number of read extensions
In the convenience functions only read as many extensions as necessary to satisfy the users request. This can greatly increase performance when one is only interested in an early subset of extensions. It adds the `_last_read_ext` keyword to the fitsopen function so it will also be accepted by the top level fits.open function, but this is an implementational detail and not documented. Closes astropygh-4589
1 parent f44e7b4 commit e526694

3 files changed

Lines changed: 92 additions & 13 deletions

File tree

‎astropy/io/fits/convenience.py‎

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,16 @@ def getdata(filename, *args, **kwargs):
191191
upper = kwargs.pop('upper', None)
192192
view = kwargs.pop('view', None)
193193

194-
hdulist, extidx = _getext(filename, mode, *args, **kwargs)
194+
# TODO this can be simplified as _getext when hdulist becomes lazy
195+
extidx = _parse_ext(args, kwargs)
196+
# return extension 1 data if extension 0 is empty
197+
# so always load at least 2 extensions
198+
if _is_int(extidx) and extidx == 0:
199+
loadext = 1
200+
else:
201+
loadext = extidx
202+
203+
hdulist = fitsopen(filename, mode=mode, _last_read_ext=loadext, **kwargs)
195204
try:
196205
hdu = hdulist[extidx]
197206
data = hdu.data
@@ -803,12 +812,13 @@ def tableload(datafile, cdfile, hfile=None):
803812
tableload.__doc__ += BinTableHDU._tdump_file_format.replace('\n', '\n ')
804813

805814

806-
def _getext(filename, mode, *args, **kwargs):
815+
def _parse_ext(args, kwargs):
807816
"""
808-
Open the input file, return the `HDUList` and the extension.
817+
parse arguments for extension selection
818+
returns integer (extname, integer) tuple
819+
modifies kwargs
809820
810-
This supports several different styles of extension selection. See the
811-
:func:`getdata()` documentation for the different possibilities.
821+
See the :func:`getdata()` documentation for the different possibilities.
812822
"""
813823

814824
ext = kwargs.pop('ext', None)
@@ -872,7 +882,18 @@ def _getext(filename, mode, *args, **kwargs):
872882
elif extver and extname is None:
873883
raise TypeError('extver alone cannot specify an extension.')
874884

875-
hdulist = fitsopen(filename, mode=mode, **kwargs)
885+
return ext
886+
887+
888+
def _getext(filename, mode, *args, **kwargs):
889+
"""
890+
Open the input file, return the `HDUList` and the extension.
891+
892+
This supports several different styles of extension selection. See the
893+
:func:`getdata()` documentation for the different possibilities.
894+
"""
895+
ext = _parse_ext(args, kwargs)
896+
hdulist = fitsopen(filename, mode=mode, _last_read_ext=ext, **kwargs)
876897

877898
return hdulist, ext
878899

‎astropy/io/fits/hdu/hdulist.py‎

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -520,17 +520,11 @@ def index_of(self, key):
520520

521521
if not isinstance(_key, string_types):
522522
raise KeyError(key)
523-
_key = (_key.strip()).upper()
524523

525524
nfound = 0
526525
found = None
527526
for idx, hdu in enumerate(self):
528-
name = hdu.name
529-
if isinstance(name, string_types):
530-
name = name.strip().upper()
531-
# 'PRIMARY' should always work as a reference to the first HDU
532-
if ((name == _key or (_key == 'PRIMARY' and idx == 0)) and
533-
(_ver is None or _ver == hdu.ver)):
527+
if self._ext_match((_key, _ver), hdu, idx):
534528
found = idx
535529
nfound += 1
536530

@@ -796,6 +790,22 @@ def filename(self):
796790
return self._file.name
797791
return None
798792

793+
@classmethod
794+
def _ext_match(cls, ext, hdu, idx):
795+
if _is_int(ext):
796+
return ext == idx
797+
# it has to be a tuple now
798+
extname, extver = ext
799+
if extname is None:
800+
return False
801+
hduname = hdu.name
802+
extname = extname.strip().upper()
803+
if isinstance(hduname, string_types):
804+
hduname = hduname.strip().upper()
805+
# 'PRIMARY' should always work as a reference to the first HDU
806+
return ((hduname == extname or (extname == 'PRIMARY' and idx == 0)) and
807+
(extver is None or extver == hdu.ver))
808+
799809
@classmethod
800810
def _readfrom(cls, fileobj=None, data=None, mode=None,
801811
memmap=None, save_backup=False, cache=True, **kwargs):
@@ -830,6 +840,10 @@ def _readfrom(cls, fileobj=None, data=None, mode=None,
830840

831841
saved_compression_enabled = compressed.COMPRESSION_ENABLED
832842

843+
# (extname, extver) tuple used by convenience functions to only load as
844+
# much as necessary
845+
last_read_ext = kwargs.pop("_last_read_ext", None)
846+
idx = 0
833847
try:
834848
if ('disable_image_compression' in kwargs and
835849
kwargs['disable_image_compression']):
@@ -874,6 +888,12 @@ def _readfrom(cls, fileobj=None, data=None, mode=None,
874888
del exc
875889
break
876890

891+
# loaded as much as required, stop loading more
892+
if (last_read_ext is not None and
893+
HDUList._ext_match(last_read_ext, hdu, idx)):
894+
break
895+
idx += 1
896+
877897
# If we're trying to read only and no header units were found,
878898
# raise and exception
879899
if mode in ('readonly', 'denywrite') and len(hdulist) == 0:

‎astropy/io/fits/tests/test_convenience.py‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,44 @@ def test_resource_warning(self):
2727
header = fits.getheader(self.data('test0.fits'))
2828
assert len(w) == 0
2929

30+
def test_getdata(self):
31+
"""
32+
Basic getdata tests
33+
"""
34+
f = self.data('test0.fits')
35+
# empty primary -> getdata loads ext 1
36+
d = fits.getdata(f)
37+
d2 = fits.getdata(f, 0)
38+
np.testing.assert_array_equal(d, d2)
39+
d2 = fits.getdata(f, 1)
40+
np.testing.assert_array_equal(d, d2)
41+
d2 = fits.getdata(f, ext=0)
42+
np.testing.assert_array_equal(d, d2)
43+
d2 = fits.getdata(f, ext=1)
44+
np.testing.assert_array_equal(d, d2)
45+
d2 = fits.getdata(f, ext=('SCI', 1))
46+
np.testing.assert_array_equal(d, d2)
47+
d2 = fits.getdata(f, extname='SCI')
48+
np.testing.assert_array_equal(d, d2)
49+
# extname case, leading and trailing space does not matter
50+
d2 = fits.getdata(f, extname=' Sci ')
51+
np.testing.assert_array_equal(d, d2)
52+
d2 = fits.getdata(f, extname='SCI', extver=1)
53+
np.testing.assert_array_equal(d, d2)
54+
55+
d2 = fits.getdata(f, ext=2)
56+
assert (not np.array_equal(d, d2))
57+
58+
np.testing.assert_array_equal(fits.getdata(f, ext=('SCI', 2)),
59+
fits.getdata(f, ext=2))
60+
61+
np.testing.assert_array_equal(fits.getdata(f, ext=('SCI', 3)),
62+
fits.getdata(f, ext=3))
63+
64+
pytest.raises(IndexError, fits.getdata, f, 5)
65+
pytest.raises(IndexError, fits.getdata, f, 'PRIMARY')
66+
67+
3068
def test_fileobj_not_closed(self):
3169
"""
3270
Tests that file-like objects are not closed after being passed

0 commit comments

Comments
 (0)