Skip to content

Commit faacd41

Browse files
committed
Remove compatibility code which forced loading all HDUs on close
Closes astropy#5582. astropy#5065 implemented lazy loading of HDUs in fits files. But it did it in a way that maintains backwards compatibility at the expense of performance, loading the remaining HDUs to allow read the headers after the file was closed. Basically this reverts 5c66c2d
1 parent 65fea6f commit faacd41

4 files changed

Lines changed: 48 additions & 92 deletions

File tree

‎CHANGES.rst‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,14 @@ API Changes
142142

143143
- ``astropy.io.fits``
144144

145+
- Angstrom, erg, G, and barn are no more reported as deprecated FITS units.
146+
[#5929]
147+
148+
- Remove compatibility code which forced loading all HDUs on close. The old
149+
behavior can be used with ``lazy_load_hdus=False``. Because of this change,
150+
trying to access the ``.data`` attribute from an HDU which is not loaded
151+
now raises a ``IndexError`` instead of a ``ValueError``. [#6082]
152+
145153
- ``astropy.io.misc``
146154

147155
- ``astropy.io.registry``

‎astropy/io/fits/convenience.py‎

Lines changed: 12 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -113,13 +113,7 @@ def getheader(filename, *args, **kwargs):
113113
hdu = hdulist[extidx]
114114
header = hdu.header
115115
finally:
116-
# Use _close instead of close to close without loading any
117-
# remaining HDUs for pre-lazy-loading backwards compatibility
118-
# In other words, when the full HDUList is opened by a user they
119-
# previously expected to be able to look at arbitrary HDUs even
120-
# after the file was closed, but for getheader and other convenience
121-
# functions this is irrelevant
122-
hdulist._close(closed=closed)
116+
hdulist.close(closed=closed)
123117

124118
return header
125119

@@ -216,8 +210,7 @@ def getdata(filename, *args, **kwargs):
216210
if header:
217211
hdr = hdu.header
218212
finally:
219-
# _close instead of close; see note in getheader
220-
hdulist._close(closed=closed)
213+
hdulist.close(closed=closed)
221214

222215
# Change case of names if requested
223216
trans = None
@@ -353,8 +346,7 @@ def setval(filename, keyword, *args, **kwargs):
353346
comment = None
354347
hdulist[extidx].header.set(keyword, value, comment, before, after)
355348
finally:
356-
# _close instead of close; see note in getheader
357-
hdulist._close(closed=closed)
349+
hdulist.close(closed=closed)
358350

359351

360352
def delval(filename, keyword, *args, **kwargs):
@@ -392,8 +384,7 @@ def delval(filename, keyword, *args, **kwargs):
392384
try:
393385
del hdulist[extidx].header[keyword]
394386
finally:
395-
# _close instead of close; see note in getheader
396-
hdulist._close(closed=closed)
387+
hdulist.close(closed=closed)
397388

398389

399390
@deprecated_renamed_argument('clobber', 'overwrite', '1.3', pending=True)
@@ -607,15 +598,14 @@ def append(filename, data, header=None, checksum=False, verify=True, **kwargs):
607598
# when writing the file.
608599
hdu._output_checksum = checksum
609600
finally:
610-
# _close instead of close; see note in getheader
611-
f._close(closed=closed)
601+
f.close(closed=closed)
612602
else:
613603
f = _File(filename, mode='append')
614604
try:
615605
hdu._output_checksum = checksum
616606
hdu._writeto(f)
617607
finally:
618-
f._close()
608+
f.close()
619609

620610

621611
def update(filename, data, *args, **kwargs):
@@ -674,8 +664,7 @@ def update(filename, data, *args, **kwargs):
674664
try:
675665
hdulist[_ext] = new_hdu
676666
finally:
677-
# _close instead of close; see note in getheader
678-
hdulist._close(closed=closed)
667+
hdulist.close(closed=closed)
679668

680669

681670
def info(filename, output=None, **kwargs):
@@ -712,8 +701,7 @@ def info(filename, output=None, **kwargs):
712701
ret = f.info(output=output)
713702
finally:
714703
if closed:
715-
# _close instead of close; see note in getheader
716-
f._close()
704+
f.close()
717705

718706
return ret
719707

@@ -795,7 +783,7 @@ def printdiff(inputa, inputb, *args, **kwargs):
795783
try:
796784
hdulistb, extidxb = _getext(inputb, modeb, *args, **extension)
797785
except Exception:
798-
hdulista._close(closed=closeda)
786+
hdulista.close(closed=closeda)
799787
raise
800788

801789
try:
@@ -805,9 +793,8 @@ def printdiff(inputa, inputb, *args, **kwargs):
805793
print(HDUDiff(hdua, hdub, **kwargs).report())
806794

807795
finally:
808-
# See note about _close in getheader function
809-
hdulista._close(closed=closeda)
810-
hdulistb._close(closed=closedb)
796+
hdulista.close(closed=closeda)
797+
hdulistb.close(closed=closedb)
811798

812799
# If input is not a string, can feed HDU objects or HDUList directly,
813800
# but can't currently handle extensions
@@ -893,8 +880,7 @@ def tabledump(filename, datafile=None, cdfile=None, hfile=None, ext=1,
893880
f[ext].dump(datafile, cdfile, hfile, overwrite)
894881
finally:
895882
if closed:
896-
# _close instead of close; see note in getheader
897-
f._close()
883+
f.close()
898884

899885
if isinstance(tabledump.__doc__, string_types):
900886
tabledump.__doc__ += BinTableHDU._tdump_file_format.replace('\n', '\n ')

‎astropy/io/fits/hdu/hdulist.py‎

Lines changed: 3 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
import os
88
import shutil
99
import sys
10-
import textwrap
1110
import warnings
1211

1312
import numpy as np
@@ -216,10 +215,6 @@ def __init__(self, hdus=[], file=None):
216215
else:
217216
self._read_all = False
218217

219-
# This is used for book-keeping for backwards compatibility of
220-
# accessing unread HDUs after the file has been closed.
221-
self._total_read = None
222-
223218
if hdus is None:
224219
hdus = []
225220

@@ -900,37 +895,8 @@ def close(self, output_verify='exception', verbose=False, closed=True):
900895
When `True`, close the underlying file object.
901896
"""
902897

903-
# If the underlying file object is not closed then we can still
904-
# technically lazy-load HDUs
905-
906-
if closed:
907-
# Get the actual current number of HDUs that have been loaded
908-
total_read = list.__len__(self)
909-
910-
# Now read in any remaining HDUs
911-
while self._read_next_hdu():
912-
pass
913-
914898
try:
915-
self._close(output_verify=output_verify, verbose=verbose,
916-
closed=closed)
917-
finally:
918-
# This will cause deprecation warnings on any future attempt
919-
# to access HDUs that were not loaded before closing
920-
if closed:
921-
self._total_read = total_read
922-
923-
def _close(self, output_verify='exception', verbose=False, closed=True):
924-
"""
925-
Internal implementation of close() that does not allow further
926-
HDUs to be loaded.
927-
928-
The HDU pre-loading behavior of the explicit close() is currently
929-
maintained only for backwards-compatibility.
930-
"""
931-
932-
try:
933-
if (self._file and self._file.mode in ['append', 'update']
899+
if (self._file and self._file.mode in ('append', 'update')
934900
and not self._file.closed):
935901
self.flush(output_verify=output_verify, verbose=verbose)
936902
finally:
@@ -1057,45 +1023,17 @@ def _readfrom(cls, fileobj=None, data=None, mode=None,
10571023

10581024
return hdulist
10591025

1060-
def _try_while_unread_hdus(self, func, index, *args, **kwargs):
1026+
def _try_while_unread_hdus(self, func, *args, **kwargs):
10611027
"""
10621028
Attempt an operation that accesses an HDU by index/name
10631029
that can fail if not all HDUs have been read yet. Keep
10641030
reading HDUs until the operation succeeds or there are no
10651031
more HDUs to read.
1066-
1067-
The first argument must always be the index of some HDU in the
1068-
HDUList, but additional arguments may be passed on.
10691032
"""
10701033

1071-
# TODO: This is a temporary hack for reporting the deprecation
1072-
# warning on the pre-lazy-loading behavior of allowing HDUs to
1073-
# be accessed after the file was closed. It can be removed once
1074-
# the deprecation period has passed.
1075-
if self._file and self._file.closed and self._total_read is not None:
1076-
if index > self._total_read - 1:
1077-
warnings.warn(textwrap.dedent(
1078-
"""\
1079-
Accessing an HDU after an HDUList is closed, where
1080-
that HDU was no read while the HDUList was open
1081-
is deprecated. That is, you did something like:
1082-
1083-
>>> hdulist.close()
1084-
>>> print(hdulist[2].header)
1085-
1086-
even though hdulist[2] had not been read yet. Instead
1087-
do:
1088-
1089-
>>> print(hdulist[2].header)
1090-
>>> hdulist.close()
1091-
1092-
or open the file with lazy_load_hdus=False to read all
1093-
the HDUs into memory immediately upon opening the file.
1094-
"""), AstropyDeprecationWarning)
1095-
10961034
while True:
10971035
try:
1098-
return func(index, *args, **kwargs)
1036+
return func(*args, **kwargs)
10991037
except Exception:
11001038
if self._read_next_hdu():
11011039
continue

‎astropy/io/fits/tests/test_image.py‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ def test_constructor_copies_header(self):
7373
# Original header should be unchanged
7474
assert phdr['FILENAME'] == 'labq01i3q_rawtag.fits'
7575

76-
@raises(ValueError)
76+
@raises(IndexError)
7777
def test_open(self):
7878
# The function "open" reads a FITS file into an HDUList object. There
7979
# are three modes to open: "readonly" (the default), "append", and
@@ -100,6 +100,30 @@ def test_open_2(self):
100100
finally:
101101
r.close()
102102

103+
def test_open_3(self):
104+
# Test that HDUs cannot be accessed after the file was closed
105+
r = fits.open(self.data('test0.fits'))
106+
r.close()
107+
with pytest.raises(IndexError):
108+
r[1]
109+
110+
# Test that HDUs can be accessed with lazy_load_hdus=False
111+
r = fits.open(self.data('test0.fits'), lazy_load_hdus=False)
112+
r.close()
113+
assert isinstance(r[1], fits.ImageHDU)
114+
assert len(r) == 5
115+
116+
# And the same with the global config item
117+
assert fits.conf.lazy_load_hdus # True by default
118+
fits.conf.lazy_load_hdus = False
119+
try:
120+
r = fits.open(self.data('test0.fits'))
121+
r.close()
122+
assert isinstance(r[1], fits.ImageHDU)
123+
assert len(r) == 5
124+
finally:
125+
fits.conf.lazy_load_hdus = True
126+
103127
def test_primary_with_extname(self):
104128
"""Regression test for https://aeon.stsci.edu/ssb/trac/pyfits/ticket/151
105129

0 commit comments

Comments
 (0)