Skip to content

Commit f096868

Browse files
committed
Merge pull request #5065 from embray/fits/lazy-load-hdus
Lazy loading of HDUs on fits.open
1 parent a45232b commit f096868

4 files changed

Lines changed: 416 additions & 122 deletions

File tree

‎astropy/io/fits/__init__.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,12 @@ class Conf(_config.ConfigNamespace):
4646
'FITS files. This generally provides better performance, especially '
4747
'for large files, but may affect performance in I/O-heavy '
4848
'applications.')
49+
lazy_load_hdus = _config.ConfigItem(
50+
True,
51+
'If True, use lazy loading of HDUs when opening FITS files by '
52+
'default; that is fits.open() will only seek for and read HDUs on '
53+
'demand rather than reading all HDUs at once. See the documentation '
54+
'for fits.open() for more datails.')
4955
enable_uint = _config.ConfigItem(
5056
True,
5157
'If True, default to recognizing the convention for representing '

‎astropy/io/fits/convenience.py‎

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,14 @@ def getheader(filename, *args, **kwargs):
110110
hdu = hdulist[extidx]
111111
header = hdu.header
112112
finally:
113-
hdulist.close(closed=closed)
113+
# Use _close instead of close to close without loading any
114+
# remaining HDUs for pre-lazy-loading backwards compatibility
115+
# In other words, when the full HDUList is opened by a user they
116+
# previously expected to be able to look at arbitrary HDUs even
117+
# after the file was closed, but for getheader and other convenience
118+
# functions this is irrelevant
119+
hdulist._close(closed=closed)
120+
114121
return header
115122

116123

@@ -206,7 +213,8 @@ def getdata(filename, *args, **kwargs):
206213
if header:
207214
hdr = hdu.header
208215
finally:
209-
hdulist.close(closed=closed)
216+
# _close instead of close; see note in getheader
217+
hdulist._close(closed=closed)
210218

211219
# Change case of names if requested
212220
trans = None
@@ -342,7 +350,8 @@ def setval(filename, keyword, *args, **kwargs):
342350
comment = None
343351
hdulist[extidx].header.set(keyword, value, comment, before, after)
344352
finally:
345-
hdulist.close(closed=closed)
353+
# _close instead of close; see note in getheader
354+
hdulist._close(closed=closed)
346355

347356

348357
def delval(filename, keyword, *args, **kwargs):
@@ -380,7 +389,8 @@ def delval(filename, keyword, *args, **kwargs):
380389
try:
381390
del hdulist[extidx].header[keyword]
382391
finally:
383-
hdulist.close(closed=closed)
392+
# _close instead of close; see note in getheader
393+
hdulist._close(closed=closed)
384394

385395

386396
@deprecated_renamed_argument('clobber', 'overwrite', '1.3')
@@ -586,14 +596,15 @@ def append(filename, data, header=None, checksum=False, verify=True, **kwargs):
586596
# when writing the file.
587597
hdu._output_checksum = checksum
588598
finally:
589-
f.close(closed=closed)
599+
# _close instead of close; see note in getheader
600+
f._close(closed=closed)
590601
else:
591602
f = _File(filename, mode='append')
592603
try:
593604
hdu._output_checksum = checksum
594605
hdu._writeto(f)
595606
finally:
596-
f.close()
607+
f._close()
597608

598609

599610
def update(filename, data, *args, **kwargs):
@@ -652,7 +663,8 @@ def update(filename, data, *args, **kwargs):
652663
try:
653664
hdulist[_ext] = new_hdu
654665
finally:
655-
hdulist.close(closed=closed)
666+
# _close instead of close; see note in getheader
667+
hdulist._close(closed=closed)
656668

657669

658670
def info(filename, output=None, **kwargs):
@@ -689,7 +701,8 @@ def info(filename, output=None, **kwargs):
689701
ret = f.info(output=output)
690702
finally:
691703
if closed:
692-
f.close()
704+
# _close instead of close; see note in getheader
705+
f._close()
693706

694707
return ret
695708

@@ -759,7 +772,8 @@ def tabledump(filename, datafile=None, cdfile=None, hfile=None, ext=1,
759772
f[ext].dump(datafile, cdfile, hfile, overwrite)
760773
finally:
761774
if closed:
762-
f.close()
775+
# _close instead of close; see note in getheader
776+
f._close()
763777

764778
if isinstance(tabledump.__doc__, string_types):
765779
tabledump.__doc__ += BinTableHDU._tdump_file_format.replace('\n', '\n ')

‎astropy/io/fits/file.py‎

Lines changed: 36 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -252,37 +252,42 @@ def readarray(self, size=None, offset=0, dtype=np.uint8, shape=None):
252252
raise ValueError('size {} is too many bytes for a {} array of '
253253
'{}'.format(size, shape, dtype))
254254

255-
if self.memmap:
256-
if self._mmap is None:
257-
# Instantiate Memmap array of the file offset at 0
258-
# (so we can return slices of it to offset anywhere else into
259-
# the file)
260-
memmap = Memmap(self._file,
261-
mode=MEMMAP_MODES[self.mode],
262-
dtype=np.uint8)
263-
264-
# Now we immediately discard the memmap array; we are really
265-
# just using it as a factory function to instantiate the mmap
266-
# object in a convenient way (may later do away with this
267-
# usage)
268-
self._mmap = memmap.base
269-
270-
# Prevent dorking with self._memmap._mmap by memmap.__del__ in
271-
# Numpy 1.6 (see
272-
# https://github.com/numpy/numpy/commit/dcc355a0b179387eeba10c95baf2e1eb21d417c7)
273-
memmap._mmap = None
274-
del memmap
275-
276-
return np.ndarray(shape=shape, dtype=dtype, offset=offset,
277-
buffer=self._mmap)
278-
else:
279-
count = reduce(operator.mul, shape)
280-
pos = self._file.tell()
281-
self._file.seek(offset)
282-
data = _array_from_file(self._file, dtype, count, '')
283-
data.shape = shape
284-
self._file.seek(pos)
285-
return data
255+
filepos = self._file.tell()
256+
257+
try:
258+
if self.memmap:
259+
if self._mmap is None:
260+
# Instantiate Memmap array of the file offset at 0 (so we
261+
# can return slices of it to offset anywhere else into the
262+
# file)
263+
memmap = Memmap(self._file, mode=MEMMAP_MODES[self.mode],
264+
dtype=np.uint8)
265+
266+
# Now we immediately discard the memmap array; we are
267+
# really just using it as a factory function to instantiate
268+
# the mmap object in a convenient way (may later do away
269+
# with this usage)
270+
self._mmap = memmap.base
271+
272+
# Prevent dorking with self._memmap._mmap by memmap.__del__
273+
# in Numpy 1.6 (see
274+
# https://github.com/numpy/numpy/commit/dcc355a0b179387eeba10c95baf2e1eb21d417c7)
275+
memmap._mmap = None
276+
del memmap
277+
278+
return np.ndarray(shape=shape, dtype=dtype, offset=offset,
279+
buffer=self._mmap)
280+
else:
281+
count = reduce(operator.mul, shape)
282+
self._file.seek(offset)
283+
data = _array_from_file(self._file, dtype, count, '')
284+
data.shape = shape
285+
return data
286+
finally:
287+
# Make sure we leave the file in the position we found it; on
288+
# some platforms (e.g. Windows) mmaping a file handle can also
289+
# reset its file pointer
290+
self._file.seek(filepos)
286291

287292
def writable(self):
288293
if self.readonly:

0 commit comments

Comments
 (0)