Skip to content

Commit e74726b

Browse files
saimnsaimn
committed
Add a fast Header parsing for the HDUs creation
This allows to quickly parse headers (only the standard keywords), just to create the HDU objects. Then, if the .header attribute is accessed, a normal Header is created.
1 parent 33db190 commit e74726b

5 files changed

Lines changed: 129 additions & 17 deletions

File tree

‎astropy/io/fits/hdu/base.py‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
# Licensed under a 3-clause BSD style license - see PYFITS.rst
22

3-
4-
53
import datetime
64
import os
75
import sys
@@ -13,10 +11,10 @@
1311

1412
from astropy.io.fits import conf
1513
from astropy.io.fits.file import _File
16-
from astropy.io.fits.header import Header, _pad_length
17-
from astropy.io.fits.util import (_is_int, _is_pseudo_unsigned, _unsigned_zero,
18-
itersubclasses, decode_ascii, _get_array_mmap, first,
19-
_free_space_check, _extract_number)
14+
from astropy.io.fits.header import Header, BasicHeader, _pad_length
15+
from astropy.io.fits.util import (
16+
_is_int, _is_pseudo_unsigned, _unsigned_zero, itersubclasses, decode_ascii,
17+
_get_array_mmap, first, _free_space_check, _extract_number)
2018
from astropy.io.fits.verify import _Verify, _ErrList
2119

2220
from astropy.utils import lazyproperty
@@ -68,6 +66,7 @@ def _hdu_class_from_header(cls, header):
6866
if not (c.__module__.startswith('astropy.io.fits.') or
6967
c in cls._hdu_registry):
7068
continue
69+
# print('try', c)
7170
if c.match_header(header):
7271
klass = c
7372
break
@@ -154,6 +153,7 @@ def __init__(self, data=None, header=None, *args, **kwargs):
154153
if header is None:
155154
header = Header()
156155
self._header = header
156+
self._header_str = None
157157
self._file = None
158158
self._buffer = None
159159
self._header_offset = None
@@ -176,6 +176,9 @@ def __init__(self, data=None, header=None, *args, **kwargs):
176176

177177
@property
178178
def header(self):
179+
if self._header is None and self._header_str is not None:
180+
self._header = Header.fromstring(self._header_str)
181+
self._header_str = None
179182
return self._header
180183

181184
@header.setter
@@ -390,7 +393,12 @@ def _readfrom_internal(cls, data, header=None, checksum=False,
390393
if isinstance(data, _File):
391394
if header is None:
392395
header_offset = data.tell()
393-
header = Header.fromfile(data, endcard=not ignore_missing_end)
396+
try:
397+
header_str, header = BasicHeader.fromfile(data)
398+
except Exception:
399+
data.seek(header_offset)
400+
header = Header.fromfile(data,
401+
endcard=not ignore_missing_end)
394402
hdu_fileobj = data
395403
data_offset = data.tell() # *after* reading the header
396404
else:
@@ -457,6 +465,10 @@ def block_iter(nbytes):
457465
if checksum and checksum != 'remove' and isinstance(hdu, _ValidHDU):
458466
hdu._verify_checksum_datasum()
459467

468+
if isinstance(hdu._header, BasicHeader):
469+
hdu._header = None
470+
hdu._header_str = header_str
471+
460472
return hdu
461473

462474
def _get_raw_data(self, shape, code, offset):

‎astropy/io/fits/hdu/compressed.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1471,6 +1471,10 @@ def header(self):
14711471
if hasattr(self, '_image_header'):
14721472
return self._image_header
14731473

1474+
if self._header is None and self._header_str is not None:
1475+
self._header = Header.fromstring(self._header_str)
1476+
self._header_str = None
1477+
14741478
# Start with a copy of the table header.
14751479
image_header = self._header.copy()
14761480

‎astropy/io/fits/hdu/image.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
import numpy as np
77

88
from .base import DELAYED, _ValidHDU, ExtensionHDU, BITPIX2DTYPE, DTYPE2BITPIX
9-
from astropy.io.fits.header import Header
9+
from astropy.io.fits.header import Header, BasicHeader
1010
from astropy.io.fits.util import _is_pseudo_unsigned, _unsigned_zero, _is_int
1111
from astropy.io.fits.verify import VerifyWarning
1212

@@ -43,7 +43,7 @@ def __init__(self, data=None, header=None, do_not_scale_image_data=False,
4343
super().__init__(data=data, header=header)
4444

4545
if header is not None:
46-
if not isinstance(header, Header):
46+
if not isinstance(header, (Header, BasicHeader)):
4747
# TODO: Instead maybe try initializing a new Header object from
4848
# whatever is passed in as the header--there are various types
4949
# of objects that could work for this...
@@ -213,6 +213,9 @@ def shape(self):
213213

214214
@property
215215
def header(self):
216+
if self._header is None and self._header_str is not None:
217+
self._header = Header.fromstring(self._header_str)
218+
self._header_str = None
216219
return self._header
217220

218221
@header.setter

‎astropy/io/fits/hdu/table.py‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,13 @@
1818
# This module may have many dependencies on astropy.io.fits.column, but
1919
# astropy.io.fits.column has fewer dependencies overall, so it's easier to
2020
# keep table/column-related utilities in astropy.io.fits.column
21-
from astropy.io.fits.column import (FITS2NUMPY, KEYWORD_NAMES, KEYWORD_TO_ATTRIBUTE,
22-
ATTRIBUTE_TO_KEYWORD, TDEF_RE, Column, ColDefs,
23-
_AsciiColDefs, _FormatP, _FormatQ, _makep,
24-
_parse_tformat, _scalar_to_format, _convert_format,
25-
_cmp_recformats)
21+
from astropy.io.fits.column import (
22+
FITS2NUMPY, KEYWORD_NAMES, KEYWORD_TO_ATTRIBUTE, ATTRIBUTE_TO_KEYWORD,
23+
TDEF_RE, Column, ColDefs, _AsciiColDefs, _FormatP, _FormatQ, _makep,
24+
_parse_tformat, _scalar_to_format, _convert_format, _cmp_recformats
25+
)
2626
from astropy.io.fits.fitsrec import FITS_rec, _get_recarray_field, _has_unicode_fields
27-
from astropy.io.fits.header import Header, _pad_length
27+
from astropy.io.fits.header import Header, BasicHeader, _pad_length
2828
from astropy.io.fits.util import _is_int, _str_to_num
2929

3030
from astropy.utils import lazyproperty
@@ -277,7 +277,8 @@ def __init__(self, data=None, header=None, name=None, uint=False, ver=None,
277277

278278
super().__init__(data=data, header=header, name=name, ver=ver)
279279

280-
if header is not None and not isinstance(header, Header):
280+
if header is not None and \
281+
not isinstance(header, (BasicHeader, Header)):
281282
raise ValueError('header must be a Header object.')
282283

283284
self._uint = uint

‎astropy/io/fits/header.py‎

Lines changed: 93 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
import re
77
import warnings
88

9-
from .card import Card, _pad, KEYWORD_LENGTH, UNDEFINED
9+
from .card import Card, _pad, KEYWORD_LENGTH, UNDEFINED, VALUE_INDICATOR, CARD_LENGTH
1010
from .file import _File
1111
from .util import encode_ascii, decode_ascii, fileobj_closed, fileobj_is_binary
1212

@@ -1928,6 +1928,98 @@ def _add_commentary(self, key, value, before=None, after=None):
19281928
collections.abc.MutableMapping.register(Header)
19291929

19301930

1931+
class BasicHeaderCards:
1932+
1933+
def __init__(self, header):
1934+
self.header = header
1935+
1936+
def __getitem__(self, key):
1937+
key = self.header._keys[key]
1938+
try:
1939+
return self.header._cards[key]
1940+
except KeyError:
1941+
card = Card.fromstring(self.header._raw_cards[key])
1942+
self.header._cards[key] = card
1943+
return card
1944+
1945+
1946+
class BasicHeader(collections.abc.Mapping):
1947+
1948+
def __init__(self, cards):
1949+
self._raw_cards = cards
1950+
self._keys = list(cards.keys())
1951+
self._cards = {}
1952+
self.cards = BasicHeaderCards(self)
1953+
1954+
def __getitem__(self, key):
1955+
if isinstance(key, int):
1956+
key = self._keys[key]
1957+
1958+
try:
1959+
return self._cards[key].value
1960+
except KeyError:
1961+
card = Card.fromstring(self._raw_cards[key])
1962+
self._cards[key] = card
1963+
return card.value
1964+
1965+
def __len__(self):
1966+
return len(self._raw_cards)
1967+
1968+
def __iter__(self):
1969+
return iter(self._raw_cards)
1970+
1971+
@classmethod
1972+
def fromfile(cls, fileobj):
1973+
1974+
close_file = False
1975+
if isinstance(fileobj, str):
1976+
fileobj = open(fileobj, 'rb')
1977+
close_file = True
1978+
1979+
cards = {}
1980+
read_blocks = []
1981+
found_end = False
1982+
1983+
try:
1984+
while True:
1985+
block = fileobj.read(BLOCK_SIZE)
1986+
if not block or len(block) < BLOCK_SIZE:
1987+
raise Exception
1988+
1989+
block = decode_ascii(block)
1990+
read_blocks.append(block)
1991+
idx = 0
1992+
while idx < BLOCK_SIZE:
1993+
end_idx = idx + CARD_LENGTH
1994+
card_image = block[idx:end_idx]
1995+
idx = end_idx
1996+
1997+
keyword = card_image[:8].strip()
1998+
keyword_upper = keyword.upper()
1999+
sep_idx = card_image.find(VALUE_INDICATOR)
2000+
2001+
# We are interested only in standard keyword, so we skip
2002+
# other cards, e.G. CONTINUE, HIERARCH, COMMENT.
2003+
if sep_idx == 8:
2004+
# ok, found standard keyword
2005+
cards[keyword_upper] = card_image
2006+
elif 0 < sep_idx < 8:
2007+
keyword = card_image[:sep_idx]
2008+
cards[keyword_upper] = card_image
2009+
elif card_image == END_CARD:
2010+
found_end = True
2011+
break
2012+
2013+
if found_end:
2014+
break
2015+
2016+
header_str = ''.join(read_blocks)
2017+
return header_str, cls(cards)
2018+
finally:
2019+
if close_file:
2020+
fileobj.close()
2021+
2022+
19312023
class _CardAccessor:
19322024
"""
19332025
This is a generic class for wrapping a Header in such a way that you can

0 commit comments

Comments
 (0)