Skip to content

Commit ef096b9

Browse files
embraypllim
authored andcommitted
Merge pull request astropy#11108 from embray/issue-5408
[astropy#5408] Don't re-verify invalid FITS cards after their values have been replaced
1 parent 02fe1e1 commit ef096b9

3 files changed

Lines changed: 127 additions & 67 deletions

File tree

‎CHANGES.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -319,6 +319,9 @@ astropy.io.ascii
319319
astropy.io.fits
320320
^^^^^^^^^^^^^^^
321321

322+
- Fix bug where manual fixes to invalid header cards were not preserved when
323+
saving a FITS file. [#11108]
324+
322325
astropy.io.misc
323326
^^^^^^^^^^^^^^^
324327

‎astropy/io/fits/card.py‎

Lines changed: 81 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -62,10 +62,15 @@ class Card(_Verify):
6262
# This regex helps delete leading zeros from numbers, otherwise
6363
# Python might evaluate them as octal values (this is not-greedy, however,
6464
# so it may not strip leading zeros from a float, which is fine)
65-
_number_FSC_RE = re.compile(r'(?P<sign>[+-])?0*?(?P<digt>{})'.format(
66-
_digits_FSC))
67-
_number_NFSC_RE = re.compile(r'(?P<sign>[+-])? *0*?(?P<digt>{})'.format(
68-
_digits_NFSC))
65+
_number_FSC_RE = re.compile(rf'(?P<sign>[+-])?0*?(?P<digt>{_digits_FSC})')
66+
_number_NFSC_RE = \
67+
re.compile(rf'(?P<sign>[+-])? *0*?(?P<digt>{_digits_NFSC})')
68+
69+
# Used in cards using the CONTINUE convention which expect a string
70+
# followed by an optional comment
71+
_strg = r'\'(?P<strg>([ -~]+?|\'\'|) *?)\'(?=$|/| )'
72+
_comm_field = r'(?P<comm_field>(?P<sepr>/ *)(?P<comm>(.|\n)*))'
73+
_strg_comment_RE = re.compile(f'({_strg})? *{_comm_field}?')
6974

7075
# FSC commentary card string which must contain printable ASCII characters.
7176
# Note: \Z matches the end of the string without allowing newlines
@@ -96,7 +101,7 @@ class Card(_Verify):
96101
# since a greedy match will find a single-quote after
97102
# the comment separator resulting in an incorrect
98103
# match.
99-
r'\'(?P<strg>([ -~]+?|\'\'|)) *?\'(?=$|/| )|'
104+
rf'{_strg}|'
100105
r'(?P<bool>[FT])|'
101106
r'(?P<numr>' + _numr_FSC + r')|'
102107
r'(?P<cplx>\( *'
@@ -111,17 +116,13 @@ class Card(_Verify):
111116
_value_NFSC_RE = re.compile(
112117
r'(?P<valu_field> *'
113118
r'(?P<valu>'
114-
r'\'(?P<strg>([ -~]+?|\'\'|) *?)\'(?=$|/| )|'
119+
rf'{_strg}|'
115120
r'(?P<bool>[FT])|'
116121
r'(?P<numr>' + _numr_NFSC + r')|'
117122
r'(?P<cplx>\( *'
118123
r'(?P<real>' + _numr_NFSC + r') *, *'
119124
r'(?P<imag>' + _numr_NFSC + r') *\))'
120-
r')? *)'
121-
r'(?P<comm_field>'
122-
r'(?P<sepr>/ *)'
123-
r'(?P<comm>(.|\n)*)'
124-
r')?$')
125+
fr')? *){_comm_field}?$')
125126

126127
_rvkc_identifier = r'[a-zA-Z_]\w*'
127128
_rvkc_field = _rvkc_identifier + r'(\.\d+)?'
@@ -726,19 +727,6 @@ def _parse_value(self):
726727
if self._check_if_rvkc(self._image):
727728
return self._value
728729

729-
if len(self._image) > self.length:
730-
values = []
731-
for card in self._itersubcards():
732-
value = card.value.rstrip().replace("''", "'")
733-
if value and value[-1] == '&':
734-
value = value[:-1]
735-
values.append(value)
736-
737-
value = ''.join(values)
738-
739-
self._valuestring = value
740-
return value
741-
742730
m = self._value_NFSC_RE.match(self._split()[1])
743731

744732
if m is None:
@@ -790,21 +778,22 @@ def _parse_comment(self):
790778
if self.keyword in Card._commentary_keywords or self._invalid:
791779
return ''
792780

793-
if len(self._image) > self.length:
794-
comments = []
795-
for card in self._itersubcards():
796-
if card.comment:
797-
comments.append(card.comment)
798-
comment = '/ ' + ' '.join(comments).rstrip()
799-
m = self._value_NFSC_RE.match(comment)
800-
else:
801-
m = self._value_NFSC_RE.match(self._split()[1])
802-
781+
valuecomment = self._split()[1]
782+
m = self._value_NFSC_RE.match(valuecomment)
783+
comment = ''
803784
if m is not None:
804-
comment = m.group('comm')
805-
if comment:
806-
return comment.rstrip()
807-
return ''
785+
# Don't combine this if statement with the one above, because
786+
# we only want the elif case to run if this was not a valid
787+
# card at all
788+
if m.group('comm'):
789+
comment = m.group('comm').rstrip()
790+
elif '/' in valuecomment:
791+
# The value in this FITS file was not in a valid/known format. In
792+
# this case the best we can do is guess that everything after the
793+
# first / was meant to be the comment
794+
comment = valuecomment.split('/', 1)[1].strip()
795+
796+
return comment
808797

809798
def _split(self):
810799
"""
@@ -818,6 +807,34 @@ def _split(self):
818807
else:
819808
image = self.image
820809

810+
# Split cards with CONTINUE cards
811+
if len(self._image) > self.length:
812+
values = []
813+
comments = []
814+
keyword = None
815+
for card in self._itersubcards():
816+
kw, vc = card._split()
817+
if keyword is None:
818+
keyword = kw
819+
820+
# Should match a string followed by a comment; if not it
821+
# might be an invalid Card, so we just take it verbatim
822+
m = self._strg_comment_RE.match(vc)
823+
if not m:
824+
return kw, vc
825+
826+
value = m.group('strg') or ''
827+
value = value.rstrip().replace("''", "'")
828+
if value and value[-1] == '&':
829+
value = value[:-1]
830+
values.append(value)
831+
comment = m.group('comm')
832+
if comment:
833+
comments.append(comment.rstrip())
834+
835+
valuecomment = f"'{''.join(values)}' / {' '.join(comments)}"
836+
return keyword, valuecomment
837+
821838
if self.keyword in self._special_keywords:
822839
keyword, valuecomment = image.split(' ', 1)
823840
else:
@@ -1052,23 +1069,19 @@ def _format_long_commentary_image(self):
10521069
return ''.join(output)
10531070

10541071
def _verify(self, option='warn'):
1055-
self._verified = True
1056-
1057-
errs = _ErrList([])
1058-
fix_text = ('Fixed {!r} card to meet the FITS '
1059-
'standard.'.format(self.keyword))
1072+
errs = []
1073+
fix_text = f'Fixed {self.keyword!r} card to meet the FITS standard.'
10601074

10611075
# Don't try to verify cards that already don't meet any recognizable
10621076
# standard
10631077
if self._invalid:
1064-
return errs
1078+
return _ErrList(errs)
10651079

10661080
# verify the equal sign position
10671081
if (self.keyword not in self._commentary_keywords and
10681082
(self._image and self._image[:9].upper() != 'HIERARCH ' and
10691083
self._image.find('=') != 8)):
1070-
errs.append(self.run_option(
1071-
option,
1084+
errs.append(dict(
10721085
err_text='Card {!r} is not FITS standard (equal sign not '
10731086
'at column 8).'.format(self.keyword),
10741087
fix_text=fix_text,
@@ -1087,10 +1100,8 @@ def _verify(self, option='warn'):
10871100
keyword = self._split()[0]
10881101
if keyword != keyword.upper():
10891102
# Keyword should be uppercase unless it's a HIERARCH card
1090-
errs.append(self.run_option(
1091-
option,
1092-
err_text='Card keyword {!r} is not upper case.'.format(
1093-
keyword),
1103+
errs.append(dict(
1104+
err_text=f'Card keyword {keyword!r} is not upper case.',
10941105
fix_text=fix_text,
10951106
fix=self._fix_keyword))
10961107

@@ -1099,8 +1110,7 @@ def _verify(self, option='warn'):
10991110
keyword = keyword.split('.', 1)[0]
11001111

11011112
if not self._keywd_FSC_RE.match(keyword):
1102-
errs.append(self.run_option(
1103-
option,
1113+
errs.append(dict(
11041114
err_text=f'Illegal keyword name {keyword!r}',
11051115
fixable=False))
11061116

@@ -1110,35 +1120,39 @@ def _verify(self, option='warn'):
11101120
# For commentary keywords all that needs to be ensured is that it
11111121
# contains only printable ASCII characters
11121122
if not self._ascii_text_re.match(valuecomment):
1113-
errs.append(self.run_option(
1114-
option,
1123+
errs.append(dict(
11151124
err_text='Unprintable string {!r}; commentary cards may '
11161125
'only contain printable ASCII characters'.format(
11171126
valuecomment),
11181127
fixable=False))
11191128
else:
1120-
m = self._value_FSC_RE.match(valuecomment)
1121-
if not m:
1122-
errs.append(self.run_option(
1123-
option,
1124-
err_text='Card {!r} is not FITS standard (invalid value '
1125-
'string: {!r}).'.format(self.keyword, valuecomment),
1126-
fix_text=fix_text,
1127-
fix=self._fix_value))
1129+
if not self._valuemodified:
1130+
m = self._value_FSC_RE.match(valuecomment)
1131+
# If the value of a card was replaced before the card was ever
1132+
# even verified, the new value can be considered valid, so we
1133+
# don't bother verifying the old value. See
1134+
# https://github.com/astropy/astropy/issues/5408
1135+
if m is None:
1136+
errs.append(dict(
1137+
err_text=f'Card {self.keyword!r} is not FITS standard '
1138+
f'(invalid value string: {valuecomment!r}).',
1139+
fix_text=fix_text,
1140+
fix=self._fix_value))
11281141

11291142
# verify the comment (string), it is never fixable
11301143
m = self._value_NFSC_RE.match(valuecomment)
11311144
if m is not None:
11321145
comment = m.group('comm')
11331146
if comment is not None:
11341147
if not self._ascii_text_re.match(comment):
1135-
errs.append(self.run_option(
1136-
option,
1137-
err_text=('Unprintable string {!r}; header comments '
1138-
'may only contain printable ASCII '
1139-
'characters'.format(comment)),
1148+
errs.append(dict(
1149+
err_text=f'Unprintable string {comment!r}; header '
1150+
'comments may only contain printable '
1151+
'ASCII characters',
11401152
fixable=False))
11411153

1154+
errs = _ErrList([self.run_option(option, **err) for err in errs])
1155+
self._verified = True
11421156
return errs
11431157

11441158
def _itersubcards(self):

‎astropy/io/fits/tests/test_header.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2401,6 +2401,49 @@ def test_set_keyword_with_space(self):
24012401
assert hdr['KEY2'] == 4
24022402
assert hdr['KEY2 '] == 4
24032403

2404+
def test_strip(self):
2405+
hdr = fits.getheader(self.data('tb.fits'), ext=1)
2406+
hdr['FOO'] = 'bar'
2407+
hdr.strip()
2408+
assert set(hdr) == {'HISTORY', 'FOO'}
2409+
2410+
hdr = fits.getheader(self.data('tb.fits'), ext=1)
2411+
hdr['FOO'] = 'bar'
2412+
hdr = hdr.copy(strip=True)
2413+
assert set(hdr) == {'HISTORY', 'FOO'}
2414+
2415+
def test_update_invalid_card(self):
2416+
"""
2417+
Regression test for https://github.com/astropy/astropy/issues/5408
2418+
2419+
Tests updating the value of a card that is malformatted (with an
2420+
invalid value literal).
2421+
2422+
This tests two ways of reproducing the problem, one working with a
2423+
Card object directly, and one when reading/writing a header containing
2424+
such an invalid card.
2425+
"""
2426+
2427+
card = fits.Card.fromstring('KW = INF / Comment')
2428+
card.value = 'FIXED'
2429+
assert tuple(card) == ('KW', 'FIXED', 'Comment')
2430+
card.verify('fix')
2431+
assert tuple(card) == ('KW', 'FIXED', 'Comment')
2432+
2433+
card = fits.Card.fromstring('KW = INF')
2434+
hdu = fits.PrimaryHDU()
2435+
# This is a loophole to write a header containing a malformatted card
2436+
card._verified = True
2437+
hdu.header.append(card)
2438+
hdu.header.tofile(self.temp('bogus.fits'))
2439+
2440+
with fits.open(self.temp('bogus.fits')) as hdul:
2441+
hdul[0].header['KW'] = -1
2442+
hdul.writeto(self.temp('bogus_fixed.fits'))
2443+
2444+
with fits.open(self.temp('bogus_fixed.fits')) as hdul:
2445+
assert hdul[0].header['KW'] == -1
2446+
24042447

24052448
class TestRecordValuedKeywordCards(FitsTestCase):
24062449
"""

0 commit comments

Comments
 (0)