Skip to content

Commit 3e0fe42

Browse files
committed
Merge branch 'master' into astropy
Merges up through r2072 Fixes #256, #309 Conflicts: astropy/io/fits/hdu/compressed.py astropy/io/fits/tests/test_core.py astropy/io/fits/tests/test_diff.py astropy/io/fits/tests/test_image.py astropy/io/fits/tests/test_table.py astropy/io/fits/util.py
1 parent cd48a70 commit 3e0fe42

14 files changed

Lines changed: 387 additions & 106 deletions

File tree

‎CHANGES.rst‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,27 @@ Bug Fixes
2929
- Fixed encoding errors that could occur when formatting coordinate objects
3030
in code using ``from __future__ import unicode_literals``. [#817]
3131

32+
- ``astropy.io.fits``
33+
34+
- Properly supports the ZQUANTIZ keyword used to support quantization
35+
level--this includes working support for lossless GZIP compression of
36+
images.
37+
38+
- Fixed support for opening gzipped FITS files in a writeable mode. [#256]
39+
40+
- Added a more helpful exception message when trying to read invalid values
41+
from a table when the required ``TNULLn`` keyword is missing. [#309]
42+
43+
- Fixed malformatting of the ``TFORMn`` keywords when writing compressed
44+
image tables (they ommitted the max array length parameter from the
45+
variable-length array format).
46+
47+
- Fixed a crash that could occur when writing a table containing multi-
48+
dimensional array columns from an existing file into a new file.
49+
50+
- Fixed a bug in fitsdiff that reported two header keywords contaning NaN
51+
as having different values.
52+
3253
- ``astropy.io.votable``
3354

3455
- Fixed links to the ``astropy.io.votable`` documentation in the VOTable

‎astropy/io/fits/diff.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1158,8 +1158,9 @@ def diff_values(a, b, tolerance=0.0):
11581158
within the given relative tolerance.
11591159
"""
11601160

1161-
# TODO: Handle ifs and nans
11621161
if isinstance(a, float) and isinstance(b, float):
1162+
if np.isnan(a) and np.isnan(b):
1163+
return False
11631164
return not np.allclose(a, b, tolerance, 0.0)
11641165
else:
11651166
return a != b

‎astropy/io/fits/file.py‎

Lines changed: 86 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -91,84 +91,11 @@ def __init__(self, fileobj=None, mode='readonly', memmap=False):
9191

9292
# Initialize the internal self.__file object
9393
if isfile(fileobj) or isinstance(fileobj, gzip.GzipFile):
94-
closed = fileobj_closed(fileobj)
95-
fmode = fileobj_mode(fileobj) or PYTHON_MODES[mode]
96-
97-
if not closed:
98-
# In some cases (like on Python 3) a file opened for
99-
# appending still shows a mode of 'r+', hence the extra
100-
# check for the append case
101-
if ((mode == 'append' and fmode not in ('ab+', 'rb+')) or
102-
(mode != 'append' and PYTHON_MODES[mode] != fmode)):
103-
raise ValueError(
104-
"Input mode '%s' (%s) does not match mode of the "
105-
"input file (%s)." % (mode, PYTHON_MODES[mode], fmode))
106-
self.__file = fileobj
107-
elif isfile(fileobj):
108-
self.__file = fileobj_open(self.name, PYTHON_MODES[mode])
109-
# Return to the beginning of the file--in Python 3 when
110-
# opening in append mode the file pointer is at the end of
111-
# the file
112-
self.__file.seek(0)
113-
else:
114-
self.__file = gzip.open(self.name, PYTHON_MODES[mode])
94+
self._open_fileobj(fileobj, mode)
11595
elif isinstance(fileobj, basestring):
116-
if os.path.exists(self.name):
117-
with fileobj_open(self.name, 'rb') as f:
118-
magic = f.read(4)
119-
else:
120-
magic = ''.encode('raw-unicode-escape')
121-
ext = os.path.splitext(self.name)[1]
122-
if ext == '.gz' or magic.startswith(GZIP_MAGIC):
123-
# Handle gzip files
124-
if mode in ['update', 'append']:
125-
raise IOError(
126-
"Writing to gzipped fits files is not currently "
127-
"supported")
128-
self.__file = gzip.open(self.name)
129-
self.compression = 'gzip'
130-
elif ext == '.zip' or magic.startswith(PKZIP_MAGIC):
131-
# Handle zip files
132-
if mode in ['update', 'append']:
133-
raise IOError(
134-
"Writing to zipped fits files is not currently "
135-
"supported")
136-
zfile = zipfile.ZipFile(self.name)
137-
namelist = zfile.namelist()
138-
if len(namelist) != 1:
139-
raise IOError(
140-
"Zip files with multiple members are not supported.")
141-
self.__file = tempfile.NamedTemporaryFile(suffix='.fits')
142-
self.__file.write(zfile.read(namelist[0]))
143-
zfile.close()
144-
self.compression = 'zip'
145-
else:
146-
self.__file = fileobj_open(self.name, PYTHON_MODES[mode])
147-
# Make certain we're back at the beginning of the file
148-
self.__file.seek(0)
96+
self._open_filename(fileobj, mode)
14997
else:
150-
# We are dealing with a file like object.
151-
# Assume it is open.
152-
self.file_like = True
153-
self.__file = fileobj
154-
155-
# If there is not seek or tell methods then set the mode to
156-
# output streaming.
157-
if (not hasattr(self.__file, 'seek') or
158-
not hasattr(self.__file, 'tell')):
159-
self.mode = mode = 'ostream'
160-
161-
if (self.mode in ('copyonwrite', 'update', 'append') and
162-
not hasattr(self.__file, 'write')):
163-
raise IOError("File-like object does not have a 'write' "
164-
"method, required for mode '%s'."
165-
% self.mode)
166-
167-
if (self.mode in ('readonly', 'denywrite') and
168-
not hasattr(self.__file, 'read')):
169-
raise IOError("File-like object does not have a 'read' "
170-
"method, required for mode %r."
171-
% self.mode)
98+
self._open_filelike(fileobj, mode)
17299

173100
if isinstance(fileobj, gzip.GzipFile):
174101
self.compression = 'gzip'
@@ -342,3 +269,86 @@ def close(self):
342269
self.__file.close()
343270

344271
self.closed = True
272+
273+
def _open_fileobj(self, fileobj, mode):
274+
"""Open a FITS file from a file object or a GzipFile object."""
275+
276+
closed = fileobj_closed(fileobj)
277+
fmode = fileobj_mode(fileobj) or PYTHON_MODES[mode]
278+
279+
if not closed:
280+
# In some cases (like on Python 3) a file opened for appending
281+
# still shows a mode of 'r+', hence the extra check for the append
282+
# case
283+
if ((mode == 'append' and fmode not in ('ab+', 'rb+')) or
284+
(mode != 'append' and PYTHON_MODES[mode] != fmode)):
285+
raise ValueError(
286+
"Input mode '%s' (%s) does not match mode of the "
287+
"input file (%s)." % (mode, PYTHON_MODES[mode], fmode))
288+
self.__file = fileobj
289+
elif isfile(fileobj):
290+
self.__file = fileobj_open(self.name, PYTHON_MODES[mode])
291+
# Return to the beginning of the file--in Python 3 when opening in
292+
# append mode the file pointer is at the end of the file
293+
self.__file.seek(0)
294+
else:
295+
self.__file = gzip.open(self.name, PYTHON_MODES[mode])
296+
297+
def _open_filelike(self, fileobj, mode):
298+
"""Open a FITS file from a file-like object, i.e. one that has
299+
read and/or write methods.
300+
"""
301+
302+
self.file_like = True
303+
self.__file = fileobj
304+
305+
# If there is not seek or tell methods then set the mode to
306+
# output streaming.
307+
if (not hasattr(self.__file, 'seek') or
308+
not hasattr(self.__file, 'tell')):
309+
self.mode = mode = 'ostream'
310+
311+
if (self.mode in ('copyonwrite', 'update', 'append') and
312+
not hasattr(self.__file, 'write')):
313+
raise IOError("File-like object does not have a 'write' "
314+
"method, required for mode '%s'."
315+
% self.mode)
316+
317+
if (self.mode in ('readonly', 'denywrite') and
318+
not hasattr(self.__file, 'read')):
319+
raise IOError("File-like object does not have a 'read' "
320+
"method, required for mode %r."
321+
% self.mode)
322+
323+
def _open_filename(self, filename, mode):
324+
"""Open a FITS file from a filename string."""
325+
326+
if os.path.exists(self.name):
327+
with fileobj_open(self.name, 'rb') as f:
328+
magic = f.read(4)
329+
else:
330+
magic = ''.encode('raw-unicode-escape')
331+
ext = os.path.splitext(self.name)[1]
332+
if ext == '.gz' or magic.startswith(GZIP_MAGIC):
333+
# Handle gzip files
334+
self.__file = gzip.open(self.name, PYTHON_MODES[mode])
335+
self.compression = 'gzip'
336+
elif ext == '.zip' or magic.startswith(PKZIP_MAGIC):
337+
# Handle zip files
338+
if mode in ['update', 'append']:
339+
raise IOError(
340+
"Writing to zipped fits files is not currently "
341+
"supported")
342+
zfile = zipfile.ZipFile(self.name)
343+
namelist = zfile.namelist()
344+
if len(namelist) != 1:
345+
raise IOError(
346+
"Zip files with multiple members are not supported.")
347+
self.__file = tempfile.NamedTemporaryFile(suffix='.fits')
348+
self.__file.write(zfile.read(namelist[0]))
349+
zfile.close()
350+
self.compression = 'zip'
351+
else:
352+
self.__file = fileobj_open(self.name, PYTHON_MODES[mode])
353+
# Make certain we're back at the beginning of the file
354+
self.__file.seek(0)

‎astropy/io/fits/fitsrec.py‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -433,7 +433,13 @@ def get_pdata(dtype, count):
433433
'E'.encode('ascii'))
434434
dummy = np.where(dummy.strip() == nullval, str(ASCIITNULL),
435435
dummy)
436-
dummy = np.array(dummy, dtype=_type)
436+
try:
437+
dummy = np.array(dummy, dtype=_type)
438+
except ValueError, e:
439+
raise ValueError(
440+
'%s; the header may be missing the necessary TNULL%d '
441+
'keyword or the table contains invalid data' %
442+
(e, indx + 1))
437443

438444
self._convert[indx] = dummy
439445
else:
@@ -667,9 +673,20 @@ def _scale_back(self):
667673
for idx in xrange(len(dummy)):
668674
val = dummy[idx]
669675
dummy[idx] = val + (pad * (itemsize - len(val)))
670-
if dummy.dtype != field.dtype:
671-
dummy = dummy.astype(field.dtype)
672-
field[:] = dummy
676+
677+
# Encode *after* handling the padding byte or else
678+
# Numpy will complain about trying to append bytes to
679+
# an array
680+
if dummy.dtype.kind == 'U':
681+
dummy = dummy.encode('ascii')
682+
683+
if field.shape == dummy.shape:
684+
field[:] = dummy
685+
else:
686+
# Reshaping the data is necessary in cases where the
687+
# TDIMn keyword was used to shape a column's entries
688+
# into arrays
689+
field[:] = dummy.ravel().view(field.dtype)
673690

674691
del dummy
675692

‎astropy/io/fits/hdu/compressed.py‎

Lines changed: 21 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,10 @@
44
import sys
55
import warnings
66

7-
from functools import reduce
8-
97
import numpy as np
108

9+
from functools import reduce
10+
1111
from .base import DELAYED, ExtensionHDU
1212
from .image import _ImageBaseHDU, ImageHDU
1313
from .table import BinTableHDU
@@ -34,9 +34,15 @@
3434
DEFAULT_BLOCK_SIZE = 32
3535
DEFAULT_BYTE_PIX = 4
3636

37+
3738
# CFITSIO version-specific features
3839
if COMPRESSION_SUPPORTED:
39-
CFITSIO_SUPPORTS_GZIPDATA = compression.CFITSIO_VERSION >= 3.28
40+
try:
41+
CFITSIO_SUPPORTS_GZIPDATA = compression.CFITSIO_VERSION >= 3.28
42+
except AttributeError:
43+
# This generally shouldn't happen unless running setup.py in an
44+
# environment where an old build of pyfits exists
45+
CFITSIO_SUPPORTS_GZIPDATA = True
4046

4147

4248
class CompImageHeader(Header):
@@ -455,9 +461,10 @@ def updateHeaderData(self, image_header,
455461
# Create the additional columns required for floating point
456462
# data and calculate the width of the output table.
457463

458-
if self._image_header['BITPIX'] < 0:
464+
if self._image_header['BITPIX'] < 0 and quantizeLevel != 0.0:
459465
# floating point image has 'COMPRESSED_DATA',
460-
# 'UNCOMPRESSED_DATA', 'ZSCALE', and 'ZZERO' columns.
466+
# 'UNCOMPRESSED_DATA', 'ZSCALE', and 'ZZERO' columns (unless using
467+
# lossless compression, per CFITSIO)
461468
ncols = 4
462469

463470
# CFITSIO 3.28 and up automatically use the GZIP_COMPRESSED_DATA
@@ -1339,10 +1346,10 @@ def updateHeader(self):
13391346

13401347
# Update TFORM for variable length columns.
13411348
for idx in range(self.compData._nfields):
1342-
format = self.compData._coldefs.formats[idx]
1349+
format = self.compData._coldefs._recformats[idx]
13431350
if isinstance(format, _FormatP):
1344-
max = self.compData.field(idx).max
1345-
format = _FormatP(format, repeat=format.repeat, max=max)
1351+
_max = self.compData.field(idx).max
1352+
format = _FormatP(format.dtype, repeat=format.repeat, max=_max)
13461353
self._header['TFORM' + str(idx + 1)] = format.tform
13471354
# Insure that for RICE_1 that the BLOCKSIZE and BYTEPIX cards
13481355
# are present and set to the hard coded values used by the
@@ -1418,18 +1425,18 @@ def scale(self, type=None, option='old', bscale=1, bzero=0):
14181425
_scale = 1
14191426
_zero = 0
14201427
else:
1421-
min = np.minimum.reduce(self.data.flat)
1422-
max = np.maximum.reduce(self.data.flat)
1428+
_min = np.minimum.reduce(self.data.flat)
1429+
_max = np.maximum.reduce(self.data.flat)
14231430
self.data.shape = dims
14241431

14251432
if _type == np.uint8: # uint8 case
1426-
_zero = min
1427-
_scale = (max - min) / (2. ** 8 - 1)
1433+
_zero = _min
1434+
_scale = (_max - _min) / (2. ** 8 - 1)
14281435
else:
1429-
_zero = (max + min) / 2.
1436+
_zero = (_max + _min) / 2.
14301437

14311438
# throw away -2^N
1432-
_scale = (max - min) / (2. ** (8 * _type.bytes) - 2)
1439+
_scale = (_max - _min) / (2. ** (8 * _type.bytes) - 2)
14331440

14341441
# Do the scaling
14351442
if _zero != 0:

‎astropy/io/fits/hdu/table.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -294,8 +294,9 @@ def _prewriteto(self, checksum=False, inplace=False):
294294
for idx in range(self.data._nfields):
295295
format = self.data._coldefs.formats[idx]
296296
if isinstance(format, _FormatP):
297-
max = self.data.field(idx).max
298-
format = _FormatP(format, repeat=format.repeat, max=max)
297+
_max = self.data.field(idx).max
298+
format = _FormatP(format.dtype, repeat=format.repeat,
299+
max=_max)
299300
self._header['TFORM' + str(idx + 1)] = format.tform
300301
return super(_TableBaseHDU, self)._prewriteto(checksum, inplace)
301302

‎astropy/io/fits/src/compressionmodule.c‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -510,6 +510,7 @@ void configure_compression(fitsfile* fileptr, PyObject* header) {
510510
char* zname;
511511
int znaxis;
512512
char* tmp;
513+
float version;
513514

514515
unsigned int idx;
515516

@@ -639,11 +640,32 @@ void configure_compression(fitsfile* fileptr, PyObject* header) {
639640
} else if (Fptr->zbitpix < 0 && 0 == strcmp(zname, "NOISEBIT")) {
640641
get_header_float(header, keyword, &(Fptr->quantize_level),
641642
DEFAULT_QUANTIZE_LEVEL);
643+
if (Fptr->quantize_level == 0.0) {
644+
/* NOISEBIT == 0 is equivalent to no quantize */
645+
Fptr->quantize_level = NO_QUANTIZE;
646+
}
642647
}
643648

644649
idx++;
645650
}
646651

652+
/* The ZQUANTIZ keyword determines the quantization algorithm; NO_QUANTIZE
653+
implies lossless compression */
654+
if (0 == get_header_string(header, "ZQUANTIZ", &tmp, "")) {
655+
/* Ugh; the fact that cfitsio defines its version as a float makes
656+
preprocessor comparison impossible */
657+
fits_get_version(&version);
658+
if ((version >= CFITSIO_LOSSLESS_COMP_SUPPORTED_VERS) &&
659+
(0 == strcmp(tmp, "NONE"))) {
660+
Fptr->quantize_level = NO_QUANTIZE;
661+
} else if (0 == strcmp(tmp, "SUBTRACTIVE_DITHER_1")) {
662+
Fptr->quantize_dither = SUBTRACTIVE_DITHER_1;
663+
} else {
664+
Fptr->quantize_dither = 0;
665+
}
666+
} else {
667+
Fptr->quantize_dither = 0;
668+
}
647669

648670
Fptr->compressimg = 1;
649671
Fptr->maxelem = imcomp_calc_max_elem(Fptr->compress_type,

0 commit comments

Comments
 (0)