Skip to content

Commit a36bb34

Browse files
saimnastrofrog
authored andcommitted
prevent creating compressed files with (u)int64 data
Fix astropy#19577 Also ref astropy#2153 If possible without overflow data is converted to (u)int32, otherwise an error is raised.
1 parent 293014d commit a36bb34

3 files changed

Lines changed: 74 additions & 1 deletion

File tree

‎astropy/io/fits/hdu/compressed/_tiled_compression.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,12 @@
66

77
import sys
88
from math import prod
9+
import warnings
910

1011
import numpy as np
1112

1213
from astropy.io.fits.hdu.base import BITPIX2DTYPE
14+
from astropy.utils.exceptions import AstropyUserWarning
1315

1416
from ._codecs import PLIO1, Gzip1, Gzip2, HCompress1, NoCompress, Rice1
1517
from ._quantization import DITHER_METHODS, QuantizationFailedException, Quantize
@@ -541,6 +543,22 @@ def compress_image_data(
541543
if not isinstance(image_data, np.ndarray):
542544
raise TypeError("Image data must be a numpy.ndarray")
543545

546+
if image_data.dtype.kind == "i" and image_data.dtype.itemsize == 8:
547+
new_dt = f"{image_data.dtype.byteorder}{image_data.dtype.kind}4"
548+
try:
549+
image_data = image_data.astype(new_dt, casting="same_value")
550+
compressed_header["ZBITPIX"] = 32
551+
warnings.warn(
552+
"compression doesn't support 64 integers, "
553+
"data has been converted to 32 bits",
554+
AstropyUserWarning,
555+
)
556+
except ValueError:
557+
raise ValueError(
558+
"compression doesn't support 64 integers, "
559+
"but data cannot be converted to 32 bits without overflow",
560+
)
561+
544562
_check_compressed_header(compressed_header)
545563

546564
# TODO: This implementation is memory inefficient as it generates all the

‎astropy/io/fits/hdu/compressed/tests/test_fitsio.py‎

Lines changed: 54 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,16 @@
99
when editing this file.
1010
"""
1111

12+
from contextlib import nullcontext
1213
import os
1314

1415
import numpy as np
1516
import pytest
1617

1718
from astropy.io import fits
19+
from astropy.utils.exceptions import AstropyUserWarning
1820

19-
from .conftest import _expand, fitsio_param_to_astropy_param
21+
from .conftest import COMPRESSION_TYPES, _expand, fitsio_param_to_astropy_param
2022

2123
# This is so that tox can force this file to be run, and not be silently
2224
# skipped on CI, but in all other test runs it's skipped if fitsio isn't present.
@@ -240,3 +242,54 @@ def test_compress(
240242

241243
with fits.open(astropy_compressed_file_path) as hdul:
242244
np.testing.assert_allclose(data, hdul[1].data, rtol=rtol, atol=atol)
245+
246+
247+
@pytest.mark.parametrize(
248+
"nbytes,overflow", [(2, False), (4, False), (8, False), (8, True)]
249+
)
250+
@pytest.mark.parametrize("compression_type", COMPRESSION_TYPES)
251+
def test_decompress_integers(nbytes, overflow, compression_type, tmp_path):
252+
253+
if compression_type == "NOCOMPRESS":
254+
pytest.xfail("fitsio does not support NOCOMPRESS")
255+
256+
testfile = tmp_path / "test.fits.fz"
257+
data = np.random.poisson(1000, size=(52, 57)).astype(f"i{nbytes}")
258+
if overflow:
259+
data += np.iinfo(np.int32).max
260+
data_hdu = fits.PrimaryHDU(data=data)
261+
compressed_hdu = fits.CompImageHDU(data=data, compression_type=compression_type)
262+
263+
if nbytes == 8 and overflow:
264+
ctx = pytest.raises(
265+
ValueError, match="compression doesn't support 64 integers.*"
266+
)
267+
elif nbytes == 8 and not overflow:
268+
ctx = pytest.warns(
269+
AstropyUserWarning, match="compression doesn't support 64 integers.*"
270+
)
271+
nbytes = 4
272+
else:
273+
ctx = nullcontext()
274+
275+
with ctx:
276+
compressed_hdu.writeto(testfile)
277+
278+
if overflow:
279+
# nothing more to test
280+
return
281+
282+
with fits.open(testfile, disable_image_compression=True) as hdul:
283+
assert hdul[1].header["ZCMPTYPE"] == compression_type
284+
if compression_type == "RICE_1":
285+
assert hdul[1].header["ZNAME2"] == "BYTEPIX"
286+
assert hdul[1].header["ZVAL2"] == nbytes
287+
288+
with fits.open(testfile) as hdul:
289+
np.testing.assert_array_equal(data, hdul[1].data)
290+
assert hdul[1].data.dtype.kind == np.dtype(data.dtype).kind
291+
assert hdul[1].data.dtype.itemsize == nbytes
292+
293+
fts = fitsio.FITS(testfile)
294+
data2 = fts[1].read()
295+
np.testing.assert_array_equal(data, data2)
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Prevent creating compressed files with (u)int64 data. If possible without
2+
overflow data is converted to (u)int32, otherwise an error is raised.

0 commit comments

Comments
 (0)