Skip to content

Commit 92b38ae

Browse files
authored
Add LZW file support to io.fits, io.ascii and utils.data (#17960)
* Add LZW file support to io.fits, io.ascii and utils.data * Clean up tests and documentation for LZW * Fix formatting * Clean up _make_lzw_file test
1 parent bafb094 commit 92b38ae

16 files changed

Lines changed: 180 additions & 20 deletions

File tree

353 Bytes
Binary file not shown.
146 Bytes
Binary file not shown.

‎astropy/io/ascii/tests/test_compressed.py‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
# Licensed under a 3-clause BSD style license - see LICENSE.rst
22
import numpy as np
33
import pytest
4+
from numpy.testing import assert_array_equal
45

56
from astropy.io.ascii import read
67

78
# NOTE: Python can be built without bz2 or lzma
8-
from astropy.utils.compat.optional_deps import HAS_BZ2, HAS_LZMA
9+
from astropy.utils.compat.optional_deps import HAS_BZ2, HAS_LZMA, HAS_UNCOMPRESSPY
910
from astropy.utils.data import get_pkg_data_filename
1011

1112

@@ -35,3 +36,12 @@ def test_xz(filename):
3536
t_uncomp = read(get_pkg_data_filename(filename.replace(".xz", "")))
3637
assert t_comp.dtype.names == t_uncomp.dtype.names
3738
assert np.all(t_comp.as_array() == t_uncomp.as_array())
39+
40+
41+
@pytest.mark.xfail(not HAS_UNCOMPRESSPY, reason="requires uncompresspy")
42+
@pytest.mark.parametrize("filename", ["data/short.rdb.Z", "data/ipac.dat.Z"])
43+
def test_lzw(filename):
44+
t_comp = read(get_pkg_data_filename(filename))
45+
t_uncomp = read(get_pkg_data_filename(filename.removesuffix(".Z")))
46+
assert t_comp.dtype.names == t_uncomp.dtype.names
47+
assert_array_equal(t_comp.as_array(), t_uncomp.as_array())

‎astropy/io/fits/file.py‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
import numpy as np
1818

1919
# NOTE: Python can be built without bz2.
20-
from astropy.utils.compat.optional_deps import HAS_BZ2, HAS_LZMA
20+
from astropy.utils.compat.optional_deps import HAS_BZ2, HAS_LZMA, HAS_UNCOMPRESSPY
2121
from astropy.utils.data import (
2222
_is_url,
2323
_requires_fsspec,
@@ -47,6 +47,10 @@
4747
if HAS_LZMA:
4848
import lzma
4949

50+
if HAS_UNCOMPRESSPY:
51+
import uncompresspy
52+
53+
5054
# Maps astropy.io.fits-specific file mode names to the appropriate file
5155
# modes to use for the underlying raw files.
5256
IO_FITS_MODES = {
@@ -101,6 +105,7 @@
101105
PKZIP_MAGIC = b"\x50\x4b\x03\x04"
102106
BZIP2_MAGIC = b"\x42\x5a"
103107
LZMA_MAGIC = b"\xfd7zXZ\x00"
108+
LZW_MAGIC = b"\x1f\x9d"
104109

105110

106111
def _is_bz2file(fileobj):
@@ -117,6 +122,13 @@ def _is_lzmafile(fileobj):
117122
return False
118123

119124

125+
def _is_lzwfile(fileobj):
126+
if HAS_UNCOMPRESSPY:
127+
return isinstance(fileobj, uncompresspy.LZWFile)
128+
else:
129+
return False
130+
131+
120132
def _normalize_fits_mode(mode):
121133
if mode is not None and mode not in IO_FITS_MODES:
122134
if TEXT_RE.match(mode):
@@ -240,6 +252,9 @@ def __init__(
240252
self.compression = "bzip2"
241253
elif _is_lzmafile(fileobj):
242254
self.compression = "lzma"
255+
elif _is_lzwfile(fileobj):
256+
self.compression = "lzw"
257+
243258
if (
244259
self.compression is not None
245260
and decompress_in_memory
@@ -581,6 +596,17 @@ def _try_read_compressed(self, obj_or_name, magic, mode, ext=""):
581596
lzma_mode = "w" if is_ostream else "r"
582597
self._file = lzma.LZMAFile(obj_or_name, mode=lzma_mode)
583598
self.compression = "lzma"
599+
elif (is_ostream and ext == ".Z") or magic.startswith(LZW_MAGIC):
600+
# Handle LZW files
601+
if mode in ["update", "append", "ostream"]:
602+
raise OSError(f"{mode} mode not supported with LZW files")
603+
if not HAS_UNCOMPRESSPY:
604+
raise ModuleNotFoundError(
605+
"The optional package uncompresspy is necessary for reading"
606+
" LZW compressed files (.Z extension)."
607+
)
608+
self._file = uncompresspy.LZWFile(obj_or_name, mode="rb")
609+
self.compression = "lzw"
584610
return self.compression is not None
585611

586612
def _open_fileobj(self, fileobj, mode, overwrite):
13.5 KB
Binary file not shown.

‎astropy/io/fits/tests/test_core.py‎

Lines changed: 76 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
from astropy.utils.compat.optional_deps import (
2525
HAS_BZ2, # NOTE: Python can be built without bz2
2626
HAS_LZMA, # NOTE: Python can be built without lzma
27+
HAS_UNCOMPRESSPY,
2728
)
2829
from astropy.utils.data import conf
2930
from astropy.utils.exceptions import AstropyUserWarning
@@ -37,6 +38,9 @@
3738
if HAS_LZMA:
3839
import lzma
3940

41+
if HAS_UNCOMPRESSPY:
42+
import uncompresspy
43+
4044

4145
class TestCore(FitsTestCase):
4246
def test_missing_file(self):
@@ -890,6 +894,69 @@ def test_writeto_lzma_filename(self):
890894
with fits.open(self.temp("testname.fits.xz")) as hdul:
891895
assert hdul[0].header == h.header
892896

897+
@pytest.mark.skipif(
898+
not HAS_UNCOMPRESSPY, reason="Optional package uncompresspy not installed"
899+
)
900+
def test_open_lzw(self):
901+
lzw_file = self._make_lzw_file()
902+
903+
arcfile = "ONTT.1991-12-30T08:55:46.000.fits"
904+
last_datapoint = 53
905+
with fits.open(lzw_file) as fits_handle:
906+
assert fits_handle._file.compression == "lzw"
907+
assert len(fits_handle) == 1
908+
assert fits_handle[0].header["ARCFILE"] == arcfile
909+
assert fits_handle[0].data[-1, -1] == last_datapoint
910+
911+
with fits.open(lzw_file, decompress_in_memory=True) as fits_handle:
912+
assert fits_handle._file.compression == "lzw"
913+
assert len(fits_handle) == 1
914+
assert fits_handle[0].header["ARCFILE"] == arcfile
915+
assert fits_handle[0].data[-1, -1] == last_datapoint
916+
917+
with fits.open(uncompresspy.LZWFile(lzw_file)) as fits_handle:
918+
assert fits_handle._file.compression == "lzw"
919+
assert len(fits_handle) == 1
920+
assert fits_handle[0].header["ARCFILE"] == arcfile
921+
assert fits_handle[0].data[-1, -1] == last_datapoint
922+
923+
@pytest.mark.skipif(
924+
not HAS_UNCOMPRESSPY, reason="Optional package uncompresspy not installed"
925+
)
926+
def test_open_lzw_from_handle(self):
927+
arcfile = "ONTT.1991-12-30T08:55:46.000.fits"
928+
last_datapoint = 53
929+
with open(self._make_lzw_file(), "rb") as handle:
930+
with fits.open(handle) as fits_handle:
931+
assert fits_handle._file.compression == "lzw"
932+
assert len(fits_handle) == 1
933+
assert fits_handle[0].header["ARCFILE"] == arcfile
934+
assert fits_handle[0].data[-1, -1] == last_datapoint
935+
936+
@pytest.mark.skipif(
937+
not HAS_UNCOMPRESSPY, reason="Optional package uncompresspy not installed"
938+
)
939+
def test_detect_lzw(self):
940+
"""Test detection of a lzw file when the extension is not .Z."""
941+
arcfile = "ONTT.1991-12-30T08:55:46.000.fits"
942+
last_datapoint = 53
943+
with fits.open(self._make_lzw_file("test0.xx")) as fits_handle:
944+
assert fits_handle._file.compression == "lzw"
945+
assert len(fits_handle) == 1
946+
assert fits_handle[0].header["ARCFILE"] == arcfile
947+
assert fits_handle[0].data[-1, -1] == last_datapoint
948+
949+
@pytest.mark.skipif(
950+
not HAS_UNCOMPRESSPY, reason="Optional package uncompresspy not installed"
951+
)
952+
def test_writeto_lzw_filename(self):
953+
"""Test writing to a LZW file by name. This should fail as writing LZW
954+
is not supported."""
955+
filename = self.temp("testname.fits.Z")
956+
h = fits.PrimaryHDU()
957+
with pytest.raises(OSError, match="mode not supported with LZW files"):
958+
h.writeto(filename)
959+
893960
def test_open_zipped(self):
894961
zip_file = self._make_zip_file()
895962

@@ -1010,7 +1077,7 @@ def test_write_read_gzip_file(self, home_is_temp):
10101077
with fits.open(self.temp("test.fits.gz")) as hdul:
10111078
assert np.all(hdul[0].data == data)
10121079

1013-
@pytest.mark.parametrize("ext", ["gz", "bz2", "zip", "xz"])
1080+
@pytest.mark.parametrize("ext", ["gz", "bz2", "zip", "xz", "Z"])
10141081
def test_compressed_ext_but_not_compressed(self, ext):
10151082
testfile = self.temp(f"test0.fits.{ext}")
10161083
shutil.copy(self.data("test0.fits"), testfile)
@@ -1409,6 +1476,14 @@ def _make_lzma_file(self, filename="test0.fits.xz"):
14091476

14101477
return lzmafile
14111478

1479+
def _make_lzw_file(self, new_filename=None):
1480+
lzwfile = "lzw.fits.Z"
1481+
self.copy_file(lzwfile)
1482+
if new_filename is not None:
1483+
shutil.move(self.temp(lzwfile), self.temp(new_filename))
1484+
return self.temp(new_filename)
1485+
return self.temp(lzwfile)
1486+
14121487
def test_simulateonly(self):
14131488
"""Write to None simulates writing."""
14141489

‎astropy/utils/compat/optional_deps.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
"scipy",
3838
"skyfield",
3939
"sortedcontainers",
40+
"uncompresspy",
4041
"lzma", # stdlib
4142
"pyarrow",
4243
"pytest_mpl",

‎astropy/utils/data.py‎

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
HAS_CERTIFI,
3333
HAS_FSSPEC,
3434
HAS_LZMA,
35+
HAS_UNCOMPRESSPY,
3536
)
3637
from astropy.utils.exceptions import AstropyDeprecationWarning, AstropyWarning
3738
from astropy.utils.introspection import find_current_module
@@ -212,7 +213,7 @@ def get_readable_fileobj(
212213
"""Yield a readable, seekable file-like object from a file or URL.
213214
214215
This supports passing filenames, URLs, and readable file-like objects,
215-
any of which can be compressed in gzip, bzip2 or lzma (xz) if the
216+
any of which can be compressed in gzip, bzip2, lzma (xz) or lzw (Z) if the
216217
appropriate compression libraries are provided by the Python installation.
217218
218219
Notes
@@ -446,10 +447,30 @@ def get_readable_fileobj(
446447
else:
447448
fileobj_new.seek(0)
448449
fileobj = fileobj_new
450+
elif signature[:2] == b"\x1f\x9d": # LZW
451+
if not HAS_UNCOMPRESSPY:
452+
for fd in close_fds:
453+
fd.close()
454+
raise ModuleNotFoundError(
455+
"The optional package uncompresspy is necessary for reading LZW"
456+
" compressed files (.Z extension)."
457+
)
458+
import uncompresspy
459+
460+
try:
461+
fileobj_new = uncompresspy.LZWFile(fileobj)
462+
fileobj_new.read(1)
463+
except ValueError:
464+
fileobj.seek(0)
465+
fileobj_new.close()
466+
else:
467+
fileobj_new.seek(0)
468+
close_fds.append(fileobj)
469+
fileobj = fileobj_new
449470

450-
# By this point, we have a file, io.FileIO, gzip.GzipFile, bz2.BZ2File
451-
# or lzma.LZMAFile instance opened in binary mode (that is, read
452-
# returns bytes). Now we need to, if requested, wrap it in a
471+
# By this point, we have a file, io.FileIO, gzip.GzipFile, bz2.BZ2File,
472+
# lzma.LZMAFile or uncompresspy.LZWFile instance opened in binary mode (that
473+
# is, read returns bytes). Now we need to, if requested, wrap it in a
453474
# io.TextIOWrapper so read will return unicode based on the
454475
# encoding parameter.
455476

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
��T��L6e@ QgN2
2+
݀��&!�2s�|a�fL6_Ȅ�拊��qs�`7cT�q3���!O�P)��&
36 Bytes
Binary file not shown.

0 commit comments

Comments
 (0)