Repository navigation
BUG: a failing io.fits test #18880
Description
pytest astropy/io/fits/tests/test_fitsdiff.py::TestFITSDiff_script::test_path -ra --run-slow
Detailled output
astropy/io/fits/tests/test_fitsdiff.py F [100%]
===================================================================================================== FAILURES =====================================================================================================
__________________________________________________________________________________________ TestFITSDiff_script.test_path ___________________________________________________________________________________________
self = <astropy.io.fits.diff.FITSDiff object at 0x1056d75c0>, a = '/Users/clm/dev/orgs/astropy-project/coordinated/astropy/astropy/io/fits/tests/data/lzw.fits.Z'
b = '/Users/clm/dev/orgs/astropy-project/coordinated/astropy/astropy/io/fits/tests/data/lzw.fits.Z', ignore_hdus = [], ignore_keywords = [], ignore_comments = [], ignore_fields = [], numdiffs = 10, rtol = 0.0
atol = 0.0, ignore_blanks = True, ignore_blank_cards = True
def __init__(
self,
a,
b,
ignore_hdus=[],
ignore_keywords=[],
ignore_comments=[],
ignore_fields=[],
numdiffs=10,
rtol=0.0,
atol=0.0,
ignore_blanks=True,
ignore_blank_cards=True,
):
"""
Parameters
----------
a : str or `HDUList`
The filename of a FITS file on disk, or an `HDUList` object.
b : str or `HDUList`
The filename of a FITS file on disk, or an `HDUList` object to
compare to the first file.
ignore_hdus : sequence, optional
HDU names to ignore when comparing two FITS files or HDU lists; the
presence of these HDUs and their contents are ignored. Wildcard
strings may also be included in the list.
ignore_keywords : sequence, optional
Header keywords to ignore when comparing two headers; the presence
of these keywords and their values are ignored. Wildcard strings
may also be included in the list.
ignore_comments : sequence, optional
A list of header keywords whose comments should be ignored in the
comparison. May contain wildcard strings as with ignore_keywords.
ignore_fields : sequence, optional
The (case-insensitive) names of any table columns to ignore if any
table data is to be compared.
numdiffs : int, optional
The number of pixel/table values to output when reporting HDU data
differences. Though the count of differences is the same either
way, this allows controlling the number of different values that
are kept in memory or output. If a negative value is given, then
numdiffs is treated as unlimited (default: 10).
rtol : float, optional
The relative difference to allow when comparing two float values
either in header values, image arrays, or table columns
(default: 0.0). Values which satisfy the expression
.. math::
\\left| a - b \\right| > \\text{atol} + \\text{rtol} \\cdot \\left| b \\right|
are considered to be different.
The underlying function used for comparison is `numpy.allclose`.
.. versionadded:: 2.0
atol : float, optional
The allowed absolute difference. See also ``rtol`` parameter.
.. versionadded:: 2.0
ignore_blanks : bool, optional
Ignore extra whitespace at the end of string values either in
headers or data. Extra leading whitespace is not ignored
(default: True).
ignore_blank_cards : bool, optional
Ignore all cards that are blank, i.e. they only contain
whitespace (default: True).
"""
if isinstance(a, (str, os.PathLike)):
try:
> a = fitsopen(a)
^^^^^^^^^^^
astropy/io/fits/diff.py:293:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
astropy/io/fits/hdu/hdulist.py:227: in fitsopen
return HDUList.fromfile(
astropy/io/fits/hdu/hdulist.py:491: in fromfile
return cls._readfrom(
astropy/io/fits/hdu/hdulist.py:1174: in _readfrom
fileobj = _File(
astropy/io/fits/file.py:240: in __init__
self._open_filename(fileobj, mode, overwrite)
astropy/io/fits/file.py:700: in _open_filename
if not self._try_read_compressed(self.name, magic, mode, ext=ext):
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <astropy.io.fits.file._File None>, obj_or_name = '/Users/clm/dev/orgs/astropy-project/coordinated/astropy/astropy/io/fits/tests/data/lzw.fits.Z', magic = b'\x1f\x9d\x90S\x924', mode = 'readonly'
ext = '.Z'
def _try_read_compressed(self, obj_or_name, magic, mode, ext=""):
"""Attempt to determine if the given file is compressed."""
is_ostream = mode == "ostream"
if (is_ostream and ext == ".gz") or magic.startswith(GZIP_MAGIC):
if mode == "append":
raise OSError(
"'append' mode is not supported with gzip files."
"Use 'update' mode instead"
)
# Handle gzip files
kwargs = {"mode": IO_FITS_MODES[mode]}
if isinstance(obj_or_name, str):
kwargs["filename"] = obj_or_name
else:
kwargs["fileobj"] = obj_or_name
self._file = gzip.GzipFile(**kwargs)
self.compression = "gzip"
elif (is_ostream and ext == ".zip") or magic.startswith(PKZIP_MAGIC):
# Handle zip files
self._open_zipfile(self.name, mode)
self.compression = "zip"
elif (is_ostream and ext == ".bz2") or magic.startswith(BZIP2_MAGIC):
# Handle bzip2 files
if mode in ["update", "append"]:
raise OSError(
"update and append modes are not supported with bzip2 files"
)
if not HAS_BZ2:
raise ModuleNotFoundError(
"This Python installation does not provide the bz2 module."
)
# bzip2 only supports 'w' and 'r' modes
bzip2_mode = "w" if is_ostream else "r"
self._file = bz2.BZ2File(obj_or_name, mode=bzip2_mode)
self.compression = "bzip2"
elif (is_ostream and ext == ".xz") or magic.startswith(LZMA_MAGIC):
# Handle lzma files
if mode in ["update", "append"]:
raise OSError(
"update and append modes are not supported with lzma files"
)
if not HAS_LZMA:
raise ModuleNotFoundError(
"This Python installation does not provide the lzma module."
)
lzma_mode = "w" if is_ostream else "r"
self._file = lzma.LZMAFile(obj_or_name, mode=lzma_mode)
self.compression = "lzma"
elif (is_ostream and ext == ".Z") or magic.startswith(LZW_MAGIC):
# Handle LZW files
if mode in ["update", "append", "ostream"]:
raise OSError(f"{mode} mode not supported with LZW files")
if not HAS_UNCOMPRESSPY:
> raise ModuleNotFoundError(
"The optional package uncompresspy is necessary for reading"
" LZW compressed files (.Z extension)."
)
E ModuleNotFoundError: The optional package uncompresspy is necessary for reading LZW compressed files (.Z extension).
astropy/io/fits/file.py:604: ModuleNotFoundError
The above exception was the direct cause of the following exception:
self = <astropy.io.fits.tests.test_fitsdiff.TestFITSDiff_script object at 0x105655790>, capsys = <_pytest.capture.CaptureFixture object at 0x1056dcc20>
@pytest.mark.slow
def test_path(self, capsys):
os.mkdir(self.temp("sub/"))
tmp_b = self.temp("sub/ascii.fits")
tmp_g = self.temp("sub/group.fits")
tmp_h = self.data("group.fits")
with hdulist.fitsopen(tmp_h) as hdu_b:
hdu_b.writeto(tmp_g)
writeto(tmp_b, np.arange(100).reshape(10, 10))
# one modified file and a directory
assert fitsdiff.main(["-q", self.data_dir, tmp_b]) == 1
assert fitsdiff.main(["-q", tmp_b, self.data_dir]) == 1
# two directories
tmp_d = self.temp("sub/")
assert fitsdiff.main(["-q", self.data_dir, tmp_d]) == 1
assert fitsdiff.main(["-q", tmp_d, self.data_dir]) == 1
> assert fitsdiff.main(["-q", self.data_dir, self.data_dir]) == 0
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
astropy/io/fits/tests/test_fitsdiff.py:259:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
astropy/io/fits/scripts/fitsdiff.py:401: in main
diff = fits.diff.FITSDiff(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <astropy.io.fits.diff.FITSDiff object at 0x1056d75c0>, a = '/Users/clm/dev/orgs/astropy-project/coordinated/astropy/astropy/io/fits/tests/data/lzw.fits.Z'
b = '/Users/clm/dev/orgs/astropy-project/coordinated/astropy/astropy/io/fits/tests/data/lzw.fits.Z', ignore_hdus = [], ignore_keywords = [], ignore_comments = [], ignore_fields = [], numdiffs = 10, rtol = 0.0
atol = 0.0, ignore_blanks = True, ignore_blank_cards = True
def __init__(
self,
a,
b,
ignore_hdus=[],
ignore_keywords=[],
ignore_comments=[],
ignore_fields=[],
numdiffs=10,
rtol=0.0,
atol=0.0,
ignore_blanks=True,
ignore_blank_cards=True,
):
"""
Parameters
----------
a : str or `HDUList`
The filename of a FITS file on disk, or an `HDUList` object.
b : str or `HDUList`
The filename of a FITS file on disk, or an `HDUList` object to
compare to the first file.
ignore_hdus : sequence, optional
HDU names to ignore when comparing two FITS files or HDU lists; the
presence of these HDUs and their contents are ignored. Wildcard
strings may also be included in the list.
ignore_keywords : sequence, optional
Header keywords to ignore when comparing two headers; the presence
of these keywords and their values are ignored. Wildcard strings
may also be included in the list.
ignore_comments : sequence, optional
A list of header keywords whose comments should be ignored in the
comparison. May contain wildcard strings as with ignore_keywords.
ignore_fields : sequence, optional
The (case-insensitive) names of any table columns to ignore if any
table data is to be compared.
numdiffs : int, optional
The number of pixel/table values to output when reporting HDU data
differences. Though the count of differences is the same either
way, this allows controlling the number of different values that
are kept in memory or output. If a negative value is given, then
numdiffs is treated as unlimited (default: 10).
rtol : float, optional
The relative difference to allow when comparing two float values
either in header values, image arrays, or table columns
(default: 0.0). Values which satisfy the expression
.. math::
\\left| a - b \\right| > \\text{atol} + \\text{rtol} \\cdot \\left| b \\right|
are considered to be different.
The underlying function used for comparison is `numpy.allclose`.
.. versionadded:: 2.0
atol : float, optional
The allowed absolute difference. See also ``rtol`` parameter.
.. versionadded:: 2.0
ignore_blanks : bool, optional
Ignore extra whitespace at the end of string values either in
headers or data. Extra leading whitespace is not ignored
(default: True).
ignore_blank_cards : bool, optional
Ignore all cards that are blank, i.e. they only contain
whitespace (default: True).
"""
if isinstance(a, (str, os.PathLike)):
try:
a = fitsopen(a)
except Exception as exc:
> raise OSError(f"error opening file a ({a})") from exc
E OSError: error opening file a (/Users/clm/dev/orgs/astropy-project/coordinated/astropy/astropy/io/fits/tests/data/lzw.fits.Z)
astropy/io/fits/diff.py:295: OSError
bisected to 92b38ae (#17960), which was merged more than 6 months ago.
While the test is rarely run, because it's marked as @pytest.mark.slow, I don't understand why I'm only seeing it in logs now.
Activity
neutrinoceros commentedon Nov 11, 2025
marking as critical for 7.2.0 for visibility even though it's not clear yet if this has any user-facing implications
neutrinoceros commentedon Nov 11, 2025
The problem seems pretty to explain actually: the test attempts to diff our test data with itself, but chokes on
astropy/io/fits/tests/data/lzw.fits.Zifuncompresspyisn't installed because it cannot read the file at all.there are several ways we could address this, tentatively sorted from easiest to hardest to implement (or from least correct to most correct, IMO)
uncompresspyis available-qis passed ?)@pytest.mark.slow?)ping @astrofrog @saimn
pllim commentedon Nov 11, 2025
My vote is (1)
@kYwzor did some work on it recently so maybe he has opinion too?
saimn commentedon Nov 11, 2025
This test is a bit annoying but given that users might run in the same error, with the current behavior of trying to open all files, I think that it would make sense to warn instead of raising an error.
kYwzor commentedon Nov 11, 2025
I agree with @neutrinoceros's explanation,
FITSDiffdepends onfitsopento open files, which in turn depends on certain packages if those files are compressed. This is technically not limited touncompresspy, as the built-inbz2andlzmapackages are also not guaranteed to exist (Python can be built without them), so I'd say we should keep that in mind when fixing this.I don't think making
uncompresspymandatory makes sense, as it is somewhat niche (again, not evenbz2orlzmaare mandatory). Option 1 would work (you could just mark skip fornot HAS_BZ2 or not HAS_LZMA or not HAS_UNCOMPRESSPY), though I would say it's a bit annoying that you'd be skipping the entire test just because of a couple of compressed files, which aren't even the point of the test.My suggestion would actually be "Option 5". Simply change the line from
assert fitsdiff.main(["-q", self.data_dir, self.data_dir]) == 0toassert fitsdiff.main(["-q", tmp_d, tmp_d]) == 0, which maintains the spirit of the test (checking that comparing a folder with itself returns 0). I think this is a good compromise because in a later step we're doingassert fitsdiff.main(["-q", self.data_dir + "/*.fits", self.data_dir]) == 0which is essentially comparingself.data_dirto itself, but skipping all the compressed files. This should also significantly speed-up the test as we're not doing any decompression (be it gz, bz2, xz or Z files). If we really care about checking thatFITSDiffworks with compressed files, we could add that as a separate test, protected by the necessaryskipifclauses.Side-note, I don't think FITSDiff should be doing that
except Exception as exconly to raise a different error. Whatever error comes fromfitsopenis probably much more explicit than the genericOSError: error opening file, so in my opinion we should just let it propagate.