Repository navigation
FITS decompression does not work if HDU is not written out #14611
Description
It looks like the decompression is not working quite right when the HDU is not written out to a file first:
In [1]: from astropy.io import fits
In [2]: import numpy as np
In [3]: data = np.arange(21 * 33).reshape((21, 33)).astype(np.int32)
In [4]: header = fits.Header()
In [5]: hdu = fits.CompImageHDU(data, header, compression_type="RICE_1", tile_shape=(5, 6))
In [6]: hdu.section[...]
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
Cell In[6], line 1
----> 1 hdu.section[...]
File ~/Code/astropy/astropy/io/fits/hdu/compressed.py:2224, in CompImageSection.__getitem__(self, index)
2222 first_tile_index = np.zeros(self._n_dim, dtype=int)
2223 last_tile_index = self._n_tiles - 1
-> 2224 data = decompress_hdu_section(self.hdu, first_tile_index, last_tile_index)
2225 return self.hdu._scale_data(data)
2227 index = simplify_basic_index(index, shape=self._data_shape)
File ~/Code/astropy/astropy/io/fits/_tiled_compression/tiled_compression.py:345, in decompress_hdu_section(hdu, first_tile_index, last_tile_index)
342 # If all the data is requested, read in all the heap.
343 if tuple(buffer_shape) == tuple(data_shape):
344 heap_cache = hdu._get_raw_data(
--> 345 hdu._header["PCOUNT"], np.uint8, hdu._data_offset + hdu._theap
346 )
347 else:
348 heap_cache = None
TypeError: unsupported operand type(s) for +: 'NoneType' and 'int'
In [7]: hdu.section[0, 1]
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
File ~/Code/astropy/astropy/io/fits/column.py:2130, in _get_index(names, key)
2129 try:
-> 2130 indx = names.index(key.rstrip())
2131 except ValueError:
2132 # try to match case-insentively,
ValueError: 'GZIP_COMPRESSED_DATA' is not in list
During handling of the above exception, another exception occurred:
KeyError Traceback (most recent call last)
Cell In[7], line 1
----> 1 hdu.section[0, 1]
File ~/Code/astropy/astropy/io/fits/hdu/compressed.py:2268, in CompImageSection.__getitem__(self, index)
2263 last_tile_index[dim] = first_tile_index[dim]
2264 final_array_index.append(
2265 idx - self._tile_shape[dim] * first_tile_index[dim]
2266 )
-> 2268 data = decompress_hdu_section(self.hdu, first_tile_index, last_tile_index)
2270 return self.hdu._scale_data(data[tuple(final_array_index)])
File ~/Code/astropy/astropy/io/fits/_tiled_compression/tiled_compression.py:362, in decompress_hdu_section(hdu, first_tile_index, last_tile_index)
359 if compressed_data_column[row_index][0] == 0:
360 if gzip_compressed_data_column is None:
361 gzip_compressed_data_column = np.array(
--> 362 hdu.compressed_data["GZIP_COMPRESSED_DATA"]
363 )
364 gzip_compressed_data_dtype = _column_dtype(hdu, "GZIP_COMPRESSED_DATA")
366 # When quantizing floating point data, sometimes the data will not
367 # quantize efficiently. In these cases the raw floating point data can
368 # be losslessly GZIP compressed and stored in the `GZIP_COMPRESSED_DATA`
369 # column.
File ~/Code/astropy/astropy/io/fits/fitsrec.py:513, in FITS_rec.__getitem__(self, key)
510 return super().__getitem__(key)
512 if isinstance(key, str):
--> 513 return self.field(key)
515 # Have to view as a recarray then back as a FITS_rec, otherwise the
516 # circular reference fix/hack in FITS_rec.field() won't preserve
517 # the slice.
518 out = self.view(np.recarray)[key]
File ~/Code/astropy/astropy/io/fits/fitsrec.py:695, in FITS_rec.field(self, key)
690 """
691 A view of a `Column`'s data as an array.
692 """
693 # NOTE: The *column* index may not be the same as the field index in
694 # the recarray, if the column is a phantom column
--> 695 column = self.columns[key]
696 name = column.name
697 format = column.format
File ~/Code/astropy/astropy/io/fits/column.py:1739, in ColDefs.__getitem__(self, key)
1737 def __getitem__(self, key):
1738 if isinstance(key, str):
-> 1739 key = _get_index(self.names, key)
1741 x = self.columns[key]
1742 if _is_int(key):
File ~/Code/astropy/astropy/io/fits/column.py:2139, in _get_index(names, key)
2137 indx = names.index(_key)
2138 elif count == 0:
-> 2139 raise KeyError(f"Key '{key}' does not exist.")
2140 else: # multiple match
2141 raise KeyError(f"Ambiguous key name '{key}'.")
KeyError: "Key 'GZIP_COMPRESSED_DATA' does not exist."It looks like hdu.compressed_data is not right:
In [8]: hdu.compressed_data
Out[8]:
FITS_rec([([ 0, 0, 0, 0, 36, 85, 85, 80, 8, 170, 170, 160, 17, 85, 85, 64, 34, 170, 170, 128, 69, 85, 85, 0],),
([ 0, 0, 0, 6, 36, 85, 85, 80, 8, 170, 170, 160, 17, 85, 85, 64, 34, 170, 170, 128, 69, 85, 85, 0],),
([ 0, 0, 0, 12, 36, 85, 85, 80, 8, 170, 170, 160, 17, 85, 85, 64, 34, 170, 170, 128, 69, 85, 85, 0],),
([ 0, 0, 0, 18, 36, 85, 85, 80, 8, 170, 170, 160, 17, 85, 85, 64, 34, 170, 170, 128, 69, 85, 85, 0],),
([ 0, 0, 0, 24, 36, 85, 85, 80, 8, 170, 170, 160, 17, 85, 85, 64, 34, 170, 170, 128, 69, 85, 85, 0],),
([ 0, 0, 0, 30, 44, 37, 33, 233, 72, 122, 82, 30, 148, 135, 165, 32],),
([ 0, 0, 0, 165, 36, 85, 85, 80, 8, 170, 170, 160, 17, 85, 85, 64, 34, 170, 170, 128, 69, 85, 85, 0],),
([ 0, 0, 0, 171, 36, 85, 85, 80, 8, 170, 170, 160, 17, 85, 85, 64, 34, 170, 170, 128, 69, 85, 85, 0],),
([ 0, 0, 0, 177, 36, 85, 85, 80, 8, 170, 170, 160, 17, 85, 85, 64, 34, 170, 170, 128, 69, 85, 85, 0],),
([ 0, 0, 0, 183, 36, 85, 85, 80, 8, 170, 170, 160, 17, 85, 85, 64, 34, 170, 170, 128, 69, 85, 85, 0],),
([ 0, 0, 0, 189, 36, 85, 85, 80, 8, 170, 170, 160, 17, 85, 85, 64, 34, 170, 170, 128, 69, 85, 85, 0],),
([ 0, 0, 0, 195, 44, 37, 33, 233, 72, 122, 82, 30, 148, 135, 165, 32],),
([ 0, 0, 1, 74, 36, 85, 85, 80, 8, 170, 170, 160, 17, 85, 85, 64, 34, 170, 170, 128, 69, 85, 85, 0],),
([ 0, 0, 1, 80, 36, 85, 85, 80, 8, 170, 170, 160, 17, 85, 85, 64, 34, 170, 170, 128, 69, 85, 85, 0],),
([ 0, 0, 1, 86, 36, 85, 85, 80, 8, 170, 170, 160, 17, 85, 85, 64, 34, 170, 170, 128, 69, 85, 85, 0],),
([ 0, 0, 1, 92, 36, 85, 85, 80, 8, 170, 170, 160, 17, 85, 85, 64, 34, 170, 170, 128, 69, 85, 85, 0],),
([ 0, 0, 1, 98, 36, 85, 85, 80, 8, 170, 170, 160, 17, 85, 85, 64, 34, 170, 170, 128, 69, 85, 85, 0],),
([ 0, 0, 1, 104, 44, 37, 33, 233, 72, 122, 82, 30, 148, 135, 165, 32],),
([ 0, 0, 1, 239, 36, 85, 85, 80, 8, 170, 170, 160, 17, 85, 85, 64, 34, 170, 170, 128, 69, 85, 85, 0],),
([ 0, 0, 1, 245, 36, 85, 85, 80, 8, 170, 170, 160, 17, 85, 85, 64, 34, 170, 170, 128, 69, 85, 85, 0],),
([ 0, 0, 1, 251, 36, 85, 85, 80, 8, 170, 170, 160, 17, 85, 85, 64, 34, 170, 170, 128, 69, 85, 85, 0],),
([ 0, 0, 2, 1, 36, 85, 85, 80, 8, 170, 170, 160, 17, 85, 85, 64, 34, 170, 170, 128, 69, 85, 85, 0],),
([ 0, 0, 2, 7, 36, 85, 85, 80, 8, 170, 170, 160, 17, 85, 85, 64, 34, 170, 170, 128, 69, 85, 85, 0],),
([ 0, 0, 2, 13, 44, 37, 33, 233, 72, 122, 82, 30, 148, 135, 165, 32],),
([ 0, 0, 2, 148, 12, 146, 72],),
([ 0, 0, 2, 154, 12, 146, 72],),
([ 0, 0, 2, 160, 12, 146, 72],),
([ 0, 0, 2, 166, 12, 146, 72],),
([ 0, 0, 2, 172, 12, 146, 72],),
([ 0, 0, 2, 178, 12, 144],)],
dtype=(numpy.record, [('COMPRESSED_DATA', '>i4', (2,))]))
Activity
astrofrog commentedon Apr 6, 2023
There are actually multiple issues here:
compressed_dataproperty,super().datawill in the above case internally return the original image data array, not the starting/empty binary table data. This appears to be a bug related to the use of@lazypropertyfordef data- somehow, the fact that we overload what.datameans inCompImageHDU(because of the inheritance ofBinTableHDU) causes issues with the lazy property. For now I'll fix this by not using@lazypropertybut the real fix is to switch the inheritance toImageHDUwhich I'll work on separately..databefore the data is written rather than trying to round-trip the compression.astrofrog commentedon Apr 6, 2023
I am working on this!
mhvk commentedon Apr 7, 2023
Maybe relevant, also in #9238, it was argued compressed images should inherit from
ImageHDU.astrofrog commentedon Apr 7, 2023
@mhvk that seems to be the only sane solution to the issue here and I am working on it!