Cannot round-trip masked value to ECSV #5337
Description
Astropy 1.2.1, Python 3.5.2, macOS 10.11.
A colleague reported being unable to load an ECSV file with a masked value into a masked Astropy table. Here is code that reproduces the exception:
from astropy import table
a = [1, 4, 5]
b = [2.0, 5.0, 8.2]
c = ['x', 'y', 'z']
t = table.Table([a, b, c], names=('a', 'b', 'c'), masked=True, dtype=('i4', 'f8', 'S1'))
t['a'].mask = [True, True, False]
t.write('./temp.ecsv', format='ascii.ecsv')
table.Table.read('./temp.ecsv', format='ascii.ecsv')The table file:
# %ECSV 0.9
# ---
# datatype:
# - {name: a, datatype: int32}
# - {name: b, datatype: float64}
# - {name: c, datatype: string}
a b c
-- 2.0 x
-- 5.0 y
5 8.2 z
The exception:
---------------------------------------------------------------------------
IndexError Traceback (most recent call last)
/.../lib/python3.5/site-packages/astropy/io/ascii/core.py in _convert_vals(self, cols)
942 try:
--> 943 converter_func, converter_type = col.converters[0]
944 if not issubclass(converter_type, col.type):
IndexError: list index out of range
During handling of the above exception, another exception occurred:
ValueError Traceback (most recent call last)
<ipython-input-5-d4804d245b66> in <module>()
8 t.write('./temp.ecsv', format='ascii.ecsv')
9
---> 10 table.Table.read('./temp.ecsv', format='ascii.ecsv')
/.../lib/python3.5/site-packages/astropy/table/table.py in read(cls, *args, **kwargs)
2330 passed through to the underlying data reader (e.g. `~astropy.io.ascii.read`).
2331 """
-> 2332 return io_registry.read(cls, *args, **kwargs)
2333
2334 def write(self, *args, **kwargs):
/.../lib/python3.5/site-packages/astropy/io/registry.py in read(cls, *args, **kwargs)
349
350 reader = get_reader(format, cls)
--> 351 data = reader(*args, **kwargs)
352
353 if not isinstance(data, cls):
/.../lib/python3.5/site-packages/astropy/io/ascii/connect.py in io_read(format, filename, **kwargs)
35 from .ui import read
36 format = re.sub(r'^ascii\.', '', format)
---> 37 return read(filename, format=format, **kwargs)
38
39
/.../lib/python3.5/site-packages/astropy/io/ascii/ui.py in read(table, guess, **kwargs)
339 ' with fast (no guessing)'})
340 else:
--> 341 dat = reader.read(table)
342 _read_trace.append({'kwargs': new_kwargs,
343 'status': 'Success with specified Reader class '
/.../lib/python3.5/site-packages/astropy/io/ascii/core.py in read(self, table)
1178
1179 self.data.masks(cols)
-> 1180 table = self.outputter(cols, self.meta)
1181 if hasattr(self.header, 'table_meta'):
1182 table.meta.update(self.header.table_meta)
/.../lib/python3.5/site-packages/astropy/io/ascii/core.py in __call__(self, cols, meta)
970
971 def __call__(self, cols, meta):
--> 972 self._convert_vals(cols)
973
974 # If there are any values that were filled and tagged with a mask bit then this
/.../lib/python3.5/site-packages/astropy/io/ascii/core.py in _convert_vals(self, cols)
957 last_err = err
958 except IndexError:
--> 959 raise ValueError('Column {} failed to convert: {}'.format(col.name, last_err))
960
961
ValueError: Column a failed to convert: invalid literal for int() with base 10: '--'
Anonymized user comments:
I'm wondering if it because the first line is masked, because I realised I have another file with a masked value and reads in ok. So in that case, indeed maybe a bug. Still, in the interest of doing science and not arsing around with error messages: bogus values for the win at the moment!
That's rather vexing, I agree, since masking entries is the Right Way To Do It and this seems to be pushing you towards using bogus sentinel values like in the
-99.99 magdaysI have no experience with the ecsv file format as such but have found masked values a bit tricky to work with in astropy…. I usually end up using bogus values too.