Skip to content

Commit 54f58bc

Browse files
taldcrofteteq
authored andcommitted
Merge pull request #11720 from taldcroft/ascii-ecsv-blank-value-subtypes
Fix parsing of blank values in ECSV subtype data
1 parent f471bcf commit 54f58bc

5 files changed

Lines changed: 308 additions & 6 deletions

File tree

‎astropy/io/ascii/core.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -733,8 +733,9 @@ def check_column_names(self, names, strict_names, guessing):
733733
or name[0] in bads or name[-1] in bads):
734734
raise InconsistentTableError(
735735
f'Column name {name!r} does not meet strict name requirements')
736-
# When guessing require at least two columns
737-
if guessing and len(self.colnames) <= 1:
736+
# When guessing require at least two columns, except for ECSV which can
737+
# reliably be guessed from the header requirements.
738+
if guessing and len(self.colnames) <= 1 and self.__class__.__name__ != 'EcsvHeader':
738739
raise ValueError('Table format guessing requires at least two columns, got {}'
739740
.format(list(self.colnames)))
740741

‎astropy/io/ascii/ecsv.py‎

Lines changed: 41 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -253,10 +253,34 @@ def _convert_vals(self, cols):
253253
# not currently supported.
254254
elif col.shape and col.shape[-1] is None:
255255
_check_dtype_is_str(col)
256-
# Remake as a 1-d object column of numpy ndarrays using the
257-
# datatype specified in the ECSV file.
258-
col_vals = [np.array(json.loads(val), dtype=col.subtype)
259-
for val in col.str_vals]
256+
257+
# Empty (blank) values in original ECSV are changed to "0"
258+
# in str_vals with corresponding col.mask being created and
259+
# set accordingly. Instead use an empty list here.
260+
if hasattr(col, 'mask'):
261+
for idx in np.nonzero(col.mask)[0]:
262+
col.str_vals[idx] = '[]'
263+
264+
# Remake as a 1-d object column of numpy ndarrays or
265+
# MaskedArray using the datatype specified in the ECSV file.
266+
col_vals = []
267+
for str_val in col.str_vals:
268+
obj_val = json.loads(str_val) # list or nested lists
269+
try:
270+
arr_val = np.array(obj_val, dtype=col.subtype)
271+
except TypeError:
272+
# obj_val has entries that are inconsistent with
273+
# dtype. For a valid ECSV file the only possibility
274+
# is None values (indicating missing values).
275+
data = np.array(obj_val, dtype=object)
276+
# Replace all the None with an appropriate fill value
277+
mask = (data == None) # noqa: E711
278+
kind = np.dtype(col.subtype).kind
279+
data[mask] = {'U': '', 'S': b''}.get(kind, 0)
280+
arr_val = np.ma.array(data.astype(col.subtype), mask=mask)
281+
282+
col_vals.append(arr_val)
283+
260284
col.shape = ()
261285
col.dtype = np.dtype(object)
262286
# np.array(col_vals_arr, dtype=object) fails ?? so this workaround:
@@ -267,6 +291,19 @@ def _convert_vals(self, cols):
267291
# might be masked.
268292
elif col.shape:
269293
_check_dtype_is_str(col)
294+
295+
# Change empty (blank) values in original ECSV to something
296+
# like "[[null, null],[null,null]]" so subsequent JSON
297+
# decoding works. Delete `col.mask` so that later code in
298+
# core TableOutputter.__call__() that deals with col.mask
299+
# does not run (since handling is done here already).
300+
if hasattr(col, 'mask'):
301+
all_none_arr = np.full(shape=col.shape, fill_value=None, dtype=object)
302+
all_none_json = json.dumps(all_none_arr.tolist())
303+
for idx in np.nonzero(col.mask)[0]:
304+
col.str_vals[idx] = all_none_json
305+
del col.mask
306+
270307
col_vals = [json.loads(val) for val in col.str_vals]
271308
# Make a numpy object array of col_vals to look for None
272309
# (masked values)
Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
# %ECSV 1.0
2+
# ---
3+
# delimiter: ','
4+
# datatype:
5+
# -
6+
# name: i_index
7+
# datatype: int32
8+
# description: Row index
9+
# -
10+
# name: s_byte
11+
# datatype: int8
12+
# -
13+
# name: s_short
14+
# datatype: int16
15+
# -
16+
# name: s_int
17+
# datatype: int32
18+
# -
19+
# name: s_long
20+
# datatype: int64
21+
# -
22+
# name: s_float
23+
# datatype: float32
24+
# -
25+
# name: s_double
26+
# datatype: float64
27+
# -
28+
# name: s_string
29+
# datatype: string
30+
# -
31+
# name: s_boolean
32+
# datatype: bool
33+
# -
34+
# name: f_byte
35+
# datatype: string
36+
# subtype: 'int8[3]'
37+
# -
38+
# name: f_short
39+
# datatype: string
40+
# subtype: 'int16[3]'
41+
# -
42+
# name: f_int
43+
# datatype: string
44+
# subtype: 'int32[3]'
45+
# -
46+
# name: f_long
47+
# datatype: string
48+
# subtype: 'int64[3]'
49+
# -
50+
# name: f_float
51+
# datatype: string
52+
# subtype: 'float32[3]'
53+
# -
54+
# name: f_double
55+
# datatype: string
56+
# subtype: 'float64[3]'
57+
# -
58+
# name: f_string
59+
# datatype: string
60+
# subtype: 'string[3]'
61+
# -
62+
# name: f_boolean
63+
# datatype: string
64+
# subtype: 'bool[3]'
65+
# -
66+
# name: v_byte
67+
# datatype: string
68+
# subtype: 'int8[null]'
69+
# -
70+
# name: v_short
71+
# datatype: string
72+
# subtype: 'int16[null]'
73+
# -
74+
# name: v_int
75+
# datatype: string
76+
# subtype: 'int32[null]'
77+
# -
78+
# name: v_long
79+
# datatype: string
80+
# subtype: 'int64[null]'
81+
# -
82+
# name: v_float
83+
# datatype: string
84+
# subtype: 'float32[null]'
85+
# -
86+
# name: v_double
87+
# datatype: string
88+
# subtype: 'float64[null]'
89+
# -
90+
# name: v_string
91+
# datatype: string
92+
# subtype: 'string[null]'
93+
# -
94+
# name: v_boolean
95+
# datatype: string
96+
# subtype: 'bool[null]'
97+
# -
98+
# name: m_int
99+
# datatype: string
100+
# subtype: 'int32[4,2]'
101+
# -
102+
# name: m_double
103+
# datatype: string
104+
# subtype: 'float64[2,3]'
105+
i_index,s_byte,s_short,s_int,s_long,s_float,s_double,s_string,s_boolean,f_byte,f_short,f_int,f_long,f_float,f_double,f_string,f_boolean,v_byte,v_short,v_int,v_long,v_float,v_double,v_string,v_boolean,m_int,m_double
106+
0,0,0,0,0,0.0,0.0,zero,False,"[0,1,2]","[0,1,2]","[0,1,2]","[0,1,2]","[0.0,null,2.5]","[0.0,null,2.5]","[""foo"",null,""zero""]","[false,false,false]","[0,1,2]","[0,1,2]","[0,1,2]","[0,1,2]","[0.0,null,2.5]","[0.0,null,2.5]","[""foo"",null,""zero""]","[false,false,false]","[[1000,1001],[2000,2001],[3000,3001],[4000,4001]]","[[0.25,0.5,0.75],[-0.25,-0.5,-0.75]]"
107+
1,,1,1,1,1.0,,one,True,,"[1,2,3]","[1,2,3]","[1,2,3]","[1.0,null,3.5]","[1.0,null,3.5]","[""foo"",null,""one""]","[true,false,false]",,"[1,2]","[1,2]","[1,2]","[1.0,null]","[1.0,null]","[""foo"",null]","[true,false]",,"[[1.25,1.5,1.75],[-1.25,-1.5,-1.75]]"
108+
2,2,,2,2,,2.0,two,False,"[2,3,4]",,"[2,3,4]","[2,3,4]","[2.0,null,4.5]","[2.0,null,4.5]","[""foo"",null,""two""]","[false,true,false]",[2],,[2],[2],[2.0],[2.0],"[""foo""]",[false],"[[1002,1003],[2002,2003],[3002,3003],[4002,4003]]",
109+
3,3,3,,3,3.0,3.0,three,True,"[3,4,5]","[3,4,5]",,"[3,4,5]","[3.0,null,5.5]","[3.0,null,5.5]","[""foo"",null,""three""]","[true,true,false]",[],[],,[],[],[],[],[],"[[1003,1004],[2003,2004],[3003,3004],[4003,4004]]","[[3.25,3.5,3.75],[-3.25,-3.5,-3.75]]"
110+
4,4,4,4,,4.0,4.0,four,False,"[4,5,6]","[4,5,6]","[4,5,6]",,"[4.0,null,6.5]","[4.0,null,6.5]","[""foo"",null,""four""]","[false,false,true]","[4,5,6]","[4,5,6]","[4,5,6]",,"[4.0,null,6.5]","[4.0,null,6.5]","[""foo"",null,""four""]","[false,false,true]","[[1004,1005],[2004,2005],[3004,3005],[4004,4005]]","[[4.25,4.5,4.75],[-4.25,-4.5,-4.75]]"
111+
5,5,5,5,5,nan,5.0,five,True,"[5,6,7]","[5,6,7]","[5,6,7]","[5,6,7]",,"[5.0,null,7.5]","[""foo"",null,""five""]","[true,false,true]","[5,6]","[5,6]","[5,6]","[5,6]",,"[5.0,null]","[""foo"",null]","[true,false]","[[1005,1006],[2005,2006],[3005,3006],[4005,4006]]","[[5.25,5.5,5.75],[-5.25,-5.5,-5.75]]"
112+
6,6,6,6,6,6.0,nan,six,False,"[6,7,8]","[6,7,8]","[6,7,8]","[6,7,8]","[6.0,null,8.5]",,"[""foo"",null,""six""]","[false,true,true]",[6],[6],[6],[6],[6.0],,"[""foo""]",[false],"[[1006,1007],[2006,2007],[3006,3007],[4006,4007]]","[[6.25,6.5,6.75],[-6.25,-6.5,-6.75]]"
113+
7,7,7,7,7,7.0,7.0,,True,"[7,8,9]","[7,8,9]","[7,8,9]","[7,8,9]","[7.0,null,9.5]","[7.0,null,9.5]",,"[true,true,true]",[],[],[],[],[],[],,[],"[[1007,1008],[2007,2008],[3007,3008],[4007,4008]]","[[7.25,7.5,7.75],[-7.25,-7.5,-7.75]]"
114+
8,8,8,8,8,8.0,8.0,"' ""\""""' ; '&<>",,"[8,9,10]","[8,9,10]","[8,9,10]","[8,9,10]","[8.0,null,10.5]","[8.0,null,10.5]","[""foo"",null,""' \""\\\""\""' ; '&<>""]",,"[8,9,10]","[8,9,10]","[8,9,10]","[8,9,10]","[8.0,null,10.5]","[8.0,null,10.5]","[""foo"",null,""' \""\\\""\""' ; '&<>""]",,"[[1008,1009],[2008,2009],[3008,3009],[4008,4009]]","[[8.25,8.5,8.75],[-8.25,-8.5,-8.75]]"
115+
9,9,9,9,9,nan,nan,,True,"[9,10,11]","[9,10,11]","[9,10,11]","[9,10,11]","[9.0,null,11.5]","[9.0,null,11.5]","[""foo"",null,""""]","[true,false,false]","[9,10]","[9,10]","[9,10]","[9,10]","[9.0,null]","[9.0,null]","[""foo"",null]","[true,false]","[[1009,1010],[2009,2010],[3009,3010],[4009,4010]]","[[9.25,9.5,9.75],[-9.25,-9.5,-9.75]]"
116+
10,-10,-10,-10,-10,-10.0,-10.0,10,False,"[10,11,12]","[10,11,12]","[10,11,12]","[10,11,12]","[10.0,null,12.5]","[10.0,null,12.5]","[""foo"",null,""10""]","[false,true,false]",[10],[10],[10],[10],[10.0],[10.0],"[""foo""]",[false],"[[1010,1011],[2010,2011],[3010,3011],[4010,4011]]","[[10.25,10.5,10.75],[-10.25,-10.5,-10.75]]"
117+
11,,-11,-11,-11,-11.0,-11.0,10 + one,True,,"[11,12,13]","[11,12,13]","[11,12,13]","[11.0,null,13.5]","[11.0,null,13.5]","[""foo"",null,""10 + one""]","[true,true,false]",,[],[],[],[],[],[],[],,"[[11.25,11.5,11.75],[-11.25,-11.5,-11.75]]"
118+
12,-12,,-12,-12,-12.0,-12.0,10 + two,False,"[12,13,14]",,"[12,13,14]","[12,13,14]","[12.0,null,14.5]","[12.0,null,14.5]","[""foo"",null,""10 + two""]","[false,false,true]","[12,13,14]",,"[12,13,14]","[12,13,14]","[12.0,null,14.5]","[12.0,null,14.5]","[""foo"",null,""10 + two""]","[false,false,true]","[[1012,1013],[2012,2013],[3012,3013],[4012,4013]]",
119+
13,-13,-13,,-13,-13.0,-13.0,10 + three,True,"[13,14,15]","[13,14,15]",,"[13,14,15]","[13.0,null,15.5]","[13.0,null,15.5]","[""foo"",null,""10 + three""]","[true,false,true]","[13,14]","[13,14]",,"[13,14]","[13.0,null]","[13.0,null]","[""foo"",null]","[true,false]","[[1013,1014],[2013,2014],[3013,3014],[4013,4014]]","[[13.25,13.5,13.75],[-13.25,-13.5,-13.75]]"
120+
14,-14,-14,-14,,-14.0,-14.0,10 + four,False,"[14,15,16]","[14,15,16]","[14,15,16]",,"[14.0,null,16.5]","[14.0,null,16.5]","[""foo"",null,""10 + four""]","[false,true,true]",[14],[14],[14],,[14.0],[14.0],"[""foo""]",[false],"[[1014,1015],[2014,2015],[3014,3015],[4014,4015]]","[[14.25,14.5,14.75],[-14.25,-14.5,-14.75]]"
121+
15,-15,-15,-15,-15,nan,-15.0,10 + five,True,"[15,16,17]","[15,16,17]","[15,16,17]","[15,16,17]",,"[15.0,null,17.5]","[""foo"",null,""10 + five""]","[true,true,true]",[],[],[],[],,[],[],[],"[[1015,1016],[2015,2016],[3015,3016],[4015,4016]]","[[15.25,15.5,15.75],[-15.25,-15.5,-15.75]]"

‎astropy/io/ascii/tests/test_ecsv.py‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
from astropy import units as u
3232
from astropy.utils.compat.optional_deps import HAS_YAML # noqa
3333

34+
from .common import TEST_DIR
3435

3536
pytestmark = pytest.mark.skipif(not HAS_YAML, reason='YAML required')
3637

@@ -920,3 +921,139 @@ def test_specialized_columns(name, col, exp):
920921
if isinstance(val1, np.ndarray):
921922
assert val1.dtype == val2.dtype
922923
assert np.all(val1 == val2)
924+
925+
926+
def test_full_subtypes():
927+
"""Read ECSV file created by M. Taylor that includes scalar, fixed array,
928+
variable array for all datatypes. This file has missing values for all
929+
columns as both per-value null and blank entries for the entire column
930+
value.
931+
932+
Note: original file was modified to include blank values in f_float and
933+
f_double columns.
934+
"""
935+
t = Table.read(os.path.join(TEST_DIR, 'data', 'subtypes.ecsv'))
936+
colnames = ('i_index,'
937+
's_byte,s_short,s_int,s_long,s_float,s_double,s_string,s_boolean,'
938+
'f_byte,f_short,f_int,f_long,f_float,f_double,f_string,f_boolean,'
939+
'v_byte,v_short,v_int,v_long,v_float,v_double,v_string,v_boolean,'
940+
'm_int,m_double').split(',')
941+
assert t.colnames == colnames
942+
943+
type_map = {'byte': 'int8',
944+
'short': 'int16',
945+
'int': 'int32',
946+
'long': 'int64',
947+
'float': 'float32',
948+
'double': 'float64',
949+
'string': 'str',
950+
'boolean': 'bool'}
951+
952+
for col in t.itercols():
953+
info = col.info
954+
if info.name == 'i_index':
955+
continue
956+
957+
assert isinstance(col, MaskedColumn)
958+
959+
type_name = info.name[2:] # short, int, etc
960+
subtype = info.name[:1]
961+
962+
if subtype == 's': # Scalar
963+
assert col.shape == (16,)
964+
965+
if subtype == 'f': # Fixed array
966+
assert col.shape == (16, 3)
967+
968+
if subtype == 'v': # Variable array
969+
assert col.shape == (16,)
970+
assert info.dtype.name == 'object'
971+
for val in col:
972+
assert isinstance(val, np.ndarray)
973+
assert val.dtype.name.startswith(type_map[type_name])
974+
assert len(val) in [0, 1, 2, 3]
975+
else:
976+
assert info.dtype.name.startswith(type_map[type_name])
977+
978+
979+
def test_masked_empty_subtypes():
980+
"""Test blank field in subtypes. Similar to previous test but with explicit
981+
checks of values"""
982+
txt = """
983+
# %ECSV 1.0
984+
# ---
985+
# datatype:
986+
# - {name: o, datatype: string, subtype: json}
987+
# - {name: f, datatype: string, subtype: 'int64[2]'}
988+
# - {name: v, datatype: string, subtype: 'int64[null]'}
989+
# schema: astropy-2.0
990+
o f v
991+
null [0,1] [1]
992+
"" "" ""
993+
[1,2] [2,3] [2,3]
994+
"""
995+
t = Table.read(txt, format='ascii.ecsv')
996+
assert np.all(t['o'] == np.array([None, -1, [1, 2]], dtype=object))
997+
assert np.all(t['o'].mask == [False, True, False])
998+
999+
exp = np.ma.array([[0, 1], [-1, -1], [2, 3]], mask=[[0, 0], [1, 1], [0, 0]])
1000+
assert np.all(t['f'] == exp)
1001+
assert np.all(t['f'].mask == exp.mask)
1002+
1003+
assert np.all(t['v'][0] == [1])
1004+
assert np.all(t['v'][2] == [2, 3])
1005+
assert np.all(t['v'].mask == [False, True, False])
1006+
1007+
1008+
def test_masked_vals_in_array_subtypes():
1009+
"""Test null values in fixed and variable array subtypes."""
1010+
t = Table()
1011+
t['f'] = np.ma.array([[1, 2], [3, 4]], mask=[[0, 1], [1, 0]], dtype=np.int64)
1012+
t['v'] = np.empty(2, dtype=object)
1013+
t['v'][0] = np.ma.array([1, 2], mask=[0, 1], dtype=np.int64)
1014+
t['v'][1] = np.ma.array([3, 4, 5], mask=[1, 0, 0], dtype=np.int64)
1015+
1016+
out = StringIO()
1017+
t.write(out, format='ascii.ecsv')
1018+
txt = """
1019+
# %ECSV 1.0
1020+
# ---
1021+
# datatype:
1022+
# - {name: f, datatype: string, subtype: 'int64[2]'}
1023+
# - {name: v, datatype: string, subtype: 'int64[null]'}
1024+
# schema: astropy-2.0
1025+
f v
1026+
[1,null] [1,null]
1027+
[null,4] [null,4,5]
1028+
"""
1029+
hdr = _get_ecsv_header_dict(out.getvalue())
1030+
hdr_exp = _get_ecsv_header_dict(txt)
1031+
assert hdr == hdr_exp
1032+
t2 = Table.read(out.getvalue(), format='ascii.ecsv')
1033+
assert t2.colnames == t.colnames
1034+
for name in t2.colnames:
1035+
assert t2[name].dtype == t[name].dtype
1036+
assert type(t2[name]) is type(t[name])
1037+
for val1, val2 in zip(t2[name], t[name]):
1038+
if isinstance(val1, np.ndarray):
1039+
assert val1.dtype == val2.dtype
1040+
if isinstance(val1, np.ma.MaskedArray):
1041+
assert np.all(val1.mask == val2.mask)
1042+
assert np.all(val1 == val2)
1043+
1044+
1045+
def test_guess_ecsv_with_one_column():
1046+
"""Except for ECSV, guessing always requires at least 2 columns"""
1047+
txt = """
1048+
# %ECSV 1.0
1049+
# ---
1050+
# datatype:
1051+
# - {name: col, datatype: string, description: hello}
1052+
# schema: astropy-2.0
1053+
col
1054+
1
1055+
2
1056+
"""
1057+
t = ascii.read(txt)
1058+
assert t['col'].dtype.kind == 'U' # would be int with basic format
1059+
assert t['col'].description == 'hello'
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
Added support for reading and writing multidimensional column data (masked and
2+
unmasked) to ECSV. Also added formal support for reading and writing object-type
3+
column data which can contain items consisting of lists, dicts, and basic scalar
4+
types. This can be used to store columns of variable-length arrays. Both of
5+
these features use JSON to convert the object to a string that is stored in the
6+
ECSV output.

0 commit comments

Comments
 (0)