Skip to content

Commit e807dbf

Browse files
committed
Add tests and fix some issues
1 parent 680fb1b commit e807dbf

2 files changed

Lines changed: 68 additions & 11 deletions

File tree

‎astropy/io/ascii/ecsv.py‎

Lines changed: 25 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@
2121

2222
ECSV_VERSION = '1.0'
2323
DELIMITERS = (' ', ',')
24+
ECSV_DATATYPES = (
25+
'bool', 'int8', 'int16', 'int32', 'int64', 'uint8', 'uint16',
26+
'uint32', 'uint64', 'float16', 'float32', 'float64',
27+
'float128', 'complex64', 'complex128', 'complex256', 'string')
2428

2529

2630
class EcsvHeader(basic.BasicHeader):
@@ -176,7 +180,11 @@ def get_cols(self, lines):
176180
for attr in ('description', 'format', 'unit', 'meta', 'shape', 'subtype'):
177181
if attr in header_cols[col.name]:
178182
setattr(col, attr, header_cols[col.name][attr])
183+
179184
col.dtype = header_cols[col.name]['datatype']
185+
if col.dtype not in ECSV_DATATYPES:
186+
raise ValueError(f'datatype {col.dtype!r} of column {col.name!r} '
187+
f'is not in allowed values {ECSV_DATATYPES}')
180188

181189
# ECSV "string" means numpy dtype.kind == 'U' AKA str in Python 3
182190
for attr in ('dtype', 'subtype'):
@@ -186,6 +194,11 @@ def get_cols(self, lines):
186194
raise TypeError('ecsv reader does not support complex number types')
187195

188196

197+
def _check_dtype_is_str(col):
198+
if col.dtype != 'str':
199+
raise ValueError(f'datatype of column {col.name!r} must be "string"')
200+
201+
189202
class EcsvOutputter(core.TableOutputter):
190203
"""
191204
After reading the input lines and processing, convert the Reader columns
@@ -220,13 +233,15 @@ def _convert_vals(self, cols):
220233
try:
221234
# 1-d or N-d object columns are serialized as JSON.
222235
if col.subtype == 'object':
236+
_check_dtype_is_str(col)
223237
col_vals = [json.loads(val) for val in col.str_vals]
224238
col.data = np.array(col_vals, dtype=col.subtype)
225239

226240
# Variable length arrays with shape (n, m, ..., *) for fixed
227241
# n, m, .. and variable in last axis. Masked values here are
228242
# not currently supported.
229243
elif col.shape and col.shape[-1] is None:
244+
_check_dtype_is_str(col)
230245
# Remake as a 1-d object column of numpy ndarrays using the
231246
# datatype specified in the ECSV file.
232247
col_vals = [np.array(json.loads(val), dtype=col.subtype)
@@ -240,11 +255,12 @@ def _convert_vals(self, cols):
240255
# Multidim columns with consistent shape (n, m, ...). These
241256
# might be masked.
242257
elif col.shape:
258+
_check_dtype_is_str(col)
243259
col_vals = [json.loads(val) for val in col.str_vals]
244260
# Make a numpy object array of col_vals to look for None
245261
# (masked values)
246262
data = np.array(col_vals, dtype=object)
247-
mask = (data == None) # noqa
263+
mask = (data == None) # noqa: E711
248264
if not np.any(mask):
249265
# No None's, just convert to required dtype
250266
col.data = data.astype(col.subtype)
@@ -253,7 +269,7 @@ def _convert_vals(self, cols):
253269
kind = np.dtype(col.subtype).kind
254270
data[mask] = {'U': '', 'S': b''}.get(kind, 0)
255271
# Finally make a MaskedArray with the filled data + mask
256-
col.data = np.ma.array(data.astype(col.dtype), mask=mask)
272+
col.data = np.ma.array(data.astype(col.subtype), mask=mask)
257273

258274
# Regular scalar value column
259275
else:
@@ -311,12 +327,14 @@ def str_vals(self):
311327
- Only replace masked values with "", not the generalized filling
312328
"""
313329
for col in self.cols:
314-
if len(col.shape) > 1:
315-
def format_col_item(idx):
316-
return json.dumps(col[idx].tolist(), separators=(',', ':'))
317-
elif col.info.dtype.kind == 'O':
330+
if len(col.shape) > 1 or col.info.dtype.kind == 'O':
318331
def format_col_item(idx):
319-
return json.dumps(col[idx], separators=(',', ':'))
332+
obj = col[idx]
333+
try:
334+
obj = obj.tolist()
335+
except AttributeError:
336+
pass
337+
return json.dumps(obj, separators=(',', ':'))
320338
else:
321339
def format_col_item(idx):
322340
return str(col[idx])

‎astropy/io/ascii/tests/test_ecsv.py‎

Lines changed: 43 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -307,6 +307,9 @@ def test_roundtrip_multidim_masked_array(serialize_method, dtype, delimiter):
307307
# TODO also test empty string with null value
308308
t = Table()
309309
col = MaskedColumn(np.arange(12).reshape(2, 3, 2), dtype=dtype)
310+
if dtype is str:
311+
# np does something funny and gives a dtype of U12.
312+
col = col.astype('U2')
310313
col.mask[0, 0, 0] = True
311314
col.mask[1, 1, 1] = True
312315
t['a'] = col
@@ -318,6 +321,7 @@ def test_roundtrip_multidim_masked_array(serialize_method, dtype, delimiter):
318321
assert t2.masked is False
319322
assert t2.colnames == t.colnames
320323
for name in t2.colnames:
324+
assert t2[name].dtype == t[name].dtype
321325
if hasattr(t[name], 'mask'):
322326
assert np.all(t2[name].mask == t[name].mask)
323327
assert np.all(t2[name] == t[name])
@@ -330,7 +334,8 @@ def test_multidim_bad_shape():
330334
# ---
331335
# datatype:
332336
# - name: a
333-
# datatype: int64
337+
# datatype: string
338+
# subtype: int64
334339
# shape: [3]
335340
# schema: astropy-2.0
336341
a
@@ -346,14 +351,16 @@ def test_object_column():
346351
This is also covered somewhat in test_serialize.
347352
"""
348353
t = Table()
349-
t['a'] = np.array([[1, 2], [1, 2, 3]], dtype=object)
354+
t['a'] = np.array([np.array([1, 2]), np.array([1, 2, 3])], dtype=object)
350355
t['b'] = np.array([{'a': 1}, [{'b': 2}, [1, 2]]], dtype=object)
351356
out = StringIO()
352357
t.write(out, format='ascii.ecsv')
353358
t2 = Table.read(out.getvalue(), format='ascii.ecsv')
354359
assert t2.colnames == t.colnames
355360
for name in t2.colnames:
356-
assert np.all(t2[name] == t[name])
361+
assert t2[name].dtype == t[name].dtype
362+
for val1, val2 in zip(t2[name], t[name]):
363+
assert np.all(val1 == val2)
357364

358365

359366
def test_write_not_json_serializable():
@@ -371,11 +378,43 @@ def test_read_not_json_serializable():
371378
# %ECSV 1.0
372379
# ---
373380
# datatype:
374-
# - {name: a, datatype: object}
381+
# - {name: a, datatype: string, subtype: object}
375382
# schema: astropy-2.0
376383
a
377384
fail
378385
[3,4]"""
379386
match = "column 'a' failed to convert: column value is not valid JSON"
380387
with pytest.raises(ValueError, match=match):
381388
Table.read(txt, format='ascii.ecsv')
389+
390+
391+
def test_read_bad_datatype_for_object_subtype():
392+
"""Test a malformed ECSV file"""
393+
txt = """\
394+
# %ECSV 1.0
395+
# ---
396+
# datatype:
397+
# - {name: a, datatype: int64, subtype: object}
398+
# schema: astropy-2.0
399+
a
400+
fail
401+
[3,4]"""
402+
match = "column 'a' failed to convert: datatype of column 'a' must be \"string\""
403+
with pytest.raises(ValueError, match=match):
404+
Table.read(txt, format='ascii.ecsv')
405+
406+
407+
def test_read_bad_datatype():
408+
"""Test a malformed ECSV file"""
409+
txt = """\
410+
# %ECSV 1.0
411+
# ---
412+
# datatype:
413+
# - {name: a, datatype: object}
414+
# schema: astropy-2.0
415+
a
416+
fail
417+
[3,4]"""
418+
match = r"column 'a' is not in allowed values \('bool', 'int8', 'int16', 'int32'"
419+
with pytest.raises(ValueError, match=match):
420+
Table.read(txt, format='ascii.ecsv')

0 commit comments

Comments
 (0)