Skip to content

Commit f2967b6

Browse files
committed
Address issue astropy#8995 by ignoring BINARY2 mask bits for string values on read.
1 parent b5da86a commit f2967b6

3 files changed

Lines changed: 76 additions & 0 deletions

File tree

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<VOTABLE version="1.3" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/v1.3">
3+
<RESOURCE type="results">
4+
<INFO name="QUERY_STATUS" value="OK"/>
5+
<INFO name="QUERY" value=""><![CDATA[
6+
select top 3 source_id, ra, dec, a_g_val, datalink_url, epoch_photometry_url
7+
from gaiadr2.gaia_source;
8+
]]></INFO>
9+
<INFO name="CAPTION" value=""><![CDATA[How to cite and acknowledge Gaia: http://gea.esac.esa.int/archive/documentation/credits.html]]></INFO>
10+
<INFO name="PAGE" value=""></INFO>
11+
<INFO name="PAGE_SIZE" value=""></INFO>
12+
<INFO name="JOBID" value="1564151313219O"><![CDATA[1564151313219O]]></INFO>
13+
<INFO name="JOBNAME" value=""></INFO>
14+
<COOSYS ID="GAIADR2" epoch="J2015.5" system="ICRS" />
15+
16+
<TABLE>
17+
<FIELD datatype="long" name="source_id" ucd="meta.id">
18+
<DESCRIPTION>Unique source identifier (unique within a particular Data Release)</DESCRIPTION>
19+
</FIELD>
20+
<FIELD datatype="double" name="ra" ref="GAIADR2" ucd="pos.eq.ra;meta.main" unit="deg" utype="Char.SpatialAxis.Coverage.Location.Coord.Position2D.Value2.C1">
21+
<DESCRIPTION>Right ascension</DESCRIPTION>
22+
</FIELD>
23+
<FIELD datatype="double" name="dec" ref="GAIADR2" ucd="pos.eq.dec;meta.main" unit="deg" utype="Char.SpatialAxis.Coverage.Location.Coord.Position2D.Value2.C2">
24+
<DESCRIPTION>Declination</DESCRIPTION>
25+
</FIELD>
26+
<FIELD datatype="float" name="a_g_val" ucd="phys.absorption.gal" unit="mag">
27+
<DESCRIPTION>line-of-sight extinction in the G band, A_G)</DESCRIPTION>
28+
</FIELD>
29+
<FIELD arraysize="*" datatype="char" name="datalink_url" ucd="meta.ref.url" utype="Acess.reference">
30+
<DESCRIPTION>datalink url</DESCRIPTION>
31+
</FIELD>
32+
<FIELD arraysize="*" datatype="char" name="epoch_photometry_url" ucd="meta.ref.url" utype="Acess.reference">
33+
<DESCRIPTION>epoch photometry url</DESCRIPTION>
34+
</FIELD>
35+
<DATA>
36+
<BINARY2>
37+
<STREAM encoding='base64'>
38+
FFLLmBAAGRWAQHAVRQ+m94jARLCaKTcsmH/AAAAAAABNaHR0cDovL2dlYWRhdGEu
39+
ZXNhYy5lc2EuaW50L2RhdGEtc2VydmVyL2RhdGFsaW5rL2xpbmtzP0lEPTU5NjYw
40+
MjkzMjU4NzA4OTY1MTIAAAAAFFLLkKkAGZWAQHAT1HglkUvARMFn9Z1IbH/AAAAA
41+
AABNaHR0cDovL2dlYWRhdGEuZXNhYy5lc2EuaW50L2RhdGEtc2VydmVyL2RhdGFs
42+
aW5rL2xpbmtzP0lEPTU5NjYwMjExODY5MDc5MDMzNjAAAAAAFFLLjfkAPRcAQHAW
43+
5p3URUPARKpfojAl2n/AAAAAAABNaHR0cDovL2dlYWRhdGEuZXNhYy5lc2EuaW50
44+
L2RhdGEtc2VydmVyL2RhdGFsaW5rL2xpbmtzP0lEPTU5NjYwMTgyMzE5NzI3MzA2
45+
MjQAAAAA
46+
</STREAM>
47+
</BINARY2>
48+
</DATA>
49+
</TABLE>
50+
</RESOURCE>
51+
</VOTABLE>

‎astropy/io/votable/tests/table_test.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
import pathlib
99
import pytest
1010
import numpy as np
11+
from numpy.testing import assert_array_equal
12+
from numpy.ma import allequal
1113

1214
from astropy.config import set_temp_config, reload_config
1315
from astropy.utils.data import get_pkg_data_filename, get_pkg_data_fileobj
@@ -185,6 +187,23 @@ def test_empty_table():
185187
table = votable.get_first_table()
186188
astropy_table = table.to_table() # noqa
187189

190+
def test_binary2_masked_strings():
191+
"""
192+
Issue #8995
193+
"""
194+
# Read a VOTable which sets the null mask bit for each empty string value.
195+
votable = parse(get_pkg_data_filename('data/binary2_masked_strings.xml'))
196+
table = votable.get_first_table()
197+
astropy_table = table.to_table()
198+
199+
# Ensure string columns have no masked values and can be written out
200+
assert not np.any(table.array.mask['epoch_photometry_url'])
201+
output = io.BytesIO()
202+
astropy_table.write(output, format='votable')
203+
204+
# TODO - Construct test cases for unicodeChar strings and fixed length
205+
# strings of both types.
206+
188207

189208
class TestVerifyOptions:
190209

‎astropy/io/votable/tree.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2626,6 +2626,12 @@ def _parse_binary(self, mode, iterator, colnumbers, fields, config, pos):
26262626
mask_bits = careful_read(int((len(fields) + 7) / 8))
26272627
row_mask_data = list(converters.bitarray_to_bool(
26282628
mask_bits, len(fields)))
2629+
2630+
# Ignore the mask for string columns (see issue 8995)
2631+
for i, f in enumerate(fields):
2632+
if f.datatype == 'char' or f.datatype == 'unicodeChar':
2633+
row_mask_data[i] = False
2634+
26292635
for i, binparse in enumerate(binparsers):
26302636
try:
26312637
value, value_mask = binparse(careful_read)

0 commit comments

Comments
 (0)