Skip to content

Commit 9f96a60

Browse files
pllimpllim
authored
Merge pull request #9057 from tomdonaldson/masked-column-writing
Address issue #8995 by ignoring BINARY2 mask bits for string values.
2 parents 3f9332a + 23cb42b commit 9f96a60

4 files changed

Lines changed: 76 additions & 0 deletions

File tree

‎CHANGES.rst‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -508,6 +508,10 @@ astropy.io.registry
508508
astropy.io.votable
509509
^^^^^^^^^^^^^^^^^^
510510

511+
- Address issue #8995 by ignoring BINARY2 null mask bits for string values
512+
on parsing a VOTable. In this way, the reader should never create masked
513+
values for string types. [#9057]
514+
511515
astropy.modeling
512516
^^^^^^^^^^^^^^^^
513517

‎astropy/io/votable/tests/data/binary2_masked_strings.xml‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<VOTABLE version="1.3" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/v1.3">
3+
<RESOURCE type="results">
4+
<INFO name="QUERY_STATUS" value="OK"/>
5+
<INFO name="QUERY" value=""><![CDATA[
6+
select top 3 source_id, ra, dec, a_g_val, datalink_url, epoch_photometry_url
7+
from gaiadr2.gaia_source;
8+
]]></INFO>
9+
<INFO name="CAPTION" value=""><![CDATA[How to cite and acknowledge Gaia: http://gea.esac.esa.int/archive/documentation/credits.html]]></INFO>
10+
<INFO name="PAGE" value=""></INFO>
11+
<INFO name="PAGE_SIZE" value=""></INFO>
12+
<INFO name="JOBID" value="1564151313219O"><![CDATA[1564151313219O]]></INFO>
13+
<INFO name="JOBNAME" value=""></INFO>
14+
<COOSYS ID="GAIADR2" epoch="J2015.5" system="ICRS" />
15+
16+
<TABLE>
17+
<FIELD datatype="long" name="source_id" ucd="meta.id">
18+
<DESCRIPTION>Unique source identifier (unique within a particular Data Release)</DESCRIPTION>
19+
</FIELD>
20+
<FIELD datatype="double" name="ra" ref="GAIADR2" ucd="pos.eq.ra;meta.main" unit="deg" utype="Char.SpatialAxis.Coverage.Location.Coord.Position2D.Value2.C1">
21+
<DESCRIPTION>Right ascension</DESCRIPTION>
22+
</FIELD>
23+
<FIELD datatype="double" name="dec" ref="GAIADR2" ucd="pos.eq.dec;meta.main" unit="deg" utype="Char.SpatialAxis.Coverage.Location.Coord.Position2D.Value2.C2">
24+
<DESCRIPTION>Declination</DESCRIPTION>
25+
</FIELD>
26+
<FIELD datatype="float" name="a_g_val" ucd="phys.absorption.gal" unit="mag">
27+
<DESCRIPTION>line-of-sight extinction in the G band, A_G)</DESCRIPTION>
28+
</FIELD>
29+
<FIELD arraysize="*" datatype="char" name="datalink_url" ucd="meta.ref.url" utype="Acess.reference">
30+
<DESCRIPTION>datalink url</DESCRIPTION>
31+
</FIELD>
32+
<FIELD arraysize="*" datatype="char" name="epoch_photometry_url" ucd="meta.ref.url" utype="Acess.reference">
33+
<DESCRIPTION>epoch photometry url</DESCRIPTION>
34+
</FIELD>
35+
<DATA>
36+
<BINARY2>
37+
<STREAM encoding='base64'>
38+
FFLLmBAAGRWAQHAVRQ+m94jARLCaKTcsmH/AAAAAAABNaHR0cDovL2dlYWRhdGEu
39+
ZXNhYy5lc2EuaW50L2RhdGEtc2VydmVyL2RhdGFsaW5rL2xpbmtzP0lEPTU5NjYw
40+
MjkzMjU4NzA4OTY1MTIAAAAAFFLLkKkAGZWAQHAT1HglkUvARMFn9Z1IbH/AAAAA
41+
AABNaHR0cDovL2dlYWRhdGEuZXNhYy5lc2EuaW50L2RhdGEtc2VydmVyL2RhdGFs
42+
aW5rL2xpbmtzP0lEPTU5NjYwMjExODY5MDc5MDMzNjAAAAAAFFLLjfkAPRcAQHAW
43+
5p3URUPARKpfojAl2n/AAAAAAABNaHR0cDovL2dlYWRhdGEuZXNhYy5lc2EuaW50
44+
L2RhdGEtc2VydmVyL2RhdGFsaW5rL2xpbmtzP0lEPTU5NjYwMTgyMzE5NzI3MzA2
45+
MjQAAAAA
46+
</STREAM>
47+
</BINARY2>
48+
</DATA>
49+
</TABLE>
50+
</RESOURCE>
51+
</VOTABLE>

‎astropy/io/votable/tests/table_test.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,21 @@ def test_empty_table():
186186
astropy_table = table.to_table() # noqa
187187

188188

189+
def test_binary2_masked_strings():
190+
"""
191+
Issue #8995
192+
"""
193+
# Read a VOTable which sets the null mask bit for each empty string value.
194+
votable = parse(get_pkg_data_filename('data/binary2_masked_strings.xml'))
195+
table = votable.get_first_table()
196+
astropy_table = table.to_table()
197+
198+
# Ensure string columns have no masked values and can be written out
199+
assert not np.any(table.array.mask['epoch_photometry_url'])
200+
output = io.BytesIO()
201+
astropy_table.write(output, format='votable')
202+
203+
189204
class TestVerifyOptions:
190205

191206
# Start off by checking the default (ignore)

‎astropy/io/votable/tree.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2626,6 +2626,12 @@ def _parse_binary(self, mode, iterator, colnumbers, fields, config, pos):
26262626
mask_bits = careful_read(int((len(fields) + 7) / 8))
26272627
row_mask_data = list(converters.bitarray_to_bool(
26282628
mask_bits, len(fields)))
2629+
2630+
# Ignore the mask for string columns (see issue 8995)
2631+
for i, f in enumerate(fields):
2632+
if row_mask_data[i] and (f.datatype == 'char' or f.datatype == 'unicodeChar'):
2633+
row_mask_data[i] = False
2634+
26292635
for i, binparse in enumerate(binparsers):
26302636
try:
26312637
value, value_mask = binparse(careful_read)

0 commit comments

Comments
 (0)