Skip to content

Commit 30bfc4a

Browse files
authored
Merge 96b3904 into db7a3e2
2 parents db7a3e2 + 96b3904 commit 30bfc4a

4 files changed

Lines changed: 341 additions & 36 deletions

File tree

‎astropy/io/votable/converters.py‎

Lines changed: 57 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,18 @@
3939
vo_warn,
4040
warn_or_raise,
4141
)
42+
from .fast_converters import (
43+
fast_binparse_bit,
44+
fast_binparse_bool,
45+
fast_binparse_double,
46+
fast_binparse_float,
47+
fast_binparse_int,
48+
fast_binparse_long,
49+
fast_binparse_short,
50+
fast_binparse_ubyte,
51+
fast_bitarray_to_bool,
52+
fast_bool_to_bitarray,
53+
)
4254

4355
__all__ = ["Converter", "get_converter", "table_column_to_votable_datatype"]
4456

@@ -104,18 +116,7 @@ def bitarray_to_bool(data, length):
104116
-------
105117
array : numpy bool array
106118
"""
107-
results = []
108-
for byte in data:
109-
for bit_no in range(7, -1, -1):
110-
bit = byte & (1 << bit_no)
111-
bit = bit != 0
112-
results.append(bit)
113-
if len(results) == length:
114-
break
115-
if len(results) == length:
116-
break
117-
118-
return np.array(results, dtype="b1")
119+
return fast_bitarray_to_bool(data, length)
119120

120121

121122
def bool_to_bitarray(value):
@@ -134,23 +135,7 @@ def bool_to_bitarray(value):
134135
significant bit in the result. The length will be `floor((N +
135136
7) / 8)` where `N` is the length of `value`.
136137
"""
137-
value = value.flat
138-
bit_no = 7
139-
byte = 0
140-
bytes = []
141-
for v in value:
142-
if v:
143-
byte |= 1 << bit_no
144-
if bit_no == 0:
145-
bytes.append(byte)
146-
bit_no = 7
147-
byte = 0
148-
else:
149-
bit_no -= 1
150-
if bit_no != 7:
151-
bytes.append(byte)
152-
153-
return struct.pack(f"{len(bytes)}B", *bytes)
138+
return fast_bool_to_bitarray(value)
154139

155140

156141
class Converter:
@@ -493,6 +478,7 @@ def _binoutput_var(self, value, mask):
493478

494479
if self.arraysize != "*" and len(value) > self.arraysize:
495480
vo_warn(W46, ("unicodeChar", self.arraysize), None, None)
481+
value = value[: self.arraysize]
496482

497483
encoded = value.encode("utf_16_be")
498484

@@ -736,6 +722,13 @@ def __init__(self, field, config=None, pos=None):
736722

737723
Numeric.__init__(self, field, config, pos)
738724

725+
if self.null is not None:
726+
self._fast_null_param = float(self.null)
727+
self._fast_has_null = True
728+
else:
729+
self._fast_null_param = 0.0
730+
self._fast_has_null = False
731+
739732
precision = field.precision
740733
width = field.width
741734

@@ -836,12 +829,15 @@ def _filter_null(self, value, mask):
836829

837830
class Double(FloatingPoint):
838831
"""
839-
Handles the double datatype. Double-precision IEEE
840-
floating-point.
832+
Handles the double datatype. Double-precision IEEE floating-point.
841833
"""
842834

843835
format = "f8"
844836

837+
def binparse(self, read):
838+
data = read(8)
839+
return fast_binparse_double(data, 0, self._fast_null_param, self._fast_has_null)
840+
845841

846842
class Float(FloatingPoint):
847843
"""
@@ -850,6 +846,10 @@ class Float(FloatingPoint):
850846

851847
format = "f4"
852848

849+
def binparse(self, read):
850+
data = read(4)
851+
return fast_binparse_float(data, 0, self._fast_null_param, self._fast_has_null)
852+
853853

854854
class Integer(Numeric):
855855
"""
@@ -861,6 +861,13 @@ class Integer(Numeric):
861861
def __init__(self, field, config=None, pos=None):
862862
Numeric.__init__(self, field, config, pos)
863863

864+
if self.null is not None:
865+
self._fast_null_param = int(self.null)
866+
self._fast_has_null = True
867+
else:
868+
self._fast_null_param = 0
869+
self._fast_has_null = False
870+
864871
def parse(self, value, config=None, pos=None):
865872
if config is None:
866873
config = {}
@@ -937,6 +944,10 @@ class UnsignedByte(Integer):
937944
val_range = (0, 255)
938945
bit_size = "8-bit unsigned"
939946

947+
def binparse(self, read):
948+
data = read(1)
949+
return fast_binparse_ubyte(data, 0, self._fast_null_param, self._fast_has_null)
950+
940951

941952
class Short(Integer):
942953
"""
@@ -947,6 +958,10 @@ class Short(Integer):
947958
val_range = (-32768, 32767)
948959
bit_size = "16-bit"
949960

961+
def binparse(self, read):
962+
data = read(2)
963+
return fast_binparse_short(data, 0, self._fast_null_param, self._fast_has_null)
964+
950965

951966
class Int(Integer):
952967
"""
@@ -957,6 +972,10 @@ class Int(Integer):
957972
val_range = (-2147483648, 2147483647)
958973
bit_size = "32-bit"
959974

975+
def binparse(self, read):
976+
data = read(4)
977+
return fast_binparse_int(data, 0, self._fast_null_param, self._fast_has_null)
978+
960979

961980
class Long(Integer):
962981
"""
@@ -967,6 +986,10 @@ class Long(Integer):
967986
val_range = (-9223372036854775808, 9223372036854775807)
968987
bit_size = "64-bit"
969988

989+
def binparse(self, read):
990+
data = read(8)
991+
return fast_binparse_long(data, 0, self._fast_null_param, self._fast_has_null)
992+
970993

971994
class ComplexArrayVarArray(VarArray):
972995
"""
@@ -1151,7 +1174,6 @@ def binparse(self, read):
11511174
def binoutput(self, value, mask):
11521175
if np.any(mask):
11531176
vo_warn(W39)
1154-
11551177
return bool_to_bitarray(value)
11561178

11571179

@@ -1192,7 +1214,7 @@ def output(self, value, mask):
11921214

11931215
def binparse(self, read):
11941216
data = read(1)
1195-
return (ord(data) & 0x8) != 0, False
1217+
return fast_binparse_bit(data)
11961218

11971219
def binoutput(self, value, mask):
11981220
if mask:
@@ -1274,8 +1296,8 @@ def output(self, value, mask):
12741296
return "F"
12751297

12761298
def binparse(self, read):
1277-
value = ord(read(1))
1278-
return self.binparse_value(value)
1299+
data = read(1)
1300+
return fast_binparse_bool(data)
12791301

12801302
_binparse_mapping = {
12811303
ord("T"): (True, False),

‎astropy/io/votable/setup_package.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
# Licensed under a 3-clause BSD style license - see LICENSE.rst
22

3+
34
from os.path import join
45

6+
import numpy as np
57
from setuptools import Extension
68

79

@@ -13,5 +15,10 @@ def get_extensions(build_type="release"):
1315
"astropy.io.votable.tablewriter",
1416
[join(VO_DIR, "tablewriter.c")],
1517
include_dirs=[VO_DIR],
16-
)
18+
),
19+
Extension(
20+
"astropy.io.votable.fast_converters",
21+
[join(VO_DIR, "fast_converters.pyx")],
22+
include_dirs=[np.get_include(), VO_DIR],
23+
),
1724
]

0 commit comments

Comments
 (0)