Skip to content

Commit 4f9d678

Browse files
authored
Merge e16c061 into 1ad7032
2 parents 1ad7032 + e16c061 commit 4f9d678

5 files changed

Lines changed: 912 additions & 11 deletions

File tree

‎astropy/io/votable/converters.py‎

Lines changed: 118 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,32 @@ def binparse(self, read):
268268
"""
269269
raise NotImplementedError("This datatype must implement a 'binparse' method.")
270270

271+
def _binparse_column(self, stored):
272+
"""
273+
Convert a whole column of fixed-size values from the BINARY_ format,
274+
giving the same values and mask as `binparse` on each value in turn.
275+
276+
Parameters
277+
----------
278+
stored : `~numpy.ndarray`
279+
One value per row as stored: big-endian numbers, or the bytes of
280+
each value as a row of a ``uint8`` array.
281+
282+
Returns
283+
-------
284+
values : `~numpy.ndarray` or list
285+
mask : `~numpy.ndarray`
286+
`True` where a value is missing.
287+
288+
Raises
289+
------
290+
Exception
291+
Where `binparse` would raise for one of the values.
292+
"""
293+
raise NotImplementedError(
294+
"This datatype does not convert a whole binary column at once."
295+
)
296+
271297
def binoutput(self, value, mask):
272298
"""
273299
Convert the object *value* in the native in-memory datatype to
@@ -335,6 +361,7 @@ def __init__(self, field, config=None, pos=None):
335361
self.binoutput = self._binoutput_var
336362
else:
337363
self.binparse = self._binparse_fixed
364+
self._binparse_column = self._binparse_fixed_column
338365
self.binoutput = self._binoutput_fixed
339366

340367
self._struct_format = f">{self.arraysize:d}s"
@@ -379,10 +406,13 @@ def _binparse_var(self, read):
379406
length = self._parse_length(read)
380407

381408
if self.arraysize != "*" and length > self.arraysize:
382-
vo_warn(W46, ("char", self.arraysize), None, None)
409+
self._warn_too_long()
383410

384411
return read(length).decode("ascii"), False
385412

413+
def _warn_too_long(self):
414+
vo_warn(W46, ("char", self.arraysize), None, None)
415+
386416
def _binparse_fixed(self, read):
387417
s = struct.unpack(self._struct_format, read(self.arraysize))[0]
388418
end = s.find(_zero_byte)
@@ -391,6 +421,44 @@ def _binparse_fixed(self, read):
391421
return s[:end], False
392422
return s, False
393423

424+
def _binparse_var_column(self, buffer, starts, stops):
425+
"""
426+
Convert a column of values from the BINARY_ format, the same as
427+
`_binparse_var` on each but without its warning, given that the
428+
characters of each value are ``buffer[start:stop]``.
429+
"""
430+
strings = [
431+
buffer[start:stop].decode("ascii") for start, stop in zip(starts, stops)
432+
]
433+
return strings, np.zeros(len(strings), dtype=bool)
434+
435+
def _binparse_fixed_column(self, stored):
436+
"""
437+
Convert a column of values from the BINARY_ format, the same as
438+
`_binparse_fixed` on each, given the bytes of each value as a row of
439+
the ``uint8`` array *stored*.
440+
"""
441+
nvalues, nbytes = stored.shape
442+
mask = np.zeros(nvalues, dtype=bool)
443+
if nbytes == 0:
444+
# arraysize="0": the numpy steps below need at least one byte
445+
return np.full(nvalues, ""), mask
446+
if nvalues and stored.max() >= 0x80:
447+
# Raise the same UnicodeDecodeError as _binparse_fixed, which
448+
# decodes every byte, even those after a NUL
449+
stored.tobytes().decode("ascii")
450+
# numpy stores unicode strings as 4-byte code points, and an ASCII
451+
# byte is its own code point
452+
codes = stored.astype(np.uint32)
453+
# Cut each value at its first NUL. numpy already drops trailing NULs,
454+
# so only values with other bytes after a NUL need cutting.
455+
is_nul = stored == 0
456+
first_nul = is_nul.argmax(axis=1)
457+
nnul = np.count_nonzero(is_nul, axis=1)
458+
cut = np.flatnonzero((nnul > 0) & (nnul != nbytes - first_nul))
459+
codes[cut] *= np.arange(nbytes) < first_nul[cut, None]
460+
return codes.view(f"U{nbytes}")[:, 0], mask
461+
394462
def _binoutput_var(self, value, mask):
395463
if mask or value is None or value == "":
396464
return _zero_int
@@ -453,6 +521,7 @@ def __init__(self, field, config=None, pos=None):
453521
self.binoutput = self._binoutput_var
454522
else:
455523
self.binparse = self._binparse_fixed
524+
self._binparse_column = self._binparse_fixed_column
456525
self.binoutput = self._binoutput_fixed
457526

458527
self._struct_format = f">{self.arraysize * 2:d}s"
@@ -471,10 +540,13 @@ def _binparse_var(self, read):
471540
length = self._parse_length(read)
472541

473542
if self.arraysize != "*" and length > self.arraysize:
474-
vo_warn(W46, ("unicodeChar", self.arraysize), None, None)
543+
self._warn_too_long()
475544

476545
return read(length * 2).decode("utf_16_be"), False
477546

547+
def _warn_too_long(self):
548+
vo_warn(W46, ("unicodeChar", self.arraysize), None, None)
549+
478550
def _binparse_fixed(self, read):
479551
s = struct.unpack(self._struct_format, read(self.arraysize * 2))[0]
480552
s = s.decode("utf_16_be")
@@ -483,6 +555,34 @@ def _binparse_fixed(self, read):
483555
return s[:end], False
484556
return s, False
485557

558+
def _binparse_var_column(self, buffer, starts, stops):
559+
"""
560+
Convert a column of values from the BINARY_ format, the same as
561+
`_binparse_var` on each but without its warning, given that the
562+
characters of each value are ``buffer[start:stop]``.
563+
"""
564+
strings = [
565+
buffer[start:stop].decode("utf_16_be") for start, stop in zip(starts, stops)
566+
]
567+
return strings, np.zeros(len(strings), dtype=bool)
568+
569+
def _binparse_fixed_column(self, stored):
570+
"""
571+
Convert a column of values from the BINARY_ format, the same as
572+
`_binparse_fixed` on each, given the bytes of each value as a row of
573+
the ``uint8`` array *stored*.
574+
"""
575+
nvalues, nbytes = stored.shape
576+
mask = np.zeros(nvalues, dtype=bool)
577+
if nbytes == 0:
578+
return [""] * nvalues, mask
579+
data = stored.tobytes()
580+
strings = [
581+
data[start : start + nbytes].decode("utf_16_be").partition("\0")[0]
582+
for start in range(0, len(data), nbytes)
583+
]
584+
return strings, mask
585+
486586
def _binoutput_var(self, value, mask):
487587
if mask or value is None or value == "":
488588
return _zero_int
@@ -716,6 +816,9 @@ def binparse(self, read):
716816
result = np.frombuffer(read(self._memsize), dtype=self._bigendian_format)
717817
return result[0], self.is_null(result[0])
718818

819+
def _binparse_column(self, stored):
820+
return stored, self.is_null(stored)
821+
719822
def _is_null(self, value):
720823
return value == self.null
721824

@@ -1227,6 +1330,10 @@ def binparse(self, read):
12271330
data = read(1)
12281331
return fast_binparse_bit(data)
12291332

1333+
def _binparse_column(self, stored):
1334+
# binary_one is 0x08, the bit that fast_binparse_bit tests
1335+
return (stored & self.binary_one[0]) != 0, np.zeros(len(stored), dtype=bool)
1336+
12301337
def binoutput(self, value, mask):
12311338
if mask:
12321339
vo_warn(W39)
@@ -1321,6 +1428,15 @@ def binparse(self, read):
13211428
ord(" "): (False, True),
13221429
ord("?"): (False, True),
13231430
}
1431+
# As in fast_binparse_bool, a code not in the mapping reads as masked
1432+
# (binparse_value, used for boolean arrays, raises E05 for it instead)
1433+
_true_codes = [code for code, (value, _) in _binparse_mapping.items() if value]
1434+
_unmasked_codes = [
1435+
code for code, (_, mask) in _binparse_mapping.items() if not mask
1436+
]
1437+
1438+
def _binparse_column(self, stored):
1439+
return np.isin(stored, self._true_codes), ~np.isin(stored, self._unmasked_codes)
13241440

13251441
def binparse_value(self, value):
13261442
try:

0 commit comments

Comments
 (0)