@@ -268,6 +268,32 @@ def binparse(self, read):
268268 """
269269 raise NotImplementedError ("This datatype must implement a 'binparse' method." )
270270
271+ def _binparse_column (self , stored ):
272+ """
273+ Convert a whole column of fixed-size values from the BINARY_ format,
274+ giving the same values and mask as `binparse` on each value in turn.
275+
276+ Parameters
277+ ----------
278+ stored : `~numpy.ndarray`
279+ One value per row as stored: big-endian numbers, or the bytes of
280+ each value as a row of a ``uint8`` array.
281+
282+ Returns
283+ -------
284+ values : `~numpy.ndarray` or list
285+ mask : `~numpy.ndarray`
286+ `True` where a value is missing.
287+
288+ Raises
289+ ------
290+ Exception
291+ Where `binparse` would raise for one of the values.
292+ """
293+ raise NotImplementedError (
294+ "This datatype does not convert a whole binary column at once."
295+ )
296+
271297 def binoutput (self , value , mask ):
272298 """
273299 Convert the object *value* in the native in-memory datatype to
@@ -335,6 +361,7 @@ def __init__(self, field, config=None, pos=None):
335361 self .binoutput = self ._binoutput_var
336362 else :
337363 self .binparse = self ._binparse_fixed
364+ self ._binparse_column = self ._binparse_fixed_column
338365 self .binoutput = self ._binoutput_fixed
339366
340367 self ._struct_format = f">{ self .arraysize :d} s"
@@ -379,10 +406,13 @@ def _binparse_var(self, read):
379406 length = self ._parse_length (read )
380407
381408 if self .arraysize != "*" and length > self .arraysize :
382- vo_warn ( W46 , ( "char" , self .arraysize ), None , None )
409+ self ._warn_too_long ( )
383410
384411 return read (length ).decode ("ascii" ), False
385412
413+ def _warn_too_long (self ):
414+ vo_warn (W46 , ("char" , self .arraysize ), None , None )
415+
386416 def _binparse_fixed (self , read ):
387417 s = struct .unpack (self ._struct_format , read (self .arraysize ))[0 ]
388418 end = s .find (_zero_byte )
@@ -391,6 +421,44 @@ def _binparse_fixed(self, read):
391421 return s [:end ], False
392422 return s , False
393423
424+ def _binparse_var_column (self , buffer , starts , stops ):
425+ """
426+ Convert a column of values from the BINARY_ format, the same as
427+ `_binparse_var` on each but without its warning, given that the
428+ characters of each value are ``buffer[start:stop]``.
429+ """
430+ strings = [
431+ buffer [start :stop ].decode ("ascii" ) for start , stop in zip (starts , stops )
432+ ]
433+ return strings , np .zeros (len (strings ), dtype = bool )
434+
435+ def _binparse_fixed_column (self , stored ):
436+ """
437+ Convert a column of values from the BINARY_ format, the same as
438+ `_binparse_fixed` on each, given the bytes of each value as a row of
439+ the ``uint8`` array *stored*.
440+ """
441+ nvalues , nbytes = stored .shape
442+ mask = np .zeros (nvalues , dtype = bool )
443+ if nbytes == 0 :
444+ # arraysize="0": the numpy steps below need at least one byte
445+ return np .full (nvalues , "" ), mask
446+ if nvalues and stored .max () >= 0x80 :
447+ # Raise the same UnicodeDecodeError as _binparse_fixed, which
448+ # decodes every byte, even those after a NUL
449+ stored .tobytes ().decode ("ascii" )
450+ # numpy stores unicode strings as 4-byte code points, and an ASCII
451+ # byte is its own code point
452+ codes = stored .astype (np .uint32 )
453+ # Cut each value at its first NUL. numpy already drops trailing NULs,
454+ # so only values with other bytes after a NUL need cutting.
455+ is_nul = stored == 0
456+ first_nul = is_nul .argmax (axis = 1 )
457+ nnul = np .count_nonzero (is_nul , axis = 1 )
458+ cut = np .flatnonzero ((nnul > 0 ) & (nnul != nbytes - first_nul ))
459+ codes [cut ] *= np .arange (nbytes ) < first_nul [cut , None ]
460+ return codes .view (f"U{ nbytes } " )[:, 0 ], mask
461+
394462 def _binoutput_var (self , value , mask ):
395463 if mask or value is None or value == "" :
396464 return _zero_int
@@ -453,6 +521,7 @@ def __init__(self, field, config=None, pos=None):
453521 self .binoutput = self ._binoutput_var
454522 else :
455523 self .binparse = self ._binparse_fixed
524+ self ._binparse_column = self ._binparse_fixed_column
456525 self .binoutput = self ._binoutput_fixed
457526
458527 self ._struct_format = f">{ self .arraysize * 2 :d} s"
@@ -471,10 +540,13 @@ def _binparse_var(self, read):
471540 length = self ._parse_length (read )
472541
473542 if self .arraysize != "*" and length > self .arraysize :
474- vo_warn ( W46 , ( "unicodeChar" , self .arraysize ), None , None )
543+ self ._warn_too_long ( )
475544
476545 return read (length * 2 ).decode ("utf_16_be" ), False
477546
547+ def _warn_too_long (self ):
548+ vo_warn (W46 , ("unicodeChar" , self .arraysize ), None , None )
549+
478550 def _binparse_fixed (self , read ):
479551 s = struct .unpack (self ._struct_format , read (self .arraysize * 2 ))[0 ]
480552 s = s .decode ("utf_16_be" )
@@ -483,6 +555,34 @@ def _binparse_fixed(self, read):
483555 return s [:end ], False
484556 return s , False
485557
558+ def _binparse_var_column (self , buffer , starts , stops ):
559+ """
560+ Convert a column of values from the BINARY_ format, the same as
561+ `_binparse_var` on each but without its warning, given that the
562+ characters of each value are ``buffer[start:stop]``.
563+ """
564+ strings = [
565+ buffer [start :stop ].decode ("utf_16_be" ) for start , stop in zip (starts , stops )
566+ ]
567+ return strings , np .zeros (len (strings ), dtype = bool )
568+
569+ def _binparse_fixed_column (self , stored ):
570+ """
571+ Convert a column of values from the BINARY_ format, the same as
572+ `_binparse_fixed` on each, given the bytes of each value as a row of
573+ the ``uint8`` array *stored*.
574+ """
575+ nvalues , nbytes = stored .shape
576+ mask = np .zeros (nvalues , dtype = bool )
577+ if nbytes == 0 :
578+ return ["" ] * nvalues , mask
579+ data = stored .tobytes ()
580+ strings = [
581+ data [start : start + nbytes ].decode ("utf_16_be" ).partition ("\0 " )[0 ]
582+ for start in range (0 , len (data ), nbytes )
583+ ]
584+ return strings , mask
585+
486586 def _binoutput_var (self , value , mask ):
487587 if mask or value is None or value == "" :
488588 return _zero_int
@@ -716,6 +816,9 @@ def binparse(self, read):
716816 result = np .frombuffer (read (self ._memsize ), dtype = self ._bigendian_format )
717817 return result [0 ], self .is_null (result [0 ])
718818
819+ def _binparse_column (self , stored ):
820+ return stored , self .is_null (stored )
821+
719822 def _is_null (self , value ):
720823 return value == self .null
721824
@@ -1227,6 +1330,10 @@ def binparse(self, read):
12271330 data = read (1 )
12281331 return fast_binparse_bit (data )
12291332
1333+ def _binparse_column (self , stored ):
1334+ # binary_one is 0x08, the bit that fast_binparse_bit tests
1335+ return (stored & self .binary_one [0 ]) != 0 , np .zeros (len (stored ), dtype = bool )
1336+
12301337 def binoutput (self , value , mask ):
12311338 if mask :
12321339 vo_warn (W39 )
@@ -1321,6 +1428,15 @@ def binparse(self, read):
13211428 ord (" " ): (False , True ),
13221429 ord ("?" ): (False , True ),
13231430 }
1431+ # As in fast_binparse_bool, a code not in the mapping reads as masked
1432+ # (binparse_value, used for boolean arrays, raises E05 for it instead)
1433+ _true_codes = [code for code , (value , _ ) in _binparse_mapping .items () if value ]
1434+ _unmasked_codes = [
1435+ code for code , (_ , mask ) in _binparse_mapping .items () if not mask
1436+ ]
1437+
1438+ def _binparse_column (self , stored ):
1439+ return np .isin (stored , self ._true_codes ), ~ np .isin (stored , self ._unmasked_codes )
13241440
13251441 def binparse_value (self , value ):
13261442 try :
0 commit comments