@@ -30,8 +30,8 @@ use crate::error::{
3030} ;
3131use crate :: { Result , ScalarValue } ;
3232use arrow:: array:: {
33- Array , ArrayRef , FixedSizeListArray , LargeListArray , ListArray , OffsetSizeTrait ,
34- cast:: AsArray ,
33+ Array , ArrayData , ArrayRef , FixedSizeListArray , LargeListArray , ListArray ,
34+ OffsetSizeTrait , cast:: AsArray , downcast_array ,
3535} ;
3636use arrow:: array:: {
3737 ArrowPrimitiveType , BooleanArray , Datum , GenericListArray , Int32Array , Int64Array ,
@@ -1544,25 +1544,38 @@ pub fn has_float_leaf(data_type: &DataType) -> bool {
15441544}
15451545
15461546/// Replace `-0.0` with `+0.0` in `Float16`, `Float32`, or `Float64` scalar
1547- /// values, including dictionary-wrapped floats. Other variants are returned
1548- /// unchanged. See [`normalize_float_zero`] for context.
1549- pub fn normalize_float_zero_scalar ( scalar : ScalarValue ) -> ScalarValue {
1550- match scalar {
1551- ScalarValue :: Float32 ( Some ( v) ) if v. to_bits ( ) << 1 == 0 => {
1552- ScalarValue :: Float32 ( Some ( 0.0 ) )
1553- }
1554- ScalarValue :: Float64 ( Some ( v) ) if v. to_bits ( ) << 1 == 0 => {
1555- ScalarValue :: Float64 ( Some ( 0.0 ) )
1556- }
1557- ScalarValue :: Float16 ( Some ( v) ) if v. to_bits ( ) << 1 == 0 => {
1558- ScalarValue :: Float16 ( Some ( half:: f16:: from_bits ( 0 ) ) )
1559- }
1560- ScalarValue :: Dictionary ( key, mut value) => {
1561- * value = normalize_float_zero_scalar ( * value) ;
1562- ScalarValue :: Dictionary ( key, value)
1547+ /// values, including floats inside nested and encoded values. Other variants
1548+ /// are returned unchanged. See [`normalize_float_zero`] for context.
1549+ pub fn normalize_float_zero_scalar ( mut scalar : ScalarValue ) -> ScalarValue {
1550+ fn normalize_array < A : Array + From < ArrayData > + ' static > ( array : & mut Arc < A > ) {
1551+ * array = Arc :: new ( downcast_array (
1552+ normalize_float_zero ( & ( Arc :: clone ( array) as ArrayRef ) ) . as_ref ( ) ,
1553+ ) ) ;
1554+ }
1555+
1556+ fn normalize ( scalar : & mut ScalarValue ) {
1557+ match scalar {
1558+ ScalarValue :: Float32 ( Some ( v) ) if v. to_bits ( ) << 1 == 0 => * v = 0.0 ,
1559+ ScalarValue :: Float64 ( Some ( v) ) if v. to_bits ( ) << 1 == 0 => * v = 0.0 ,
1560+ ScalarValue :: Float16 ( Some ( v) ) if v. to_bits ( ) << 1 == 0 => {
1561+ * v = half:: f16:: from_bits ( 0 ) ;
1562+ }
1563+ ScalarValue :: FixedSizeList ( array) => normalize_array ( array) ,
1564+ ScalarValue :: List ( array) => normalize_array ( array) ,
1565+ ScalarValue :: LargeList ( array) => normalize_array ( array) ,
1566+ ScalarValue :: ListView ( array) => normalize_array ( array) ,
1567+ ScalarValue :: LargeListView ( array) => normalize_array ( array) ,
1568+ ScalarValue :: Struct ( array) => normalize_array ( array) ,
1569+ ScalarValue :: Map ( array) => normalize_array ( array) ,
1570+ ScalarValue :: Union ( Some ( ( _, value) ) , _, _)
1571+ | ScalarValue :: Dictionary ( _, value)
1572+ | ScalarValue :: RunEndEncoded ( _, _, value) => normalize ( value) ,
1573+ _ => { }
15631574 }
1564- other => other,
15651575 }
1576+
1577+ normalize ( & mut scalar) ;
1578+ scalar
15661579}
15671580
15681581/// Apply a struct's nulls to one of its fields.
@@ -1723,6 +1736,77 @@ mod tests {
17231736 Ok ( ( ) )
17241737 }
17251738
1739+ #[ test]
1740+ fn normalize_float_zero_scalar_preserves_union_and_run_metadata ( ) -> Result < ( ) > {
1741+ use arrow:: datatypes:: { UnionFields , UnionMode } ;
1742+
1743+ let run_ends = Arc :: new ( Field :: new ( "ends" , DataType :: Int32 , false ) ) ;
1744+ let values = Arc :: new ( Field :: new ( "samples" , DataType :: Float64 , true ) ) ;
1745+ let fields = UnionFields :: try_new (
1746+ [ 7 , 42 ] ,
1747+ [
1748+ Field :: new (
1749+ "runs" ,
1750+ DataType :: RunEndEncoded ( Arc :: clone ( & run_ends) , Arc :: clone ( & values) ) ,
1751+ true ,
1752+ ) ,
1753+ Field :: new ( "other" , DataType :: Int32 , true ) ,
1754+ ] ,
1755+ ) ?;
1756+ let nan = f64:: from_bits ( 0x7ff8_0000_0000_0001 ) ;
1757+ for mode in [ UnionMode :: Sparse , UnionMode :: Dense ] {
1758+ let wrap = |value| {
1759+ ScalarValue :: Union (
1760+ Some ( (
1761+ 7 ,
1762+ Box :: new ( ScalarValue :: RunEndEncoded (
1763+ Arc :: clone ( & run_ends) ,
1764+ Arc :: clone ( & values) ,
1765+ Box :: new ( ScalarValue :: Float64 ( value) ) ,
1766+ ) ) ,
1767+ ) ) ,
1768+ fields. clone ( ) ,
1769+ mode,
1770+ )
1771+ } ;
1772+ for ( value, expected) in [
1773+ ( Some ( -0.0 ) , Some ( 0.0 ) ) ,
1774+ ( Some ( nan) , Some ( nan) ) ,
1775+ ( None , None ) ,
1776+ ] {
1777+ assert_eq ! ( normalize_float_zero_scalar( wrap( value) ) , wrap( expected) ) ;
1778+ }
1779+ let null = ScalarValue :: Union ( None , fields. clone ( ) , mode) ;
1780+ assert_eq ! ( normalize_float_zero_scalar( null. clone( ) ) , null) ;
1781+ }
1782+ Ok ( ( ) )
1783+ }
1784+
1785+ #[ test]
1786+ fn normalize_float_zero_scalar_preserves_sliced_list ( ) {
1787+ let nan = f64:: from_bits ( 0x7ff8_0000_0000_0001 ) ;
1788+ let list = Arc :: new (
1789+ ListArray :: from_iter_primitive :: < Float64Type , _ , _ > ( [
1790+ Some ( vec ! [ Some ( 99.0 ) ] ) ,
1791+ Some ( vec ! [ Some ( -0.0 ) , Some ( nan) , None ] ) ,
1792+ Some ( vec ! [ Some ( 42.0 ) ] ) ,
1793+ ] )
1794+ . slice ( 1 , 1 ) ,
1795+ ) ;
1796+ let ScalarValue :: List ( normalized) =
1797+ normalize_float_zero_scalar ( ScalarValue :: List ( list) )
1798+ else {
1799+ panic ! ( "normalization must preserve the scalar variant" ) ;
1800+ } ;
1801+ assert_eq ! ( normalized. len( ) , 1 ) ;
1802+ let child = normalized. value ( 0 ) ;
1803+ let child = child. as_primitive :: < Float64Type > ( ) ;
1804+ assert_eq ! ( child. len( ) , 3 ) ;
1805+ assert_eq ! ( child. value( 0 ) . to_bits( ) , 0.0_f64 . to_bits( ) ) ;
1806+ assert_eq ! ( child. value( 1 ) . to_bits( ) , nan. to_bits( ) ) ;
1807+ assert ! ( child. is_null( 2 ) ) ;
1808+ }
1809+
17261810 #[ test]
17271811 fn test_bisect_linear_left_and_right ( ) -> Result < ( ) > {
17281812 let arrays: Vec < ArrayRef > = vec ! [
0 commit comments