@@ -19,6 +19,7 @@ use arrow::compute::kernels::boolean::{not, or_kleene};
1919use arrow:: compute:: kernels:: cmp:: eq as arrow_eq;
2020use arrow:: datatypes:: * ;
2121
22+ use datafusion_common:: float_semantics:: normalize_floats;
2223use datafusion_common:: tree_node:: TreeNode ;
2324use datafusion_common:: {
2425 DFSchema , Result , ScalarValue , assert_or_internal_err, exec_err,
@@ -71,6 +72,13 @@ fn supports_arrow_eq(dt: &DataType) -> bool {
7172 }
7273}
7374
75+ fn normalize_in_list_value ( value : ColumnarValue ) -> ColumnarValue {
76+ match value {
77+ ColumnarValue :: Array ( array) => ColumnarValue :: Array ( normalize_floats ( & array) ) ,
78+ ColumnarValue :: Scalar ( scalar) => ColumnarValue :: Scalar ( scalar. normalize ( ) ) ,
79+ }
80+ }
81+
7482/// Evaluates the list of expressions into an array, flattening any dictionaries
7583fn evaluate_list (
7684 list : & [ Arc < dyn PhysicalExpr > ] ,
@@ -368,12 +376,14 @@ impl PhysicalExpr for InListExpr {
368376 // Use Arrow's vectorized eq kernel for types it supports (primitive,
369377 // boolean, string, binary, dictionary), falling back to row-by-row
370378 // comparator for unsupported types (nested, RunEndEncoded, etc.).
371- let value = value. into_array ( num_rows) ?;
379+ // Normalize the left side once for the whole list. Doing this
380+ // outside `compare_one` avoids rescanning it for every item.
381+ let value = normalize_in_list_value ( value) . into_array ( num_rows) ?;
372382 let lhs_supports_arrow_eq = supports_arrow_eq ( value. data_type ( ) ) ;
373383
374384 // Helper: compare value against a single list expression
375385 let compare_one = |expr : & Arc < dyn PhysicalExpr > | -> Result < BooleanArray > {
376- match expr. evaluate ( batch) ? {
386+ match normalize_in_list_value ( expr. evaluate ( batch) ?) {
377387 ColumnarValue :: Array ( array) => {
378388 if lhs_supports_arrow_eq
379389 && supports_arrow_eq ( array. data_type ( ) )
@@ -1287,20 +1297,28 @@ mod tests {
12871297 Arc :: new ( schema. clone ( ) ) ,
12881298 vec ! [ Arc :: new( Int32Array :: from( vec![ 0 , 0 , 0 ] ) ) ] ,
12891299 ) ?;
1290- let needle = lit ( ScalarValue :: Dictionary (
1291- Box :: new ( DataType :: Int16 ) ,
1292- Box :: new ( ScalarValue :: Dictionary (
1293- Box :: new ( DataType :: Int8 ) ,
1294- Box :: new ( ScalarValue :: Int32 ( Some ( 2 ) ) ) ,
1295- ) ) ,
1296- ) ) ;
1300+ for ( value, list) in [
1301+ ( ScalarValue :: Int32 ( Some ( 2 ) ) , vec ! [ lit( 1_i32 ) , lit( 2_i32 ) ] ) ,
1302+ (
1303+ ScalarValue :: Float64 ( Some ( -0.0 ) ) ,
1304+ vec ! [ lit( 0.0_f64 ) , lit( 1.0_f64 ) ] ,
1305+ ) ,
1306+ ] {
1307+ let needle = lit ( ScalarValue :: Dictionary (
1308+ Box :: new ( DataType :: Int16 ) ,
1309+ Box :: new ( ScalarValue :: Dictionary (
1310+ Box :: new ( DataType :: Int8 ) ,
1311+ Box :: new ( value) ,
1312+ ) ) ,
1313+ ) ) ;
12971314
1298- let expr = in_list ( needle, vec ! [ lit( 1_i32 ) , lit( 2_i32 ) ] , & false , & schema) ?;
1299- let result = expr. evaluate ( & batch) ?. into_array ( batch. num_rows ( ) ) ?;
1300- assert_eq ! (
1301- as_boolean_array( & result) ,
1302- & BooleanArray :: from( vec![ true , true , true ] )
1303- ) ;
1315+ let expr = in_list ( needle, list, & false , & schema) ?;
1316+ let result = expr. evaluate ( & batch) ?. into_array ( batch. num_rows ( ) ) ?;
1317+ assert_eq ! (
1318+ as_boolean_array( & result) ,
1319+ & BooleanArray :: from( vec![ true , true , true ] )
1320+ ) ;
1321+ }
13041322
13051323 Ok ( ( ) )
13061324 }
@@ -3391,6 +3409,44 @@ mod tests {
33913409 Ok ( ( ) )
33923410 }
33933411
3412+ #[ test]
3413+ fn test_in_list_with_columns_float_signed_zero ( ) -> Result < ( ) > {
3414+ use arrow:: compute:: cast;
3415+
3416+ for ( data_type, dictionary) in [
3417+ ( DataType :: Float32 , false ) ,
3418+ ( DataType :: Float64 , false ) ,
3419+ ( DataType :: Float64 , true ) ,
3420+ ] {
3421+ let mut left = cast ( & Float64Array :: from ( vec ! [ 0.0 , -0.0 , 1.0 ] ) , & data_type) ?;
3422+ let mut right = cast ( & Float64Array :: from ( vec ! [ -0.0 , 0.0 , 2.0 ] ) , & data_type) ?;
3423+ if dictionary {
3424+ left = wrap_in_dict ( left) ;
3425+ right = wrap_in_dict ( right) ;
3426+ }
3427+ let scalar = lit ( ScalarValue :: try_from_array ( left. as_ref ( ) , 1 ) ?) ;
3428+ let batch = RecordBatch :: try_from_iter ( [ ( "a" , left) , ( "b" , right) ] ) ?;
3429+ let schema = batch. schema ( ) ;
3430+
3431+ // Exercise array and scalar values on both sides, including scalar
3432+ // normalization before the left-hand side is broadcast.
3433+ for ( left, right) in [
3434+ ( col ( "a" , & schema) ?, col ( "b" , & schema) ?) ,
3435+ ( col ( "a" , & schema) ?, Arc :: clone ( & scalar) ) ,
3436+ ( scalar, col ( "a" , & schema) ?) ,
3437+ ] {
3438+ let expr = make_in_list_with_columns ( left, vec ! [ right] , false ) ;
3439+ let result = expr. evaluate ( & batch) ?. into_array ( batch. num_rows ( ) ) ?;
3440+ assert_eq ! (
3441+ as_boolean_array( & result) ,
3442+ & BooleanArray :: from( vec![ true , true , false ] ) ,
3443+ "{data_type:?}, dictionary={dictionary}: {expr}"
3444+ ) ;
3445+ }
3446+ }
3447+ Ok ( ( ) )
3448+ }
3449+
33943450 /// Tests that short-circuit evaluation produces correct results.
33953451 /// When all rows match after the first list item, remaining items
33963452 /// should be skipped without affecting correctness.
@@ -3916,6 +3972,23 @@ mod tests {
39163972 Ok ( ( ) )
39173973 }
39183974
3975+ #[ test]
3976+ fn test_try_new_from_array_dict_haystack_float64_signed_zero ( ) -> Result < ( ) > {
3977+ // One value beyond the branchless limit selects the hash-set strategy.
3978+ let list_len =
3979+ <Float64Type as branchless_filter:: BranchlessFilterType >:: MAX_LIST_LEN + 1 ;
3980+ let haystack = make_f64_dict_array ( vec ! [ Some ( -0.0 ) ; list_len] ) ;
3981+ let needles: ArrayRef = Arc :: new ( Float64Array :: from ( vec ! [ 0.0 ] ) ) ;
3982+ for needles in [ Arc :: clone ( & needles) , wrap_in_dict ( needles) ] {
3983+ assert_eq ! (
3984+ eval_in_list_from_array( needles, Arc :: clone( & haystack) ) ?,
3985+ BooleanArray :: from( vec![ true ] )
3986+ ) ;
3987+ }
3988+
3989+ Ok ( ( ) )
3990+ }
3991+
39193992 #[ test]
39203993 fn test_try_new_from_array_type_mismatch_rejects ( ) -> Result < ( ) > {
39213994 let schema = Schema :: new ( vec ! [ Field :: new( "a" , DataType :: Int32 , false ) ] ) ;
0 commit comments