@@ -222,13 +222,16 @@ impl PhysicalExpr for NestedPredicate {
222222pub enum FloatOperands {
223223 /// Normalize them, so that the comparison follows Spark's SQL ordering.
224224 Normalize ,
225- /// Leave a Float32 or Float64 column compared with a literal as it is, and normalize every
226- /// other operand. Only a scan's pushed-down data filters use this, and only when the Parquet
227- /// reader prunes with them but does not filter rows: Parquet pruning recognizes a column
228- /// compared with a literal but not a normalized column, and Spark's Filter above the scan
229- /// applies Spark's semantics to every row. With row-level pushdown the reader would drop the
230- /// rows such a comparison rejects, including a stored NaN whose bits differ from the
231- /// normalized literal, so the data filters use [`FloatOperands::Normalize`] there instead.
225+ /// Leave a Float32 or Float64 column compared with a literal other than NaN as it is, and
226+ /// normalize every other operand. Only a scan's pushed-down data filters use this, and only
227+ /// when the Parquet reader prunes with them but does not filter rows: Parquet pruning
228+ /// recognizes a column compared with a literal but not a normalized column, and Spark's
229+ /// Filter above the scan applies Spark's semantics to every row. A NaN literal is normalized
230+ /// with the column, giving up pruning, because a stored NaN can have other bits than the
231+ /// literal and a writer can leave NaNs out of the column statistics. With row-level pushdown
232+ /// the reader would drop the rows such a comparison rejects, including a stored NaN with the
233+ /// sign bit set, which Arrow orders below every other value, so the data filters use
234+ /// [`FloatOperands::Normalize`] there instead.
232235 Raw ,
233236}
234237
@@ -269,18 +272,69 @@ pub fn spark_comparison(
269272 membership : false ,
270273 } ) ) ;
271274 }
272- let raw = float_operands == FloatOperands :: Raw ;
273- let ( left, right) = if raw && is_float_column ( & left, schema) && is_literal ( & right) {
274- ( left, normalize_comparison_operand ( right, schema) ?)
275- } else if raw && is_literal ( & left) && is_float_column ( & right, schema) {
276- ( normalize_comparison_operand ( left, schema) ?, right)
277- } else {
278- (
279- normalize_comparison_operand ( left, schema) ?,
280- normalize_comparison_operand ( right, schema) ?,
281- )
275+ if float_operands == FloatOperands :: Raw {
276+ if let Some ( comparison) = raw_float_comparison ( & left, op, & right, schema) {
277+ return Ok ( comparison) ;
278+ }
279+ }
280+ Ok ( Arc :: new ( BinaryExpr :: new (
281+ normalize_comparison_operand ( left, schema) ?,
282+ op,
283+ normalize_comparison_operand ( right, schema) ?,
284+ ) ) )
285+ }
286+
287+ /// The comparison that [`FloatOperands::Raw`] builds for a Float32 or Float64 column compared with
288+ /// a literal other than NaN: the column and the literal as they are. `None` for any other operands.
289+ ///
290+ /// Statistics pruning compares `-0.0` and `0.0` as equal, but a bloom filter probe hashes the
291+ /// literal's bits, so `=` against a zero literal becomes `=` against either zero.
292+ fn raw_float_comparison (
293+ left : & Arc < dyn PhysicalExpr > ,
294+ op : Operator ,
295+ right : & Arc < dyn PhysicalExpr > ,
296+ schema : & Schema ,
297+ ) -> Option < Arc < dyn PhysicalExpr > > {
298+ let ( literal, literal_on_left) = match (
299+ left. downcast_ref :: < Literal > ( ) ,
300+ right. downcast_ref :: < Literal > ( ) ,
301+ ) {
302+ ( None , Some ( literal) ) if is_float_column ( left, schema) => ( literal, false ) ,
303+ ( Some ( literal) , None ) if is_float_column ( right, schema) => ( literal, true ) ,
304+ _ => return None ,
282305 } ;
283- Ok ( Arc :: new ( BinaryExpr :: new ( left, op, right) ) )
306+ let ( negative_zero, positive_zero) = match literal. value ( ) {
307+ ScalarValue :: Float32 ( Some ( v) ) if v. is_nan ( ) => return None ,
308+ ScalarValue :: Float64 ( Some ( v) ) if v. is_nan ( ) => return None ,
309+ ScalarValue :: Float32 ( Some ( v) ) if op == Operator :: Eq && * v == 0.0 => (
310+ ScalarValue :: Float32 ( Some ( -0.0 ) ) ,
311+ ScalarValue :: Float32 ( Some ( 0.0 ) ) ,
312+ ) ,
313+ ScalarValue :: Float64 ( Some ( v) ) if op == Operator :: Eq && * v == 0.0 => (
314+ ScalarValue :: Float64 ( Some ( -0.0 ) ) ,
315+ ScalarValue :: Float64 ( Some ( 0.0 ) ) ,
316+ ) ,
317+ _ => {
318+ return Some ( Arc :: new ( BinaryExpr :: new (
319+ Arc :: clone ( left) ,
320+ op,
321+ Arc :: clone ( right) ,
322+ ) ) )
323+ }
324+ } ;
325+ let equal_to = |zero : ScalarValue | -> Arc < dyn PhysicalExpr > {
326+ let zero: Arc < dyn PhysicalExpr > = Arc :: new ( Literal :: new ( zero) ) ;
327+ if literal_on_left {
328+ Arc :: new ( BinaryExpr :: new ( zero, op, Arc :: clone ( right) ) )
329+ } else {
330+ Arc :: new ( BinaryExpr :: new ( Arc :: clone ( left) , op, zero) )
331+ }
332+ } ;
333+ Some ( Arc :: new ( BinaryExpr :: new (
334+ equal_to ( negative_zero) ,
335+ Operator :: Or ,
336+ equal_to ( positive_zero) ,
337+ ) ) )
284338}
285339
286340/// Whether `expr` is a Float32 or Float64 column, the operand that Parquet pruning reads.
@@ -292,10 +346,6 @@ fn is_float_column(expr: &Arc<dyn PhysicalExpr>, schema: &Schema) -> bool {
292346 )
293347}
294348
295- fn is_literal ( expr : & Arc < dyn PhysicalExpr > ) -> bool {
296- expr. downcast_ref :: < Literal > ( ) . is_some ( )
297- }
298-
299349fn validate_types (
300350 value : & Arc < dyn PhysicalExpr > ,
301351 candidates : & [ Arc < dyn PhysicalExpr > ] ,
@@ -965,8 +1015,8 @@ mod tests {
9651015 }
9661016
9671017 /// A literal operand is normalized while planning, so the comparison stays `column op
968- /// literal`. With `FloatOperands::Raw`, as for a scan's data filters, only the column of that
969- /// shape is left unwrapped .
1018+ /// literal`. With `FloatOperands::Raw`, as for a scan's data filters, the column and the
1019+ /// literal of that shape are left as they are .
9701020 #[ test]
9711021 fn float_operand_shapes ( ) -> Result < ( ) > {
9721022 use crate :: float_semantics:: NormalizeNaNAndZero ;
@@ -992,8 +1042,8 @@ mod tests {
9921042 let folded = binary. right ( ) . downcast_ref :: < Literal > ( ) . unwrap ( ) ;
9931043 assert ! ( matches!( folded. value( ) , ScalarValue :: Float64 ( Some ( v) ) if v. to_bits( ) == 0 ) ) ;
9941044
995- // With `FloatOperands::Raw`, a float column compared with a literal keeps the column, on
996- // either side, and the literal is still normalized .
1045+ // With `FloatOperands::Raw`, a float column compared with a literal keeps both as they
1046+ // are, on either side .
9971047 let expr = spark_comparison (
9981048 Arc :: new ( Column :: new ( "a" , 0 ) ) ,
9991049 Operator :: Lt ,
@@ -1003,8 +1053,7 @@ mod tests {
10031053 ) ?;
10041054 let binary = expr. downcast_ref :: < BinaryExpr > ( ) . unwrap ( ) ;
10051055 assert ! ( binary. left( ) . downcast_ref:: <Column >( ) . is_some( ) ) ;
1006- let folded = binary. right ( ) . downcast_ref :: < Literal > ( ) . unwrap ( ) ;
1007- assert ! ( matches!( folded. value( ) , ScalarValue :: Float64 ( Some ( v) ) if v. to_bits( ) == 0 ) ) ;
1056+ assert ! ( Arc :: ptr_eq( binary. right( ) , & negative_zero) ) ;
10081057 let expr = spark_comparison (
10091058 Arc :: clone ( & negative_zero) ,
10101059 Operator :: Lt ,
@@ -1013,6 +1062,7 @@ mod tests {
10131062 FloatOperands :: Raw ,
10141063 ) ?;
10151064 let binary = expr. downcast_ref :: < BinaryExpr > ( ) . unwrap ( ) ;
1065+ assert ! ( Arc :: ptr_eq( binary. left( ) , & negative_zero) ) ;
10161066 assert ! ( binary. right( ) . downcast_ref:: <Column >( ) . is_some( ) ) ;
10171067
10181068 // Any other shape is normalized even with `FloatOperands::Raw`, because a reader with
@@ -1049,6 +1099,81 @@ mod tests {
10491099 Ok ( ( ) )
10501100 }
10511101
1102+ /// A bloom filter holds the bits of each value, so with `FloatOperands::Raw` an `=` against
1103+ /// either zero probes the filter for both zeros, while other operators keep the literal. A
1104+ /// NaN literal is normalized together with the column instead, because no single literal
1105+ /// stands for every NaN a file can hold.
1106+ #[ test]
1107+ fn raw_float_comparison_literals ( ) -> Result < ( ) > {
1108+ use crate :: float_semantics:: NormalizeNaNAndZero ;
1109+ use datafusion:: physical_expr:: utils:: { Guarantee , LiteralGuarantee } ;
1110+ use std:: collections:: HashSet ;
1111+ let schema = Schema :: new ( vec ! [
1112+ Field :: new( "f" , DataType :: Float32 , true ) ,
1113+ Field :: new( "d" , DataType :: Float64 , true ) ,
1114+ ] ) ;
1115+ let scalar = |value : f64 , data_type : & DataType | match data_type {
1116+ DataType :: Float32 => ScalarValue :: Float32 ( Some ( value as f32 ) ) ,
1117+ _ => ScalarValue :: Float64 ( Some ( value) ) ,
1118+ } ;
1119+ for ( index, name, data_type) in [ ( 0 , "f" , DataType :: Float32 ) , ( 1 , "d" , DataType :: Float64 ) ] {
1120+ let column: Arc < dyn PhysicalExpr > = Arc :: new ( Column :: new ( name, index) ) ;
1121+ for zero in [ -0.0 , 0.0 ] {
1122+ let literal: Arc < dyn PhysicalExpr > =
1123+ Arc :: new ( Literal :: new ( scalar ( zero, & data_type) ) ) ;
1124+ for ( left, right) in [ ( & column, & literal) , ( & literal, & column) ] {
1125+ let expr = spark_comparison (
1126+ Arc :: clone ( left) ,
1127+ Operator :: Eq ,
1128+ Arc :: clone ( right) ,
1129+ & schema,
1130+ FloatOperands :: Raw ,
1131+ ) ?;
1132+ let guarantees = LiteralGuarantee :: analyze ( & expr) ;
1133+ assert_eq ! ( guarantees. len( ) , 1 , "{expr}" ) ;
1134+ assert_eq ! ( guarantees[ 0 ] . guarantee, Guarantee :: In , "{expr}" ) ;
1135+ assert_eq ! ( guarantees[ 0 ] . column. name( ) , name, "{expr}" ) ;
1136+ assert_eq ! (
1137+ guarantees[ 0 ] . literals,
1138+ HashSet :: from( [ scalar( -0.0 , & data_type) , scalar( 0.0 , & data_type) ] ) ,
1139+ "{expr}"
1140+ ) ;
1141+ }
1142+ let expr = spark_comparison (
1143+ Arc :: clone ( & column) ,
1144+ Operator :: GtEq ,
1145+ Arc :: clone ( & literal) ,
1146+ & schema,
1147+ FloatOperands :: Raw ,
1148+ ) ?;
1149+ let binary = expr. downcast_ref :: < BinaryExpr > ( ) . unwrap ( ) ;
1150+ assert ! ( Arc :: ptr_eq( binary. left( ) , & column) ) ;
1151+ assert ! ( Arc :: ptr_eq( binary. right( ) , & literal) ) ;
1152+ }
1153+ }
1154+
1155+ let column: Arc < dyn PhysicalExpr > = Arc :: new ( Column :: new ( "d" , 1 ) ) ;
1156+ for nan in [ f64:: NAN , f64:: from_bits ( 0xfff8_0000_0000_0000 ) ] {
1157+ let expr = spark_comparison (
1158+ Arc :: clone ( & column) ,
1159+ Operator :: Eq ,
1160+ Arc :: new ( Literal :: new ( ScalarValue :: Float64 ( Some ( nan) ) ) ) ,
1161+ & schema,
1162+ FloatOperands :: Raw ,
1163+ ) ?;
1164+ let binary = expr. downcast_ref :: < BinaryExpr > ( ) . unwrap ( ) ;
1165+ assert ! ( binary
1166+ . left( )
1167+ . downcast_ref:: <NormalizeNaNAndZero >( )
1168+ . is_some( ) ) ;
1169+ let folded = binary. right ( ) . downcast_ref :: < Literal > ( ) . unwrap ( ) ;
1170+ assert ! (
1171+ matches!( folded. value( ) , ScalarValue :: Float64 ( Some ( v) ) if v. to_bits( ) == f64 :: NAN . to_bits( ) )
1172+ ) ;
1173+ }
1174+ Ok ( ( ) )
1175+ }
1176+
10521177 /// Ordering and null-safe comparisons of lists and structs match `spark_comparator`, which
10531178 /// compares float leaves in Spark's order without normalizing them.
10541179 #[ test]
0 commit comments