Skip to content

Commit 9822fa7

Browse files
committed
fix: keep short float IN lists consistent with = for signed zero
Short Float32/Float64 IN lists are kept as membership expressions, which compare floats by bit pattern, so -0.0 did not match 0.0. Fall back to the equality chain when the list contains a zero literal.
1 parent 70f2ed0 commit 9822fa7

3 files changed

Lines changed: 48 additions & 13 deletions

File tree

‎datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4961,8 +4961,6 @@ mod tests {
49614961
};
49624962
let left_list = literals(&[0.0, 1.0, 2.0, 3.0])?;
49634963
let right_list = literals(&[-0.0, 4.0, 5.0, 6.0])?;
4964-
let retains_short_lists =
4965-
matches!(data_type, DataType::Float32 | DataType::Float64);
49664964
let schema = Arc::new(Schema::new(vec![Field::new("x", data_type, false)]));
49674965
let batch = RecordBatch::try_new(schema, vec![values])?;
49684966
let schema = batch.schema().to_dfschema_ref()?;
@@ -4987,7 +4985,7 @@ mod tests {
49874985
(left(true).or(right(true)), vec![false, false]),
49884986
] {
49894987
let simplified = simplifier.simplify(expr.clone())?;
4990-
if list_len > THRESHOLD_INLINE_INLIST || retains_short_lists {
4988+
if list_len > THRESHOLD_INLINE_INLIST {
49914989
// Floating-point literals must bypass structural set rewrites.
49924990
assert_eq!(simplified, expr);
49934991
} else {

‎datafusion/optimizer/src/simplify_expressions/inlist_simplifier.rs‎

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ use super::THRESHOLD_INLINE_INLIST;
88

99
use arrow::datatypes::DataType;
1010
use datafusion_common::tree_node::{Transformed, TreeNodeRewriter};
11-
use datafusion_common::{DFSchema, Result};
11+
use datafusion_common::{DFSchema, Result, ScalarValue};
1212
use datafusion_expr::expr::InList;
1313
use datafusion_expr::{Expr, ExprSchemable};
1414

@@ -32,15 +32,25 @@ impl<'a> ShortenInListSimplifier<'a> {
3232
let Ok(data_type) = expr.get_type(self.schema) else {
3333
return false;
3434
};
35-
matches!(
36-
data_type,
37-
DataType::Int32
38-
| DataType::Int64
39-
| DataType::UInt64
40-
| DataType::Float32
41-
| DataType::Float64
42-
| DataType::FixedSizeBinary(1 | 2 | 4 | 8 | 16)
43-
) && list.iter().all(|expr| expr.as_literal().is_some())
35+
let is_float = matches!(data_type, DataType::Float32 | DataType::Float64);
36+
(is_float
37+
|| matches!(
38+
data_type,
39+
DataType::Int32
40+
| DataType::Int64
41+
| DataType::UInt64
42+
| DataType::FixedSizeBinary(1 | 2 | 4 | 8 | 16)
43+
))
44+
&& list.iter().all(|expr| match expr.as_literal() {
45+
// The membership filters compare floats by bit pattern, so
46+
// -0.0 and +0.0 would not match each other, while `=` treats
47+
// them as equal. Keep the comparison chain when a zero is in
48+
// the list so both forms agree.
49+
Some(ScalarValue::Float32(Some(v))) if is_float => *v != 0.0,
50+
Some(ScalarValue::Float64(Some(v))) if is_float => *v != 0.0,
51+
Some(_) => true,
52+
None => false,
53+
})
4454
}
4555
}
4656

‎datafusion/sqllogictest/test_files/in_list.slt‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1147,3 +1147,30 @@ true
11471147

11481148
statement ok
11491149
DROP TABLE in_list_nulls;
1150+
1151+
# Short float IN lists must treat -0.0 and +0.0 as equal, matching `=`
1152+
# https://github.com/apache/datafusion/issues/27121
1153+
statement ok
1154+
CREATE TABLE signed_zeros(id BIGINT, x DOUBLE, y FLOAT) AS VALUES
1155+
(1, -0.0, -0.0),
1156+
(2, 0.0, 0.0),
1157+
(3, 5.0, 5.0);
1158+
1159+
query IBBBBBB
1160+
SELECT
1161+
id,
1162+
x = 0.0,
1163+
x IN (0.0, 5.0),
1164+
x NOT IN (0.0, 5.0),
1165+
x IN (-0.0, 7.0),
1166+
y IN (CAST(0 AS FLOAT), CAST(9 AS FLOAT)),
1167+
y NOT IN (CAST(0 AS FLOAT), CAST(9 AS FLOAT))
1168+
FROM signed_zeros
1169+
ORDER BY id;
1170+
----
1171+
1 true true false true true false
1172+
2 true true false true true false
1173+
3 false true false false false true
1174+
1175+
statement ok
1176+
DROP TABLE signed_zeros;

0 commit comments

Comments
 (0)