Skip to content

Commit 8aea24b

Browse files
test: simplify short IN list coverage
1 parent 076b2e3 commit 8aea24b

3 files changed

Lines changed: 53 additions & 72 deletions

File tree

‎datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs‎

Lines changed: 46 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -5106,24 +5106,30 @@ mod tests {
51065106
}
51075107

51085108
#[test]
5109-
fn simplify_short_inlist_measured_representations() {
5110-
for data_type in [
5111-
DataType::Float32,
5112-
DataType::Float64,
5113-
DataType::FixedSizeBinary(8),
5114-
DataType::Float16,
5115-
DataType::Int32,
5116-
DataType::Int64,
5117-
DataType::UInt64,
5118-
DataType::FixedSizeBinary(1),
5119-
DataType::FixedSizeBinary(2),
5120-
DataType::FixedSizeBinary(3),
5121-
DataType::FixedSizeBinary(4),
5122-
DataType::FixedSizeBinary(16),
5123-
DataType::Utf8View,
5124-
DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Float64)),
5109+
fn simplify_short_inlist_types() {
5110+
for (data_type, retain_short) in [
5111+
(DataType::Int32, true),
5112+
(DataType::Int64, true),
5113+
(DataType::UInt64, true),
5114+
(DataType::Float32, true),
5115+
(DataType::Float64, true),
5116+
(DataType::FixedSizeBinary(1), true),
5117+
(DataType::FixedSizeBinary(2), true),
5118+
(DataType::FixedSizeBinary(4), true),
5119+
(DataType::FixedSizeBinary(8), true),
5120+
(DataType::FixedSizeBinary(16), true),
5121+
(DataType::Float16, false),
5122+
(DataType::FixedSizeBinary(3), false),
5123+
(DataType::Utf8View, false),
5124+
(
5125+
DataType::Dictionary(
5126+
Box::new(DataType::Int8),
5127+
Box::new(DataType::Float64),
5128+
),
5129+
false,
5130+
),
51255131
] {
5126-
let values = (1..=4)
5132+
let values = (1..=3)
51275133
.map(|value| {
51285134
lit(match &data_type {
51295135
DataType::FixedSizeBinary(width) => ScalarValue::FixedSizeBinary(
@@ -5139,65 +5145,39 @@ mod tests {
51395145
})
51405146
})
51415147
.collect::<Vec<_>>();
5142-
let retain_short = matches!(
5143-
data_type,
5144-
DataType::Int32
5145-
| DataType::Int64
5146-
| DataType::UInt64
5147-
| DataType::Float32
5148-
| DataType::Float64
5149-
| DataType::FixedSizeBinary(1 | 2 | 4 | 8 | 16)
5150-
);
5151-
let schema = Schema::new(vec![
5152-
Field::new("value", data_type.clone(), true),
5153-
Field::new("other", data_type, true),
5154-
])
5155-
.to_dfschema_ref()
5156-
.unwrap();
5148+
let schema = Schema::new(vec![Field::new("value", data_type.clone(), true)])
5149+
.to_dfschema_ref()
5150+
.unwrap();
51575151
let simplifier = ExprSimplifier::new(
51585152
SimplifyContext::builder().with_schema(schema).build(),
51595153
);
5160-
for list_len in 1..=4 {
5154+
for list_len in [2, 3] {
51615155
for negated in [false, true] {
5162-
let list = values[..list_len].to_vec();
5163-
let expr = in_list(col("value"), list.clone(), negated);
5164-
let expected = if list_len > THRESHOLD_INLINE_INLIST
5165-
|| list_len > 1 && retain_short
5166-
{
5167-
expr.clone()
5156+
let expr =
5157+
in_list(col("value"), values[..list_len].to_vec(), negated);
5158+
let simplified = simplifier.simplify(expr.clone()).unwrap();
5159+
if retain_short {
5160+
assert_eq!(simplified, expr, "{data_type:?}");
51685161
} else {
5169-
let comparisons = list.into_iter().map(|value| {
5170-
if negated {
5171-
col("value").not_eq(value)
5172-
} else {
5173-
col("value").eq(value)
5174-
}
5175-
});
5176-
comparisons
5177-
.reduce(|left, right| {
5178-
if negated {
5179-
left.and(right)
5180-
} else {
5181-
left.or(right)
5182-
}
5183-
})
5184-
.unwrap()
5185-
};
5186-
assert_eq!(simplifier.simplify(expr).unwrap(), expected);
5162+
let op = if negated { Operator::And } else { Operator::Or };
5163+
assert!(
5164+
matches!(simplified, Expr::BinaryExpr(binary) if binary.op == op),
5165+
"{data_type:?}, list_len={list_len}, negated={negated}"
5166+
);
5167+
}
51875168
}
51885169
}
5189-
5190-
// A column-valued list keeps the comparison path even for a
5191-
// representation whose static lists use membership filters.
5192-
let expr =
5193-
in_list(col("value"), vec![values[0].clone(), col("other")], false);
5194-
let expected = col("value")
5195-
.eq(values[0].clone())
5196-
.or(col("value").eq(col("other")));
5197-
assert_eq!(simplifier.simplify(expr).unwrap(), expected);
51985170
}
51995171
}
52005172

5173+
#[test]
5174+
fn simplify_short_inlist_mismatched_literals() {
5175+
// c3 is Int64; these Int32 literals have not been coerced.
5176+
let expr = in_list(col("c3"), vec![lit(1_i32), lit(2_i32)], false);
5177+
let expected = col("c3").eq(lit(1_i32)).or(col("c3").eq(lit(2_i32)));
5178+
assert_eq!(simplify(expr), expected);
5179+
}
5180+
52015181
#[test]
52025182
fn simplify_inlist_set_operation_propagates_nullability_errors() {
52035183
let info = SimplifyContext::builder()

‎datafusion/optimizer/src/simplify_expressions/inlist_simplifier.rs‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,7 @@ impl<'a> ShortenInListSimplifier<'a> {
3434
Self { schema }
3535
}
3636

37-
/// The paired `in_list_rewrite` benchmark favors membership for these
38-
/// representations. For integers, prefer the gains on mixed and miss-heavy
39-
/// batches despite the comparison chain's advantage on first-item matches.
37+
/// Keep short literal lists for types with efficient membership filters.
4038
fn should_retain_short_inlist(&self, expr: &Expr, list: &[Expr]) -> bool {
4139
let Ok(data_type) = expr.get_type(self.schema) else {
4240
return false;

‎datafusion/sqllogictest/test_files/in_list.slt‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -263,14 +263,17 @@ nulls NULL NULL NULL NULL NULL NULL
263263
query TT
264264
EXPLAIN SELECT i32 IN (arrow_cast(3, 'Int32'), arrow_cast(11, 'Int32')) AS i32_in_two,
265265
i64 NOT IN (arrow_cast(3, 'Int64'), arrow_cast(4, 'Int64'), arrow_cast(11, 'Int64')) AS i64_not_in_three,
266-
u64 IN (arrow_cast(3, 'UInt64'), arrow_cast(4, 'UInt64'), arrow_cast(11, 'UInt64')) AS u64_in_three
266+
u64 IN (arrow_cast(3, 'UInt64'), arrow_cast(4, 'UInt64'), arrow_cast(11, 'UInt64')) AS u64_in_three,
267+
i64 IN (arrow_cast(1, 'Int64')) AS i64_in_one,
268+
i64 IN (arrow_cast(3, 'Int64'), i32) AS i64_in_column,
269+
i64 NOT IN (arrow_cast(3, 'Int64'), i32) AS i64_not_in_column
267270
FROM in_list_ints_nullable;
268271
----
269272
logical_plan
270-
01)Projection: in_list_ints_nullable.i32 IN ([Int32(3), Int32(11)]) AS i32_in_two, in_list_ints_nullable.i64 NOT IN ([Int64(3), Int64(4), Int64(11)]) AS i64_not_in_three, in_list_ints_nullable.u64 IN ([UInt64(3), UInt64(4), UInt64(11)]) AS u64_in_three
273+
01)Projection: in_list_ints_nullable.i32 IN ([Int32(3), Int32(11)]) AS i32_in_two, in_list_ints_nullable.i64 NOT IN ([Int64(3), Int64(4), Int64(11)]) AS i64_not_in_three, in_list_ints_nullable.u64 IN ([UInt64(3), UInt64(4), UInt64(11)]) AS u64_in_three, in_list_ints_nullable.i64 = Int64(1) AS i64_in_one, in_list_ints_nullable.i64 = Int64(3) OR in_list_ints_nullable.i64 = CAST(in_list_ints_nullable.i32 AS Int64) AS i64_in_column, in_list_ints_nullable.i64 != Int64(3) AND in_list_ints_nullable.i64 != CAST(in_list_ints_nullable.i32 AS Int64) AS i64_not_in_column
271274
02)--TableScan: in_list_ints_nullable projection=[i32, i64, u64]
272275
physical_plan
273-
01)ProjectionExec: expr=[i32@0 IN (SET) ([3, 11]) as i32_in_two, i64@1 NOT IN (SET) ([3, 4, 11]) as i64_not_in_three, u64@2 IN (SET) ([3, 4, 11]) as u64_in_three]
276+
01)ProjectionExec: expr=[i32@0 IN (SET) ([3, 11]) as i32_in_two, i64@1 NOT IN (SET) ([3, 4, 11]) as i64_not_in_three, u64@2 IN (SET) ([3, 4, 11]) as u64_in_three, i64@1 = 1 as i64_in_one, i64@1 = 3 OR i64@1 = CAST(i32@0 AS Int64) as i64_in_column, i64@1 != 3 AND i64@1 != CAST(i32@0 AS Int64) as i64_not_in_column]
274277
02)--DataSourceExec: partitions=1, partition_sizes=[1]
275278

276279
# Null input values return NULL when the IN list has no nulls.

0 commit comments

Comments
 (0)