Skip to content

Commit 9ceca30

Browse files
ranflarionjayzhan211
authored andcommitted
fix: build the IN list static filter only from literal-only items (apache#26083)
## Which issue does this PR close? - Closes apache#26082. ## Rationale for this change `n IN (2, 100, 101, CASE WHEN d > 5 THEN 1 ELSE d END * 5)` returns wrong results on every row (repro in apache#26082). `try_evaluate_constant_list` decides that a list is constant by evaluating it on an empty batch, and `CaseExpr` returns its THEN branch as a scalar for a zero-length WHEN mask, so the CASE item is frozen to `5` in the static filter and never evaluated per row. ## What changes are included in this PR? `try_evaluate_constant_list` returns `None` unless every leaf of every list item is a `Literal`, so any other list takes the per-batch evaluation `InListExpr` already uses for items that return arrays. Lists of literals, casts of literals and other expressions over literals still build the static filter from their empty-batch evaluation, as before. Checking leaves rather than `collect_columns` keeps the rule independent of which leaf types read the input (`Column`, `LambdaVariable`, or a leaf defined outside DataFusion). The cost is that an item with a non-literal leaf that is nonetheless constant, such as a stable function the optimizer did not fold, is now evaluated per batch instead of once. ## What is the testing strategy for this PR? A new section in `in_list.slt` runs the repro as a projection and as a filter, and `test_in_list_case_item_reading_a_column` builds the same expression through `in_list()`. Both fail on main and pass with this change. The other `in_list` unit tests, the `physical-expr` lib tests, the full sqllogictest suite and the `filter_pushdown` integration tests pass unchanged. ## Are there any user-facing changes? No API changes. IN lists whose items have a non-literal leaf return correct results, evaluated per batch rather than through the static filter. --------- Co-authored-by: Jay Zhan <[email protected]>
1 parent c4e72de commit 9ceca30

2 files changed

Lines changed: 72 additions & 2 deletions

File tree

‎datafusion/physical-expr/src/expressions/in_list.rs‎

Lines changed: 41 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ use std::hash::{Hash, Hasher};
2222
use std::sync::Arc;
2323

2424
use crate::PhysicalExpr;
25+
use crate::expressions::Literal;
2526
use crate::physical_expr::physical_exprs_bag_equal;
2627

2728
use arrow::array::*;
@@ -31,6 +32,7 @@ use arrow::compute::kernels::boolean::{not, or_kleene};
3132
use arrow::compute::kernels::cmp::eq as arrow_eq;
3233
use arrow::datatypes::*;
3334

35+
use datafusion_common::tree_node::TreeNode;
3436
use datafusion_common::{
3537
DFSchema, Result, ScalarValue, assert_or_internal_err, exec_err,
3638
};
@@ -107,7 +109,7 @@ fn evaluate_list(
107109
/// Try to evaluate a list of expressions as constants.
108110
///
109111
/// Returns:
110-
/// - `Ok(Some(ArrayRef))` if all expressions are constants (can be evaluated on an empty RecordBatch)
112+
/// - `Ok(Some(ArrayRef))` if all expressions are constants (built only from literals)
111113
/// - `Ok(None)` if the list contains non-constant expressions
112114
/// - `Err(...)` only for actual errors (not for non-constant expressions)
113115
///
@@ -117,6 +119,15 @@ fn try_evaluate_constant_list(
117119
list: &[Arc<dyn PhysicalExpr>],
118120
schema: &Schema,
119121
) -> Result<Option<ArrayRef>> {
122+
// An item can read a column yet return a scalar on an empty batch (a CASE whose
123+
// WHEN matches all zero rows returns its THEN), so require literal leaves.
124+
for expr in list {
125+
if expr.exists(|e| {
126+
Ok(e.is_volatile_node() || (e.children().is_empty() && !e.is::<Literal>()))
127+
})? {
128+
return Ok(None);
129+
}
130+
}
120131
let batch = RecordBatch::new_empty(Arc::new(schema.clone()));
121132
match evaluate_list(list, &batch) {
122133
Ok(array) => Ok(Some(array)),
@@ -539,9 +550,10 @@ pub fn in_list(
539550
#[cfg(test)]
540551
mod tests {
541552
use super::*;
542-
use crate::expressions::{col, lit, try_cast};
553+
use crate::expressions::{binary, case, col, lit, try_cast};
543554
use arrow::datatypes::{IntervalDayTime, IntervalMonthDayNano, i256};
544555
use datafusion_common::plan_err;
556+
use datafusion_expr::Operator;
545557
use datafusion_expr::type_coercion::binary::comparison_coercion;
546558
use datafusion_physical_expr_common::physical_expr::fmt_sql;
547559
use insta::assert_snapshot;
@@ -3176,6 +3188,33 @@ mod tests {
31763188
Ok(())
31773189
}
31783190

3191+
#[test]
3192+
fn test_in_list_case_item_reading_a_column() -> Result<()> {
3193+
// n IN (2, CASE WHEN d > 5 THEN 1 ELSE d END)
3194+
let schema = Schema::new(vec![
3195+
Field::new("n", DataType::Int32, false),
3196+
Field::new("d", DataType::Int32, false),
3197+
]);
3198+
let d = col("d", &schema)?;
3199+
let when = binary(Arc::clone(&d), Operator::Gt, lit(5i32), &schema)?;
3200+
let item = case(None, vec![(when, lit(1i32))], Some(d))?;
3201+
let expr = in_list(col("n", &schema)?, vec![lit(2i32), item], &false, &schema)?;
3202+
let batch = RecordBatch::try_new(
3203+
Arc::new(schema),
3204+
vec![
3205+
Arc::new(Int32Array::from(vec![2, 4, 1])),
3206+
Arc::new(Int32Array::from(vec![0, 4, 3])),
3207+
],
3208+
)?;
3209+
3210+
let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;
3211+
assert_eq!(
3212+
as_boolean_array(&result),
3213+
&BooleanArray::from(vec![true, true, false])
3214+
);
3215+
Ok(())
3216+
}
3217+
31793218
/// Helper: creates an InListExpr with `static_filter = None`
31803219
/// to force the column-reference evaluation path.
31813220
fn make_in_list_with_columns(

‎datafusion/sqllogictest/test_files/in_list.slt‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -827,3 +827,34 @@ ORDER BY label
827827
match true false
828828
no_match false true
829829
nulls NULL NULL
830+
831+
####
832+
## List items that read a column
833+
####
834+
835+
statement ok
836+
CREATE TABLE in_list_case_items (n INT, d INT) AS VALUES (2, 0), (10, 2), (5, 2), (7, 1);
837+
838+
# A CASE item with a literal THEN is still evaluated per row, not once on an empty batch.
839+
query IB
840+
SELECT n, n IN (2, 100, 101, CASE WHEN d > 5 THEN 1 ELSE d END * 5)
841+
FROM in_list_case_items
842+
ORDER BY n
843+
----
844+
2 true
845+
5 false
846+
7 false
847+
10 true
848+
849+
query I
850+
SELECT n
851+
FROM in_list_case_items
852+
WHERE n IN (2, 100, 101, CASE WHEN d > 5 THEN 1 ELSE d END * 5)
853+
ORDER BY n
854+
----
855+
2
856+
10
857+
858+
# Cleanup
859+
statement ok
860+
DROP TABLE in_list_case_items;

0 commit comments

Comments
 (0)