Skip to content

Commit 707d861

Browse files
timsaucerclaude
andauthored
fix: simplifier on leaf nodes returns null (apache#22368)
## Which issue does this PR close? Addresses the immediate problem identified in apache/datafusion-python#1551 and the first recommendation in apache#22367 ## Rationale for this change We expect Literal and Column expressions and they get `Transform::no` as expected. If we have other PhysicalExpr that similarly should not get through because they have no children then they should likewise get `Transform::no`. This was found when a FFI wrapped Column expression failed in simplification. ## What changes are included in this PR? Essentially a two line change to see if it is a leaf node (no children) instead of just checking that all of the children are literals. ## Are these changes tested? Unit test added. ## Are there any user-facing changes? No user facing changes. Co-authored-by: Claude Opus 4.7 <[email protected]>
1 parent 199fbc8 commit 707d861

2 files changed

Lines changed: 81 additions & 3 deletions

File tree

‎datafusion/physical-expr/src/simplifier/const_evaluator.rs‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -107,9 +107,18 @@ pub(crate) fn simplify_const_expr_immediate(
107107
// Since transform visits bottom-up, children have already been simplified.
108108
// If all children are now Literals, this node can be const-evaluated.
109109
// This is O(k) where k = number of children, instead of O(subtree).
110-
let all_children_literal = expr.children().iter().all(|child| child.is::<Literal>());
111-
112-
if !all_children_literal {
110+
//
111+
// Leaf nodes (zero children) are rejected here. Const-folding is only
112+
// sound for a node whose value is fully determined by its child literals;
113+
// a leaf has no children, so there is nothing to derive constness from.
114+
// The known leaves that are constant (`Literal`) or known-non-constant
115+
// (`Column`, volatile) are handled by the dedicated checks above. Any
116+
// other leaf is opaque to the simplifier and must be preserved as-is,
117+
// otherwise `all` over an empty child list would vacuously hold and the
118+
// node would be evaluated against the dummy batch, producing a value
119+
// unrelated to its real runtime semantics.
120+
let children = expr.children();
121+
if children.is_empty() || !children.iter().all(|c| c.is::<Literal>()) {
113122
return Ok(Transformed::no(expr));
114123
}
115124

‎datafusion/physical-expr/src/simplifier/mod.rs‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -590,6 +590,75 @@ mod tests {
590590
assert_eq!(left_literal.value(), &ScalarValue::Int32(Some(3)));
591591
}
592592

593+
/// Regression test for https://github.com/apache/datafusion/issues/22367.
594+
///
595+
/// A leaf `PhysicalExpr` that is neither a `Literal` nor a `Column`
596+
/// (nor volatile) must not be const-folded: it has no children to
597+
/// derive constness from, and evaluating it against the dummy batch
598+
/// produces a value unrelated to its real runtime semantics. Without
599+
/// the zero-children guard, `all(empty)` would vacuously hold and the
600+
/// node would be replaced with whatever scalar fell out of the dummy
601+
/// evaluation. Verify the node is left untouched.
602+
#[test]
603+
fn test_no_simplify_opaque_leaf_expr() {
604+
use arrow::array::ArrayRef;
605+
use arrow::array::Int32Array;
606+
use arrow::record_batch::RecordBatch;
607+
use datafusion_expr_common::columnar_value::ColumnarValue;
608+
use datafusion_physical_expr_common::physical_expr::PhysicalExpr as PhysicalExprTrait;
609+
use std::fmt;
610+
611+
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
612+
struct OpaqueLeaf;
613+
614+
impl fmt::Display for OpaqueLeaf {
615+
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
616+
write!(f, "OpaqueLeaf")
617+
}
618+
}
619+
620+
impl PhysicalExprTrait for OpaqueLeaf {
621+
fn data_type(&self, _input_schema: &Schema) -> Result<DataType> {
622+
Ok(DataType::Int32)
623+
}
624+
fn nullable(&self, _input_schema: &Schema) -> Result<bool> {
625+
Ok(true)
626+
}
627+
fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> {
628+
// Simulate the broken FFI Column path: when handed a dummy
629+
// batch, return whatever scalar happens to materialize. If
630+
// the simplifier ever reaches this branch for a leaf node,
631+
// the predicate has already been silently corrupted.
632+
let arr: ArrayRef = Arc::new(Int32Array::from(vec![0; batch.num_rows()]));
633+
Ok(ColumnarValue::Array(arr))
634+
}
635+
fn children(&self) -> Vec<&Arc<dyn PhysicalExprTrait>> {
636+
vec![]
637+
}
638+
fn with_new_children(
639+
self: Arc<Self>,
640+
_children: Vec<Arc<dyn PhysicalExprTrait>>,
641+
) -> Result<Arc<dyn PhysicalExprTrait>> {
642+
Ok(self)
643+
}
644+
fn fmt_sql(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
645+
write!(f, "OpaqueLeaf")
646+
}
647+
}
648+
649+
let schema = Schema::empty();
650+
let simplifier = PhysicalExprSimplifier::new(&schema);
651+
652+
let opaque: Arc<dyn PhysicalExpr> = Arc::new(OpaqueLeaf);
653+
let result = simplifier.simplify(Arc::clone(&opaque)).unwrap();
654+
655+
assert!(
656+
result.downcast_ref::<Literal>().is_none(),
657+
"opaque leaf must not be rewritten to a Literal, got: {result}"
658+
);
659+
assert_eq!(&result, &opaque);
660+
}
661+
593662
#[test]
594663
fn test_simplify_literal_string_concat() {
595664
let schema = Schema::empty();

0 commit comments

Comments
 (0)