@@ -56,6 +56,7 @@ use datafusion_expr::{
5656
5757use crate :: extract_leaf_expressions:: is_pure_extraction_projection;
5858use crate :: optimizer:: ApplyOrder ;
59+ use crate :: projection_inliner:: { DuplicationCost , ProjectionInliner } ;
5960use crate :: simplify_expressions:: { reorder_predicates, simplify_predicates} ;
6061use crate :: utils:: {
6162 ColumnReference , has_all_column_refs, is_restrict_null_predicate, schema_columns,
@@ -1544,33 +1545,36 @@ fn rewrite_projection(
15441545 ) ) ;
15451546 }
15461547
1547- // Partition projection expressions into non-pushable vs pushable.
1548- // Non-pushable expressions are volatile (must not be duplicated) or
1549- // MoveTowardsLeafNodes (cheap expressions like get_field where re-inlining
1550- // into a filter causes optimizer instability — ExtractLeafExpressions will
1551- // undo the push-down, creating an infinite loop that runs until the
1552- // iteration limit is hit).
1553- //
1554- // Keyed by the output column rather than its flat name, for the reason
1555- // given on `replace_cols_by_column`. `EliminateGroupByConstant` can leave a
1556- // projection that outputs both `t.a` and `CAST(t.a AS ...)` under that one
1557- // name.
1558- let ( non_pushable_map, pushable_map) : ( HashMap < _ , _ > , HashMap < _ , _ > ) = projection
1559- . schema
1560- . iter ( )
1561- . zip ( projection. expr . iter ( ) )
1562- . map ( |( ( qualifier, field) , expr) | {
1563- ( Column :: new ( qualifier. cloned ( ) , field. name ( ) ) , unalias ( expr) )
1564- } )
1565- . partition ( |( _, value) | {
1566- value. is_volatile ( )
1567- || value. placement ( ) == ExpressionPlacement :: MoveTowardsLeafNodes
1568- } ) ;
1569-
1548+ let inliner = ProjectionInliner :: new ( & projection) ;
1549+
1550+ // A predicate is kept above the projection when it references:
1551+ // - a volatile definition. The projection still computes it, so a second
1552+ // evaluation below would see a different value.
1553+ // - an expensive definition that the predicates reference more than one
1554+ // time. Pushing would evaluate it more than one time, and
1555+ // `CommonSubexprEliminate` would create a projection for it again below
1556+ // the filter, so the two rules would never converge (issue #25329). A
1557+ // single reference is pushed: if the columns above still use the
1558+ // definition, that is the usual cost of a filter push-down.
1559+ // - a `MoveTowardsLeafNodes` definition (a cheap expression like
1560+ // `get_field`). Re-inlining such an expression into a filter causes
1561+ // optimizer instability: `ExtractLeafExpressions` undoes the push-down,
1562+ // which creates a loop that runs until the iteration limit is hit.
1563+ let no_other_consumers: [ & Expr ; 0 ] = [ ] ;
1564+ let pinned = inliner. pinned_for_known_consumers (
1565+ & predicates,
1566+ no_other_consumers,
1567+ DuplicationCost :: Cheap ,
1568+ ) ;
15701569 let mut push_predicates = vec ! [ ] ;
15711570 let mut keep_predicates = vec ! [ ] ;
15721571 for expr in predicates {
1573- if contains_column ( & expr, & non_pushable_map) {
1572+ let references_leaf_expr = expr. column_refs ( ) . into_iter ( ) . any ( |col| {
1573+ inliner. definition ( col) . is_some_and ( |definition| {
1574+ definition. placement ( ) == ExpressionPlacement :: MoveTowardsLeafNodes
1575+ } )
1576+ } ) ;
1577+ if references_leaf_expr || inliner. references_pinned ( & expr, & pinned) {
15741578 keep_predicates. push ( expr) ;
15751579 } else {
15761580 push_predicates. push ( expr) ;
@@ -1581,7 +1585,7 @@ fn rewrite_projection(
15811585 // re-write all filters based on this projection
15821586 // E.g. in `Filter: b\n Projection: a > 1 as b`, we can swap them, but the filter must be "a > 1"
15831587 projection. input = Arc :: new ( LogicalPlan :: Filter ( Filter :: new (
1584- replace_cols_by_column ( expr, & pushable_map ) ?,
1588+ inliner . substitute ( expr) ?,
15851589 projection. input ,
15861590 ) ) ) ;
15871591
@@ -1672,12 +1676,6 @@ fn unalias(expr: &Expr) -> &Expr {
16721676 }
16731677}
16741678
1675- /// check whether the expression uses the columns in `check_map`.
1676- fn contains_column < T > ( e : & Expr , check_map : & HashMap < Column , T > ) -> bool {
1677- e. exists ( |expr| Ok ( matches ! ( expr, Expr :: Column ( c) if check_map. contains_key( c) ) ) )
1678- . unwrap ( )
1679- }
1680-
16811679fn with_filters ( predicates : Vec < Expr > , plan : LogicalPlan ) -> LogicalPlan {
16821680 if let Some ( predicate) = conjunction ( predicates) {
16831681 LogicalPlan :: Filter ( Filter :: new ( predicate, Arc :: new ( plan) ) )
@@ -1767,7 +1765,7 @@ mod tests {
17671765 use crate :: assert_optimized_plan_eq_snapshot;
17681766 use crate :: optimizer:: Optimizer ;
17691767 use crate :: simplify_expressions:: SimplifyExpressions ;
1770- use crate :: test:: udfs:: leaf_udf_expr;
1768+ use crate :: test:: udfs:: { PlacementTestUDF , get_field_like , leaf_udf_expr} ;
17711769 use crate :: test:: * ;
17721770 use datafusion_expr:: test:: function_stub:: { count, sum} ;
17731771 use insta:: assert_snapshot;
@@ -4959,6 +4957,90 @@ mod tests {
49594957 )
49604958 }
49614959
4960+ fn keep_in_place_udf ( arg : Expr ) -> Expr {
4961+ ScalarUDF :: new_from_impl (
4962+ PlacementTestUDF :: new ( ) . with_placement ( ExpressionPlacement :: KeepInPlace ) ,
4963+ )
4964+ . call ( vec ! [ arg] )
4965+ }
4966+
4967+ /// Issue #25329: `CommonSubexprEliminate` computes `c1` one time for
4968+ /// predicates that reference it twice. Pushing the predicates would compute
4969+ /// it twice, and CSE would then create the projection again below the
4970+ /// filter, so the two rules never converge.
4971+ #[ test]
4972+ fn filter_with_repeated_expensive_reference_not_pushed_through_projection ( )
4973+ -> Result < ( ) > {
4974+ let plan = LogicalPlanBuilder :: from ( test_table_scan ( ) ?)
4975+ . project ( vec ! [ keep_in_place_udf( col( "a" ) ) . alias( "c1" ) , col( "b" ) ] ) ?
4976+ . filter (
4977+ col ( "c1" )
4978+ . is_not_null ( )
4979+ . and ( get_field_like ( col ( "c1" ) , "x" ) . is_null ( ) )
4980+ . and ( col ( "b" ) . gt ( lit ( 1 ) ) ) ,
4981+ ) ?
4982+ . build ( ) ?;
4983+
4984+ assert_optimized_plan_equal ! (
4985+ plan,
4986+ @r#"
4987+ Filter: c1 IS NOT NULL AND get_field_like(c1, Utf8("x")) IS NULL
4988+ Projection: keep_in_place_udf(test.a) AS c1, test.b
4989+ TableScan: test, full_filters=[test.b > Int32(1)]
4990+ "#
4991+ )
4992+ }
4993+
4994+ /// A single reference to an expensive definition is pushed, like any
4995+ /// other computed column of a projection.
4996+ #[ test]
4997+ fn filter_with_single_expensive_reference_pushed_through_projection ( ) -> Result < ( ) > {
4998+ let plan = LogicalPlanBuilder :: from ( test_table_scan ( ) ?)
4999+ . project ( vec ! [ keep_in_place_udf( col( "a" ) ) . alias( "c1" ) , col( "b" ) ] ) ?
5000+ . filter ( col ( "c1" ) . is_not_null ( ) ) ?
5001+ . build ( ) ?;
5002+
5003+ assert_optimized_plan_equal ! (
5004+ plan,
5005+ @r"
5006+ Projection: keep_in_place_udf(test.a) AS c1, test.b
5007+ TableScan: test, full_filters=[keep_in_place_udf(test.a) IS NOT NULL]
5008+ "
5009+ )
5010+ }
5011+
5012+ /// The shape of draft https://github.com/apache/datafusion/pull/25388:
5013+ /// a filter over a projection that reads an expensive column of the
5014+ /// projection below it. The narrow policy pushes `c1 IS NOT NULL`,
5015+ /// because the predicates reference `c1` one time. `c2 IS NULL` stays
5016+ /// above, because `c2` is a `MoveTowardsLeafNodes` definition.
5017+ #[ test]
5018+ fn filter_not_pushed_through_nested_computed_projection ( ) -> Result < ( ) > {
5019+ let inner = LogicalPlanBuilder :: from ( test_table_scan ( ) ?)
5020+ . project ( vec ! [ keep_in_place_udf( col( "a" ) ) . alias( "c1" ) , col( "b" ) ] ) ?
5021+ . build ( ) ?;
5022+ let outer = LogicalPlanBuilder :: from ( inner)
5023+ . project ( vec ! [
5024+ get_field_like( col( "c1" ) , "x" ) . alias( "c2" ) ,
5025+ col( "c1" ) ,
5026+ col( "b" ) ,
5027+ ] ) ?
5028+ . build ( ) ?;
5029+ let plan = LogicalPlanBuilder :: from ( outer)
5030+ . filter ( col ( "c1" ) . is_not_null ( ) . and ( col ( "c2" ) . is_null ( ) ) ) ?
5031+ . build ( ) ?;
5032+
5033+ assert_optimized_plan_equal ! (
5034+ plan,
5035+ @r#"
5036+ Filter: c2 IS NULL
5037+ Projection: get_field_like(c1, Utf8("x")) AS c2, c1, test.b
5038+ Projection: keep_in_place_udf(test.a) AS c1, test.b
5039+ TableScan: test, full_filters=[keep_in_place_udf(test.a) IS NOT NULL]
5040+ "#
5041+ )
5042+ }
5043+
49625044 #[ test]
49635045 fn filter_not_pushed_down_through_table_scan_with_fetch ( ) -> Result < ( ) > {
49645046 let scan = test_table_scan ( ) ?;
0 commit comments