@@ -43,6 +43,7 @@ use datafusion_expr::{
4343} ;
4444
4545use crate :: optimizer:: ApplyOrder ;
46+ use crate :: projection_inliner:: { DuplicationCost , ProjectionInliner } ;
4647use crate :: simplify_expressions:: { reorder_predicates, simplify_predicates} ;
4748use crate :: utils:: {
4849 ColumnReference , has_all_column_refs, is_restrict_null_predicate, schema_columns,
@@ -1388,28 +1389,36 @@ fn rewrite_projection(
13881389 predicates : Vec < Expr > ,
13891390 mut projection : Projection ,
13901391) -> Result < ( Transformed < LogicalPlan > , Vec < Expr > ) > {
1391- // Partition projection expressions into non-pushable vs pushable.
1392- // Non-pushable expressions are volatile (must not be duplicated) or
1393- // MoveTowardsLeafNodes (cheap expressions like get_field where re-inlining
1394- // into a filter causes optimizer instability — ExtractLeafExpressions will
1395- // undo the push-down, creating an infinite loop that runs until the
1396- // iteration limit is hit).
1397- let ( non_pushable_map, pushable_map) = projection
1398- . schema
1399- . iter ( )
1400- . zip ( projection. expr . iter ( ) )
1401- . map ( |( ( qualifier, field) , expr) | {
1402- ( qualified_name ( qualifier, field. name ( ) ) , unalias ( expr) )
1403- } )
1404- . partition ( |( _, value) | {
1405- value. is_volatile ( )
1406- || value. placement ( ) == ExpressionPlacement :: MoveTowardsLeafNodes
1407- } ) ;
1408-
1392+ let inliner = ProjectionInliner :: new ( & projection) ;
1393+
1394+ // A predicate is kept above the projection when it references:
1395+ // - a volatile definition. The projection still computes it, so a second
1396+ // evaluation below would see a different value.
1397+ // - an expensive definition that the predicates reference more than one
1398+ // time. Pushing would evaluate it more than one time, and
1399+ // `CommonSubexprEliminate` would create a projection for it again below
1400+ // the filter, so the two rules would never converge (issue #25329). A
1401+ // single reference is pushed: if the columns above still use the
1402+ // definition, that is the usual cost of a filter push-down.
1403+ // - a `MoveTowardsLeafNodes` definition (a cheap expression like
1404+ // `get_field`). Re-inlining such an expression into a filter causes
1405+ // optimizer instability: `ExtractLeafExpressions` undoes the push-down,
1406+ // which creates a loop that runs until the iteration limit is hit.
1407+ let no_other_consumers: [ & Expr ; 0 ] = [ ] ;
1408+ let pinned = inliner. pinned_for_known_consumers (
1409+ & predicates,
1410+ no_other_consumers,
1411+ DuplicationCost :: Cheap ,
1412+ ) ;
14091413 let mut push_predicates = vec ! [ ] ;
14101414 let mut keep_predicates = vec ! [ ] ;
14111415 for expr in predicates {
1412- if contain ( & expr, & non_pushable_map) {
1416+ let references_leaf_expr = expr. column_refs ( ) . into_iter ( ) . any ( |col| {
1417+ inliner. definition ( col) . is_some_and ( |definition| {
1418+ definition. placement ( ) == ExpressionPlacement :: MoveTowardsLeafNodes
1419+ } )
1420+ } ) ;
1421+ if references_leaf_expr || inliner. references_pinned ( & expr, & pinned) {
14131422 keep_predicates. push ( expr) ;
14141423 } else {
14151424 push_predicates. push ( expr) ;
@@ -1420,7 +1429,7 @@ fn rewrite_projection(
14201429 // re-write all filters based on this projection
14211430 // E.g. in `Filter: b\n Projection: a > 1 as b`, we can swap them, but the filter must be "a > 1"
14221431 projection. input = Arc :: new ( LogicalPlan :: Filter ( Filter :: new (
1423- replace_cols_by_name ( expr, & pushable_map ) ?,
1432+ inliner . substitute ( expr) ?,
14241433 projection. input ,
14251434 ) ) ) ;
14261435
@@ -1489,14 +1498,6 @@ fn unalias(expr: &Expr) -> &Expr {
14891498 }
14901499}
14911500
1492- /// check whether the expression uses the columns in `check_map`.
1493- fn contain < T > ( e : & Expr , check_map : & HashMap < String , T > ) -> bool {
1494- e. exists ( |expr| {
1495- Ok ( matches ! ( expr, Expr :: Column ( c) if check_map. contains_key( & c. flat_name( ) ) ) )
1496- } )
1497- . unwrap ( )
1498- }
1499-
15001501fn with_filters ( predicates : Vec < Expr > , plan : LogicalPlan ) -> LogicalPlan {
15011502 if let Some ( predicate) = conjunction ( predicates) {
15021503 LogicalPlan :: Filter ( Filter :: new ( predicate, Arc :: new ( plan) ) )
@@ -1537,7 +1538,7 @@ mod tests {
15371538 use crate :: assert_optimized_plan_eq_snapshot;
15381539 use crate :: optimizer:: Optimizer ;
15391540 use crate :: simplify_expressions:: SimplifyExpressions ;
1540- use crate :: test:: udfs:: leaf_udf_expr;
1541+ use crate :: test:: udfs:: { PlacementTestUDF , get_field_like , leaf_udf_expr} ;
15411542 use crate :: test:: * ;
15421543 use datafusion_expr:: test:: function_stub:: sum;
15431544 use insta:: assert_snapshot;
@@ -4546,6 +4547,90 @@ mod tests {
45464547 )
45474548 }
45484549
4550+ fn keep_in_place_udf ( arg : Expr ) -> Expr {
4551+ ScalarUDF :: new_from_impl (
4552+ PlacementTestUDF :: new ( ) . with_placement ( ExpressionPlacement :: KeepInPlace ) ,
4553+ )
4554+ . call ( vec ! [ arg] )
4555+ }
4556+
4557+ /// Issue #25329: `CommonSubexprEliminate` computes `c1` one time for
4558+ /// predicates that reference it twice. Pushing the predicates would compute
4559+ /// it twice, and CSE would then create the projection again below the
4560+ /// filter, so the two rules never converge.
4561+ #[ test]
4562+ fn filter_with_repeated_expensive_reference_not_pushed_through_projection ( )
4563+ -> Result < ( ) > {
4564+ let plan = LogicalPlanBuilder :: from ( test_table_scan ( ) ?)
4565+ . project ( vec ! [ keep_in_place_udf( col( "a" ) ) . alias( "c1" ) , col( "b" ) ] ) ?
4566+ . filter (
4567+ col ( "c1" )
4568+ . is_not_null ( )
4569+ . and ( get_field_like ( col ( "c1" ) , "x" ) . is_null ( ) )
4570+ . and ( col ( "b" ) . gt ( lit ( 1 ) ) ) ,
4571+ ) ?
4572+ . build ( ) ?;
4573+
4574+ assert_optimized_plan_equal ! (
4575+ plan,
4576+ @r#"
4577+ Filter: c1 IS NOT NULL AND get_field_like(c1, Utf8("x")) IS NULL
4578+ Projection: keep_in_place_udf(test.a) AS c1, test.b
4579+ TableScan: test, full_filters=[test.b > Int32(1)]
4580+ "#
4581+ )
4582+ }
4583+
4584+ /// A single reference to an expensive definition is pushed, like any
4585+ /// other computed column of a projection.
4586+ #[ test]
4587+ fn filter_with_single_expensive_reference_pushed_through_projection ( ) -> Result < ( ) > {
4588+ let plan = LogicalPlanBuilder :: from ( test_table_scan ( ) ?)
4589+ . project ( vec ! [ keep_in_place_udf( col( "a" ) ) . alias( "c1" ) , col( "b" ) ] ) ?
4590+ . filter ( col ( "c1" ) . is_not_null ( ) ) ?
4591+ . build ( ) ?;
4592+
4593+ assert_optimized_plan_equal ! (
4594+ plan,
4595+ @r"
4596+ Projection: keep_in_place_udf(test.a) AS c1, test.b
4597+ TableScan: test, full_filters=[keep_in_place_udf(test.a) IS NOT NULL]
4598+ "
4599+ )
4600+ }
4601+
4602+ /// The shape of draft https://github.com/apache/datafusion/pull/25388:
4603+ /// a filter over a projection that reads an expensive column of the
4604+ /// projection below it. The narrow policy pushes `c1 IS NOT NULL`,
4605+ /// because the predicates reference `c1` one time. `c2 IS NULL` stays
4606+ /// above, because `c2` is a `MoveTowardsLeafNodes` definition.
4607+ #[ test]
4608+ fn filter_not_pushed_through_nested_computed_projection ( ) -> Result < ( ) > {
4609+ let inner = LogicalPlanBuilder :: from ( test_table_scan ( ) ?)
4610+ . project ( vec ! [ keep_in_place_udf( col( "a" ) ) . alias( "c1" ) , col( "b" ) ] ) ?
4611+ . build ( ) ?;
4612+ let outer = LogicalPlanBuilder :: from ( inner)
4613+ . project ( vec ! [
4614+ get_field_like( col( "c1" ) , "x" ) . alias( "c2" ) ,
4615+ col( "c1" ) ,
4616+ col( "b" ) ,
4617+ ] ) ?
4618+ . build ( ) ?;
4619+ let plan = LogicalPlanBuilder :: from ( outer)
4620+ . filter ( col ( "c1" ) . is_not_null ( ) . and ( col ( "c2" ) . is_null ( ) ) ) ?
4621+ . build ( ) ?;
4622+
4623+ assert_optimized_plan_equal ! (
4624+ plan,
4625+ @r#"
4626+ Filter: c2 IS NULL
4627+ Projection: get_field_like(c1, Utf8("x")) AS c2, c1, test.b
4628+ Projection: keep_in_place_udf(test.a) AS c1, test.b
4629+ TableScan: test, full_filters=[keep_in_place_udf(test.a) IS NOT NULL]
4630+ "#
4631+ )
4632+ }
4633+
45494634 #[ test]
45504635 fn filter_not_pushed_down_through_table_scan_with_fetch ( ) -> Result < ( ) > {
45514636 let scan = test_table_scan ( ) ?;
0 commit comments