@@ -1532,6 +1532,16 @@ fn push_down_into_table_scan(
15321532 let LogicalPlan :: TableScan ( mut scan) = Arc :: unwrap_or_clone ( filter. input ) else {
15331533 return internal_err ! ( "the input of the filter is not a table scan" ) ;
15341534 } ;
1535+ // The fetch of a scan applies after its filters, so a filter above a scan
1536+ // with a fetch cannot move into it without changing the result.
1537+ if scan. fetch . is_some ( ) {
1538+ filter. input = Arc :: new ( LogicalPlan :: TableScan ( scan) ) ;
1539+ return Ok ( Transformed :: new (
1540+ LogicalPlan :: Filter ( filter) ,
1541+ changed,
1542+ TreeNodeRecursion :: Continue ,
1543+ ) ) ;
1544+ }
15351545 let filter_predicates = split_conjunction ( & filter. predicate ) ;
15361546 // Filters containing scalar subqueries cannot be pushed to
15371547 // providers because the subquery result is not available
@@ -1587,9 +1597,11 @@ fn push_down_into_table_scan(
15871597 . cloned ( )
15881598 . collect ( ) ;
15891599
1600+ // Nothing changes when no predicate leaves the filter and the scan already
1601+ // records every accepted predicate.
15901602 if supported_filters
15911603 . iter ( )
1592- . all ( |res| res == & TableProviderFilterPushDown :: Inexact )
1604+ . all ( |res| res != & TableProviderFilterPushDown :: Exact )
15931605 && scan. filters == new_scan_filters
15941606 {
15951607 filter. input = Arc :: new ( LogicalPlan :: TableScan ( scan) ) ;
@@ -1642,46 +1654,54 @@ fn push_down_through_extraction_projection_into_scan(
16421654 let pass_through: HashSet < & Column > = projection
16431655 . expr
16441656 . iter ( )
1645- . filter_map ( |expr| match expr {
1646- Expr :: Column ( column) => Some ( column) ,
1647- _ => None ,
1648- } )
1657+ . filter_map ( Expr :: try_as_col)
16491658 . collect ( ) ;
1650- let ( scan_predicates, mut keep_predicates) : ( Vec < Expr > , Vec < Expr > ) =
1651- predicates. into_iter ( ) . partition ( |predicate| {
1652- predicate
1653- . column_refs ( )
1654- . iter ( )
1655- . all ( |column| pass_through. contains ( column) )
1656- } ) ;
1659+ let reads_only_pass_through = |predicate : & Expr | {
1660+ predicate
1661+ . column_refs ( )
1662+ . iter ( )
1663+ . all ( |column| pass_through. contains ( column) )
1664+ } ;
16571665
1658- let Some ( scan_predicate) = conjunction ( scan_predicates. clone ( ) ) else {
1666+ let Some ( scan_predicate) = conjunction (
1667+ predicates
1668+ . iter ( )
1669+ . filter ( |predicate| reads_only_pass_through ( predicate) )
1670+ . cloned ( ) ,
1671+ ) else {
16591672 return Ok ( (
16601673 Transformed :: no ( LogicalPlan :: Projection ( projection) ) ,
1661- keep_predicates ,
1674+ predicates ,
16621675 ) ) ;
16631676 } ;
16641677 let pushed = push_down_into_table_scan (
16651678 Filter :: try_new ( scan_predicate, Arc :: clone ( & projection. input ) ) ?,
16661679 false ,
16671680 ) ?;
16681681 if !pushed. transformed {
1669- keep_predicates. extend ( scan_predicates) ;
16701682 return Ok ( (
16711683 Transformed :: no ( LogicalPlan :: Projection ( projection) ) ,
1672- keep_predicates ,
1684+ predicates ,
16731685 ) ) ;
16741686 }
16751687
16761688 // The predicates the scan does not apply exactly come back in a filter
16771689 // above it. They stay above the projection.
1678- let scan = match pushed. data {
1679- LogicalPlan :: Filter ( residual) => {
1680- keep_predicates . extend ( split_conjunction_owned ( residual. predicate ) ) ;
1681- Arc :: unwrap_or_clone ( residual. input )
1682- }
1683- scan => scan,
1690+ let ( scan, residual ) = match pushed. data {
1691+ LogicalPlan :: Filter ( residual) => (
1692+ Arc :: unwrap_or_clone ( residual. input ) ,
1693+ split_conjunction_owned ( residual. predicate ) ,
1694+ ) ,
1695+ scan => ( scan, vec ! [ ] ) ,
16841696 } ;
1697+ // Keep the predicates in their original order, which `reorder_predicates`
1698+ // chose by cost.
1699+ let keep_predicates = predicates
1700+ . into_iter ( )
1701+ . filter ( |predicate| {
1702+ !reads_only_pass_through ( predicate) || residual. contains ( predicate)
1703+ } )
1704+ . collect ( ) ;
16851705 projection. input = Arc :: new ( scan) ;
16861706 Ok ( (
16871707 Transformed :: yes ( LogicalPlan :: Projection ( projection) ) ,
@@ -4981,11 +5001,12 @@ mod tests {
49815001 {
49825002 // The scan records the pass-through predicate. Both predicates stay in
49835003 // the filter above the projection: one because the source applies it
4984- // inexactly, the other because it reads an extracted alias.
5004+ // inexactly, the other because it reads an extracted alias. They keep
5005+ // their order.
49855006 assert_optimized_plan_equal ! (
49865007 filter_above_extraction_projection_over_inexact_source( ) ?,
49875008 @r"
4988- Filter: __datafusion_extracted_1 = Int64(1 ) AND b > Int64(5 )
5009+ Filter: b > Int64(5 ) AND __datafusion_extracted_1 = Int64(1 )
49895010 Projection: leaf_udf(a) AS __datafusion_extracted_1, b
49905011 TableScan: test, partial_filters=[b > Int64(5)]
49915012 "
@@ -5033,6 +5054,37 @@ mod tests {
50335054 )
50345055 }
50355056
5057+ /// The fetch of a scan applies after its filters, so a predicate above a
5058+ /// pure extraction projection must not reach a scan with a fetch.
5059+ #[ test]
5060+ fn predicate_above_pure_extraction_projection_skips_scan_with_fetch ( ) -> Result < ( ) > {
5061+ let LogicalPlan :: TableScan ( scan) = test_table_scan ( ) ? else {
5062+ unreachable ! ( )
5063+ } ;
5064+ let scan_with_fetch = LogicalPlan :: TableScan ( TableScan {
5065+ fetch : Some ( 10 ) ,
5066+ ..scan
5067+ } ) ;
5068+ let proj = LogicalPlanBuilder :: from ( scan_with_fetch)
5069+ . project ( vec ! [
5070+ leaf_udf_expr( col( "a" ) ) . alias( "__datafusion_extracted_1" ) ,
5071+ col( "b" ) ,
5072+ ] ) ?
5073+ . build ( ) ?;
5074+ let plan = LogicalPlanBuilder :: from ( proj)
5075+ . filter ( col ( "b" ) . gt ( lit ( 5i64 ) ) ) ?
5076+ . build ( ) ?;
5077+
5078+ assert_optimized_plan_equal ! (
5079+ plan,
5080+ @r"
5081+ Filter: test.b > Int64(5)
5082+ Projection: leaf_udf(test.a) AS __datafusion_extracted_1, test.b
5083+ TableScan: test, fetch=10
5084+ "
5085+ )
5086+ }
5087+
50365088 /// Runs the default optimizer and returns, for each pass, the names of the
50375089 /// rules that changed the plan in that pass.
50385090 fn rules_that_changed_plan_per_pass ( plan : LogicalPlan ) -> Result < Vec < Vec < String > > > {
0 commit comments