Skip to content

Commit f8c53ed

Browse files
adriangbclaude
andcommitted
fix: guard scan fetch, preserve predicate order, and avoid spurious Transformed in extraction-projection pushdown
Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 6e86b54 commit f8c53ed

1 file changed

Lines changed: 76 additions & 24 deletions

File tree

‎datafusion/optimizer/src/push_down_filter.rs‎

Lines changed: 76 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1532,6 +1532,16 @@ fn push_down_into_table_scan(
15321532
let LogicalPlan::TableScan(mut scan) = Arc::unwrap_or_clone(filter.input) else {
15331533
return internal_err!("the input of the filter is not a table scan");
15341534
};
1535+
// The fetch of a scan applies after its filters, so a filter above a scan
1536+
// with a fetch cannot move into it without changing the result.
1537+
if scan.fetch.is_some() {
1538+
filter.input = Arc::new(LogicalPlan::TableScan(scan));
1539+
return Ok(Transformed::new(
1540+
LogicalPlan::Filter(filter),
1541+
changed,
1542+
TreeNodeRecursion::Continue,
1543+
));
1544+
}
15351545
let filter_predicates = split_conjunction(&filter.predicate);
15361546
// Filters containing scalar subqueries cannot be pushed to
15371547
// providers because the subquery result is not available
@@ -1587,9 +1597,11 @@ fn push_down_into_table_scan(
15871597
.cloned()
15881598
.collect();
15891599

1600+
// Nothing changes when no predicate leaves the filter and the scan already
1601+
// records every accepted predicate.
15901602
if supported_filters
15911603
.iter()
1592-
.all(|res| res == &TableProviderFilterPushDown::Inexact)
1604+
.all(|res| res != &TableProviderFilterPushDown::Exact)
15931605
&& scan.filters == new_scan_filters
15941606
{
15951607
filter.input = Arc::new(LogicalPlan::TableScan(scan));
@@ -1642,46 +1654,54 @@ fn push_down_through_extraction_projection_into_scan(
16421654
let pass_through: HashSet<&Column> = projection
16431655
.expr
16441656
.iter()
1645-
.filter_map(|expr| match expr {
1646-
Expr::Column(column) => Some(column),
1647-
_ => None,
1648-
})
1657+
.filter_map(Expr::try_as_col)
16491658
.collect();
1650-
let (scan_predicates, mut keep_predicates): (Vec<Expr>, Vec<Expr>) =
1651-
predicates.into_iter().partition(|predicate| {
1652-
predicate
1653-
.column_refs()
1654-
.iter()
1655-
.all(|column| pass_through.contains(column))
1656-
});
1659+
let reads_only_pass_through = |predicate: &Expr| {
1660+
predicate
1661+
.column_refs()
1662+
.iter()
1663+
.all(|column| pass_through.contains(column))
1664+
};
16571665

1658-
let Some(scan_predicate) = conjunction(scan_predicates.clone()) else {
1666+
let Some(scan_predicate) = conjunction(
1667+
predicates
1668+
.iter()
1669+
.filter(|predicate| reads_only_pass_through(predicate))
1670+
.cloned(),
1671+
) else {
16591672
return Ok((
16601673
Transformed::no(LogicalPlan::Projection(projection)),
1661-
keep_predicates,
1674+
predicates,
16621675
));
16631676
};
16641677
let pushed = push_down_into_table_scan(
16651678
Filter::try_new(scan_predicate, Arc::clone(&projection.input))?,
16661679
false,
16671680
)?;
16681681
if !pushed.transformed {
1669-
keep_predicates.extend(scan_predicates);
16701682
return Ok((
16711683
Transformed::no(LogicalPlan::Projection(projection)),
1672-
keep_predicates,
1684+
predicates,
16731685
));
16741686
}
16751687

16761688
// The predicates the scan does not apply exactly come back in a filter
16771689
// above it. They stay above the projection.
1678-
let scan = match pushed.data {
1679-
LogicalPlan::Filter(residual) => {
1680-
keep_predicates.extend(split_conjunction_owned(residual.predicate));
1681-
Arc::unwrap_or_clone(residual.input)
1682-
}
1683-
scan => scan,
1690+
let (scan, residual) = match pushed.data {
1691+
LogicalPlan::Filter(residual) => (
1692+
Arc::unwrap_or_clone(residual.input),
1693+
split_conjunction_owned(residual.predicate),
1694+
),
1695+
scan => (scan, vec![]),
16841696
};
1697+
// Keep the predicates in their original order, which `reorder_predicates`
1698+
// chose by cost.
1699+
let keep_predicates = predicates
1700+
.into_iter()
1701+
.filter(|predicate| {
1702+
!reads_only_pass_through(predicate) || residual.contains(predicate)
1703+
})
1704+
.collect();
16851705
projection.input = Arc::new(scan);
16861706
Ok((
16871707
Transformed::yes(LogicalPlan::Projection(projection)),
@@ -4981,11 +5001,12 @@ mod tests {
49815001
{
49825002
// The scan records the pass-through predicate. Both predicates stay in
49835003
// the filter above the projection: one because the source applies it
4984-
// inexactly, the other because it reads an extracted alias.
5004+
// inexactly, the other because it reads an extracted alias. They keep
5005+
// their order.
49855006
assert_optimized_plan_equal!(
49865007
filter_above_extraction_projection_over_inexact_source()?,
49875008
@r"
4988-
Filter: __datafusion_extracted_1 = Int64(1) AND b > Int64(5)
5009+
Filter: b > Int64(5) AND __datafusion_extracted_1 = Int64(1)
49895010
Projection: leaf_udf(a) AS __datafusion_extracted_1, b
49905011
TableScan: test, partial_filters=[b > Int64(5)]
49915012
"
@@ -5033,6 +5054,37 @@ mod tests {
50335054
)
50345055
}
50355056

5057+
/// The fetch of a scan applies after its filters, so a predicate above a
5058+
/// pure extraction projection must not reach a scan with a fetch.
5059+
#[test]
5060+
fn predicate_above_pure_extraction_projection_skips_scan_with_fetch() -> Result<()> {
5061+
let LogicalPlan::TableScan(scan) = test_table_scan()? else {
5062+
unreachable!()
5063+
};
5064+
let scan_with_fetch = LogicalPlan::TableScan(TableScan {
5065+
fetch: Some(10),
5066+
..scan
5067+
});
5068+
let proj = LogicalPlanBuilder::from(scan_with_fetch)
5069+
.project(vec![
5070+
leaf_udf_expr(col("a")).alias("__datafusion_extracted_1"),
5071+
col("b"),
5072+
])?
5073+
.build()?;
5074+
let plan = LogicalPlanBuilder::from(proj)
5075+
.filter(col("b").gt(lit(5i64)))?
5076+
.build()?;
5077+
5078+
assert_optimized_plan_equal!(
5079+
plan,
5080+
@r"
5081+
Filter: test.b > Int64(5)
5082+
Projection: leaf_udf(test.a) AS __datafusion_extracted_1, test.b
5083+
TableScan: test, fetch=10
5084+
"
5085+
)
5086+
}
5087+
50365088
/// Runs the default optimizer and returns, for each pass, the names of the
50375089
/// rules that changed the plan in that pass.
50385090
fn rules_that_changed_plan_per_pass(plan: LogicalPlan) -> Result<Vec<Vec<String>>> {

0 commit comments

Comments
 (0)