Skip to content

Commit 684da9c

Browse files
adriangbclaude
andcommitted
fix: apply the TopK pushdown when the Limit above a Sort with a fetch is dropped
When the Sort already has an equal or smaller fetch, PushDownLimit also drops the Limit and returns the Sort, so the Sort's own visit is skipped in that pass. Both skip = 0 paths now share one branch that calls push_topk_through_join. The result is always marked transformed, because the Limit is removed even when the TopK does not move. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent d918553 commit 684da9c

2 files changed

Lines changed: 82 additions & 14 deletions

File tree

‎datafusion/optimizer/src/push_down_limit.rs‎

Lines changed: 31 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -163,24 +163,24 @@ fn rewrite_limit(mut limit: Limit) -> Result<Transformed<LogicalPlan>> {
163163
let sort_fetch = skip + fetch;
164164
Some(sort.fetch.map(|f| f.min(sort_fetch)).unwrap_or(sort_fetch))
165165
};
166-
if new_fetch == sort.fetch {
167-
if skip > 0 {
168-
original_limit(skip, fetch, LogicalPlan::Sort(sort))
169-
} else {
170-
Ok(Transformed::yes(LogicalPlan::Sort(sort)))
171-
}
172-
} else {
173-
sort.fetch = new_fetch;
174-
if skip > 0 {
166+
let fetch_changed = new_fetch != sort.fetch;
167+
sort.fetch = new_fetch;
168+
if skip > 0 {
169+
if fetch_changed {
175170
limit.input = Arc::new(LogicalPlan::Sort(sort));
176171
Ok(Transformed::yes(LogicalPlan::Limit(limit)))
177172
} else {
178-
// With `skip = 0` the Sort's fetch already caps the output,
179-
// so drop the Limit now instead of on the next pass. The
180-
// Sort then replaces the Limit as the visited node and its
181-
// own visit is skipped, so apply the TopK pushdown here.
182-
Ok(Transformed::yes(push_topk_through_join(sort)?.data))
173+
original_limit(skip, fetch, LogicalPlan::Sort(sort))
183174
}
175+
} else {
176+
// With `skip = 0` the Sort's fetch already caps the output, so
177+
// drop the Limit now instead of on the next pass. The Sort then
178+
// replaces the Limit as the visited node and its own visit is
179+
// skipped, so apply the TopK pushdown here. The plan changed
180+
// (the Limit is gone) even if the TopK does not move.
181+
let mut result = push_topk_through_join(sort)?;
182+
result.transformed = true;
183+
Ok(result)
184184
}
185185
}
186186
LogicalPlan::Projection(mut proj) => {
@@ -698,6 +698,23 @@ mod test {
698698
Ok(())
699699
}
700700

701+
/// Dropping the `Limit` above a `Sort` changes the plan even when the
702+
/// TopK has no join to move through, so the rule must report it.
703+
#[test]
704+
fn limit_push_down_sort_reports_transformed() -> Result<()> {
705+
for sort_fetch in [None, Some(10), Some(5)] {
706+
let plan = LogicalPlanBuilder::from(test_table_scan()?)
707+
.sort_with_limit(vec![col("a").sort(true, false)], sort_fetch)?
708+
.limit(0, Some(10))?
709+
.build()?;
710+
711+
let result = PushDownLimit::new().rewrite(plan, &OptimizerContext::new())?;
712+
assert!(result.transformed, "sort fetch {sort_fetch:?}");
713+
assert!(matches!(result.data, LogicalPlan::Sort(_)));
714+
}
715+
Ok(())
716+
}
717+
701718
#[test]
702719
fn limit_push_down_sort_skip() -> Result<()> {
703720
let table_scan = test_table_scan()?;

‎datafusion/optimizer/src/push_down_limit/topk_through_join.rs‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1417,6 +1417,57 @@ mod test {
14171417
)
14181418
}
14191419

1420+
/// A `Limit` over a `Sort` that already has an equal or smaller fetch is
1421+
/// dropped, and the `Sort` replaces it as the visited node. The TopK must
1422+
/// still move through the join in the same pass.
1423+
fn optimize_limit_over_sort_with_fetch(limit_fetch: usize) -> Result<LogicalPlan> {
1424+
let t1 = test_table_scan_with_name("t1")?;
1425+
let t2 = test_table_scan_with_name("t2")?;
1426+
1427+
let plan = LogicalPlanBuilder::from(t1)
1428+
.join(
1429+
LogicalPlanBuilder::from(t2).build()?,
1430+
JoinType::Left,
1431+
(vec!["a"], vec!["a"]),
1432+
None,
1433+
)?
1434+
.sort_with_limit(vec![col("t1.b").sort(true, false)], Some(3))?
1435+
.limit(0, Some(limit_fetch))?
1436+
.build()?;
1437+
1438+
let optimizer_ctx = OptimizerContext::new().with_max_passes(1);
1439+
crate::Optimizer::with_rules(vec![Arc::new(PushDownLimit::new())]).optimize(
1440+
plan,
1441+
&optimizer_ctx,
1442+
|_, _| {},
1443+
)
1444+
}
1445+
1446+
#[test]
1447+
fn topk_pushed_through_limit_over_sort_with_same_fetch_in_one_pass() -> Result<()> {
1448+
insta::assert_snapshot!(optimize_limit_over_sort_with_fetch(3)?, @r"
1449+
Sort: t1.b ASC NULLS LAST, fetch=3
1450+
Left Join: t1.a = t2.a
1451+
Sort: t1.b ASC NULLS LAST, fetch=3
1452+
TableScan: t1
1453+
TableScan: t2
1454+
");
1455+
Ok(())
1456+
}
1457+
1458+
#[test]
1459+
fn topk_pushed_through_limit_over_sort_with_smaller_fetch_in_one_pass() -> Result<()>
1460+
{
1461+
insta::assert_snapshot!(optimize_limit_over_sort_with_fetch(5)?, @r"
1462+
Sort: t1.b ASC NULLS LAST, fetch=3
1463+
Left Join: t1.a = t2.a
1464+
Sort: t1.b ASC NULLS LAST, fetch=3
1465+
TableScan: t1
1466+
TableScan: t2
1467+
");
1468+
Ok(())
1469+
}
1470+
14201471
/// Running the optimizer a second time on its own output must be a
14211472
/// no-op: a rule that keeps rewriting an already-optimized plan (e.g.
14221473
/// oscillating between two equivalent forms) would never let the

0 commit comments

Comments
 (0)