Skip to content

Commit 6339fcb

Browse files
[branch-56] fix: give each SortExec reset_state copy its own TopK dynamic filter (backport #27142) (#27174)
## Which issue does this PR close? - part of #24461 - Backport of #27142 to `branch-56` - Fixes the 56 regression reported in #27130. ## What changes are included in this PR? Clean cherry-pick of #27142: copies made by `SortExec::reset_state()` each get their own TopK dynamic filter again (reverting #26670), and the old filter is reset so a plan that is run, reset and run again doesn't keep the previous bound in the scan. ## Are these changes tested? Same tests as #27142. On `branch-56`: `datafusion-physical-plan` sort tests 164/164, sqllogictest 544/544, clippy and fmt clean. 🤖 Generated with [Claude Code](https://claude.com/claude-code)
1 parent 7711800 commit 6339fcb

2 files changed

Lines changed: 84 additions & 84 deletions

File tree

  • datafusion

‎datafusion/physical-plan/src/sorts/sort.rs‎

Lines changed: 84 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1755,8 +1755,8 @@ impl ExecutionPlan for SortExec {
17551755
// we need to reset.
17561756
if let Some(filter) = &new_sort.filter {
17571757
filter.read().expr().reset();
1758+
new_sort.filter = Some(new_sort.create_filter());
17581759
}
1759-
new_sort.rebuild_filter_for_current_partitioning();
17601760
new_sort.metrics_set = ExecutionPlanMetricsSet::new();
17611761
new_sort.sort_concurrency = Arc::new(OnceLock::new());
17621762

@@ -4759,4 +4759,87 @@ mod tests {
47594759
assert_eq!(num_rows(&topk(false), None)?, Precision::Exact(10));
47604760
Ok(())
47614761
}
4762+
4763+
/// Regression test for <https://github.com/apache/datafusion/issues/27130>
4764+
///
4765+
/// Plans derived from the same TopK plan via `reset_state` must not share
4766+
/// dynamic filter state. This models a cached `LIMIT 1` plan that is reset
4767+
/// and executed by two queries at the same time: both copies are created
4768+
/// before either executes, and the first one to finish must not cause the
4769+
/// other to return an empty result.
4770+
#[tokio::test]
4771+
async fn test_topk_reset_state_copies_are_independent() -> Result<()> {
4772+
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
4773+
let batch = RecordBatch::try_new(
4774+
Arc::clone(&schema),
4775+
vec![Arc::new(Int32Array::from(vec![3, 1, 2]))],
4776+
)?;
4777+
let input =
4778+
TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?;
4779+
let sort: Arc<dyn ExecutionPlan> = Arc::new(
4780+
SortExec::new(
4781+
[PhysicalSortExpr::new_default(Arc::new(Column::new("a", 0)))].into(),
4782+
input,
4783+
)
4784+
.with_fetch(Some(1)),
4785+
);
4786+
4787+
// Both executions are prepared before either runs
4788+
let first = Arc::clone(&sort).reset_state()?;
4789+
let second = Arc::clone(&sort).reset_state()?;
4790+
4791+
let task_ctx = Arc::new(TaskContext::default());
4792+
let first_result = collect(first, Arc::clone(&task_ctx)).await?;
4793+
let second_result = collect(second, task_ctx).await?;
4794+
4795+
assert_snapshot!(batches_to_string(&first_result), @r"
4796+
+---+
4797+
| a |
4798+
+---+
4799+
| 1 |
4800+
+---+
4801+
");
4802+
assert_snapshot!(batches_to_string(&second_result), @r"
4803+
+---+
4804+
| a |
4805+
+---+
4806+
| 1 |
4807+
+---+
4808+
");
4809+
4810+
Ok(())
4811+
}
4812+
4813+
/// A scan below a TopK keeps the dynamic filter it was given at planning
4814+
/// time. After the plan has run once, `reset_state` must clear that
4815+
/// filter's bounds, or running the reset plan again can return too few rows.
4816+
#[tokio::test]
4817+
async fn test_topk_reset_state_clears_pushed_down_filter() -> Result<()> {
4818+
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
4819+
let batch = RecordBatch::try_new(
4820+
Arc::clone(&schema),
4821+
vec![Arc::new(Int32Array::from(vec![3, 1, 2]))],
4822+
)?;
4823+
let input =
4824+
TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?;
4825+
let sort: Arc<dyn ExecutionPlan> = Arc::new(
4826+
SortExec::new(
4827+
[PhysicalSortExpr::new_default(Arc::new(Column::new("a", 0)))].into(),
4828+
input,
4829+
)
4830+
.with_fetch(Some(1)),
4831+
);
4832+
// The filter a scan would have received during filter pushdown
4833+
let pushed_down = sort.dynamic_expressions_produced();
4834+
assert_eq!(pushed_down.len(), 1);
4835+
let pushed_down = Arc::clone(&pushed_down[0]);
4836+
4837+
collect(Arc::clone(&sort), Arc::new(TaskContext::default())).await?;
4838+
assert_snapshot!(pushed_down.to_string(), @"DynamicFilter [ a@0 IS NULL OR a@0 < 1 ]");
4839+
4840+
let _reset = Arc::clone(&sort).reset_state()?;
4841+
assert_snapshot!(pushed_down.to_string(), @"DynamicFilter [ true ]");
4842+
4843+
Ok(())
4844+
}
47624845
}

‎datafusion/sqllogictest/test_files/cte.slt‎

Lines changed: 0 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -1993,86 +1993,3 @@ SET datafusion.execution.target_partitions = 4;
19931993

19941994
statement ok
19951995
RESET datafusion.execution.parquet.pushdown_filters;
1996-
1997-
# TopK must reset its dynamic filter in place so the Parquet scan keeps receiving
1998-
# updated bounds after each recursive iteration (#26642). The minimum is in the
1999-
# first row group, so each iteration should scan only 10 rows instead of 100.
2000-
statement ok
2001-
SET datafusion.execution.target_partitions = 1;
2002-
2003-
statement ok
2004-
SET datafusion.execution.batch_size = 10;
2005-
2006-
statement ok
2007-
SET datafusion.execution.parquet.pushdown_filters = true;
2008-
2009-
statement ok
2010-
COPY (
2011-
SELECT CASE WHEN value = 1 THEN 1 ELSE 102 - value END AS k
2012-
FROM generate_series(1, 100)
2013-
) TO 'test_files/scratch/cte/topk_dynamic_filter.parquet'
2014-
OPTIONS ('format.max_row_group_size' '10');
2015-
2016-
statement ok
2017-
CREATE EXTERNAL TABLE topk_dynamic_filter STORED AS PARQUET
2018-
LOCATION 'test_files/scratch/cte/topk_dynamic_filter.parquet';
2019-
2020-
query I
2021-
WITH RECURSIVE w(n) AS (
2022-
SELECT 1
2023-
UNION ALL
2024-
SELECT w.n + s.k
2025-
FROM w CROSS JOIN (
2026-
SELECT k FROM topk_dynamic_filter ORDER BY k LIMIT 1
2027-
) s
2028-
WHERE w.n < 4
2029-
)
2030-
SELECT * FROM w ORDER BY n;
2031-
----
2032-
1
2033-
2
2034-
3
2035-
4
2036-
2037-
statement ok
2038-
SET datafusion.explain.analyze_categories = 'rows';
2039-
2040-
query TT
2041-
EXPLAIN ANALYZE
2042-
WITH RECURSIVE w(n) AS (
2043-
SELECT 1
2044-
UNION ALL
2045-
SELECT w.n + s.k
2046-
FROM w CROSS JOIN (
2047-
SELECT k FROM topk_dynamic_filter ORDER BY k LIMIT 1
2048-
) s
2049-
WHERE w.n < 4
2050-
)
2051-
SELECT * FROM w ORDER BY n;
2052-
----
2053-
Plan with Metrics
2054-
01)SortExec: expr=[n@0 ASC NULLS LAST], preserve_partitioning=[false], metrics=[output_rows=4, output_batches=1, spill_count=0, spilled_rows=0]
2055-
02)--RecursiveQueryExec: name=w, is_distinct=false, metrics=[output_rows=4, output_batches=4]
2056-
03)----ProjectionExec: expr=[CAST(1 AS Int64;N) as n], metrics=[output_rows=1, output_batches=1]
2057-
04)------PlaceholderRowExec, metrics=[]
2058-
05)----ProjectionExec: expr=[n@0 + k@1 as n], metrics=[]
2059-
06)------CrossJoinExec, metrics=[]
2060-
07)--------FilterExec: n@0 < 4, metrics=[]
2061-
08)----------WorkTableExec: name=w, metrics=[]
2062-
09)--------SortExec: TopK(fetch=1), expr=[k@0 ASC NULLS LAST], preserve_partitioning=[false], metrics=[]
2063-
10)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/cte/topk_dynamic_filter.parquet]]}, projection=[k], file_type=parquet, predicate=DynamicFilter [ true ], sort_order_for_reorder=[k@0 ASC NULLS LAST], dynamic_rg_pruning=eligible, metrics=[output_rows=30, output_batches=3, files_ranges_pruned_statistics=3 total → 3 matched, row_groups_pruned_statistics=30 total → 30 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=3, files_processed=3, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=30, pushdown_rows_pruned=0, predicate_cache_inner_records=30, predicate_cache_records=30, scan_efficiency_ratio=9.96% (279/2.80 K)]
2064-
2065-
statement ok
2066-
DROP TABLE topk_dynamic_filter;
2067-
2068-
statement ok
2069-
RESET datafusion.explain.analyze_categories;
2070-
2071-
statement ok
2072-
SET datafusion.execution.target_partitions = 4;
2073-
2074-
statement ok
2075-
RESET datafusion.execution.batch_size;
2076-
2077-
statement ok
2078-
RESET datafusion.execution.parquet.pushdown_filters;

0 commit comments

Comments
 (0)