Skip to content

Commit 54557c3

Browse files
authored
Fix topk emitted batches metric (apache#24481)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123. --> - Closes apache#24468 - Closes apache#24470 ## Rationale for this change Fixes some issues around metrics and TopK execution nodes ## What changes are included in this PR? Adjusts to include metrics in PartitionedTopKExec and also fixes the batch count in TopK ## Are these changes tested? Yes new tests added. ## Are there any user-facing changes? No new traits, etc.. just fixes some bugs on the physical plan side
1 parent 1f5be0e commit 54557c3

3 files changed

Lines changed: 180 additions & 8 deletions

File tree

‎datafusion/core/tests/physical_optimizer/window_topn.rs‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
use std::sync::Arc;
2121

2222
use arrow::datatypes::{DataType, Field, Schema};
23+
use datafusion::prelude::{SessionConfig, SessionContext};
2324
use datafusion_common::Result;
2425
use datafusion_common::ScalarValue;
2526
use datafusion_common::config::ConfigOptions;
@@ -32,10 +33,13 @@ use datafusion_physical_expr::window::StandardWindowExpr;
3233
use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};
3334
use datafusion_physical_optimizer::PhysicalOptimizerRule;
3435
use datafusion_physical_optimizer::window_topn::WindowTopN;
36+
use datafusion_physical_plan::collect;
3537
use datafusion_physical_plan::displayable;
3638
use datafusion_physical_plan::filter::FilterExec;
39+
use datafusion_physical_plan::metrics::MetricValue;
3740
use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;
3841
use datafusion_physical_plan::projection::ProjectionExec;
42+
use datafusion_physical_plan::sorts::partitioned_topk::PartitionedTopKExec;
3943
use datafusion_physical_plan::sorts::sort::SortExec;
4044
use datafusion_physical_plan::windows::{BoundedWindowAggExec, create_udwf_window_expr};
4145
use datafusion_physical_plan::{ExecutionPlan, InputOrderMode};
@@ -658,3 +662,68 @@ fn predicate_lt_1_no_change() -> Result<()> {
658662
}
659663
Ok(())
660664
}
665+
666+
// ----------------------------------------------------------------------
667+
// Execution: metrics exposure
668+
// ----------------------------------------------------------------------
669+
670+
/// Recursively finds the `PartitionedTopKExec` node in `plan`.
671+
fn find_partitioned_topk(
672+
plan: &Arc<dyn ExecutionPlan>,
673+
) -> Option<Arc<dyn ExecutionPlan>> {
674+
if plan.is::<PartitionedTopKExec>() {
675+
return Some(Arc::clone(plan));
676+
}
677+
plan.children().into_iter().find_map(find_partitioned_topk)
678+
}
679+
680+
/// Regression test for #24470: `PartitionedTopKExec` builds an
681+
/// `ExecutionPlanMetricsSet` and hands it to the top-K state, but used to omit
682+
/// `ExecutionPlan::metrics()`, so nothing it recorded was ever observable (and
683+
/// `EXPLAIN ANALYZE` showed no metrics for the operator).
684+
///
685+
/// The `output_batches` assertion also covers the emit-side counting fixed for
686+
/// #24468: the per-partition heaps are coalesced into `batch_size` batches, so
687+
/// the metric must count the coalesced output, not the per-partition inputs.
688+
#[tokio::test]
689+
async fn partitioned_topk_exec_exposes_metrics() -> Result<()> {
690+
let mut config = SessionConfig::new()
691+
.with_batch_size(10)
692+
.with_target_partitions(1);
693+
config.options_mut().optimizer.enable_window_topn = true;
694+
let ctx = SessionContext::new_with_config(config);
695+
696+
// 10 partition keys x 5 rows each; top-5 per key keeps all 50 rows, which
697+
// at batch_size 10 must be emitted as 5 batches.
698+
ctx.sql(
699+
"CREATE TABLE t AS
700+
SELECT v % 10 AS pk, v AS val FROM (SELECT unnest(range(0, 50)) AS v)",
701+
)
702+
.await?
703+
.collect()
704+
.await?;
705+
706+
let df = ctx
707+
.sql(
708+
"SELECT pk, val FROM (
709+
SELECT *, ROW_NUMBER() OVER (PARTITION BY pk ORDER BY val) AS rn FROM t
710+
) WHERE rn <= 5",
711+
)
712+
.await?;
713+
let plan = df.create_physical_plan().await?;
714+
let batches = collect(Arc::clone(&plan), ctx.task_ctx()).await?;
715+
assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 50);
716+
717+
let topk = find_partitioned_topk(&plan).expect("plan has a PartitionedTopKExec");
718+
let metrics = topk
719+
.metrics()
720+
.expect("PartitionedTopKExec should expose metrics");
721+
assert_eq!(metrics.output_rows(), Some(50));
722+
let output_batches = metrics
723+
.sum(|m| matches!(m.value(), MetricValue::OutputBatches(_)))
724+
.expect("output_batches metric")
725+
.as_usize();
726+
assert_eq!(output_batches, 5);
727+
728+
Ok(())
729+
}

‎datafusion/physical-plan/src/sorts/partitioned_topk.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ use futures::StreamExt;
4646
use futures::TryStreamExt;
4747

4848
use crate::execution_plan::{Boundedness, EmissionType};
49-
use crate::metrics::ExecutionPlanMetricsSet;
49+
use crate::metrics::{ExecutionPlanMetricsSet, MetricsSet};
5050
use crate::topk::{
5151
PartitionedTopK, PartitionedTopKDenseRank, PartitionedTopKRank, build_sort_fields,
5252
};
@@ -463,6 +463,10 @@ impl ExecutionPlan for PartitionedTopKExec {
463463
stream,
464464
)))
465465
}
466+
467+
fn metrics(&self) -> Option<MetricsSet> {
468+
Some(self.metrics_set.clone_inner())
469+
}
466470
}
467471

468472
/// Read all input, feed each batch into a per-partition top-K state

‎datafusion/physical-plan/src/topk/mod.rs‎

Lines changed: 106 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -810,14 +810,15 @@ impl TopK {
810810
// break into record batches as needed
811811
let mut batches = vec![];
812812
if let Some(mut batch) = heap.emit()? {
813-
(&batch).record_output(&metrics.baseline);
814-
815813
loop {
816814
if batch.num_rows() <= batch_size {
815+
(&batch).record_output(&metrics.baseline);
817816
batches.push(Ok(batch));
818817
break;
819818
} else {
820-
batches.push(Ok(batch.slice(0, batch_size)));
819+
let head = batch.slice(0, batch_size);
820+
(&head).record_output(&metrics.baseline);
821+
batches.push(Ok(head));
821822
let remaining_length = batch.num_rows() - batch_size;
822823
batch = batch.slice(batch_size, remaining_length);
823824
}
@@ -1430,14 +1431,14 @@ impl PartitionedTopK {
14301431
for pk in sorted_pks {
14311432
let mut heap = heaps.remove(&pk).expect("key from heaps.keys()");
14321433
if let Some(batch) = heap.emit()? {
1433-
(&batch).record_output(&metrics.baseline);
14341434
coalescer.push_batch(batch)?;
14351435
}
14361436
}
14371437
coalescer.finish_buffered_batch()?;
14381438

14391439
let mut out: Vec<Result<RecordBatch>> = Vec::new();
14401440
while let Some(b) = coalescer.next_completed_batch() {
1441+
(&b).record_output(&metrics.baseline);
14411442
out.push(Ok(b));
14421443
}
14431444

@@ -1775,20 +1776,19 @@ impl PartitionedTopKRank {
17751776
let RankPartitionState { mut heap, ties } =
17761777
states.remove(&pk).expect("key from states.keys()");
17771778
if let Some(batch) = heap.emit()? {
1778-
(&batch).record_output(&metrics.baseline);
17791779
coalescer.push_batch(batch)?;
17801780
}
17811781
for tie in ties {
17821782
let indices = UInt32Array::from(tie.row_indices);
17831783
let tie_batch = take_record_batch(&tie.batch, &indices)?;
1784-
(&tie_batch).record_output(&metrics.baseline);
17851784
coalescer.push_batch(tie_batch)?;
17861785
}
17871786
}
17881787
coalescer.finish_buffered_batch()?;
17891788

17901789
let mut out: Vec<Result<RecordBatch>> = Vec::new();
17911790
while let Some(b) = coalescer.next_completed_batch() {
1791+
(&b).record_output(&metrics.baseline);
17921792
out.push(Ok(b));
17931793
}
17941794

@@ -2206,7 +2206,6 @@ impl PartitionedTopKDenseRank {
22062206
.batch;
22072207
let indices = UInt32Array::from(entry.row_indices);
22082208
let sub = take_record_batch(batch, &indices)?;
2209-
(&sub).record_output(&metrics.baseline);
22102209
coalescer.push_batch(sub)?;
22112210
}
22122211
}
@@ -2215,6 +2214,7 @@ impl PartitionedTopKDenseRank {
22152214

22162215
let mut out: Vec<Result<RecordBatch>> = Vec::new();
22172216
while let Some(b) = coalescer.next_completed_batch() {
2217+
(&b).record_output(&metrics.baseline);
22182218
out.push(Ok(b));
22192219
}
22202220

@@ -2258,6 +2258,7 @@ impl PartitionedTopKDenseRank {
22582258
#[cfg(test)]
22592259
mod tests {
22602260
use super::*;
2261+
use crate::metrics::MetricValue;
22612262
use arrow::array::{BooleanArray, Float64Array, Int32Array, StringArray};
22622263
use arrow::datatypes::{DataType, Field, Schema};
22632264
use arrow_schema::SortOptions;
@@ -2361,6 +2362,54 @@ mod tests {
23612362
)
23622363
}
23632364

2365+
/// Reads the `output_batches` and `output_rows` metrics recorded by an emit.
2366+
fn output_batches_and_rows(metrics: &ExecutionPlanMetricsSet) -> (usize, usize) {
2367+
let metrics = metrics.clone_inner();
2368+
let batches = metrics
2369+
.sum(|m| matches!(m.value(), MetricValue::OutputBatches(_)))
2370+
.expect("output_batches metric")
2371+
.as_usize();
2372+
(batches, metrics.output_rows().expect("output_rows metric"))
2373+
}
2374+
2375+
/// Regression test for #24468: `emit` splits the heap's single batch into
2376+
/// `batch_size` chunks, so `output_batches` must count each emitted chunk
2377+
/// rather than the one pre-split batch.
2378+
#[tokio::test]
2379+
async fn test_topk_output_batches_metric_counts_emitted_batches() -> Result<()> {
2380+
let schema = make_ab_schema();
2381+
let metrics = ExecutionPlanMetricsSet::new();
2382+
let sort_expr = PhysicalSortExpr {
2383+
expr: col("a", schema.as_ref())?,
2384+
options: SortOptions::default(),
2385+
};
2386+
// k = 5 with batch_size = 2 => emitted batches of [2, 2, 1]
2387+
let mut topk = TopK::try_new(
2388+
0,
2389+
Arc::clone(&schema),
2390+
vec![],
2391+
LexOrdering::from([sort_expr]),
2392+
5,
2393+
2,
2394+
Arc::new(RuntimeEnv::default()),
2395+
&metrics,
2396+
make_topk_filter(),
2397+
)?;
2398+
2399+
topk.insert_batch(make_ab_batch(
2400+
Arc::clone(&schema),
2401+
&[Some(5), Some(4), Some(3), Some(2), Some(1), Some(0)],
2402+
&[0.0, 1.0, 2.0, 3.0, 4.0, 5.0],
2403+
)?)?;
2404+
2405+
let results: Vec<_> = topk.emit()?.try_collect().await?;
2406+
let row_counts: Vec<usize> = results.iter().map(|b| b.num_rows()).collect();
2407+
assert_eq!(row_counts, vec![2, 2, 1]);
2408+
assert_eq!(output_batches_and_rows(&metrics), (3, 5));
2409+
2410+
Ok(())
2411+
}
2412+
23642413
fn make_ab_batch(
23652414
schema: SchemaRef,
23662415
a: &[Option<i32>],
@@ -2980,6 +3029,56 @@ mod tests {
29803029
)?)
29813030
}
29823031

3032+
/// Companion to [`test_topk_output_batches_metric_counts_emitted_batches`] for
3033+
/// the partitioned emit path: rows from several per-partition heaps are
3034+
/// coalesced into `batch_size` batches, so the metric must count the
3035+
/// coalesced output rather than the per-partition inputs.
3036+
#[tokio::test]
3037+
async fn test_partitioned_topk_output_batches_metric_counts_emitted_batches()
3038+
-> Result<()> {
3039+
let schema = Arc::new(Schema::new(vec![
3040+
Field::new("pk", DataType::Int32, false),
3041+
Field::new("val", DataType::Int32, false),
3042+
]));
3043+
let pk_expr: Arc<dyn PhysicalExpr> = col("pk", schema.as_ref())?;
3044+
let partition_sort_fields = build_sort_fields(
3045+
&[PhysicalSortExpr {
3046+
expr: Arc::clone(&pk_expr),
3047+
options: SortOptions::default(),
3048+
}],
3049+
&schema,
3050+
)?;
3051+
let metrics = ExecutionPlanMetricsSet::new();
3052+
// 3 per-partition heaps of 2 rows each, coalesced into 2 batches of 3
3053+
let mut state = PartitionedTopK::try_new(
3054+
0,
3055+
Arc::clone(&schema),
3056+
vec![pk_expr],
3057+
partition_sort_fields,
3058+
LexOrdering::from([PhysicalSortExpr {
3059+
expr: col("val", schema.as_ref())?,
3060+
options: SortOptions::default(),
3061+
}]),
3062+
2,
3063+
3, // batch_size
3064+
&Arc::new(RuntimeEnv::default()),
3065+
&metrics,
3066+
)?;
3067+
3068+
state.insert_batch(&pk_val_batch(
3069+
&schema,
3070+
vec![1, 1, 2, 2, 3, 3],
3071+
vec![10, 5, 20, 15, 30, 25],
3072+
)?)?;
3073+
3074+
let results: Vec<_> = state.emit()?.try_collect().await?;
3075+
let row_counts: Vec<usize> = results.iter().map(|b| b.num_rows()).collect();
3076+
assert_eq!(row_counts, vec![3, 3]);
3077+
assert_eq!(output_batches_and_rows(&metrics), (2, 6));
3078+
3079+
Ok(())
3080+
}
3081+
29833082
/// Multiple distinct partition keys interleaved within a single
29843083
/// input batch — grouping rows by partition key, per-partition heap eviction,
29853084
/// and partition-key-ordered emit must all behave correctly.

0 commit comments

Comments
 (0)