@@ -810,14 +810,15 @@ impl TopK {
810810 // break into record batches as needed
811811 let mut batches = vec ! [ ] ;
812812 if let Some ( mut batch) = heap. emit ( ) ? {
813- ( & batch) . record_output ( & metrics. baseline ) ;
814-
815813 loop {
816814 if batch. num_rows ( ) <= batch_size {
815+ ( & batch) . record_output ( & metrics. baseline ) ;
817816 batches. push ( Ok ( batch) ) ;
818817 break ;
819818 } else {
820- batches. push ( Ok ( batch. slice ( 0 , batch_size) ) ) ;
819+ let head = batch. slice ( 0 , batch_size) ;
820+ ( & head) . record_output ( & metrics. baseline ) ;
821+ batches. push ( Ok ( head) ) ;
821822 let remaining_length = batch. num_rows ( ) - batch_size;
822823 batch = batch. slice ( batch_size, remaining_length) ;
823824 }
@@ -1430,14 +1431,14 @@ impl PartitionedTopK {
14301431 for pk in sorted_pks {
14311432 let mut heap = heaps. remove ( & pk) . expect ( "key from heaps.keys()" ) ;
14321433 if let Some ( batch) = heap. emit ( ) ? {
1433- ( & batch) . record_output ( & metrics. baseline ) ;
14341434 coalescer. push_batch ( batch) ?;
14351435 }
14361436 }
14371437 coalescer. finish_buffered_batch ( ) ?;
14381438
14391439 let mut out: Vec < Result < RecordBatch > > = Vec :: new ( ) ;
14401440 while let Some ( b) = coalescer. next_completed_batch ( ) {
1441+ ( & b) . record_output ( & metrics. baseline ) ;
14411442 out. push ( Ok ( b) ) ;
14421443 }
14431444
@@ -1775,20 +1776,19 @@ impl PartitionedTopKRank {
17751776 let RankPartitionState { mut heap, ties } =
17761777 states. remove ( & pk) . expect ( "key from states.keys()" ) ;
17771778 if let Some ( batch) = heap. emit ( ) ? {
1778- ( & batch) . record_output ( & metrics. baseline ) ;
17791779 coalescer. push_batch ( batch) ?;
17801780 }
17811781 for tie in ties {
17821782 let indices = UInt32Array :: from ( tie. row_indices ) ;
17831783 let tie_batch = take_record_batch ( & tie. batch , & indices) ?;
1784- ( & tie_batch) . record_output ( & metrics. baseline ) ;
17851784 coalescer. push_batch ( tie_batch) ?;
17861785 }
17871786 }
17881787 coalescer. finish_buffered_batch ( ) ?;
17891788
17901789 let mut out: Vec < Result < RecordBatch > > = Vec :: new ( ) ;
17911790 while let Some ( b) = coalescer. next_completed_batch ( ) {
1791+ ( & b) . record_output ( & metrics. baseline ) ;
17921792 out. push ( Ok ( b) ) ;
17931793 }
17941794
@@ -2206,7 +2206,6 @@ impl PartitionedTopKDenseRank {
22062206 . batch ;
22072207 let indices = UInt32Array :: from ( entry. row_indices ) ;
22082208 let sub = take_record_batch ( batch, & indices) ?;
2209- ( & sub) . record_output ( & metrics. baseline ) ;
22102209 coalescer. push_batch ( sub) ?;
22112210 }
22122211 }
@@ -2215,6 +2214,7 @@ impl PartitionedTopKDenseRank {
22152214
22162215 let mut out: Vec < Result < RecordBatch > > = Vec :: new ( ) ;
22172216 while let Some ( b) = coalescer. next_completed_batch ( ) {
2217+ ( & b) . record_output ( & metrics. baseline ) ;
22182218 out. push ( Ok ( b) ) ;
22192219 }
22202220
@@ -2258,6 +2258,7 @@ impl PartitionedTopKDenseRank {
22582258#[ cfg( test) ]
22592259mod tests {
22602260 use super :: * ;
2261+ use crate :: metrics:: MetricValue ;
22612262 use arrow:: array:: { BooleanArray , Float64Array , Int32Array , StringArray } ;
22622263 use arrow:: datatypes:: { DataType , Field , Schema } ;
22632264 use arrow_schema:: SortOptions ;
@@ -2361,6 +2362,54 @@ mod tests {
23612362 )
23622363 }
23632364
2365+ /// Reads the `output_batches` and `output_rows` metrics recorded by an emit.
2366+ fn output_batches_and_rows ( metrics : & ExecutionPlanMetricsSet ) -> ( usize , usize ) {
2367+ let metrics = metrics. clone_inner ( ) ;
2368+ let batches = metrics
2369+ . sum ( |m| matches ! ( m. value( ) , MetricValue :: OutputBatches ( _) ) )
2370+ . expect ( "output_batches metric" )
2371+ . as_usize ( ) ;
2372+ ( batches, metrics. output_rows ( ) . expect ( "output_rows metric" ) )
2373+ }
2374+
2375+ /// Regression test for #24468: `emit` splits the heap's single batch into
2376+ /// `batch_size` chunks, so `output_batches` must count each emitted chunk
2377+ /// rather than the one pre-split batch.
2378+ #[ tokio:: test]
2379+ async fn test_topk_output_batches_metric_counts_emitted_batches ( ) -> Result < ( ) > {
2380+ let schema = make_ab_schema ( ) ;
2381+ let metrics = ExecutionPlanMetricsSet :: new ( ) ;
2382+ let sort_expr = PhysicalSortExpr {
2383+ expr : col ( "a" , schema. as_ref ( ) ) ?,
2384+ options : SortOptions :: default ( ) ,
2385+ } ;
2386+ // k = 5 with batch_size = 2 => emitted batches of [2, 2, 1]
2387+ let mut topk = TopK :: try_new (
2388+ 0 ,
2389+ Arc :: clone ( & schema) ,
2390+ vec ! [ ] ,
2391+ LexOrdering :: from ( [ sort_expr] ) ,
2392+ 5 ,
2393+ 2 ,
2394+ Arc :: new ( RuntimeEnv :: default ( ) ) ,
2395+ & metrics,
2396+ make_topk_filter ( ) ,
2397+ ) ?;
2398+
2399+ topk. insert_batch ( make_ab_batch (
2400+ Arc :: clone ( & schema) ,
2401+ & [ Some ( 5 ) , Some ( 4 ) , Some ( 3 ) , Some ( 2 ) , Some ( 1 ) , Some ( 0 ) ] ,
2402+ & [ 0.0 , 1.0 , 2.0 , 3.0 , 4.0 , 5.0 ] ,
2403+ ) ?) ?;
2404+
2405+ let results: Vec < _ > = topk. emit ( ) ?. try_collect ( ) . await ?;
2406+ let row_counts: Vec < usize > = results. iter ( ) . map ( |b| b. num_rows ( ) ) . collect ( ) ;
2407+ assert_eq ! ( row_counts, vec![ 2 , 2 , 1 ] ) ;
2408+ assert_eq ! ( output_batches_and_rows( & metrics) , ( 3 , 5 ) ) ;
2409+
2410+ Ok ( ( ) )
2411+ }
2412+
23642413 fn make_ab_batch (
23652414 schema : SchemaRef ,
23662415 a : & [ Option < i32 > ] ,
@@ -2980,6 +3029,56 @@ mod tests {
29803029 ) ?)
29813030 }
29823031
3032+ /// Companion to [`test_topk_output_batches_metric_counts_emitted_batches`] for
3033+ /// the partitioned emit path: rows from several per-partition heaps are
3034+ /// coalesced into `batch_size` batches, so the metric must count the
3035+ /// coalesced output rather than the per-partition inputs.
3036+ #[ tokio:: test]
3037+ async fn test_partitioned_topk_output_batches_metric_counts_emitted_batches ( )
3038+ -> Result < ( ) > {
3039+ let schema = Arc :: new ( Schema :: new ( vec ! [
3040+ Field :: new( "pk" , DataType :: Int32 , false ) ,
3041+ Field :: new( "val" , DataType :: Int32 , false ) ,
3042+ ] ) ) ;
3043+ let pk_expr: Arc < dyn PhysicalExpr > = col ( "pk" , schema. as_ref ( ) ) ?;
3044+ let partition_sort_fields = build_sort_fields (
3045+ & [ PhysicalSortExpr {
3046+ expr : Arc :: clone ( & pk_expr) ,
3047+ options : SortOptions :: default ( ) ,
3048+ } ] ,
3049+ & schema,
3050+ ) ?;
3051+ let metrics = ExecutionPlanMetricsSet :: new ( ) ;
3052+ // 3 per-partition heaps of 2 rows each, coalesced into 2 batches of 3
3053+ let mut state = PartitionedTopK :: try_new (
3054+ 0 ,
3055+ Arc :: clone ( & schema) ,
3056+ vec ! [ pk_expr] ,
3057+ partition_sort_fields,
3058+ LexOrdering :: from ( [ PhysicalSortExpr {
3059+ expr : col ( "val" , schema. as_ref ( ) ) ?,
3060+ options : SortOptions :: default ( ) ,
3061+ } ] ) ,
3062+ 2 ,
3063+ 3 , // batch_size
3064+ & Arc :: new ( RuntimeEnv :: default ( ) ) ,
3065+ & metrics,
3066+ ) ?;
3067+
3068+ state. insert_batch ( & pk_val_batch (
3069+ & schema,
3070+ vec ! [ 1 , 1 , 2 , 2 , 3 , 3 ] ,
3071+ vec ! [ 10 , 5 , 20 , 15 , 30 , 25 ] ,
3072+ ) ?) ?;
3073+
3074+ let results: Vec < _ > = state. emit ( ) ?. try_collect ( ) . await ?;
3075+ let row_counts: Vec < usize > = results. iter ( ) . map ( |b| b. num_rows ( ) ) . collect ( ) ;
3076+ assert_eq ! ( row_counts, vec![ 3 , 3 ] ) ;
3077+ assert_eq ! ( output_batches_and_rows( & metrics) , ( 2 , 6 ) ) ;
3078+
3079+ Ok ( ( ) )
3080+ }
3081+
29833082 /// Multiple distinct partition keys interleaved within a single
29843083 /// input batch — grouping rows by partition key, per-partition heap eviction,
29853084 /// and partition-key-ordered emit must all behave correctly.
0 commit comments