3434//! reserved against the task's memory pool. The operator still cannot spill, so a window
3535//! partition that does not fit fails the task with a memory error instead of growing
3636//! untracked. A window partition spread over several batches needs up to twice its size,
37- //! because the copy is reserved while the batches it was made from still are. The consumer
38- //! registers with the pool when the first batch is buffered, so that it does not shrink the
39- //! share of a sort below it while the sort reads its input.
37+ //! because the copy is reserved while the batches it was made from still are.
4038//!
4139//! Once DataFusion's `WindowAggExec` does both, this operator can be removed.
4240
@@ -253,6 +251,11 @@ struct CometWindowAggStream {
253251}
254252
255253impl CometWindowAggStream {
254+ /// Reserves `bytes` more, registering the consumer with the pool on first use.
255+ fn grow ( & mut self , bytes : usize ) -> Result < ( ) > {
256+ self . reservation ( ) . try_grow ( bytes) . map_err ( with_oom_context)
257+ }
258+
256259 fn reservation ( & mut self ) -> & mut MemoryReservation {
257260 self . reservation . get_or_insert_with ( || {
258261 MemoryConsumer :: new ( self . consumer_name . clone ( ) ) . register ( & self . pool )
@@ -273,9 +276,7 @@ impl CometWindowAggStream {
273276 return Ok ( None ) ;
274277 }
275278 let added = self . buffered_memory . count_batch ( & batch) ;
276- self . reservation ( )
277- . try_grow ( added)
278- . map_err ( with_oom_context) ?;
279+ self . grow ( added) ?;
279280 if self . partition_by_sort_keys . is_empty ( ) {
280281 self . buffered . push ( batch) ;
281282 return Ok ( None ) ;
@@ -360,9 +361,7 @@ impl CometWindowAggStream {
360361 // which is not copied. That is exact, where estimating the copy from each batch would
361362 // charge every batch sliced from one list array for all of its values.
362363 let copy = self . buffered_memory . count_batch ( & batch) ;
363- self . reservation ( )
364- . try_grow ( copy)
365- . map_err ( with_oom_context) ?;
364+ self . grow ( copy) ?;
366365 drop ( batches) ;
367366
368367 let mut partition_results = vec ! [ ] ;
@@ -478,7 +477,9 @@ mod tests {
478477 use datafusion:: common:: ScalarValue ;
479478 use datafusion:: datasource:: memory:: MemorySourceConfig ;
480479 use datafusion:: datasource:: source:: DataSourceExec ;
481- use datafusion:: execution:: memory_pool:: { GreedyMemoryPool , TrackConsumersPool } ;
480+ use datafusion:: execution:: memory_pool:: {
481+ GreedyMemoryPool , MemoryConsumerMetrics , TrackConsumersPool ,
482+ } ;
482483 use datafusion:: execution:: runtime_env:: { RuntimeEnv , RuntimeEnvBuilder } ;
483484 use datafusion:: execution:: FunctionRegistry ;
484485 use datafusion:: logical_expr:: {
@@ -680,12 +681,16 @@ mod tests {
680681 ( runtime, context)
681682 }
682683
683- fn registered_consumers ( runtime : & RuntimeEnv ) -> Vec < String > {
684+ fn consumer_metrics ( runtime : & RuntimeEnv ) -> Vec < MemoryConsumerMetrics > {
684685 runtime
685686 . memory_pool
686687 . downcast_ref :: < TrackConsumersPool < GreedyMemoryPool > > ( )
687688 . unwrap ( )
688689 . metrics ( )
690+ }
691+
692+ fn registered_consumers ( runtime : & RuntimeEnv ) -> Vec < String > {
693+ consumer_metrics ( runtime)
689694 . into_iter ( )
690695 . map ( |consumer| consumer. name )
691696 . collect ( )
@@ -703,11 +708,7 @@ mod tests {
703708 output. push ( batch?) ;
704709 }
705710 // Read the consumer before dropping the stream unregisters it.
706- let consumer = runtime
707- . memory_pool
708- . downcast_ref :: < TrackConsumersPool < GreedyMemoryPool > > ( )
709- . unwrap ( )
710- . metrics ( )
711+ let consumer = consumer_metrics ( & runtime)
711712 . into_iter ( )
712713 . find ( |consumer| consumer. name == CONSUMER )
713714 . unwrap ( ) ;
0 commit comments