@@ -22,8 +22,8 @@ use datafusion_execution::memory_pool::{
2222 GreedyMemoryPool , MemoryConsumer , MemoryLimit , MemoryPool , MemoryReservation ,
2323} ;
2424use datafusion_execution:: runtime_env:: { RuntimeEnv , RuntimeEnvBuilder } ;
25- use datafusion_physical_expr:: expressions:: Column ;
26- use datafusion_physical_expr:: { LexOrdering , PhysicalSortExpr } ;
25+ use datafusion_physical_expr:: expressions:: { CastExpr , Column } ;
26+ use datafusion_physical_expr:: { LexOrdering , PhysicalExpr , PhysicalSortExpr } ;
2727use futures:: TryStreamExt ;
2828use std:: fmt:: { Display , Formatter } ;
2929use std:: sync:: { Arc , Mutex } ;
@@ -721,10 +721,17 @@ async fn check_merge_keys_larger_than_batches() -> Result<()> {
721721 . await ?;
722722 }
723723 // Views into 1024 dictionary values share their bytes, as a
724- // dictionary-encoded Parquet column produces. Sorting the buffered batches
725- // before a spill grows past the pool, on `main` too.
724+ // dictionary-encoded Parquet column produces. A computed key counts as
725+ // larger than the run, so only the shared views keep these runs at
726+ // `batch_size`. Sorting the buffered batches before a spill grows past the
727+ // pool, on `main` too.
726728 let fields = [ & int_fields[ ..] , & [ ( "c" , DataType :: Utf8View ) ] ] . concat ( ) ;
727- check_merge_keys ( & fields, 2 , 16 , false , |batch| {
729+ let a: Arc < dyn PhysicalExpr > = Arc :: new ( Column :: new ( "a" , 0 ) ) ;
730+ let keys = vec ! [
731+ Arc :: new( CastExpr :: new( a, DataType :: Int64 , None ) ) as Arc <dyn PhysicalExpr >,
732+ Arc :: new( Column :: new( "b" , 1 ) ) ,
733+ ] ;
734+ check_merge_exprs ( & fields, keys, 16 , false , |batch| {
728735 let values: StringArray = ( 0 ..1024 )
729736 . map ( |v| Some ( format ! ( "{batch:>4}-{v:0>59}" ) ) )
730737 . collect ( ) ;
@@ -775,6 +782,20 @@ async fn check_merge_keys(
775782 buffered : usize ,
776783 within_pool : bool ,
777784 columns : impl Fn ( i32 ) -> Result < Vec < ArrayRef > > ,
785+ ) -> Result < ( ) > {
786+ let keys = ( 0 ..keys)
787+ . map ( |i| Arc :: new ( Column :: new ( fields[ i] . 0 , i) ) as Arc < dyn PhysicalExpr > )
788+ . collect ( ) ;
789+ check_merge_exprs ( fields, keys, buffered, within_pool, columns) . await
790+ }
791+
792+ /// Like [`check_merge_keys`], sorting by `keys`.
793+ async fn check_merge_exprs (
794+ fields : & [ ( & str , DataType ) ] ,
795+ keys : Vec < Arc < dyn PhysicalExpr > > ,
796+ buffered : usize ,
797+ within_pool : bool ,
798+ columns : impl Fn ( i32 ) -> Result < Vec < ArrayRef > > ,
778799) -> Result < ( ) > {
779800 let schema = Arc :: new ( Schema :: new (
780801 fields
@@ -785,10 +806,7 @@ async fn check_merge_keys(
785806 let batches = ( 0 ..64 )
786807 . map ( |batch| Ok ( RecordBatch :: try_new ( Arc :: clone ( & schema) , columns ( batch) ?) ?) )
787808 . collect :: < Result < Vec < _ > > > ( ) ?;
788- let ordering =
789- LexOrdering :: new ( ( 0 ..keys) . map ( |i| {
790- PhysicalSortExpr :: new_default ( Arc :: new ( Column :: new ( fields[ i] . 0 , i) ) )
791- } ) )
809+ let ordering = LexOrdering :: new ( keys. into_iter ( ) . map ( PhysicalSortExpr :: new_default) )
792810 . expect ( "at least one sort key" ) ;
793811 let capacity =
794812 MERGE_BYTES + buffered * get_reserved_bytes_for_record_batch ( & batches[ 0 ] ) ?;
0 commit comments