@@ -721,14 +721,14 @@ async fn check_merge_keys_larger_than_batches() -> Result<()> {
721721 . await ?;
722722 }
723723 // Views into 1024 dictionary values share their bytes, as a
724- // dictionary-encoded Parquet column produces. A computed key counts as
725- // larger than the run, so only the shared views keep these runs at
726- // `batch_size`. Sorting the buffered batches before a spill grows past the
727- // pool, on `main` too.
724+ // dictionary-encoded Parquet column produces. A computed key without a
725+ // fixed width counts as larger than the run, so only the shared views keep
726+ // these runs at `batch_size`. Sorting the buffered batches before a spill
727+ // grows past the pool, on `main` too.
728728 let fields = [ & int_fields[ ..] , & [ ( "c" , DataType :: Utf8View ) ] ] . concat ( ) ;
729729 let a: Arc < dyn PhysicalExpr > = Arc :: new ( Column :: new ( "a" , 0 ) ) ;
730730 let keys = vec ! [
731- Arc :: new( CastExpr :: new( a, DataType :: Int64 , None ) ) as Arc <dyn PhysicalExpr >,
731+ Arc :: new( CastExpr :: new( a, DataType :: Utf8 , None ) ) as Arc <dyn PhysicalExpr >,
732732 Arc :: new( Column :: new( "b" , 1 ) ) ,
733733 ] ;
734734 check_merge_exprs ( & fields, keys, 16 , false , |batch| {
@@ -856,20 +856,21 @@ async fn test_merge_keys_larger_than_batches_multithreaded() -> Result<()> {
856856fn test_merge_run_batch_size_follows_key_size ( ) -> Result < ( ) > {
857857 let batch_size = 8192 ;
858858 let rows = batch_size as i32 ;
859- let check = |payload : Option < ArrayRef > , expected : usize | -> Result < ( ) > {
859+ // Sorts by `first`, then `b`
860+ let check = |first : Arc < dyn PhysicalExpr > ,
861+ payload : Option < ArrayRef > ,
862+ expected : usize |
863+ -> Result < ( ) > {
860864 let mut columns: Vec < ( & str , ArrayRef ) > = vec ! [
861865 ( "a" , Arc :: new( Int64Array :: from_iter_values( 0 ..rows as i64 ) ) ) ,
862866 ( "b" , Arc :: new( Int64Array :: from_iter_values( 0 ..rows as i64 ) ) ) ,
863867 ] ;
864868 columns. extend ( payload. map ( |payload| ( "c" , payload) ) ) ;
865869 let batch = RecordBatch :: try_from_iter ( columns) ?;
866- let ordering = LexOrdering :: new ( ( 0 ..2 ) . map ( |i| {
867- PhysicalSortExpr :: new_default ( Arc :: new ( Column :: new (
868- batch. schema ( ) . field ( i) . name ( ) ,
869- i,
870- ) ) )
871- } ) )
872- . expect ( "two sort keys" ) ;
870+ let b: Arc < dyn PhysicalExpr > = Arc :: new ( Column :: new ( "b" , 1 ) ) ;
871+ let ordering =
872+ LexOrdering :: new ( [ first, b] . into_iter ( ) . map ( PhysicalSortExpr :: new_default) )
873+ . expect ( "two sort keys" ) ;
873874 let runtime = RuntimeEnvBuilder :: new ( )
874875 . with_memory_limit ( 1024 * 1024 * 1024 , 1.0 )
875876 . build_arc ( ) ?;
@@ -887,14 +888,21 @@ fn test_merge_run_batch_size_follows_key_size() -> Result<()> {
887888 assert_eq ! ( sorter. merge_run_batch_size( & batch) ?, expected) ;
888889 Ok ( ( ) )
889890 } ;
891+ let a = || Arc :: new ( Column :: new ( "a" , 0 ) ) as Arc < dyn PhysicalExpr > ;
890892 // Encoded keys: 26 bytes a row, against 16 bytes a row
891- check ( None , batch_size / 8 ) ?;
893+ check ( a ( ) , None , batch_size / 8 ) ?;
892894 // Against 16 + 16 bytes a row: inlined views add more than the keys need
893895 let short: StringViewArray = ( 0 ..rows) . map ( |i| Some ( format ! ( "{i}" ) ) ) . collect ( ) ;
894- check ( Some ( Arc :: new ( short) ) , batch_size) ?;
896+ check ( a ( ) , Some ( Arc :: new ( short) ) , batch_size) ?;
895897 // Against 16 + 16 + 64 bytes a row
896898 let long: StringViewArray = ( 0 ..rows) . map ( |i| Some ( format ! ( "{i:0>64}" ) ) ) . collect ( ) ;
897- check ( Some ( Arc :: new ( long) ) , batch_size)
899+ let long: ArrayRef = Arc :: new ( long) ;
900+ check ( a ( ) , Some ( Arc :: clone ( & long) ) , batch_size) ?;
901+ // A computed key is sized from its type if that has a fixed width, and
902+ // otherwise counts as larger than the run
903+ let cast = |to| Arc :: new ( CastExpr :: new ( a ( ) , to, None ) ) as Arc < dyn PhysicalExpr > ;
904+ check ( cast ( DataType :: Int64 ) , Some ( Arc :: clone ( & long) ) , batch_size) ?;
905+ check ( cast ( DataType :: Utf8 ) , Some ( long) , batch_size / 8 )
898906}
899907
900908#[ tokio:: test]
0 commit comments