44
55//! Define the `InProgressSpillFile` struct, which represents an in-progress spill file used for writing `RecordBatch`es to disk, created by `SpillManager`.
66
7+ use datafusion_common:: config:: SpillCompression ;
78use datafusion_common:: { Result , internal_datafusion_err} ;
89use std:: sync:: Arc ;
910
@@ -89,7 +90,11 @@ impl InProgressSpillFile {
8990 ) ) ;
9091 }
9192
92- let pieces = SpillPieces :: new ( batch, self . max_batch_bytes ) ?;
93+ let pieces = SpillPieces :: new (
94+ batch,
95+ self . max_batch_bytes ,
96+ self . spill_manager . compression != SpillCompression :: Uncompressed ,
97+ ) ?;
9398
9499 if self . writer . is_none ( ) {
95100 // Use the SpillManager's declared schema rather than the batch's schema.
@@ -147,7 +152,11 @@ impl InProgressSpillFile {
147152 ) ) ;
148153 }
149154
150- let pieces = SpillPieces :: new ( batch, self . max_batch_bytes ) ?;
155+ let pieces = SpillPieces :: new (
156+ batch,
157+ self . max_batch_bytes ,
158+ self . spill_manager . compression != SpillCompression :: Uncompressed ,
159+ ) ?;
151160
152161 if self . writer . is_none ( ) {
153162 let schema = self . spill_manager . schema ( ) ;
@@ -286,7 +295,7 @@ mod tests {
286295 use arrow:: array:: {
287296 Array , ArrayRef , AsArray , BinaryViewArray , DictionaryArray , Int32Array ,
288297 Int64Array , ListBuilder , MapBuilder , StringArray , StringBuilder , StringViewArray ,
289- StructArray , UInt32Array ,
298+ StructArray , UInt8Array , UInt32Array ,
290299 } ;
291300 use arrow_schema:: { DataType , Field , Fields , Schema , SchemaRef } ;
292301 use datafusion_common:: utils:: memory:: get_record_batch_memory_size;
@@ -673,8 +682,8 @@ mod tests {
673682 ) ] ) ) ,
674683 vec ! [ Arc :: new( dictionary) ] ,
675684 ) ?;
676- let pieces =
677- SpillPieces :: new ( & batch , Some ( 1024 * 1024 ) ) ? . collect :: < Result < Vec < _ > > > ( ) ?;
685+ let pieces = SpillPieces :: new ( & batch , Some ( 1024 * 1024 ) , false ) ?
686+ . collect :: < Result < Vec < _ > > > ( ) ?;
678687 assert_eq ! ( pieces. len( ) , 1 ) ;
679688 assert_eq ! ( pieces[ 0 ] . 0 , batch) ;
680689 Ok ( ( ) )
@@ -733,8 +742,8 @@ mod tests {
733742 false ,
734743 ) ] ) ) ;
735744 let batch = RecordBatch :: try_new ( schema, vec ! [ list_view] ) ?;
736- let pieces =
737- SpillPieces :: new ( & batch , Some ( 16 * 1024 ) ) ? . collect :: < Result < Vec < _ > > > ( ) ?;
745+ let pieces = SpillPieces :: new ( & batch , Some ( 16 * 1024 ) , false ) ?
746+ . collect :: < Result < Vec < _ > > > ( ) ?;
738747 assert_eq ! ( pieces. len( ) , 1 ) ;
739748 assert_eq ! ( pieces[ 0 ] . 0 , batch) ;
740749 Ok ( ( ) )
@@ -760,6 +769,25 @@ mod tests {
760769 Ok ( ( ) )
761770 }
762771
772+ /// With many narrow columns the IPC padding of each buffer is larger than the data, so
773+ /// the recorded size must include it or a reader decodes more than was reserved.
774+ #[ tokio:: test]
775+ async fn recorded_size_includes_ipc_padding ( ) -> Result < ( ) > {
776+ let schema = Arc :: new ( Schema :: new (
777+ ( 0 ..4096 )
778+ . map ( |i| Field :: new ( format ! ( "c{i}" ) , DataType :: UInt8 , false ) )
779+ . collect :: < Vec < _ > > ( ) ,
780+ ) ) ;
781+ let columns = ( 0 ..4096 )
782+ . map ( |_| Arc :: new ( UInt8Array :: from_iter_values ( 0 ..17 ) ) as ArrayRef )
783+ . collect ( ) ;
784+ let batch = RecordBatch :: try_new ( schema, columns) ?;
785+ // `round_trip` checks every batch read back against the recorded size.
786+ let ( read, _) = round_trip ( std:: slice:: from_ref ( & batch) , 64 * 1024 ) . await ?;
787+ assert_same_rows ( & read, std:: slice:: from_ref ( & batch) ) ;
788+ Ok ( ( ) )
789+ }
790+
763791 /// A piece whose dictionary keys are all null has no used values. After the unused
764792 /// values are dropped the dictionary is empty, which must not panic when measured, and
765793 /// must read back as nulls.
@@ -778,7 +806,7 @@ mod tests {
778806 // Pieces of at most 4 rows: the first ones have only null keys.
779807 let budget = 4 * WIDE + WIDE / 2 ;
780808 let pieces =
781- SpillPieces :: new ( & batch, Some ( budget) ) ?. collect :: < Result < Vec < _ > > > ( ) ?;
809+ SpillPieces :: new ( & batch, Some ( budget) , false ) ?. collect :: < Result < Vec < _ > > > ( ) ?;
782810 assert ! ( pieces. len( ) > 1 ) ;
783811 assert ! (
784812 pieces. iter( ) . any( |( piece, _) | piece
@@ -788,7 +816,7 @@ mod tests {
788816 . is_empty( ) ) ,
789817 "expected a piece with an empty dictionary"
790818 ) ;
791- SpillPieces :: new ( & batch. slice ( 0 , 0 ) , Some ( budget) ) ?
819+ SpillPieces :: new ( & batch. slice ( 0 , 0 ) , Some ( budget) , false ) ?
792820 . collect :: < Result < Vec < _ > > > ( ) ?;
793821
794822 let ( read, _) = round_trip ( std:: slice:: from_ref ( & batch) , budget) . await ?;
@@ -967,3 +995,99 @@ mod tests {
967995 Ok ( ( ) )
968996 }
969997}
998+
999+ #[ cfg( test) ]
1000+ /// The recorded size bounds what a reader decodes, for every codec and for buffer types
1001+ /// whose IPC padding or decompressed allocation is larger than their data.
1002+ mod decoded_size_bound_tests {
1003+ use super :: * ;
1004+ use arrow:: array:: * ;
1005+ use arrow_schema:: { DataType , Field , Schema } ;
1006+ use datafusion_common:: config:: SpillCompression ;
1007+ use datafusion_common:: utils:: memory:: get_record_batch_memory_size;
1008+ use datafusion_execution:: runtime_env:: RuntimeEnvBuilder ;
1009+ use datafusion_physical_expr_common:: metrics:: {
1010+ ExecutionPlanMetricsSet , SpillMetrics ,
1011+ } ;
1012+ use futures:: TryStreamExt ;
1013+
1014+ fn cols ( n : usize , rows : usize ) -> Vec < ( String , RecordBatch ) > {
1015+ let mk = |name : & str , dt : DataType , f : & dyn Fn ( ) -> ArrayRef | {
1016+ let schema = Arc :: new ( Schema :: new (
1017+ ( 0 ..n)
1018+ . map ( |i| Field :: new ( format ! ( "c{i}" ) , dt. clone ( ) , true ) )
1019+ . collect :: < Vec < _ > > ( ) ,
1020+ ) ) ;
1021+ (
1022+ name. to_string ( ) ,
1023+ RecordBatch :: try_new ( schema, ( 0 ..n) . map ( |_| f ( ) ) . collect ( ) ) . unwrap ( ) ,
1024+ )
1025+ } ;
1026+ vec ! [
1027+ mk( "u8" , DataType :: UInt8 , & || {
1028+ Arc :: new( UInt8Array :: from_iter_values( ( 0 ..rows) . map( |i| i as u8 ) ) )
1029+ } ) ,
1030+ mk( "i64_nulls" , DataType :: Int64 , & || {
1031+ Arc :: new( Int64Array :: from_iter(
1032+ ( 0 ..rows) . map( |i| ( i % 2 == 0 ) . then_some( i as i64 ) ) ,
1033+ ) )
1034+ } ) ,
1035+ mk( "bool" , DataType :: Boolean , & || {
1036+ Arc :: new( BooleanArray :: from_iter( ( 0 ..rows) . map( |i| Some ( i % 2 == 0 ) ) ) )
1037+ } ) ,
1038+ mk( "utf8" , DataType :: Utf8 , & || {
1039+ Arc :: new( StringArray :: from_iter_values(
1040+ ( 0 ..rows) . map( |i| format!( "v{i}" ) ) ,
1041+ ) )
1042+ } ) ,
1043+ mk( "utf8view" , DataType :: Utf8View , & || {
1044+ Arc :: new( StringViewArray :: from_iter_values(
1045+ ( 0 ..rows) . map( |i| format!( "a-long-string-value-{i}" ) ) ,
1046+ ) )
1047+ } ) ,
1048+ mk( "dec256" , DataType :: Decimal256 ( 40 , 2 ) , & || {
1049+ Arc :: new(
1050+ Decimal256Array :: from_iter_values(
1051+ ( 0 ..rows) . map( |i| arrow:: datatypes:: i256:: from_i128( i as i128 ) ) ,
1052+ )
1053+ . with_precision_and_scale( 40 , 2 )
1054+ . unwrap( ) ,
1055+ )
1056+ } ) ,
1057+ ]
1058+ }
1059+
1060+ #[ tokio:: test]
1061+ async fn recorded_size_bounds_decoded_batch ( ) -> Result < ( ) > {
1062+ for comp in [
1063+ SpillCompression :: Uncompressed ,
1064+ SpillCompression :: Lz4Frame ,
1065+ SpillCompression :: Zstd ,
1066+ ] {
1067+ for ( n, rows) in [ ( 256usize , 17usize ) , ( 4 , 4000 ) ] {
1068+ for ( name, batch) in cols ( n, rows) {
1069+ let runtime = Arc :: new ( RuntimeEnvBuilder :: new ( ) . build ( ) ?) ;
1070+ let metrics = SpillMetrics :: new ( & ExecutionPlanMetricsSet :: new ( ) , 0 ) ;
1071+ let m = Arc :: new (
1072+ SpillManager :: new ( runtime, metrics, batch. schema ( ) )
1073+ . with_compression_type ( comp) ,
1074+ ) ;
1075+ let mut f = m
1076+ . create_in_progress_file ( "t" ) ?
1077+ . with_max_batch_bytes ( Some ( 1 << 30 ) ) ;
1078+ let rec = f. append_batch ( & batch) ?;
1079+ let file = f. finish ( ) ?. unwrap ( ) ;
1080+ let read: Vec < RecordBatch > =
1081+ m. read_spill_as_stream ( file, None ) ?. try_collect ( ) . await ?;
1082+ let actual =
1083+ read. iter ( ) . map ( get_record_batch_memory_size) . max ( ) . unwrap ( ) ;
1084+ assert ! (
1085+ actual <= rec + crate :: spill:: SPILL_BATCH_MEMORY_MARGIN ,
1086+ "{comp:?} {name} cols={n} rows={rows}: recorded {rec}, decoded {actual}"
1087+ ) ;
1088+ }
1089+ }
1090+ }
1091+ Ok ( ( ) )
1092+ }
1093+ }
0 commit comments