1616// under the License.
1717
1818use arrow:: array:: builder:: { Date32Builder , Decimal128Builder , Int32Builder } ;
19- use arrow:: array:: { builder:: StringBuilder , Array , Int32Array , RecordBatch } ;
19+ use arrow:: array:: {
20+ builder:: StringBuilder , Array , ArrayRef , BooleanArray , Decimal128Array , Int32Array , Int64Array ,
21+ RecordBatch ,
22+ } ;
2023use arrow:: datatypes:: { DataType , Field , Schema } ;
2124use arrow:: row:: { RowConverter , SortField } ;
2225use criterion:: { criterion_group, criterion_main, BatchSize , Bencher , Criterion } ;
@@ -188,6 +191,86 @@ fn criterion_benchmark(c: &mut Criterion) {
188191 high_partition_group. finish ( ) ;
189192}
190193
194+ // Include decimal SUM's non-null is_empty state, nullable Boolean-heavy rows,
195+ // and an Int64-only control. Both low and high partition counts matter because
196+ // short runs are more common with fewer hash partitions.
197+ fn boolean_state_benchmark ( c : & mut Criterion ) {
198+ let mut group = c. benchmark_group ( "shuffle_writer_boolean_state" ) ;
199+ group. sample_size ( 10 ) ;
200+ for shape in [ "decimal_sum" , "nullable_booleans" , "int64_control" ] {
201+ for partitions in [ 4 , 200 ] {
202+ for key_run in [ 1 , 64 ] {
203+ // Build distinct batches so the gather reads independent buffers and the
204+ // writer does not encode and compress the same rows 100 times.
205+ let batches = ( 0 ..100 )
206+ . map ( |batch_index| boolean_state_batch ( 8192 , shape, key_run, batch_index) )
207+ . collect :: < Vec < _ > > ( ) ;
208+ group. bench_function (
209+ format ! ( "{shape}/partitions={partitions}/key_run={key_run}" ) ,
210+ |b| {
211+ bench_end_to_end ( b, || {
212+ create_shuffle_writer_from_batches (
213+ CompressionCodec :: Lz4Frame ,
214+ CometPartitioning :: Hash (
215+ vec ! [ Arc :: new( Column :: new( "key" , 0 ) ) ] ,
216+ partitions,
217+ ) ,
218+ batches. clone ( ) ,
219+ )
220+ } )
221+ } ,
222+ ) ;
223+ }
224+ }
225+ }
226+ group. finish ( ) ;
227+ }
228+
229+ fn boolean_state_batch (
230+ rows : usize ,
231+ shape : & str ,
232+ key_run : usize ,
233+ batch_index : usize ,
234+ ) -> RecordBatch {
235+ let first_row = batch_index * rows;
236+ let row_range = first_row..first_row + rows;
237+ let mut fields = vec ! [ Field :: new( "key" , DataType :: Int64 , false ) ] ;
238+ let mut columns: Vec < ArrayRef > = vec ! [ Arc :: new( Int64Array :: from_iter_values(
239+ row_range. clone( ) . map( |row| ( row / key_run) as i64 ) ,
240+ ) ) ] ;
241+ if shape == "decimal_sum" {
242+ fields. push ( Field :: new ( "sum" , DataType :: Decimal128 ( 38 , 2 ) , true ) ) ;
243+ fields. push ( Field :: new ( "is_empty" , DataType :: Boolean , false ) ) ;
244+ columns. push ( Arc :: new (
245+ Decimal128Array :: from_iter (
246+ row_range
247+ . clone ( )
248+ . map ( |row| ( row % 7 != 0 ) . then_some ( row as i128 ) ) ,
249+ )
250+ . with_precision_and_scale ( 38 , 2 )
251+ . unwrap ( ) ,
252+ ) ) ;
253+ columns. push ( Arc :: new ( BooleanArray :: from_iter (
254+ row_range. clone ( ) . map ( |row| Some ( row % 7 == 0 ) ) ,
255+ ) ) ) ;
256+ } else {
257+ for column in 0 ..8 {
258+ if shape == "nullable_booleans" {
259+ fields. push ( Field :: new ( format ! ( "b{column}" ) , DataType :: Boolean , true ) ) ;
260+ columns. push ( Arc :: new ( BooleanArray :: from_iter ( row_range. clone ( ) . map (
261+ |row| ( ( row + column) % 5 != 0 ) . then_some ( ( row + column) % 3 == 0 ) ,
262+ ) ) ) ) ;
263+ } else {
264+ fields. push ( Field :: new ( format ! ( "i{column}" ) , DataType :: Int64 , false ) ) ;
265+ columns. push ( Arc :: new ( Int64Array :: from_iter_values (
266+ row_range. clone ( ) . map ( |row| ( row + column) as i64 ) ,
267+ ) ) ) ;
268+ }
269+ }
270+ }
271+ RecordBatch :: try_new ( Arc :: new ( Schema :: new ( fields) ) , columns) . unwrap ( )
272+ }
273+
191274/// Times one execution of a freshly built writer per iteration. A `ShuffleWriterExec`
192275/// publishes its partition offsets once, so it cannot be re-executed; building it is
193276/// setup and stays outside the measurement.
@@ -211,6 +294,14 @@ fn create_shuffle_writer_exec(
211294 num_batches : usize ,
212295) -> ShuffleWriterExec {
213296 let batches = create_batches ( rows_per_batch, num_batches) ;
297+ create_shuffle_writer_from_batches ( compression_codec, partitioning, batches)
298+ }
299+
300+ fn create_shuffle_writer_from_batches (
301+ compression_codec : CompressionCodec ,
302+ partitioning : CometPartitioning ,
303+ batches : Vec < RecordBatch > ,
304+ ) -> ShuffleWriterExec {
214305 let schema = batches[ 0 ] . schema ( ) ;
215306 let partitions = & [ batches] ;
216307 ShuffleWriterExec :: try_new (
@@ -593,6 +684,6 @@ fn config() -> Criterion {
593684criterion_group ! {
594685 name = benches;
595686 config = config( ) ;
596- targets = criterion_benchmark, partitioning_benchmark, schema_encoding_benchmark, ipc_context_reuse_benchmark
687+ targets = criterion_benchmark, boolean_state_benchmark , partitioning_benchmark, schema_encoding_benchmark, ipc_context_reuse_benchmark
597688}
598689criterion_main ! ( benches) ;
0 commit comments