Skip to content

Commit 3b4425c

Browse files
committed
perf: reuse per-partition scratch in the shuffle write path
The multi-partition write path pays two allocation costs per partition per event: BufBatchWriter starts from an empty byte buffer and regrows it toward the 1MB write-buffer size, and the interleave iterator materializes the whole partition index list as a fresh usize-pair vector (16 bytes per row) plus a batch-ref vector over all buffered batches. The task-level writer now owns one byte buffer and recycles it through the sequential partition loops, drained between partitions with its capacity kept, so an event allocates the buffer once instead of once per partition and holds exactly one regardless of partition count. The interleave path builds the batch-ref list once per write cycle and converts indices per output chunk into a small reusable scratch instead of widening the full list up front. Wire format and produced batches are unchanged, pinned by byte-level tests. Write time drops ~27% on a 2000-partition zstd shuffle bench; the larger effect is allocator pressure under concurrent tasks, which a single-task bench understates. Part of #5002.
1 parent 199a910 commit 3b4425c

6 files changed

Lines changed: 211 additions & 23 deletions

File tree

‎native/shuffle/src/partitioners/multi_partition.rs‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -567,12 +567,17 @@ impl<T: PartitionWriter> MultiPartitionShuffleRepartitioner<T> {
567567
with_trace("shuffle_spill", self.tracing_enabled, || {
568568
let num_output_partitions = self.partition_indices.len();
569569
let write_result = {
570-
let mut partitioned_batches = self.partitioned_batches();
570+
let partitioned_batches = self.partitioned_batches();
571+
// Build the batch-ref slice once and share it across all partitions.
572+
let batch_refs = partitioned_batches.batch_refs();
571573
(0..num_output_partitions).try_for_each(|partition_id| {
572574
self.partition_writer.write(
573575
partition_id,
574-
&mut partitioned_batches
575-
.produce(partition_id, &self.metrics.interleave_time),
576+
&mut partitioned_batches.produce(
577+
&batch_refs,
578+
partition_id,
579+
&self.metrics.interleave_time,
580+
),
576581
&self.metrics,
577582
)
578583
})
@@ -630,15 +635,17 @@ impl<T: PartitionWriter> ShufflePartitioner for MultiPartitionShuffleRepartition
630635
with_trace("shuffle_write", self.tracing_enabled, || {
631636
let start_time = Instant::now();
632637

633-
let mut partitioned_batches = self.partitioned_batches();
638+
let partitioned_batches = self.partitioned_batches();
634639
self.pinned_buffers.clear();
635640
let num_output_partitions = self.partition_indices.len();
636641

642+
// Build the batch-ref slice once and share it across all partitions.
643+
let batch_refs = partitioned_batches.batch_refs();
637644
#[allow(clippy::needless_range_loop)]
638645
for i in 0..num_output_partitions {
639646
self.partition_writer.finish_partition(
640647
i,
641-
&mut partitioned_batches.produce(i, &self.metrics.interleave_time),
648+
&mut partitioned_batches.produce(&batch_refs, i, &self.metrics.interleave_time),
642649
&self.metrics,
643650
)?;
644651
}

‎native/shuffle/src/partitioners/partitioned_batch_iterator.rs‎

Lines changed: 103 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -42,14 +42,22 @@ impl PartitionedBatchesProducer {
4242
}
4343
}
4444

45+
/// References to all buffered batches. Build this once per write cycle and share it
46+
/// across every partition's [`Self::produce`] call instead of rebuilding a fresh
47+
/// `Vec<&RecordBatch>` over all buffered batches for each partition.
48+
pub(super) fn batch_refs(&self) -> Vec<&RecordBatch> {
49+
self.buffered_batches.iter().collect()
50+
}
51+
4552
pub(super) fn produce<'a>(
46-
&'a mut self,
53+
&'a self,
54+
refs: &'a [&'a RecordBatch],
4755
partition_id: usize,
4856
interleave_time: &'a Time,
4957
) -> PartitionedBatchIterator<'a> {
5058
PartitionedBatchIterator::new(
5159
&self.partition_indices[partition_id],
52-
&self.buffered_batches,
60+
refs,
5361
self.batch_size,
5462
interleave_time,
5563
)
@@ -58,39 +66,40 @@ impl PartitionedBatchesProducer {
5866

5967
/// Iterates over the shuffled record batches belonging to a single output partition.
6068
pub(crate) struct PartitionedBatchIterator<'a> {
61-
record_batches: Vec<&'a RecordBatch>,
69+
record_batches: &'a [&'a RecordBatch],
6270
batch_size: usize,
63-
indices: Vec<(usize, usize)>,
71+
indices: &'a [(u32, u32)],
72+
/// Scratch for the current chunk's indices widened to what `interleave_record_batch`
73+
/// expects. Reused across chunks so each partition costs one small allocation
74+
/// (capacity at most `batch_size`) rather than re-materializing its whole index list.
75+
chunk_scratch: Vec<(usize, usize)>,
6476
pos: usize,
6577
interleave_time: &'a Time,
6678
}
6779

6880
impl<'a> PartitionedBatchIterator<'a> {
6981
fn new(
7082
indices: &'a [(u32, u32)],
71-
buffered_batches: &'a [RecordBatch],
83+
record_batches: &'a [&'a RecordBatch],
7284
batch_size: usize,
7385
interleave_time: &'a Time,
7486
) -> Self {
7587
if indices.is_empty() {
7688
// Avoid unnecessary allocations when the partition is empty
7789
return Self {
78-
record_batches: vec![],
90+
record_batches: &[],
7991
batch_size,
80-
indices: vec![],
92+
indices: &[],
93+
chunk_scratch: vec![],
8194
pos: 0,
8295
interleave_time,
8396
};
8497
}
85-
let record_batches = buffered_batches.iter().collect::<Vec<_>>();
86-
let current_indices = indices
87-
.iter()
88-
.map(|(i_batch, i_row)| (*i_batch as usize, *i_row as usize))
89-
.collect::<Vec<_>>();
9098
Self {
9199
record_batches,
92100
batch_size,
93-
indices: current_indices,
101+
indices,
102+
chunk_scratch: Vec::with_capacity(batch_size.min(indices.len())),
94103
pos: 0,
95104
interleave_time,
96105
}
@@ -106,9 +115,14 @@ impl Iterator for PartitionedBatchIterator<'_> {
106115
}
107116

108117
let indices_end = std::cmp::min(self.pos + self.batch_size, self.indices.len());
109-
let indices = &self.indices[self.pos..indices_end];
118+
self.chunk_scratch.clear();
119+
self.chunk_scratch.extend(
120+
self.indices[self.pos..indices_end]
121+
.iter()
122+
.map(|(i_batch, i_row)| (*i_batch as usize, *i_row as usize)),
123+
);
110124
let mut timer = self.interleave_time.timer();
111-
let result = interleave_record_batch(&self.record_batches, indices);
125+
let result = interleave_record_batch(self.record_batches, &self.chunk_scratch);
112126
timer.stop();
113127
match result {
114128
Ok(batch) => {
@@ -122,3 +136,77 @@ impl Iterator for PartitionedBatchIterator<'_> {
122136
}
123137
}
124138
}
139+
140+
#[cfg(test)]
141+
mod tests {
142+
use super::*;
143+
use arrow::array::Int32Array;
144+
use arrow::datatypes::{DataType, Field, Schema};
145+
use std::sync::Arc;
146+
147+
fn batches() -> Vec<RecordBatch> {
148+
let schema = Arc::new(Schema::new(vec![Field::new("v", DataType::Int32, false)]));
149+
(0..3)
150+
.map(|b| {
151+
let values: Vec<i32> = (0..5).map(|r| b * 100 + r).collect();
152+
RecordBatch::try_new(
153+
Arc::clone(&schema),
154+
vec![Arc::new(Int32Array::from(values))],
155+
)
156+
.unwrap()
157+
})
158+
.collect()
159+
}
160+
161+
/// Chunked index conversion must interleave exactly like converting the whole partition's
162+
/// index list up front, including the short tail chunk, and share one batch-ref slice
163+
/// across partitions.
164+
#[test]
165+
fn chunked_interleave_matches_full_conversion() {
166+
let buffered = batches();
167+
let indices: Vec<(u32, u32)> = vec![
168+
(0, 0),
169+
(2, 4),
170+
(1, 1),
171+
(0, 3),
172+
(2, 0),
173+
(1, 4),
174+
(0, 1),
175+
(2, 2),
176+
(1, 0),
177+
(0, 4),
178+
];
179+
let batch_size = 4; // chunks of 4, 4, and a tail of 2
180+
let producer = PartitionedBatchesProducer::new(
181+
buffered.clone(),
182+
vec![indices.clone(), Vec::new()],
183+
batch_size,
184+
);
185+
let refs = producer.batch_refs();
186+
let time = Time::default();
187+
188+
let produced: Vec<RecordBatch> = producer
189+
.produce(&refs, 0, &time)
190+
.collect::<datafusion::common::Result<_>>()
191+
.unwrap();
192+
193+
let expected_refs: Vec<&RecordBatch> = buffered.iter().collect();
194+
let full: Vec<(usize, usize)> = indices
195+
.iter()
196+
.map(|(b, r)| (*b as usize, *r as usize))
197+
.collect();
198+
let expected: Vec<RecordBatch> = full
199+
.chunks(batch_size)
200+
.map(|chunk| interleave_record_batch(&expected_refs, chunk).unwrap())
201+
.collect();
202+
203+
assert_eq!(produced, expected);
204+
assert_eq!(produced.last().unwrap().num_rows(), 2, "tail chunk");
205+
206+
let empty: Vec<RecordBatch> = producer
207+
.produce(&refs, 1, &time)
208+
.collect::<datafusion::common::Result<_>>()
209+
.unwrap();
210+
assert!(empty.is_empty());
211+
}
212+
}

‎native/shuffle/src/shuffle_writer.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1170,6 +1170,7 @@ mod test {
11701170
Cursor::new(&mut coalesced_output),
11711171
1024 * 1024,
11721172
8192,
1173+
Vec::new(),
11731174
);
11741175
for batch in &small_batches {
11751176
buf_writer.write(batch, &encode_time, &write_time).unwrap();
@@ -1186,6 +1187,7 @@ mod test {
11861187
Cursor::new(&mut uncoalesced_output),
11871188
1024 * 1024,
11881189
1,
1190+
Vec::new(),
11891191
);
11901192
for batch in &small_batches {
11911193
buf_writer.write(batch, &encode_time, &write_time).unwrap();
@@ -1290,6 +1292,7 @@ mod test {
12901292
Cursor::new(&mut output),
12911293
1024 * 1024,
12921294
batch_size as usize,
1295+
Vec::new(),
12931296
);
12941297
for batch in &inputs {
12951298
buf_writer.write(batch, &encode_time, &write_time).unwrap();

‎native/shuffle/src/writers/buf_batch_writer.rs‎

Lines changed: 73 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,23 +47,34 @@ pub(crate) struct BufBatchWriter<S: Borrow<ShuffleBlockWriter>, W: Write> {
4747
}
4848

4949
impl<S: Borrow<ShuffleBlockWriter>, W: Write> BufBatchWriter<S, W> {
50+
/// `buffer` is the caller-owned byte buffer to serialize into. Passing a buffer recovered
51+
/// from a previous writer's [`Self::into_buffer`] reuses its capacity instead of regrowing
52+
/// a fresh allocation toward `buffer_max_size` for every writer.
5053
pub(crate) fn new(
5154
shuffle_block_writer: S,
5255
writer: W,
5356
buffer_max_size: usize,
5457
batch_size: usize,
58+
mut buffer: Vec<u8>,
5559
) -> Self {
60+
buffer.clear();
5661
Self {
5762
shuffle_block_writer,
5863
writer,
59-
buffer: vec![],
64+
buffer,
6065
buffer_max_size,
6166
compression_context: CompressionContext::default(),
6267
coalescer: None,
6368
batch_size,
6469
}
6570
}
6671

72+
/// Recover the byte buffer so its capacity can be recycled into the next writer.
73+
/// The buffer is drained (empty) if the writer was flushed.
74+
pub(crate) fn into_buffer(self) -> Vec<u8> {
75+
self.buffer
76+
}
77+
6778
pub(crate) fn write(
6879
&mut self,
6980
batch: &RecordBatch,
@@ -160,3 +171,64 @@ impl<S: Borrow<ShuffleBlockWriter>, W: Write + Seek> BufBatchWriter<S, W> {
160171
self.writer.stream_position().map_err(Into::into)
161172
}
162173
}
174+
175+
#[cfg(test)]
176+
mod tests {
177+
use super::*;
178+
use crate::{read_ipc_compressed, CompressionCodec};
179+
use arrow::array::Int64Array;
180+
use arrow::datatypes::{DataType, Field, Schema};
181+
use std::sync::Arc;
182+
183+
fn test_batch(seed: i64) -> RecordBatch {
184+
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
185+
let values: Vec<i64> = (0..100).map(|i| seed * 1_000 + i).collect();
186+
RecordBatch::try_new(schema, vec![Arc::new(Int64Array::from(values))]).unwrap()
187+
}
188+
189+
fn write_one_partition(seed: i64, buffer: Vec<u8>) -> (Vec<u8>, Vec<u8>) {
190+
let batch = test_batch(seed);
191+
let block_writer =
192+
ShuffleBlockWriter::try_new(batch.schema().as_ref(), CompressionCodec::Zstd(1))
193+
.unwrap();
194+
let mut output = Vec::new();
195+
let time = Time::default();
196+
let mut writer = BufBatchWriter::new(block_writer, &mut output, 1 << 20, 8192, buffer);
197+
writer.write(&batch, &time, &time).unwrap();
198+
writer.flush(&time, &time).unwrap();
199+
let buffer = writer.into_buffer();
200+
(output, buffer)
201+
}
202+
203+
/// A buffer recycled across partitions must produce byte-identical output to fresh
204+
/// per-partition buffers, come back empty, and keep its grown capacity.
205+
#[test]
206+
#[cfg_attr(miri, ignore)] // miri can't call zstd's C FFI
207+
fn recycled_buffer_matches_fresh_buffers_and_keeps_capacity() {
208+
let fresh: Vec<Vec<u8>> = (0..3)
209+
.map(|p| write_one_partition(p, Vec::new()).0)
210+
.collect();
211+
212+
let mut scratch = Vec::new();
213+
let mut recycled = Vec::new();
214+
for p in 0..3 {
215+
let (output, returned) = write_one_partition(p, scratch);
216+
assert!(
217+
returned.is_empty(),
218+
"recycled buffer must come back drained"
219+
);
220+
scratch = returned;
221+
recycled.push(output);
222+
}
223+
224+
assert_eq!(fresh, recycled);
225+
assert!(
226+
scratch.capacity() > 0,
227+
"capacity grown in one partition must survive into the next"
228+
);
229+
for output in &recycled {
230+
let decoded = read_ipc_compressed(&output[16..]).unwrap();
231+
assert_eq!(decoded.num_rows(), 100);
232+
}
233+
}
234+
}

0 commit comments

Comments
 (0)