Skip to content

Commit 08ec7be

Browse files
committed
refactor: borrow shuffle write scratch per call and cap retained capacity
The write scratch is now threaded through write and flush as a borrow instead of moving through the constructor, so one convention covers all task-scoped shuffle scratch, and an error mid-partition can no longer strand the buffer inside a dropped writer. The writer asserts a fresh scratch is drained and that every call passes the same buffer, since a swapped buffer would silently lose unflushed bytes. Flush shrinks the scratch back to the configured write-buffer size, bounding retained capacity that could previously reach the buffer size plus the largest block. The producer asserts the batch-ref slice covers every buffered batch, and the criterion suite gains a high-partition group so partition scaling stays visible to benchmark checks.
1 parent 6f93521 commit 08ec7be

6 files changed

Lines changed: 275 additions & 73 deletions

File tree

‎native/shuffle/benches/shuffle_writer.rs‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,35 @@ fn criterion_benchmark(c: &mut Criterion) {
174174
},
175175
);
176176
}
177+
group.finish();
178+
179+
// High partition counts stress the per-partition write path (one short-lived
180+
// buffered writer per partition), which low counts barely exercise; compression
181+
// is disabled to isolate it. Few samples: each iteration is a full end-to-end
182+
// write across thousands of partitions.
183+
let mut high_partition_group = c.benchmark_group("shuffle_writer_high_partition");
184+
high_partition_group.sample_size(10);
185+
for num_partitions in [200usize, 2000, 8000] {
186+
high_partition_group.bench_function(
187+
format!("shuffle_writer: end to end (partitions={num_partitions}, compression=None)"),
188+
|b| {
189+
let ctx = SessionContext::new();
190+
let exec = create_shuffle_writer_exec(
191+
CompressionCodec::None,
192+
CometPartitioning::Hash(vec![Arc::new(Column::new("a", 0))], num_partitions),
193+
8192,
194+
10,
195+
);
196+
b.iter(|| {
197+
let task_ctx = ctx.task_ctx();
198+
let stream = exec.execute(0, task_ctx).unwrap();
199+
let rt = Runtime::new().unwrap();
200+
rt.block_on(collect(stream)).unwrap();
201+
});
202+
},
203+
);
204+
}
205+
high_partition_group.finish();
177206
}
178207

179208
fn create_shuffle_writer_exec(

‎native/shuffle/src/partitioners/partitioned_batch_iterator.rs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,13 @@ impl PartitionedBatchesProducer {
5555
partition_id: usize,
5656
interleave_time: &'a Time,
5757
) -> PartitionedBatchIterator<'a> {
58+
// Partition indices index into `buffered_batches`; a refs slice built from a
59+
// different producer would silently interleave wrong rows.
60+
debug_assert_eq!(
61+
refs.len(),
62+
self.buffered_batches.len(),
63+
"refs slice must cover every buffered batch"
64+
);
5865
PartitionedBatchIterator::new(
5966
&self.partition_indices[partition_id],
6067
refs,
@@ -209,4 +216,18 @@ mod tests {
209216
.unwrap();
210217
assert!(empty.is_empty());
211218
}
219+
220+
/// A refs slice that does not cover every buffered batch (e.g. built from a different
221+
/// producer) must fail fast in debug builds instead of interleaving wrong rows.
222+
#[cfg(debug_assertions)]
223+
#[test]
224+
#[should_panic(expected = "refs slice must cover every buffered batch")]
225+
fn produce_rejects_mismatched_refs() {
226+
let buffered = batches();
227+
let producer = PartitionedBatchesProducer::new(buffered, vec![vec![(0, 0), (2, 1)]], 4);
228+
let refs = producer.batch_refs();
229+
let truncated = &refs[..refs.len() - 1];
230+
let time = Time::default();
231+
let _ = producer.produce(truncated, 0, &time);
232+
}
212233
}

‎native/shuffle/src/shuffle_writer.rs‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1170,12 +1170,16 @@ mod test {
11701170
Cursor::new(&mut coalesced_output),
11711171
1024 * 1024,
11721172
8192,
1173-
Vec::new(),
11741173
);
1174+
let mut scratch = Vec::new();
11751175
for batch in &small_batches {
1176-
buf_writer.write(batch, &encode_time, &write_time).unwrap();
1176+
buf_writer
1177+
.write(batch, &mut scratch, &encode_time, &write_time)
1178+
.unwrap();
11771179
}
1178-
buf_writer.flush(&encode_time, &write_time).unwrap();
1180+
buf_writer
1181+
.flush(&mut scratch, &encode_time, &write_time)
1182+
.unwrap();
11791183
}
11801184

11811185
// Write without coalescing (batch_size=1)
@@ -1187,12 +1191,16 @@ mod test {
11871191
Cursor::new(&mut uncoalesced_output),
11881192
1024 * 1024,
11891193
1,
1190-
Vec::new(),
11911194
);
1195+
let mut scratch = Vec::new();
11921196
for batch in &small_batches {
1193-
buf_writer.write(batch, &encode_time, &write_time).unwrap();
1197+
buf_writer
1198+
.write(batch, &mut scratch, &encode_time, &write_time)
1199+
.unwrap();
11941200
}
1195-
buf_writer.flush(&encode_time, &write_time).unwrap();
1201+
buf_writer
1202+
.flush(&mut scratch, &encode_time, &write_time)
1203+
.unwrap();
11961204
}
11971205

11981206
// Coalesced output should be smaller due to fewer IPC schema blocks
@@ -1292,12 +1300,16 @@ mod test {
12921300
Cursor::new(&mut output),
12931301
1024 * 1024,
12941302
batch_size as usize,
1295-
Vec::new(),
12961303
);
1304+
let mut scratch = Vec::new();
12971305
for batch in &inputs {
1298-
buf_writer.write(batch, &encode_time, &write_time).unwrap();
1306+
buf_writer
1307+
.write(batch, &mut scratch, &encode_time, &write_time)
1308+
.unwrap();
12991309
}
1300-
buf_writer.flush(&encode_time, &write_time).unwrap();
1310+
buf_writer
1311+
.flush(&mut scratch, &encode_time, &write_time)
1312+
.unwrap();
13011313
}
13021314

13031315
let blocks = read_all_ipc_batches(&output);

0 commit comments

Comments
 (0)