Skip to content

Commit 3194f1a

Browse files
committed
refactor: decide offsets as lengths once per shuffle block writer
ShuffleBlockWriter::try_new_with_offsets_as_lengths replaces the builder. The writer now decides when it is built: it checks once, on an empty batch, that arrow-ipc writes the buffers the mapped roles describe, keeps plain offsets if not, and sets the codec-tag flag in its header. That removes the per-block flag plumbing through every codec arm and the header rewrite. The writer and reader share one buffer-range helper and one rewrite loop. The reader rebuilds offsets before freezing the body. The two Scala tests share a cometShuffleMetric helper.
1 parent c317d54 commit 3194f1a

8 files changed

Lines changed: 298 additions & 269 deletions

File tree

‎native/shuffle/src/ipc.rs‎

Lines changed: 42 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,9 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
use crate::offsets_as_lengths::{buffer_roles, lengths_to_offsets, LENGTHS_TAG_FLAG};
18+
use crate::offsets_as_lengths::{
19+
buffer_ranges, buffer_roles, lengths_to_offsets, LENGTHS_TAG_FLAG,
20+
};
1921
use arrow::array::{ArrayRef, RecordBatch};
2022
use arrow::buffer::{Buffer, MutableBuffer};
2123
use arrow::datatypes::{Schema, SchemaRef};
@@ -323,7 +325,7 @@ fn read_single_batch<'b, S: BlockSource<'b>>(
323325
let dictionary = message
324326
.header_as_dictionary_batch()
325327
.ok_or_else(|| decode_error("unable to read dictionary batch"))?;
326-
let body = source.body(body_len)?;
328+
let body = Buffer::from(source.body(body_len)?);
327329
read_dictionary_impl(
328330
&body,
329331
dictionary,
@@ -347,8 +349,9 @@ fn read_single_batch<'b, S: BlockSource<'b>>(
347349
.ok_or_else(|| decode_error("unable to read record batch"))?;
348350
let mut body = source.body(body_len)?;
349351
if lengths {
350-
body = restore_offsets(body, &record_batch, &schema)?;
352+
restore_offsets(body.as_slice_mut(), &record_batch, &schema)?;
351353
}
354+
let body = Buffer::from(body);
352355
batch = Some(
353356
RecordBatchDecoder::try_new(
354357
&body,
@@ -378,38 +381,21 @@ fn read_single_batch<'b, S: BlockSource<'b>>(
378381
Ok(batch)
379382
}
380383

381-
/// Turns the lengths a writer stored back into offsets, in the body buffer itself: both sources
382-
/// hand out a freshly allocated body, so this does not copy it again.
384+
/// Turns the lengths a writer stored back into offsets, in the body itself.
383385
fn restore_offsets(
384-
body: Buffer,
386+
body: &mut [u8],
385387
record_batch: &arrow::ipc::RecordBatch<'_>,
386388
schema: &Schema,
387-
) -> Result<Buffer> {
389+
) -> Result<()> {
388390
let roles = buffer_roles(schema).ok_or_else(|| {
389391
decode_error("offsets stored as lengths for a schema with an unmapped buffer layout")
390392
})?;
391-
let buffers = record_batch
392-
.buffers()
393-
.ok_or_else(|| decode_error("record batch has no buffers"))?;
394-
let mut body = body.into_mutable().unwrap_or_else(|shared| {
395-
let mut copy = MutableBuffer::with_capacity(shared.len());
396-
copy.extend_from_slice(shared.as_slice());
397-
copy
398-
});
399-
// an offset or length that does not fit lands outside the body and is rejected there
400-
let range = |value: i64| usize::try_from(value).unwrap_or(usize::MAX);
401-
lengths_to_offsets(
402-
body.as_slice_mut(),
403-
buffers
404-
.iter()
405-
.map(|b| (range(b.offset()), range(b.length()))),
406-
&roles,
407-
)?;
408-
Ok(body.into())
393+
lengths_to_offsets(body, buffer_ranges(record_batch)?, &roles)
409394
}
410395

411396
/// Where a block's IPC messages come from. Metadata is borrowed one message at a time; bodies
412-
/// become exactly sized buffers that the decoded arrays keep.
397+
/// become exactly sized buffers that the decoded arrays keep, still mutable so offsets stored as
398+
/// lengths can be rebuilt in place first.
413399
///
414400
/// `'b` is the lifetime of an in-memory block, so [`Sliced`] can hand out metadata without
415401
/// copying it; a streamed source uses `'static` and copies metadata into the caller's scratch.
@@ -421,7 +407,7 @@ trait BlockSource<'b> {
421407
'b: 'a;
422408

423409
/// The next message's body, `len` bytes long.
424-
fn body(&mut self, len: usize) -> Result<Buffer>;
410+
fn body(&mut self, len: usize) -> Result<MutableBuffer>;
425411

426412
/// Errors unless every byte of the block has been consumed.
427413
fn expect_exhausted(&mut self) -> Result<()>;
@@ -485,10 +471,10 @@ impl<R: Read> BlockSource<'static> for Streamed<R> {
485471
Ok(Some(scratch.as_slice()))
486472
}
487473

488-
fn body(&mut self, len: usize) -> Result<Buffer> {
474+
fn body(&mut self, len: usize) -> Result<MutableBuffer> {
489475
let mut body = MutableBuffer::from_len_zeroed(len);
490476
self.read_exact(&mut body, "truncated IPC body")?;
491-
Ok(body.into())
477+
Ok(body)
492478
}
493479

494480
fn expect_exhausted(&mut self) -> Result<()> {
@@ -541,9 +527,12 @@ impl<'b> BlockSource<'b> for Sliced<'b> {
541527
Ok(Some(self.take(len, "truncated IPC metadata")?))
542528
}
543529

544-
fn body(&mut self, len: usize) -> Result<Buffer> {
530+
fn body(&mut self, len: usize) -> Result<MutableBuffer> {
545531
// an exactly sized copy, with no zero fill before it
546-
Ok(Buffer::from(self.take(len, "truncated IPC body")?))
532+
let bytes = self.take(len, "truncated IPC body")?;
533+
let mut body = MutableBuffer::with_capacity(bytes.len());
534+
body.extend_from_slice(bytes);
535+
Ok(body)
547536
}
548537

549538
fn expect_exhausted(&mut self) -> Result<()> {
@@ -578,11 +567,13 @@ impl<R: Read> Read for RequireLz4EndMark<R> {
578567
mod tests {
579568
use super::{
580569
read_ipc_compressed, read_ipc_compressed_validated, reset_schema_cache, schema_cache_stats,
581-
scratch_capacity, RequireLz4EndMark, SchemaCacheStats, SCHEMA_CACHE_CAPACITY,
582-
SCHEMA_CACHE_RETAIN_LIMIT, SCRATCH_RETAIN_LIMIT,
570+
scratch_capacity, RequireLz4EndMark, SchemaCacheStats, LENGTHS_TAG_FLAG,
571+
SCHEMA_CACHE_CAPACITY, SCHEMA_CACHE_RETAIN_LIMIT, SCRATCH_RETAIN_LIMIT,
583572
};
584573
use crate::writers::rss::tests::allocations;
585-
use arrow::array::{Array, DictionaryArray, Int32Array, RecordBatch, StringArray};
574+
use arrow::array::{
575+
Array, ArrayRef, DictionaryArray, Int32Array, RecordBatch, StringArray, StringViewArray,
576+
};
586577
use arrow::datatypes::{DataType, Field, Int32Type, Schema};
587578
use arrow::ipc::reader::StreamReader;
588579
use arrow::ipc::writer::StreamWriter;
@@ -650,6 +641,23 @@ mod tests {
650641
encode(codec, &ipc_bytes(batch))
651642
}
652643

644+
/// A flagged block whose schema has no mapped buffer layout cannot have come from Comet's
645+
/// writer; decoding it must fail rather than guess at which buffers to rebuild.
646+
#[test]
647+
fn lengths_flag_on_an_unmapped_layout_is_an_error() {
648+
let views = StringViewArray::from(vec!["a", "bb", "ccc"]);
649+
let batch = RecordBatch::try_from_iter(vec![("v", Arc::new(views) as ArrayRef)]).unwrap();
650+
let mut block = block_for(&batch, b"NONE");
651+
assert_eq!(read_ipc_compressed(&block).unwrap(), batch);
652+
653+
block[0] |= LENGTHS_TAG_FLAG;
654+
let error = read_ipc_compressed_validated(&block).unwrap_err();
655+
assert!(
656+
error.to_string().contains("unmapped buffer layout"),
657+
"{error}"
658+
);
659+
}
660+
653661
fn mixed_batch() -> RecordBatch {
654662
let schema = Arc::new(Schema::new(vec![
655663
Field::new("i", DataType::Int32, true),

‎native/shuffle/src/offsets_as_lengths.rs‎

Lines changed: 63 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -106,19 +106,35 @@ pub(crate) fn has_offsets(roles: &[BufferRole]) -> bool {
106106
roles.iter().any(|role| *role != BufferRole::Other)
107107
}
108108

109+
/// The (offset, length) of every body buffer of a record-batch message, in body order. A value
110+
/// that does not fit a `usize` becomes one that lies outside any body, so the range check that
111+
/// follows rejects it.
112+
pub(crate) fn buffer_ranges<'a>(
113+
batch: &arrow::ipc::RecordBatch<'a>,
114+
) -> Result<impl ExactSizeIterator<Item = (usize, usize)> + 'a> {
115+
let buffers = batch.buffers().ok_or_else(|| {
116+
DataFusionError::Execution("Shuffle block record batch has no buffers".to_string())
117+
})?;
118+
let range = |value: i64| usize::try_from(value).unwrap_or(usize::MAX);
119+
Ok(buffers
120+
.iter()
121+
.map(move |buffer| (range(buffer.offset()), range(buffer.length()))))
122+
}
123+
109124
/// Rewrites every offsets buffer of `body` as byte planes of lengths, in place. `buffers` are the
110125
/// record-batch message's (offset, length) pairs in body order, one per entry of `roles`.
111126
pub(crate) fn offsets_to_lengths(
112127
body: &mut [u8],
113128
buffers: impl ExactSizeIterator<Item = (usize, usize)>,
114129
roles: &[BufferRole],
115130
) -> Result<()> {
116-
let (mut scratch32, mut scratch64) = (Vec::new(), Vec::new());
117-
for_each_offsets_buffer(body, buffers, roles, |bytes, role| match role {
118-
BufferRole::Offsets32 => to_length_planes_32(bytes, &mut scratch32),
119-
BufferRole::Offsets64 => to_length_planes_64(bytes, &mut scratch64),
120-
BufferRole::Other => {}
121-
})
131+
rewrite(
132+
body,
133+
buffers,
134+
roles,
135+
to_length_planes_32,
136+
to_length_planes_64,
137+
)
122138
}
123139

124140
/// Undoes [`offsets_to_lengths`], in place.
@@ -127,19 +143,21 @@ pub(crate) fn lengths_to_offsets(
127143
buffers: impl ExactSizeIterator<Item = (usize, usize)>,
128144
roles: &[BufferRole],
129145
) -> Result<()> {
130-
let (mut scratch32, mut scratch64) = (Vec::new(), Vec::new());
131-
for_each_offsets_buffer(body, buffers, roles, |bytes, role| match role {
132-
BufferRole::Offsets32 => from_length_planes_32(bytes, &mut scratch32),
133-
BufferRole::Offsets64 => from_length_planes_64(bytes, &mut scratch64),
134-
BufferRole::Other => {}
135-
})
146+
rewrite(
147+
body,
148+
buffers,
149+
roles,
150+
from_length_planes_32,
151+
from_length_planes_64,
152+
)
136153
}
137154

138-
fn for_each_offsets_buffer(
155+
fn rewrite(
139156
body: &mut [u8],
140157
buffers: impl ExactSizeIterator<Item = (usize, usize)>,
141158
roles: &[BufferRole],
142-
mut rewrite: impl FnMut(&mut [u8], BufferRole),
159+
rewrite32: fn(&mut [u8], &mut Vec<u32>),
160+
rewrite64: fn(&mut [u8], &mut Vec<u64>),
143161
) -> Result<()> {
144162
if buffers.len() != roles.len() {
145163
return Err(DataFusionError::Execution(format!(
@@ -148,40 +166,48 @@ fn for_each_offsets_buffer(
148166
roles.len()
149167
)));
150168
}
151-
let body_len = body.len();
169+
let (mut scratch32, mut scratch64) = (Vec::new(), Vec::new());
152170
for ((offset, len), role) in buffers.zip(roles) {
153-
if *role == BufferRole::Other {
154-
continue;
171+
match role {
172+
BufferRole::Other => {}
173+
BufferRole::Offsets32 => rewrite32(buffer_mut(body, offset, len)?, &mut scratch32),
174+
BufferRole::Offsets64 => rewrite64(buffer_mut(body, offset, len)?, &mut scratch64),
155175
}
156-
let bytes = offset
157-
.checked_add(len)
158-
.and_then(|end| body.get_mut(offset..end))
159-
.ok_or_else(|| {
160-
DataFusionError::Execution(format!(
161-
"Shuffle block offsets buffer at {offset} with length {len} is outside \
162-
its {body_len}-byte body"
163-
))
164-
})?;
165-
rewrite(bytes, *role);
166176
}
167177
Ok(())
168178
}
169179

180+
/// `body[offset..offset + len]`, or an error when that range is not inside `body`.
181+
fn buffer_mut(body: &mut [u8], offset: usize, len: usize) -> Result<&mut [u8]> {
182+
let body_len = body.len();
183+
offset
184+
.checked_add(len)
185+
.and_then(|end| body.get_mut(offset..end))
186+
.ok_or_else(|| {
187+
DataFusionError::Execution(format!(
188+
"Shuffle block offsets buffer at {offset} with length {len} is outside its \
189+
{body_len}-byte body"
190+
))
191+
})
192+
}
193+
170194
/// Lengths in byte planes for one offset width. The first entry is a length from zero. One pass
171195
/// computes the lengths, then one contiguous pass per plane moves their bytes, so the loops
172196
/// vectorize; decoding gathers the planes back the same way and keeps a running sum. Wrapping
173197
/// arithmetic makes every bit pattern round-trip. Only whole entries move: the trailing bytes of a
174198
/// buffer that is not a whole number of entries stay where they are.
175199
macro_rules! length_planes {
176-
($to_planes:ident, $from_planes:ident, $word:ty, $width:literal) => {
200+
($to_planes:ident, $from_planes:ident, $word:ty) => {
177201
fn $to_planes(bytes: &mut [u8], scratch: &mut Vec<$word>) {
178-
let count = bytes.len() / $width;
202+
const WIDTH: usize = size_of::<$word>();
203+
let count = bytes.len() / WIDTH;
179204
if count == 0 {
180205
return;
181206
}
182-
let entries = &mut bytes[..count * $width];
183-
let words = entries.as_chunks::<$width>().0;
207+
let entries = &mut bytes[..count * WIDTH];
208+
let words = entries.as_chunks::<WIDTH>().0;
184209
scratch.clear();
210+
scratch.reserve(count);
185211
scratch.push(<$word>::from_le_bytes(words[0]));
186212
scratch.extend(words.windows(2).map(|pair| {
187213
<$word>::from_le_bytes(pair[1]).wrapping_sub(<$word>::from_le_bytes(pair[0]))
@@ -195,11 +221,12 @@ macro_rules! length_planes {
195221
}
196222

197223
fn $from_planes(bytes: &mut [u8], scratch: &mut Vec<$word>) {
198-
let count = bytes.len() / $width;
224+
const WIDTH: usize = size_of::<$word>();
225+
let count = bytes.len() / WIDTH;
199226
if count == 0 {
200227
return;
201228
}
202-
let entries = &mut bytes[..count * $width];
229+
let entries = &mut bytes[..count * WIDTH];
203230
scratch.clear();
204231
scratch.resize(count, 0);
205232
for (plane, input) in entries.chunks_exact(count).enumerate() {
@@ -209,7 +236,7 @@ macro_rules! length_planes {
209236
}
210237
}
211238
let mut running: $word = 0;
212-
let words = entries.as_chunks_mut::<$width>().0;
239+
let words = entries.as_chunks_mut::<WIDTH>().0;
213240
for (entry, length) in words.iter_mut().zip(scratch.iter()) {
214241
running = running.wrapping_add(*length);
215242
*entry = running.to_le_bytes();
@@ -218,8 +245,8 @@ macro_rules! length_planes {
218245
};
219246
}
220247

221-
length_planes!(to_length_planes_32, from_length_planes_32, u32, 4);
222-
length_planes!(to_length_planes_64, from_length_planes_64, u64, 8);
248+
length_planes!(to_length_planes_32, from_length_planes_32, u32);
249+
length_planes!(to_length_planes_64, from_length_planes_64, u64);
223250

224251
#[cfg(test)]
225252
mod tests {

‎native/shuffle/src/shuffle_writer.rs‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -128,8 +128,7 @@ pub struct ShuffleWriterExec {
128128
write_buffer_size: usize,
129129
/// Maximum bytes buffered in memory before spilling; `None` disables the limit
130130
max_buffer_bytes: Option<usize>,
131-
/// Whether local blocks store offsets as lengths; see
132-
/// [`ShuffleBlockWriter::with_offsets_as_lengths`].
131+
/// Whether local blocks store offsets as lengths
133132
offsets_as_lengths: bool,
134133
}
135134

@@ -202,8 +201,8 @@ impl ShuffleWriterExec {
202201
})
203202
}
204203

205-
/// Store offsets as lengths in local shuffle blocks, where the codec takes it; see
206-
/// [`ShuffleBlockWriter::with_offsets_as_lengths`]. Remote (RSS) blocks are unaffected.
204+
/// Store offsets as lengths in local shuffle blocks where the writer can; see
205+
/// [`ShuffleBlockWriter::try_new_with_offsets_as_lengths`]. Remote (RSS) blocks keep offsets.
207206
pub fn with_offsets_as_lengths(mut self, enabled: bool) -> Self {
208207
self.offsets_as_lengths = enabled;
209208
self
@@ -336,8 +335,11 @@ async fn external_shuffle(
336335
output_data_file,
337336
partition_offsets,
338337
} => {
339-
let shuffle_block_writer = ShuffleBlockWriter::try_new(schema.as_ref(), codec.clone())?
340-
.with_offsets_as_lengths(offsets_as_lengths);
338+
let shuffle_block_writer = ShuffleBlockWriter::try_new_with_offsets_as_lengths(
339+
schema.as_ref(),
340+
codec.clone(),
341+
offsets_as_lengths,
342+
)?;
341343
let writer = LocalPartitionWriter::try_new(
342344
output_data_file,
343345
partition_offsets,

‎native/shuffle/src/spark_unsafe/row.rs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1375,7 +1375,6 @@ pub fn process_sorted_row_partition(
13751375
// inside the loop within the method across batches.
13761376
initial_checksum: Option<u32>,
13771377
codec: &CompressionCodec,
1378-
// see `ShuffleBlockWriter::with_offsets_as_lengths`
13791378
offsets_as_lengths: bool,
13801379
) -> Result<(i64, Option<u32>, i64), CometError> {
13811380
// The current row number we are reading
@@ -1452,8 +1451,11 @@ pub fn process_sorted_row_partition(
14521451
.as_ref()
14531452
.is_none_or(|(cached, _)| cached != batch.schema_ref())
14541453
{
1455-
let writer = ShuffleBlockWriter::try_new(batch.schema_ref(), codec.clone())?
1456-
.with_offsets_as_lengths(offsets_as_lengths);
1454+
let writer = ShuffleBlockWriter::try_new_with_offsets_as_lengths(
1455+
batch.schema_ref(),
1456+
codec.clone(),
1457+
offsets_as_lengths,
1458+
)?;
14571459
block_writer = Some((batch.schema(), writer));
14581460
}
14591461
let (_, writer) = block_writer.as_ref().expect("block writer is set above");

0 commit comments

Comments
 (0)