Skip to content

Commit c6678f3

Browse files
authored
move Buffer/BufferMut panic helpers into separate functions (#9927)
Continuation of #9903. Buffer/BufferMut are used everywhere, so inlining their panic handlers both increases code size and degrades performance. Signed-off-by: Mikhail Kot <[email protected]>
1 parent 7eca46d commit c6678f3

4 files changed

Lines changed: 99 additions & 60 deletions

File tree

‎vortex-array/src/arrays/primitive/compute/cast.rs‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -377,20 +377,22 @@ where
377377
} else {
378378
(-(scale as i16)) as u32
379379
};
380-
let ten = <T as BigCast>::from(10i8).ok_or_else(
381-
|| vortex_err!(Compute: "Cannot create decimal scale factor for scale {scale}"),
382-
)?;
383-
let mut factor = <T as BigCast>::from(1i8).ok_or_else(
384-
|| vortex_err!(Compute: "Cannot create decimal scale factor for scale {scale}"),
385-
)?;
380+
let ten = <T as BigCast>::from(10i8).ok_or_else(|| scale_factor_error(scale))?;
381+
let mut factor = <T as BigCast>::from(1i8).ok_or_else(|| scale_factor_error(scale))?;
386382
for _ in 0..exponent {
387-
factor = factor.checked_mul(&ten).ok_or_else(
388-
|| vortex_err!(Compute: "Cannot create decimal scale factor for scale {scale}"),
389-
)?;
383+
factor = factor
384+
.checked_mul(&ten)
385+
.ok_or_else(|| scale_factor_error(scale))?;
390386
}
391387
Ok(factor)
392388
}
393389

390+
#[cold]
391+
#[inline(never)]
392+
fn scale_factor_error(scale: i8) -> VortexError {
393+
vortex_err!(Compute: "Cannot create decimal scale factor for scale {scale}")
394+
}
395+
394396
fn decimal_value_fits_precision<T: NativeDecimalType>(
395397
value: T,
396398
decimal_dtype: DecimalDType,
@@ -431,6 +433,8 @@ where
431433
Ok(buffer.freeze())
432434
}
433435

436+
#[cold]
437+
#[inline(never)]
434438
fn primitive_to_decimal_cast_error<S>(value: S, decimal_dtype: DecimalDType) -> VortexError
435439
where
436440
S: IntegerPType + ToI256,

‎vortex-array/src/arrays/varbin/builder.rs‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -460,12 +460,7 @@ impl<O: OffsetBuilderPType> VarBinBuilder<O> {
460460
fn push_value(&mut self, value: &[u8]) {
461461
self.offsets
462462
.push(O::from(self.data.len() + value.len()).unwrap_or_else(|| {
463-
vortex_panic!(
464-
"Failed to convert sum of {} and {} to offset of type {}",
465-
self.data.len(),
466-
value.len(),
467-
std::any::type_name::<O>()
468-
)
463+
offset_overflow(self.data.len(), value.len(), std::any::type_name::<O>())
469464
}));
470465
self.data.extend_from_slice(value);
471466
}
@@ -708,6 +703,14 @@ macro_rules! __match_varbin_builder_arms {
708703
};
709704
}
710705

706+
#[cold]
707+
#[inline(never)]
708+
fn offset_overflow(data_len: usize, value_len: usize, offset_type: &'static str) -> ! {
709+
vortex_panic!(
710+
"Failed to convert sum of {data_len} and {value_len} to offset of type {offset_type}"
711+
)
712+
}
713+
711714
/// Running totals of `lengths`, wrapping so a corrupt lengths child is rejected rather than
712715
/// panicking; [`VarBinBuilder::extend_offsets`] catches the resulting non-monotonic offsets.
713716
#[inline]

‎vortex-buffer/src/buffer.rs‎

Lines changed: 70 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -253,11 +253,7 @@ impl<T> Buffer<T> {
253253
pub fn empty_aligned(alignment: Alignment) -> Self {
254254
const { assert!(size_of::<T>() != 0, "ZSTs are not supported") };
255255
if !alignment.is_aligned_to(Alignment::of::<T>()) {
256-
vortex_panic!(
257-
"Alignment {} must align to the scalar type's alignment {}",
258-
alignment,
259-
Alignment::of::<T>(),
260-
);
256+
misaligned_scalar_type(alignment, Alignment::of::<T>());
261257
}
262258
Self {
263259
ptr: empty_ptr(),
@@ -303,21 +299,13 @@ impl<T> Buffer<T> {
303299
pub fn from_byte_buffer_aligned(buffer: ByteBuffer, alignment: Alignment) -> Self {
304300
const { assert!(size_of::<T>() != 0, "ZSTs are not supported") };
305301
if !alignment.is_aligned_to(Alignment::of::<T>()) {
306-
vortex_panic!(
307-
"Alignment {} must be compatible with the scalar type's alignment {}",
308-
alignment,
309-
Alignment::of::<T>(),
310-
);
302+
incompatible_scalar_alignment(alignment, Alignment::of::<T>());
311303
}
312304
if !alignment.is_ptr_aligned(buffer.as_ptr()) {
313-
vortex_panic!("Buffer must align to the requested alignment {}", alignment);
305+
buffer_not_aligned(alignment);
314306
}
315307
if !buffer.len().is_multiple_of(size_of::<T>()) {
316-
vortex_panic!(
317-
"Buffer length {} must be a multiple of the scalar type's size {}",
318-
buffer.len(),
319-
size_of::<T>()
320-
);
308+
buffer_len_not_multiple(buffer.len(), size_of::<T>());
321309
}
322310
Self {
323311
ptr: buffer.ptr.cast(),
@@ -336,24 +324,13 @@ impl<T> Buffer<T> {
336324
pub fn from_bytes_aligned(bytes: Bytes, alignment: Alignment) -> Self {
337325
const { assert!(size_of::<T>() != 0, "ZSTs are not supported") };
338326
if !alignment.is_aligned_to(Alignment::of::<T>()) {
339-
vortex_panic!(
340-
"Alignment {} must be compatible with the scalar type's alignment {}",
341-
alignment,
342-
Alignment::of::<T>(),
343-
);
327+
incompatible_scalar_alignment(alignment, Alignment::of::<T>());
344328
}
345329
if !alignment.is_ptr_aligned(bytes.as_ptr()) {
346-
vortex_panic!(
347-
"Bytes alignment must align to the requested alignment {}",
348-
alignment,
349-
);
330+
bytes_not_aligned(alignment);
350331
}
351332
if !bytes.len().is_multiple_of(size_of::<T>()) {
352-
vortex_panic!(
353-
"Bytes length {} must be a multiple of the scalar type's size {}",
354-
bytes.len(),
355-
size_of::<T>()
356-
);
333+
bytes_len_not_multiple(bytes.len(), size_of::<T>());
357334
}
358335
Self::from_bytes(bytes, alignment)
359336
}
@@ -504,14 +481,10 @@ impl<T> Buffer<T> {
504481
};
505482

506483
if begin > end {
507-
vortex_panic!(
508-
"range start must not be greater than end: {:?} <= {:?}",
509-
begin,
510-
end
511-
);
484+
slice_range_inverted(begin, end);
512485
}
513486
if end > len {
514-
vortex_panic!("range end out of bounds: {:?} > {:?}", end, len);
487+
slice_range_out_of_bounds(end, len);
515488
}
516489

517490
if end == begin {
@@ -522,10 +495,7 @@ impl<T> Buffer<T> {
522495

523496
let begin_byte = begin * size_of::<T>();
524497
if !alignment.is_offset_aligned(begin_byte) {
525-
vortex_panic!(
526-
"range start must be aligned to {alignment:?}, byte {}",
527-
begin_byte
528-
);
498+
slice_start_misaligned(alignment, begin_byte);
529499
}
530500
if !alignment.is_aligned_to(Alignment::of::<T>()) {
531501
vortex_panic!("Slice alignment must at least align to type T")
@@ -972,6 +942,66 @@ impl<T> From<BufferMut<T>> for Buffer<T> {
972942
}
973943
}
974944

945+
// If panic helpers are inlined, this makes callers of BufferMut's functions
946+
// not inline-eligible. This is more ugly than direct panics but helps with
947+
// performance
948+
949+
#[cold]
950+
#[inline(never)]
951+
fn misaligned_scalar_type(alignment: Alignment, scalar_align: Alignment) -> ! {
952+
vortex_panic!("Alignment {alignment} must align to the scalar type's alignment {scalar_align}")
953+
}
954+
955+
#[cold]
956+
#[inline(never)]
957+
fn incompatible_scalar_alignment(alignment: Alignment, scalar_align: Alignment) -> ! {
958+
vortex_panic!(
959+
"Alignment {alignment} must be compatible with the scalar type's alignment {scalar_align}"
960+
)
961+
}
962+
963+
#[cold]
964+
#[inline(never)]
965+
fn buffer_not_aligned(alignment: Alignment) -> ! {
966+
vortex_panic!("Buffer must align to the requested alignment {alignment}")
967+
}
968+
969+
#[cold]
970+
#[inline(never)]
971+
fn bytes_not_aligned(alignment: Alignment) -> ! {
972+
vortex_panic!("Bytes alignment must align to the requested alignment {alignment}")
973+
}
974+
975+
#[cold]
976+
#[inline(never)]
977+
fn buffer_len_not_multiple(len: usize, scalar_size: usize) -> ! {
978+
vortex_panic!("Buffer length {len} must be a multiple of the scalar type's size {scalar_size}")
979+
}
980+
981+
#[cold]
982+
#[inline(never)]
983+
fn bytes_len_not_multiple(len: usize, scalar_size: usize) -> ! {
984+
vortex_panic!("Bytes length {len} must be a multiple of the scalar type's size {scalar_size}")
985+
}
986+
987+
#[cold]
988+
#[inline(never)]
989+
fn slice_range_inverted(begin: usize, end: usize) -> ! {
990+
vortex_panic!("range start must not be greater than end: {begin:?} <= {end:?}")
991+
}
992+
993+
#[cold]
994+
#[inline(never)]
995+
fn slice_range_out_of_bounds(end: usize, len: usize) -> ! {
996+
vortex_panic!("range end out of bounds: {end:?} > {len:?}")
997+
}
998+
999+
#[cold]
1000+
#[inline(never)]
1001+
fn slice_start_misaligned(alignment: Alignment, begin_byte: usize) -> ! {
1002+
vortex_panic!("range start must be aligned to {alignment:?}, byte {begin_byte}")
1003+
}
1004+
9751005
#[cfg(test)]
9761006
mod test {
9771007
use std::mem::align_of;

‎vortex-buffer/src/buffer_mut.rs‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -125,11 +125,7 @@ impl<T> BufferMut<T> {
125125
);
126126

127127
if !alignment.is_aligned_to(Alignment::of::<T>()) {
128-
vortex_panic!(
129-
"Alignment {} must align to the scalar type's alignment {}",
130-
alignment,
131-
align_of::<T>()
132-
);
128+
misaligned_scalar_type(alignment, Alignment::of::<T>());
133129
}
134130

135131
let size = capacity
@@ -1034,6 +1030,12 @@ impl<T> FromIterator<T> for BufferMut<T> {
10341030
}
10351031
}
10361032

1033+
#[cold]
1034+
#[inline(never)]
1035+
fn misaligned_scalar_type(alignment: Alignment, scalar_align: Alignment) -> ! {
1036+
vortex_panic!("Alignment {alignment} must align to the scalar type's alignment {scalar_align}")
1037+
}
1038+
10371039
#[cfg(test)]
10381040
mod tests {
10391041
use std::cell::Cell;

0 commit comments

Comments
 (0)