Skip to content

Commit 402fd1d

Browse files
authored
fix(buffer): reject ZSTs and address allocator review comments (#9807)
Reject zero-sized buffer elements at construction with compile-time assertions. ZST support was previously inconsistent; rejecting these types prevents division-by-zero and pointer-distance failures without adding iterator or growth special cases. Empty buffers of non-ZST types remain supported. Six compile-fail doctests cover the independent construction paths. Remove the generic raw-owner constructor. Typed Vec adoption takes its pointer and element count directly from the Vec and retains that Vec for destruction. External byte owners continue to use `Bytes::from_owner` at the call site, without adding a buffer API. Address the allocator-stack review comments with field documentation, an explanation of the optional allocator, and tests for owner lifetime, allocator retention, empty/shared ownership, sliced-buffer growth, and alignment. The allocator representation, buffer layout, alignment policy, and allocation benchmarks are retained. Both iterator-extension implementations and the owned iterator match the merged baseline. This is a focused alternative to #9745. Validation: - `cargo nextest run -p vortex-buffer --all-features`: 874 tests passed. - `cargo test --doc -p vortex-buffer --all-features`: 8 ordinary and 6 compile-fail doctests passed. - `cargo clippy -p vortex-buffer --all-targets --all-features`: passed. - Nightly formatting and `git diff --check`: passed. - Workspace clippy was blocked locally by the CUDA nvCOMP download; full CI is running on this revision. Performance validation: - Three serial alternating local comparisons against the allocator-stack baseline place all 20 measured interleave, batched i8 filter, and multiplication cases within 2% of baseline. - The final Graviton NEON run matches develop: u64 multiplication 15.1 vs 15.3 µs; i64 and 16,384-row multiplication 17.1 vs approximately 17.3 µs. The earlier NEON slowdowns disappeared with the ZST ban. - The complete final CodSpeed report passed with no reported regressions. --------- Signed-off-by: Nicholas Gates <[email protected]>
1 parent 803a3dc commit 402fd1d

3 files changed

Lines changed: 165 additions & 65 deletions

File tree

‎vortex-buffer/src/allocation.rs‎

Lines changed: 60 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
//! Allocator-backed storage for Vortex buffers.
55
66
use std::alloc::Layout;
7+
use std::any::Any;
78
use std::fmt;
89
use std::fmt::Debug;
910
use std::mem::ManuallyDrop;
@@ -27,8 +28,15 @@ pub trait BufferAllocator: Allocator + Debug + Send + Sync + 'static {}
2728
impl<A> BufferAllocator for A where A: Allocator + Debug + Send + Sync + 'static {}
2829

2930
/// A shared reference to a buffer allocator.
31+
///
32+
/// The static allocator does not need shared ownership, so it is stored without an [`Arc`]. This
33+
/// makes cloning the common static allocator a simple value copy.
3034
#[derive(Clone)]
31-
pub struct BufferAllocatorRef(Option<Arc<dyn BufferAllocator>>);
35+
pub struct BufferAllocatorRef(
36+
// `None` selects the static allocator without allocating or updating an Arc reference count.
37+
// `Some` keeps a custom allocator alive for as long as its buffers need it.
38+
Option<Arc<dyn BufferAllocator>>,
39+
);
3240

3341
impl BufferAllocatorRef {
3442
/// Wrap an allocator in a shared reference.
@@ -333,32 +341,13 @@ impl Drop for Allocation {
333341
}
334342
}
335343

336-
pub(crate) trait BufferOwner: Send + Sync + 'static {
337-
fn as_ptr(&self) -> *const u8;
338-
339-
fn len(&self) -> usize;
340-
}
341-
342-
impl<T> BufferOwner for T
343-
where
344-
T: AsRef<[u8]> + Send + Sync + 'static,
345-
{
346-
fn as_ptr(&self) -> *const u8 {
347-
self.as_ref().as_ptr()
348-
}
349-
350-
fn len(&self) -> usize {
351-
self.as_ref().len()
352-
}
353-
}
354-
355344
pub(crate) enum BufferBacking {
356345
Owned(Allocation),
357346
Bytes(bytes::Bytes),
358347
#[cfg(feature = "arrow")]
359348
Arrow(arrow_buffer::Buffer),
360349
External {
361-
_owner: Box<dyn BufferOwner>,
350+
_owner: Box<dyn Any + Send + Sync>,
362351
},
363352
}
364353

@@ -386,9 +375,13 @@ mod tests {
386375
use allocator_api2::alloc::AllocError;
387376
use allocator_api2::alloc::Allocator;
388377
use allocator_api2::alloc::Global;
378+
use rstest::rstest;
379+
use vortex_error::VortexResult;
380+
use vortex_error::vortex_err;
389381

390382
use crate::Alignment;
391383
use crate::BufferAllocatorRef;
384+
use crate::BufferMut;
392385

393386
#[derive(Clone, Debug, Default)]
394387
struct TrackingAllocator {
@@ -462,15 +455,18 @@ mod tests {
462455
assert_eq!(state.deallocations.load(Ordering::Relaxed), 1);
463456
}
464457

465-
#[test]
466-
fn buffer_growth_uses_allocator_grow() {
458+
#[rstest]
459+
fn buffer_growth_uses_allocator_grow(#[values(4, 64, 4096)] alignment: usize) {
467460
let allocator = TrackingAllocator::default();
468461
let state = Arc::clone(&allocator.state);
469-
let mut buffer = BufferAllocatorRef::new(allocator).with_capacity::<u32>(1);
462+
let alignment = Alignment::new(alignment);
463+
let mut buffer =
464+
BufferAllocatorRef::new(allocator).with_capacity_aligned::<u32>(1, alignment);
470465
let initial_capacity = buffer.capacity();
471466
buffer.extend(std::iter::repeat_n(7, initial_capacity));
472467

473468
buffer.push(u32::MAX);
469+
assert!(alignment.is_ptr_aligned(buffer.as_ptr()));
474470

475471
assert_eq!(&buffer[..initial_capacity], vec![7; initial_capacity]);
476472
assert_eq!(buffer[initial_capacity], u32::MAX);
@@ -498,4 +494,44 @@ mod tests {
498494
assert_eq!(state.allocations.load(Ordering::Relaxed), 1);
499495
assert_eq!(state.grows.load(Ordering::Relaxed), 0);
500496
}
497+
498+
#[test]
499+
fn empty_buffers_preserve_allocator_without_allocating() -> VortexResult<()> {
500+
let allocator = TrackingAllocator::default();
501+
let state = Arc::clone(&allocator.state);
502+
let allocator = BufferAllocatorRef::new(allocator);
503+
let buffer = BufferMut::<u32>::zeroed_in(0, allocator.clone());
504+
let buffer = buffer.freeze();
505+
let copy = buffer.clone().into_mut();
506+
assert!(copy.allocator().ptr_eq(&allocator));
507+
let mut buffer = buffer
508+
.try_into_mut()
509+
.map_err(|_| vortex_err!("unique buffer"))?;
510+
buffer.reserve(0);
511+
assert!(buffer.is_empty());
512+
assert!(buffer.allocator().ptr_eq(&allocator));
513+
drop((copy, buffer));
514+
assert_eq!(state.allocations.load(Ordering::Relaxed), 0);
515+
assert_eq!(state.grows.load(Ordering::Relaxed), 0);
516+
assert_eq!(state.deallocations.load(Ordering::Relaxed), 0);
517+
Ok(())
518+
}
519+
520+
#[test]
521+
fn shared_into_mut_preserves_allocator() {
522+
let allocator = TrackingAllocator::default();
523+
let state = Arc::clone(&allocator.state);
524+
let allocator = BufferAllocatorRef::new(allocator);
525+
let original = allocator.copy_from([1u32, 2, 3]).freeze();
526+
let mut copy = original.clone().into_mut();
527+
assert!(copy.allocator().ptr_eq(&allocator));
528+
copy[0] = 42;
529+
assert_eq!(original.as_slice(), [1, 2, 3]);
530+
assert_eq!(copy.as_slice(), [42, 2, 3]);
531+
assert_eq!(state.allocations.load(Ordering::Relaxed), 2);
532+
drop(copy);
533+
assert_eq!(state.deallocations.load(Ordering::Relaxed), 1);
534+
drop(original);
535+
assert_eq!(state.deallocations.load(Ordering::Relaxed), 2);
536+
}
501537
}

‎vortex-buffer/src/buffer.rs‎

Lines changed: 83 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,38 @@ use crate::debug::TruncatedDebug;
2828
use crate::trusted_len::TrustedLen;
2929

3030
/// An immutable buffer of items of `T`.
31+
///
32+
/// Zero-sized element types are rejected at compile time when constructing a buffer.
33+
///
34+
/// ```compile_fail
35+
/// use vortex_buffer::Buffer;
36+
/// let _ = Buffer::<()>::empty();
37+
/// ```
38+
///
39+
/// ```compile_fail
40+
/// use vortex_buffer::Buffer;
41+
/// let _ = Buffer::from(vec![(); 3]);
42+
/// ```
43+
///
44+
/// ```compile_fail
45+
/// use vortex_buffer::{Buffer, ByteBuffer};
46+
/// let _ = Buffer::<()>::from_byte_buffer(ByteBuffer::empty());
47+
/// ```
48+
///
49+
/// ```compile_fail
50+
/// use bytes::Bytes;
51+
/// use vortex_buffer::{Alignment, Buffer};
52+
/// let _ = Buffer::<()>::from_bytes_aligned(Bytes::new(), Alignment::none());
53+
/// ```
3154
#[derive(Clone)]
3255
pub struct Buffer<T> {
56+
/// The first element in this view; may dangle for an empty buffer.
3357
pub(crate) ptr: NonNull<T>,
58+
/// The number of initialized `T` values visible from `ptr`.
3459
pub(crate) length: usize,
60+
/// The minimum alignment promised for `ptr` and preserved by aligned slices.
3561
pub(crate) alignment: Alignment,
62+
/// Shared ownership of the storage containing `ptr`, if any; `Buffer::empty` has no backing.
3663
pub(crate) backing: Option<Arc<BufferBacking>>,
3764
}
3865

@@ -100,22 +127,6 @@ impl<T> Buffer<T> {
100127
}
101128
}
102129

103-
fn from_owner(owner: impl crate::BufferOwner, alignment: Alignment) -> Self {
104-
let owner: Box<dyn crate::BufferOwner> = Box::new(owner);
105-
let length = owner.len() / size_of::<T>();
106-
let ptr = if length == 0 {
107-
empty_ptr()
108-
} else {
109-
NonNull::new(owner.as_ptr().cast_mut().cast()).vortex_expect("owner pointer is null")
110-
};
111-
Self {
112-
ptr,
113-
length,
114-
alignment,
115-
backing: Some(Arc::new(BufferBacking::External { _owner: owner })),
116-
}
117-
}
118-
119130
fn from_bytes(bytes: Bytes, alignment: Alignment) -> Self {
120131
let length = bytes.len() / size_of::<T>();
121132
if length == 0 {
@@ -228,6 +239,7 @@ impl<T> Buffer<T> {
228239
///
229240
/// This does not allocate. Empty buffers use an aligned dangling pointer.
230241
pub fn empty_aligned(alignment: Alignment) -> Self {
242+
const { assert!(size_of::<T>() != 0, "ZSTs are not supported") };
231243
if !alignment.is_aligned_to(Alignment::of::<T>()) {
232244
vortex_panic!(
233245
"Alignment {} must align to the scalar type's alignment {}",
@@ -277,6 +289,7 @@ impl<T> Buffer<T> {
277289
/// Panics if the buffer is not aligned to the given alignment, if the length is not a multiple
278290
/// of the size of `T`, or if the given alignment is not aligned to that of `T`.
279291
pub fn from_byte_buffer_aligned(buffer: ByteBuffer, alignment: Alignment) -> Self {
292+
const { assert!(size_of::<T>() != 0, "ZSTs are not supported") };
280293
if !alignment.is_aligned_to(Alignment::of::<T>()) {
281294
vortex_panic!(
282295
"Alignment {} must be compatible with the scalar type's alignment {}",
@@ -309,6 +322,7 @@ impl<T> Buffer<T> {
309322
/// Panics if the buffer is not aligned to the size of `T`, or the length is not a multiple of
310323
/// the size of `T`.
311324
pub fn from_bytes_aligned(bytes: Bytes, alignment: Alignment) -> Self {
325+
const { assert!(size_of::<T>() != 0, "ZSTs are not supported") };
312326
if !alignment.is_aligned_to(Alignment::of::<T>()) {
313327
vortex_panic!(
314328
"Alignment {} must be compatible with the scalar type's alignment {}",
@@ -802,29 +816,25 @@ impl<T> FromIterator<T> for Buffer<T> {
802816
}
803817
}
804818

805-
// Helper struct that preserves drop glue for non-native Vec elements.
806-
#[repr(transparent)]
807-
struct Wrapper<T>(Vec<T>);
808-
809-
impl<T: Send + Sync + 'static> crate::BufferOwner for Wrapper<T> {
810-
fn as_ptr(&self) -> *const u8 {
811-
self.0.as_ptr().cast()
812-
}
813-
814-
fn len(&self) -> usize {
815-
self.0.len() * size_of::<T>()
816-
}
817-
}
818-
819819
impl<T> From<Vec<T>> for Buffer<T>
820820
where
821821
T: Send + Sync + 'static,
822822
{
823823
fn from(value: Vec<T>) -> Self {
824+
const { assert!(size_of::<T>() != 0, "ZSTs are not supported") };
824825
let length = value.len();
825826
let alignment = Alignment::of::<T>();
826827
if std::mem::needs_drop::<T>() {
827-
Self::from_owner(Wrapper(value), alignment)
828+
// Keep the typed owner so its elements are dropped.
829+
Self {
830+
ptr: NonNull::new(value.as_ptr().cast_mut())
831+
.vortex_expect("a Vec always has a non-null pointer"),
832+
length,
833+
alignment,
834+
backing: Some(Arc::new(BufferBacking::External {
835+
_owner: Box::new(value),
836+
})),
837+
}
828838
} else {
829839
Self::from_allocation(Allocation::from_vec(value), 0, length, alignment)
830840
}
@@ -1094,6 +1104,42 @@ mod test {
10941104
assert_eq!(buffer.allocation.alignment(), align_of::<u32>());
10951105
}
10961106

1107+
#[test]
1108+
fn byte_owner_preserves_slice_and_lifetime() {
1109+
struct Owner {
1110+
values: Vec<u8>,
1111+
drops: Arc<AtomicUsize>,
1112+
}
1113+
1114+
impl AsRef<[u8]> for Owner {
1115+
fn as_ref(&self) -> &[u8] {
1116+
&self.values[1..4]
1117+
}
1118+
}
1119+
1120+
impl Drop for Owner {
1121+
fn drop(&mut self) {
1122+
self.drops.fetch_add(1, Ordering::Relaxed);
1123+
}
1124+
}
1125+
1126+
let drops = Arc::new(AtomicUsize::new(0));
1127+
let owner = Owner {
1128+
values: vec![0, 1, 2, 3, 4],
1129+
drops: Arc::clone(&drops),
1130+
};
1131+
let ptr = owner.as_ref().as_ptr();
1132+
let buffer = ByteBuffer::from(Bytes::from_owner(owner));
1133+
assert_eq!(buffer.as_ptr(), ptr);
1134+
assert_eq!(buffer.as_slice(), [1, 2, 3]);
1135+
let view = buffer.slice(1..);
1136+
drop(buffer);
1137+
assert_eq!(drops.load(Ordering::Relaxed), 0);
1138+
assert_eq!(view.as_slice(), [2, 3]);
1139+
drop(view);
1140+
assert_eq!(drops.load(Ordering::Relaxed), 1);
1141+
}
1142+
10971143
#[test]
10981144
fn bytes_round_trip_reuses_owner() {
10991145
let bytes = Bytes::from_static(&[1, 2, 3, 4]);
@@ -1151,9 +1197,15 @@ mod test {
11511197
let Ok(mut sliced) = sliced.try_into_mut() else {
11521198
panic!("uniquely owned slice should become mutable")
11531199
};
1200+
let ptr = sliced.as_ptr();
11541201
let capacity = sliced.capacity();
11551202
sliced.push_n(0, capacity - sliced.len());
11561203
assert_eq!(sliced.len(), capacity);
1204+
assert_eq!(sliced.as_ptr(), ptr);
1205+
sliced.push(42);
1206+
assert_eq!(&sliced[..32], (64u32..96).collect::<Vec<_>>());
1207+
assert_eq!(&sliced[32..capacity], vec![0; capacity - 32]);
1208+
assert_eq!(sliced[capacity], 42);
11571209
}
11581210

11591211
#[test]

‎vortex-buffer/src/buffer_mut.rs‎

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,30 @@ use crate::debug::TruncatedDebug;
2323
use crate::trusted_len::TrustedLen;
2424

2525
/// A mutable buffer that maintains a runtime-defined alignment through resizing operations.
26+
///
27+
/// Zero-sized element types are rejected at compile time when constructing a buffer.
28+
///
29+
/// ```compile_fail
30+
/// use vortex_buffer::BufferMut;
31+
/// let _ = BufferMut::<()>::empty();
32+
/// ```
33+
///
34+
/// ```compile_fail
35+
/// use vortex_buffer::BufferMut;
36+
/// let _ = BufferMut::<()>::zeroed(3);
37+
/// ```
2638
pub struct BufferMut<T> {
39+
/// The owned allocation, including any bytes before `ptr` used for alignment.
2740
pub(crate) allocation: Allocation,
41+
/// The first element, aligned to `alignment`; it may dangle for an empty buffer.
2842
pub(crate) ptr: std::ptr::NonNull<T>,
43+
/// The number of initialized `T` values starting at `ptr`.
2944
pub(crate) length: usize,
45+
/// The number of `T` values that fit from `ptr`.
3046
pub(crate) capacity: usize,
47+
/// The minimum alignment maintained for `ptr` across reallocations.
3148
pub(crate) alignment: Alignment,
49+
/// Marks the buffer as logically owning values of `T` despite storing an erased allocation.
3250
pub(crate) _marker: std::marker::PhantomData<T>,
3351
}
3452

@@ -100,6 +118,7 @@ impl<T> BufferMut<T> {
100118
preferred_alignment: Option<Alignment>,
101119
allocator: BufferAllocatorRef,
102120
) -> Self {
121+
const { assert!(size_of::<T>() != 0, "ZSTs are not supported") };
103122
let actual = max(
104123
alignment,
105124
preferred_alignment.unwrap_or(Alignment::of::<u8>()),
@@ -131,11 +150,7 @@ impl<T> BufferMut<T> {
131150
let offset = allocation.ptr().as_ptr().align_offset(actual.as_usize());
132151
// SAFETY: the allocation includes enough padding to reach this aligned pointer.
133152
let ptr = unsafe { allocation.ptr().add(offset).cast() };
134-
let capacity = if size_of::<T>() == 0 {
135-
capacity
136-
} else {
137-
(allocation.size() - offset) / size_of::<T>()
138-
};
153+
let capacity = (allocation.size() - offset) / size_of::<T>();
139154
Self {
140155
allocation,
141156
ptr,
@@ -204,6 +219,7 @@ impl<T> BufferMut<T> {
204219
preferred_alignment: Option<Alignment>,
205220
allocator: BufferAllocatorRef,
206221
) -> Self {
222+
const { assert!(size_of::<T>() != 0, "ZSTs are not supported") };
207223
let preferred_alignment = preferred_alignment.unwrap_or(Alignment::of::<u8>());
208224
let actual_alignment = max(preferred_alignment, alignment);
209225
let size = len
@@ -226,11 +242,7 @@ impl<T> BufferMut<T> {
226242
.align_offset(actual_alignment.as_usize());
227243
// SAFETY: the allocation includes enough padding to reach this aligned pointer.
228244
let ptr = unsafe { allocation.ptr().add(offset).cast() };
229-
let capacity = if size_of::<T>() == 0 {
230-
len
231-
} else {
232-
(allocation.size() - offset) / size_of::<T>()
233-
};
245+
let capacity = (allocation.size() - offset) / size_of::<T>();
234246
Self {
235247
allocation,
236248
ptr,

0 commit comments

Comments
 (0)