Skip to content

Commit c3ade49

Browse files
authored
fix: alignment and padding of sliced CUDA Arrow bitmaps (#9926)
## Summary Fix CUDA Arrow export of sliced BOOL values and validity bitmaps so their storage is safe for cuDF’s word-based reads. Correct logical values and offsets are not sufficient: - Byte slicing can misalign device pointers. - Word-sized reads can extend beyond the logical bitmap bytes into cuDF’s padded mask extent. Export now reuses suitably aligned, zero-padded storage or copies/repacks it when necessary. Repacking aligns the input view within its backing allocation and uses bounded byte loads only for the partial tail. Kernel-produced bitmaps are padded up front to avoid export copies. Focused regression tests cover sliced values and validity, zeroed tails, nonzero-offset reuse, and capped-grid repacking and null counting. Signed-off-by: Alexander Droste <[email protected]>
1 parent 2c8f160 commit c3ade49

8 files changed

Lines changed: 570 additions & 285 deletions

File tree

‎vortex-cuda/benches/arrow_validity_cuda.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ const INPUT_OFFSET: usize = 5;
4141
const ARROW_OFFSET: usize = 3;
4242
const EXPORT_BENCH_SIZES: &[(usize, &str)] = &[(100_000_000, "100M")];
4343

44-
fn validity_bitmap_byte_len(len: usize, bit_offset: usize) -> usize {
44+
fn arrow_bitmap_byte_len(len: usize, bit_offset: usize) -> usize {
4545
(bit_offset + len).div_ceil(8)
4646
}
4747

@@ -98,7 +98,7 @@ fn benchmark_arrow_validity_export(c: &mut Criterion) {
9898
[("device_bitmap", 0), ("device_bitmap_repack", INPUT_OFFSET)]
9999
{
100100
group.throughput(Throughput::Bytes(
101-
validity_bitmap_byte_len(len, validity_offset) as u64,
101+
arrow_bitmap_byte_len(len, validity_offset) as u64,
102102
));
103103
group.bench_with_input(
104104
BenchmarkId::new(format!("cuda/arrow_validity/export/{case}"), len_label),
@@ -148,7 +148,7 @@ fn benchmark_arrow_validity_repack(c: &mut Criterion) {
148148

149149
for &(len, len_label) in bench_config::BENCH_SIZES {
150150
group.throughput(Throughput::Bytes(
151-
validity_bitmap_byte_len(len, INPUT_OFFSET) as u64,
151+
arrow_bitmap_byte_len(len, INPUT_OFFSET) as u64
152152
));
153153
group.bench_with_input(
154154
BenchmarkId::new("cuda/arrow_validity/repack", len_label),
@@ -165,7 +165,7 @@ fn benchmark_arrow_validity_repack(c: &mut Criterion) {
165165
.vortex_expect("failed to create validity fixture");
166166

167167
for _ in 0..iters {
168-
let output = test_harness::repack_arrow_validity_buffer(
168+
let output = test_harness::repack_arrow_bitmap(
169169
&input_buffer,
170170
input_offset,
171171
len,
@@ -190,7 +190,7 @@ fn benchmark_arrow_validity_count_nulls(c: &mut Criterion) {
190190

191191
for &(len, len_label) in bench_config::BENCH_SIZES {
192192
group.throughput(Throughput::Bytes(
193-
validity_bitmap_byte_len(len, ARROW_OFFSET) as u64,
193+
arrow_bitmap_byte_len(len, ARROW_OFFSET) as u64
194194
));
195195
group.bench_with_input(
196196
BenchmarkId::new("cuda/arrow_validity/count_nulls", len_label),

‎vortex-cuda/kernels/src/arrow_validity.cu‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,18 +18,21 @@ __device__ uint64_t load_input_word(const uint8_t *const input, int64_t word_idx
1818
if (byte_idx >= input_bytes) {
1919
return 0;
2020
}
21-
if (byte_idx + sizeof(uint64_t) <= input_bytes) {
22-
return reinterpret_cast<const uint64_t *>(input)[word_idx];
21+
const uint64_t available_bytes = input_bytes - byte_idx;
22+
// The host aligns input down within its allocation and adjusts the bit offset.
23+
if (available_bytes >= sizeof(uint64_t)) {
24+
return reinterpret_cast<const uint64_t *>(input + byte_idx)[0];
2325
}
24-
// Trailing partial word: assemble byte-by-byte to avoid reading past the buffer.
26+
// Only the final partial word needs byte loads; never overread the logical tail.
27+
const uint64_t min_bytes = min(static_cast<uint64_t>(sizeof(uint64_t)), available_bytes);
2528
uint64_t word = 0;
26-
for (uint64_t i = byte_idx; i < input_bytes; i++) {
27-
word |= static_cast<uint64_t>(input[i]) << ((i - byte_idx) * 8);
29+
for (uint64_t i = 0; i < min_bytes; i++) {
30+
word |= static_cast<uint64_t>(input[byte_idx + i]) << (i * 8);
2831
}
2932
return word;
3033
}
3134

32-
// Build one output word for sliced validity. The row bits are the same, but
35+
// Build one output word for a sliced bitmap. The row bits are the same, but
3336
// row 0 may live at a different bit position in the source and Arrow bitmaps.
3437
// For example, `input_offset = 5` and `arrow_offset = 0` shifts row0 from bit 5
3538
// in the input bitmap to bit 0 in the Arrow bitmap.
@@ -39,7 +42,7 @@ __device__ uint64_t load_input_word(const uint8_t *const input, int64_t word_idx
3942
// Arrow bitmap: [ row0 ][ row1 ][ row2 ]....
4043
// ^ arrow_offset
4144
//
42-
// Padding bits are cleared so word-sized validity readers can safely over-read.
45+
// Padding bits are cleared for word-sized bitmap readers.
4346
__device__ uint64_t repack_word(const uint8_t *const input,
4447
uint64_t word_idx,
4548
int64_t shift,
@@ -137,7 +140,7 @@ __device__ uint64_t block_sum_to_thread_zero(uint64_t value, uint64_t *const war
137140

138141
} // namespace
139142

140-
// Repack sliced validity when the source bitmap offset does not match the
143+
// Repack a sliced bitmap when the source bit offset does not match the
141144
// Arrow array offset. Each thread writes independent output words.
142145
//
143146
// thread 0 -> output word 0, word N, ...

0 commit comments

Comments
 (0)