Skip to content

Commit 3be09d1

Browse files
connortsui200ax1
authored andcommitted
Execute Widen arrays on CUDA
Device-resident Widen arrays cannot use the CPU fallback. Restore their logical width on the device so narrow integer and decimal children can execute through CUDA. Progress towards #11284. Signed-off-by: Connor Tsui <[email protected]> Signed-off-by: Alexander Droste <[email protected]>
1 parent 23147cf commit 3be09d1

10 files changed

Lines changed: 576 additions & 81 deletions

File tree

‎vortex-cuda/kernels/src/decimal_cast.cu‎

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,17 +6,16 @@
66
#include <stdint.h>
77
#include <type_traits>
88

9-
// Arrow decimal schemas fix the physical values buffer width:
10-
// - Decimal32: 4 bytes per value.
11-
// - Decimal64: 8 bytes per value.
12-
// - Decimal128: 16 bytes per value.
13-
// - Decimal256: 32 bytes per value.
9+
// Widen decimal values buffers to a wider physical width. Two callers use these kernels:
10+
// - The Arrow export casts to the width that the Arrow schema fixes: 4 bytes for Decimal32,
11+
// 8 bytes for Decimal64, 16 bytes for Decimal128, and 32 bytes for Decimal256.
12+
// - The Widen executor casts a narrow decimal child to the canonical width of its decimal
13+
// dtype, from 2 to 32 bytes.
1414
//
15-
// Vortex storage width can differ, so export casts to the schema-implied width.
16-
// Rust-side export rejects narrowing casts because detecting overflow on-device
17-
// would require synchronizing an overflow flag back to the host.
15+
// The Rust side rejects narrowing casts, because detecting overflow on the device would require
16+
// synchronizing an overflow flag back to the host.
1817

19-
// Low 64-bit conversion for Decimal32/64 outputs.
18+
// Low 64-bit conversion for 16-, 32-, and 64-bit outputs.
2019
template <typename Input>
2120
__device__ __forceinline__ int64_t decimal_to_i64(Input value) {
2221
if constexpr (std::is_same_v<Input, int128_t>) {
@@ -42,10 +41,12 @@ __device__ __forceinline__ int128_t decimal_to_i128(Input value) {
4241
}
4342
}
4443

45-
// Convert one value to the Arrow schema's physical width.
44+
// Convert one value to the output physical width.
4645
template <typename Output, typename Input>
4746
__device__ __forceinline__ Output decimal_cast_value(Input value) {
48-
if constexpr (std::is_same_v<Output, int32_t>) {
47+
if constexpr (std::is_same_v<Output, int16_t>) {
48+
return static_cast<int16_t>(decimal_to_i64(value));
49+
} else if constexpr (std::is_same_v<Output, int32_t>) {
4950
return static_cast<int32_t>(decimal_to_i64(value));
5051
} else if constexpr (std::is_same_v<Output, int64_t>) {
5152
return decimal_to_i64(value);
@@ -63,7 +64,7 @@ __device__ __forceinline__ Output decimal_cast_value(Input value) {
6364
}
6465
}
6566

66-
// Cast a contiguous values buffer to the Arrow schema's physical width.
67+
// Cast a contiguous values buffer to the output physical width.
6768
template <typename Input, typename Output>
6869
__device__ void
6970
decimal_cast_device(const Input *__restrict input, Output *__restrict output, uint64_t array_len) {
@@ -76,8 +77,13 @@ decimal_cast_device(const Input *__restrict input, Output *__restrict output, ui
7677
}
7778
}
7879

79-
// Generate Decimal32/64/128/256 cast kernels for one input storage type.
80+
// Generate the i16, i32, i64, i128, and i256 cast kernels for one input storage type.
8081
#define GENERATE_DECIMAL_CAST_KERNELS(input_suffix, InputType) \
82+
extern "C" __global__ void decimal_cast_##input_suffix##_i16(const InputType *__restrict input, \
83+
int16_t *__restrict output, \
84+
uint64_t array_len) { \
85+
decimal_cast_device(input, output, array_len); \
86+
} \
8187
extern "C" __global__ void decimal_cast_##input_suffix##_i32(const InputType *__restrict input, \
8288
int32_t *__restrict output, \
8389
uint64_t array_len) { \

‎vortex-cuda/kernels/src/widen.cu‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
#include "scalar_kernel.cuh"
5+
#include "types.cuh"
6+
#include <type_traits>
7+
8+
template <typename InputT, typename OutputT>
9+
struct WidenOp {
10+
static_assert(sizeof(InputT) < sizeof(OutputT));
11+
static_assert(std::is_signed_v<InputT> == std::is_signed_v<OutputT>);
12+
13+
__device__ OutputT operator()(InputT value) const {
14+
return static_cast<OutputT>(value);
15+
}
16+
};
17+
18+
#define GENERATE_WIDEN(input_suffix, InputT, output_suffix, OutputT) \
19+
extern "C" __global__ void widen_##input_suffix##_##output_suffix(const InputT *__restrict input, \
20+
OutputT *__restrict output, \
21+
uint64_t array_len) { \
22+
scalar_kernel(input, output, array_len, WidenOp<InputT, OutputT> {}); \
23+
}
24+
25+
GENERATE_WIDEN(i8, int8_t, i16, int16_t)
26+
GENERATE_WIDEN(i8, int8_t, i32, int32_t)
27+
GENERATE_WIDEN(i8, int8_t, i64, int64_t)
28+
GENERATE_WIDEN(i16, int16_t, i32, int32_t)
29+
GENERATE_WIDEN(i16, int16_t, i64, int64_t)
30+
GENERATE_WIDEN(i32, int32_t, i64, int64_t)
31+
GENERATE_WIDEN(u8, uint8_t, u16, uint16_t)
32+
GENERATE_WIDEN(u8, uint8_t, u32, uint32_t)
33+
GENERATE_WIDEN(u8, uint8_t, u64, uint64_t)
34+
GENERATE_WIDEN(u16, uint16_t, u32, uint32_t)
35+
GENERATE_WIDEN(u16, uint16_t, u64, uint64_t)
36+
GENERATE_WIDEN(u32, uint32_t, u64, uint64_t)

‎vortex-cuda/src/arrow/canonical.rs‎

Lines changed: 3 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -44,16 +44,13 @@ use vortex::array::arrays::varbinview::VarBinViewDataParts;
4444
use vortex::array::buffer::BufferHandle;
4545
use vortex::array::buffer::DeviceBuffer;
4646
use vortex::array::builtins::ArrayBuiltins;
47-
use vortex::array::match_each_decimal_value_type;
4847
use vortex::array::optimizer::ArrayOptimizer;
4948
use vortex::array::validity::Validity;
5049
use vortex::buffer::Buffer;
5150
use vortex::dtype::DType;
5251
use vortex::dtype::DecimalType;
53-
use vortex::dtype::NativeDecimalType;
5452
use vortex::dtype::Nullability;
5553
use vortex::dtype::PType;
56-
use vortex::dtype::i256;
5754
use vortex::encodings::fsst::FSST;
5855
use vortex::error::VortexResult;
5956
use vortex::error::vortex_bail;
@@ -62,7 +59,6 @@ use vortex::error::vortex_ensure_eq;
6259
use vortex::error::vortex_err;
6360
use vortex::extension::datetime::AnyTemporal;
6461
use vortex_cuda_driver::CudaSlice;
65-
use vortex_cuda_driver::DeviceRepr;
6662
use vortex_cuda_driver::LaunchConfig;
6763
use vortex_cuda_driver::PushKernelArg;
6864
use vortex_onpair::OnPair;
@@ -92,6 +88,7 @@ use crate::executor::execute_validity_cuda;
9288
use crate::kernel::DecodedVarBin;
9389
use crate::kernel::decode_fsst_varbin;
9490
use crate::kernel::decode_onpair_varbin;
91+
use crate::kernel::widen_decimal_values;
9592
use crate::stream::zero_padding;
9693

9794
const BITMAP_THREADS_PER_BLOCK: u32 = 256;
@@ -475,73 +472,11 @@ async fn export_decimal_values(
475472
"cannot export decimal values from {values_type} storage to Arrow {target_type}: narrowing would require a device-to-host overflow check",
476473
);
477474
}
478-
let values = ctx.ensure_on_device(values).await?;
479475
if values_type == target_type {
480-
return Ok(values);
476+
return ctx.ensure_on_device(values).await;
481477
}
482478

483-
match_each_decimal_value_type!(values_type, |S| {
484-
export_decimal_values_from::<S>(values, target_type, len, ctx).await
485-
})
486-
}
487-
488-
/// Dispatch from a concrete storage type `S` to the Arrow-required output width.
489-
async fn export_decimal_values_from<S>(
490-
values: BufferHandle,
491-
target_type: DecimalType,
492-
len: usize,
493-
ctx: &mut CudaExecutionCtx,
494-
) -> VortexResult<BufferHandle>
495-
where
496-
S: NativeDecimalType + DeviceRepr,
497-
{
498-
match target_type {
499-
DecimalType::I32 => decimal_cast::<S, i32>(values, len, ctx).await,
500-
DecimalType::I64 => decimal_cast::<S, i64>(values, len, ctx).await,
501-
DecimalType::I128 => decimal_cast::<S, i128>(values, len, ctx).await,
502-
DecimalType::I256 => decimal_cast::<S, i256>(values, len, ctx).await,
503-
target_type => {
504-
vortex_bail!("cannot export DecimalArray as Arrow decimal value type {target_type}")
505-
}
506-
}
507-
}
508-
509-
/// Launches the CUDA kernel that casts from Vortex storage type `S` to Arrow output type `D`.
510-
///
511-
/// The caller must ensure this is not a narrowing cast.
512-
async fn decimal_cast<S, D>(
513-
values: BufferHandle,
514-
len: usize,
515-
ctx: &mut CudaExecutionCtx,
516-
) -> VortexResult<BufferHandle>
517-
where
518-
S: NativeDecimalType + DeviceRepr,
519-
D: NativeDecimalType + DeviceRepr,
520-
{
521-
if len == 0 {
522-
return ctx
523-
.ensure_on_device(BufferHandle::new_host(
524-
Buffer::<D>::empty().into_byte_buffer(),
525-
))
526-
.await;
527-
}
528-
529-
let mut output_buffer = ctx.device_alloc::<D>(len)?;
530-
531-
let values_view = values.cuda_view::<S>()?;
532-
let len_u64 = len as u64;
533-
let cuda_function = ctx.load_function_with_suffixes(
534-
"decimal_cast",
535-
&[&S::DECIMAL_TYPE.to_string(), &D::DECIMAL_TYPE.to_string()],
536-
)?;
537-
538-
ctx.launch_kernel(&cuda_function, len, |args| {
539-
args.arg(&values_view).arg(&mut output_buffer).arg(&len_u64);
540-
})?;
541-
542-
Ok(BufferHandle::new_device(Arc::new(CudaDeviceBuffer::new(
543-
output_buffer,
544-
)?)))
479+
widen_decimal_values(values, values_type, target_type, len, ctx).await
545480
}
546481

547482
/// Export Vortex binary views as an Arrow Device array with `Utf8View`/`BinaryView` layout.

‎vortex-cuda/src/dynamic_dispatch/plan_builder.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,8 @@ fn is_dyn_dispatch_compatible(array: &ArrayRef) -> bool {
133133
_ => false,
134134
};
135135
}
136+
// TODO(connor): Add Widen fusion after fixing sign extension of signed narrow BitPacked and
137+
// FoR children in dynamic_dispatch.cu. Those paths currently zero-extend to the output width.
136138
(id == FoR.id() && array.as_::<FoR>().references().is_global())
137139
|| id == ZigZag.id()
138140
|| id == Primitive.id()
Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
//! Widening casts of decimal values buffers on the device.
5+
//!
6+
//! The Arrow export widens decimal values to the width that the Arrow schema fixes, and the
7+
//! Widen executor widens a narrow decimal child to the canonical width of its decimal dtype. Both
8+
//! launch the `decimal_cast` kernels through this module.
9+
//!
10+
//! These casts only widen. A narrowing cast can overflow, and to detect that on the device, the
11+
//! kernel would need an overflow flag that synchronizes with the host. So this module rejects a
12+
//! narrowing cast, and each caller decides how to handle values that are wider than its target.
13+
14+
use std::sync::Arc;
15+
16+
use vortex::array::buffer::BufferHandle;
17+
use vortex::array::match_each_decimal_value_type;
18+
use vortex::buffer::Buffer;
19+
use vortex::dtype::DecimalType;
20+
use vortex::dtype::NativeDecimalType;
21+
use vortex::dtype::i256;
22+
use vortex::error::VortexResult;
23+
use vortex::error::vortex_ensure;
24+
use vortex::error::vortex_ensure_eq;
25+
use vortex::error::vortex_panic;
26+
use vortex_cuda_driver::DeviceRepr;
27+
use vortex_cuda_driver::PushKernelArg;
28+
29+
use crate::CudaBufferExt;
30+
use crate::CudaDeviceBuffer;
31+
use crate::CudaExecutionCtx;
32+
33+
/// Widen `len` decimal values from `values_type` to the wider `target_type` on the device.
34+
///
35+
/// The values can be on the host or on the device, and the result is on the device. Each value is
36+
/// sign-extended, so the result holds the same values as the input.
37+
///
38+
/// # Errors
39+
///
40+
/// Returns an error if `target_type` is not wider than `values_type`, if the buffer does not
41+
/// contain `len` complete values, or if the copy to the device or the kernel launch fails.
42+
pub(crate) async fn widen_decimal_values(
43+
values: BufferHandle,
44+
values_type: DecimalType,
45+
target_type: DecimalType,
46+
len: usize,
47+
ctx: &mut CudaExecutionCtx,
48+
) -> VortexResult<BufferHandle> {
49+
vortex_ensure!(
50+
values_type < target_type,
51+
"a decimal cast on the GPU must widen to a type wider than {values_type}, got \
52+
{target_type}"
53+
);
54+
vortex_ensure_eq!(values.len(), len * values_type.byte_width());
55+
56+
let values = ctx.ensure_on_device(values).await?;
57+
58+
match_each_decimal_value_type!(values_type, |S| {
59+
decimal_cast_to::<S>(values, target_type, len, ctx).await
60+
})
61+
}
62+
63+
/// Dispatch [`decimal_cast`] from `S` on the runtime `target_type`.
64+
async fn decimal_cast_to<S>(
65+
values: BufferHandle,
66+
target_type: DecimalType,
67+
len: usize,
68+
ctx: &mut CudaExecutionCtx,
69+
) -> VortexResult<BufferHandle>
70+
where
71+
S: NativeDecimalType + DeviceRepr,
72+
{
73+
match target_type {
74+
DecimalType::I8 => vortex_panic!(
75+
"widen_decimal_values ensures a target wider than the values type, and no decimal \
76+
type is narrower than i8, got {target_type}"
77+
),
78+
DecimalType::I16 => decimal_cast::<S, i16>(values, len, ctx).await,
79+
DecimalType::I32 => decimal_cast::<S, i32>(values, len, ctx).await,
80+
DecimalType::I64 => decimal_cast::<S, i64>(values, len, ctx).await,
81+
DecimalType::I128 => decimal_cast::<S, i128>(values, len, ctx).await,
82+
DecimalType::I256 => decimal_cast::<S, i256>(values, len, ctx).await,
83+
}
84+
}
85+
86+
/// Launch the `decimal_cast_<S>_<D>` kernel, which casts device values of type `S` to type `D`.
87+
///
88+
/// The cast must widen. A narrowing cast truncates each value with no overflow check.
89+
async fn decimal_cast<S, D>(
90+
values: BufferHandle,
91+
len: usize,
92+
ctx: &mut CudaExecutionCtx,
93+
) -> VortexResult<BufferHandle>
94+
where
95+
S: NativeDecimalType + DeviceRepr,
96+
D: NativeDecimalType + DeviceRepr,
97+
{
98+
if len == 0 {
99+
return ctx
100+
.ensure_on_device(BufferHandle::new_host(
101+
Buffer::<D>::empty().into_byte_buffer(),
102+
))
103+
.await;
104+
}
105+
106+
let mut output_buffer = ctx.device_alloc::<D>(len)?;
107+
108+
let values_view = values.cuda_view::<S>()?;
109+
let len_u64 = len as u64;
110+
let cuda_function = ctx.load_function_with_suffixes(
111+
"decimal_cast",
112+
&[&S::DECIMAL_TYPE.to_string(), &D::DECIMAL_TYPE.to_string()],
113+
)?;
114+
115+
ctx.launch_kernel(&cuda_function, len, |args| {
116+
args.arg(&values_view).arg(&mut output_buffer).arg(&len_u64);
117+
})?;
118+
119+
Ok(BufferHandle::new_device(Arc::new(CudaDeviceBuffer::new(
120+
output_buffer,
121+
)?)))
122+
}

‎vortex-cuda/src/kernel/encodings/mod.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
// SPDX-License-Identifier: Apache-2.0
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

4+
//! CUDA executors for encoded arrays.
5+
//!
6+
//! Executors decode children through the CUDA execution context and return canonical arrays.
7+
48
mod alp;
59
mod bitpacked;
610
mod date_time_parts;
@@ -11,6 +15,7 @@ mod fsst;
1115
mod onpair;
1216
mod runend;
1317
mod sequence;
18+
mod widen;
1419
mod zigzag;
1520
mod zstd;
1621
mod zstd_buffers;
@@ -29,6 +34,7 @@ pub(crate) use onpair::OnPairExecutor;
2934
pub(crate) use onpair::decode_onpair_varbin;
3035
pub(crate) use runend::RunEndExecutor;
3136
pub(crate) use sequence::SequenceExecutor;
37+
pub(crate) use widen::WidenExecutor;
3238
pub(crate) use zigzag::ZigZagExecutor;
3339
pub(crate) use zstd::ZstdExecutor;
3440
pub use zstd::ZstdKernelPrep;

0 commit comments

Comments
 (0)