Skip to content

Commit ec0fd92

Browse files
authored
feat(cuda): Execute Widen arrays on CUDA (#11303)
## Summary - Tracking Issue: #10701 Progress towards #11284 (part 1). CUDA has no kernel for `vortex.widen`, and the CPU fallback only runs when every buffer is on the host, so executing a device-resident `Widen` fails with "CPU fallback with device-resident buffers is not supported". After CD5, every DecimalByteParts array with a narrow MSP reads from a file with an integer `Widen` as its MSP, so CD5 can't merge until GPU reads can execute it. ## Changes `Widen` is not fused into the dynamic dispatch path. Dynamic dispatch zero-extends signed narrow BitPacked and FoR children, so fusing a signed `Widen` there needs care to sign-extend instead. The `TODO` in `plan_builder.rs` tracks that. Until then, `WidenExecutor` executes the child on the device and widens it in a separate kernel. Decimal storage that is wider than the `Widen`'s logical type is an error instead of a narrowing copy on the device. Narrowing needs either an overflow check or trust in the stored precision, and that choice is the open question in part 2 of #11284. The decimal path reuses the kernels in `decimal_cast.cu`, and the Arrow export now shares `widen_decimal_values` with it, so there is one copy of the widening code. The tests compare every integer and decimal widening pair against the CPU `Widen` (with nulls, negative values, a FoR child, and the kernel's block boundaries). They only run in the GPU CI. Signed-off-by: Alexander Droste <[email protected]>
1 parent ced8ec9 commit ec0fd92

10 files changed

Lines changed: 608 additions & 140 deletions

File tree

‎vortex-cuda/kernels/src/decimal_cast.cu‎

Lines changed: 49 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -6,64 +6,43 @@
66
#include <stdint.h>
77
#include <type_traits>
88

9-
// Arrow decimal schemas fix the physical values buffer width:
10-
// - Decimal32: 4 bytes per value.
11-
// - Decimal64: 8 bytes per value.
12-
// - Decimal128: 16 bytes per value.
13-
// - Decimal256: 32 bytes per value.
9+
// Widen decimal values buffers to a wider physical width. Two callers use these kernels:
10+
// - The Arrow export casts to the width that the Arrow schema fixes: 4 bytes for Decimal32,
11+
// 8 bytes for Decimal64, 16 bytes for Decimal128, and 32 bytes for Decimal256.
12+
// - The Widen executor casts a narrow decimal child to the canonical width of its decimal
13+
// dtype, from 2 to 32 bytes.
1414
//
15-
// Vortex storage width can differ, so export casts to the schema-implied width.
16-
// Rust-side export rejects narrowing casts because detecting overflow on-device
17-
// would require synchronizing an overflow flag back to the host.
15+
// Only widening pairs are instantiated. A narrowing cast can overflow, and detecting that on the
16+
// device would require synchronizing an overflow flag back to the host, so the Rust side rejects
17+
// narrowing before it loads a kernel.
1818

19-
// Low 64-bit conversion for Decimal32/64 outputs.
20-
template <typename Input>
21-
__device__ __forceinline__ int64_t decimal_to_i64(Input value) {
22-
if constexpr (std::is_same_v<Input, int128_t>) {
23-
return value.lo;
24-
} else if constexpr (std::is_same_v<Input, int256_t>) {
25-
return value.parts[0];
26-
} else {
27-
return static_cast<int64_t>(value);
28-
}
29-
}
30-
31-
// 128-bit conversion for Decimal128 outputs.
32-
template <typename Input>
33-
__device__ __forceinline__ int128_t decimal_to_i128(Input value) {
34-
if constexpr (std::is_same_v<Input, int128_t>) {
35-
return value;
36-
} else if constexpr (std::is_same_v<Input, int256_t>) {
37-
return int128_t {value.parts[0], value.parts[1]};
38-
} else {
39-
const int64_t lo = static_cast<int64_t>(value);
40-
const int64_t hi = value < 0 ? -1 : 0;
41-
return int128_t {lo, hi};
42-
}
43-
}
44-
45-
// Convert one value to the Arrow schema's physical width.
19+
// Sign-extend one value to the wider output width in a single step, with no intermediate width.
20+
// Native outputs use one `static_cast`. The 128-bit and 256-bit outputs keep the input's words
21+
// and fill the upper words with its sign.
4622
template <typename Output, typename Input>
4723
__device__ __forceinline__ Output decimal_cast_value(Input value) {
48-
if constexpr (std::is_same_v<Output, int32_t>) {
49-
return static_cast<int32_t>(decimal_to_i64(value));
50-
} else if constexpr (std::is_same_v<Output, int64_t>) {
51-
return decimal_to_i64(value);
52-
} else if constexpr (std::is_same_v<Output, int128_t>) {
53-
return decimal_to_i128(value);
24+
static_assert(sizeof(Input) < sizeof(Output), "decimal casts on the GPU only widen");
25+
26+
if constexpr (std::is_same_v<Output, int128_t>) {
27+
static_assert(std::is_integral_v<Input> && std::is_signed_v<Input>);
28+
const int64_t lo = static_cast<int64_t>(value);
29+
return int128_t {lo, lo < 0 ? -1 : 0};
30+
} else if constexpr (std::is_same_v<Output, int256_t> && std::is_same_v<Input, int128_t>) {
31+
const int64_t sign = value.hi < 0 ? -1 : 0;
32+
return int256_t {{value.lo, value.hi, sign, sign}};
33+
} else if constexpr (std::is_same_v<Output, int256_t>) {
34+
static_assert(std::is_integral_v<Input> && std::is_signed_v<Input>);
35+
const int64_t lo = static_cast<int64_t>(value);
36+
const int64_t sign = lo < 0 ? -1 : 0;
37+
return int256_t {{lo, sign, sign, sign}};
5438
} else {
55-
static_assert(std::is_same_v<Output, int256_t>);
56-
if constexpr (std::is_same_v<Input, int256_t>) {
57-
return value;
58-
} else {
59-
const int128_t value128 = decimal_to_i128(value);
60-
const int64_t sign = value128.hi < 0 ? -1 : 0;
61-
return int256_t {{value128.lo, value128.hi, sign, sign}};
62-
}
39+
static_assert(std::is_integral_v<Input> && std::is_signed_v<Input>);
40+
static_assert(std::is_integral_v<Output> && std::is_signed_v<Output>);
41+
return static_cast<Output>(value);
6342
}
6443
}
6544

66-
// Cast a contiguous values buffer to the Arrow schema's physical width.
45+
// Cast a contiguous values buffer to the output physical width.
6746
template <typename Input, typename Output>
6847
__device__ void
6948
decimal_cast_device(const Input *__restrict input, Output *__restrict output, uint64_t array_len) {
@@ -76,28 +55,26 @@ decimal_cast_device(const Input *__restrict input, Output *__restrict output, ui
7655
}
7756
}
7857

79-
// Generate Decimal32/64/128/256 cast kernels for one input storage type.
80-
#define GENERATE_DECIMAL_CAST_KERNELS(input_suffix, InputType) \
81-
extern "C" __global__ void decimal_cast_##input_suffix##_i32(const InputType *__restrict input, \
82-
int32_t *__restrict output, \
83-
uint64_t array_len) { \
84-
decimal_cast_device(input, output, array_len); \
85-
} \
86-
extern "C" __global__ void decimal_cast_##input_suffix##_i64(const InputType *__restrict input, \
87-
int64_t *__restrict output, \
88-
uint64_t array_len) { \
89-
decimal_cast_device(input, output, array_len); \
90-
} \
91-
extern "C" __global__ void decimal_cast_##input_suffix##_i128(const InputType *__restrict input, \
92-
int128_t *__restrict output, \
93-
uint64_t array_len) { \
94-
decimal_cast_device(input, output, array_len); \
95-
} \
96-
extern "C" __global__ void decimal_cast_##input_suffix##_i256(const InputType *__restrict input, \
97-
int256_t *__restrict output, \
98-
uint64_t array_len) { \
58+
#define GENERATE_DECIMAL_CAST(input_suffix, InputType, output_suffix, OutputType) \
59+
extern "C" __global__ void decimal_cast_##input_suffix##_##output_suffix( \
60+
const InputType *__restrict input, \
61+
OutputType *__restrict output, \
62+
uint64_t array_len) { \
9963
decimal_cast_device(input, output, array_len); \
10064
}
10165

102-
FOR_EACH_SIGNED_INT(GENERATE_DECIMAL_CAST_KERNELS)
103-
FOR_EACH_LARGE_DECIMAL(GENERATE_DECIMAL_CAST_KERNELS)
66+
GENERATE_DECIMAL_CAST(i8, int8_t, i16, int16_t)
67+
GENERATE_DECIMAL_CAST(i8, int8_t, i32, int32_t)
68+
GENERATE_DECIMAL_CAST(i8, int8_t, i64, int64_t)
69+
GENERATE_DECIMAL_CAST(i8, int8_t, i128, int128_t)
70+
GENERATE_DECIMAL_CAST(i8, int8_t, i256, int256_t)
71+
GENERATE_DECIMAL_CAST(i16, int16_t, i32, int32_t)
72+
GENERATE_DECIMAL_CAST(i16, int16_t, i64, int64_t)
73+
GENERATE_DECIMAL_CAST(i16, int16_t, i128, int128_t)
74+
GENERATE_DECIMAL_CAST(i16, int16_t, i256, int256_t)
75+
GENERATE_DECIMAL_CAST(i32, int32_t, i64, int64_t)
76+
GENERATE_DECIMAL_CAST(i32, int32_t, i128, int128_t)
77+
GENERATE_DECIMAL_CAST(i32, int32_t, i256, int256_t)
78+
GENERATE_DECIMAL_CAST(i64, int64_t, i128, int128_t)
79+
GENERATE_DECIMAL_CAST(i64, int64_t, i256, int256_t)
80+
GENERATE_DECIMAL_CAST(i128, int128_t, i256, int256_t)

‎vortex-cuda/kernels/src/widen.cu‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
#include "scalar_kernel.cuh"
5+
#include "types.cuh"
6+
#include <type_traits>
7+
8+
template <typename InputT, typename OutputT>
9+
struct WidenOp {
10+
static_assert(sizeof(InputT) < sizeof(OutputT));
11+
static_assert(std::is_signed_v<InputT> == std::is_signed_v<OutputT>);
12+
13+
__device__ OutputT operator()(InputT value) const {
14+
return static_cast<OutputT>(value);
15+
}
16+
};
17+
18+
#define GENERATE_WIDEN(input_suffix, InputT, output_suffix, OutputT) \
19+
extern "C" __global__ void widen_##input_suffix##_##output_suffix(const InputT *__restrict input, \
20+
OutputT *__restrict output, \
21+
uint64_t array_len) { \
22+
scalar_kernel(input, output, array_len, WidenOp<InputT, OutputT> {}); \
23+
}
24+
25+
GENERATE_WIDEN(i8, int8_t, i16, int16_t)
26+
GENERATE_WIDEN(i8, int8_t, i32, int32_t)
27+
GENERATE_WIDEN(i8, int8_t, i64, int64_t)
28+
GENERATE_WIDEN(i16, int16_t, i32, int32_t)
29+
GENERATE_WIDEN(i16, int16_t, i64, int64_t)
30+
GENERATE_WIDEN(i32, int32_t, i64, int64_t)
31+
GENERATE_WIDEN(u8, uint8_t, u16, uint16_t)
32+
GENERATE_WIDEN(u8, uint8_t, u32, uint32_t)
33+
GENERATE_WIDEN(u8, uint8_t, u64, uint64_t)
34+
GENERATE_WIDEN(u16, uint16_t, u32, uint32_t)
35+
GENERATE_WIDEN(u16, uint16_t, u64, uint64_t)
36+
GENERATE_WIDEN(u32, uint32_t, u64, uint64_t)

‎vortex-cuda/src/arrow/canonical.rs‎

Lines changed: 3 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -44,16 +44,13 @@ use vortex::array::arrays::varbinview::VarBinViewDataParts;
4444
use vortex::array::buffer::BufferHandle;
4545
use vortex::array::buffer::DeviceBuffer;
4646
use vortex::array::builtins::ArrayBuiltins;
47-
use vortex::array::match_each_decimal_value_type;
4847
use vortex::array::optimizer::ArrayOptimizer;
4948
use vortex::array::validity::Validity;
5049
use vortex::buffer::Buffer;
5150
use vortex::dtype::DType;
5251
use vortex::dtype::DecimalType;
53-
use vortex::dtype::NativeDecimalType;
5452
use vortex::dtype::Nullability;
5553
use vortex::dtype::PType;
56-
use vortex::dtype::i256;
5754
use vortex::encodings::fsst::FSST;
5855
use vortex::error::VortexResult;
5956
use vortex::error::vortex_bail;
@@ -62,7 +59,6 @@ use vortex::error::vortex_ensure_eq;
6259
use vortex::error::vortex_err;
6360
use vortex::extension::datetime::AnyTemporal;
6461
use vortex_cuda_driver::CudaSlice;
65-
use vortex_cuda_driver::DeviceRepr;
6662
use vortex_cuda_driver::LaunchConfig;
6763
use vortex_cuda_driver::PushKernelArg;
6864
use vortex_onpair::OnPair;
@@ -92,6 +88,7 @@ use crate::executor::execute_validity_cuda;
9288
use crate::kernel::DecodedVarBin;
9389
use crate::kernel::decode_fsst_varbin;
9490
use crate::kernel::decode_onpair_varbin;
91+
use crate::kernel::widen_decimal_values;
9592
use crate::stream::zero_padding;
9693

9794
const BITMAP_THREADS_PER_BLOCK: u32 = 256;
@@ -475,73 +472,11 @@ async fn export_decimal_values(
475472
"cannot export decimal values from {values_type} storage to Arrow {target_type}: narrowing would require a device-to-host overflow check",
476473
);
477474
}
478-
let values = ctx.ensure_on_device(values).await?;
479475
if values_type == target_type {
480-
return Ok(values);
476+
return ctx.ensure_on_device(values).await;
481477
}
482478

483-
match_each_decimal_value_type!(values_type, |S| {
484-
export_decimal_values_from::<S>(values, target_type, len, ctx).await
485-
})
486-
}
487-
488-
/// Dispatch from a concrete storage type `S` to the Arrow-required output width.
489-
async fn export_decimal_values_from<S>(
490-
values: BufferHandle,
491-
target_type: DecimalType,
492-
len: usize,
493-
ctx: &mut CudaExecutionCtx,
494-
) -> VortexResult<BufferHandle>
495-
where
496-
S: NativeDecimalType + DeviceRepr,
497-
{
498-
match target_type {
499-
DecimalType::I32 => decimal_cast::<S, i32>(values, len, ctx).await,
500-
DecimalType::I64 => decimal_cast::<S, i64>(values, len, ctx).await,
501-
DecimalType::I128 => decimal_cast::<S, i128>(values, len, ctx).await,
502-
DecimalType::I256 => decimal_cast::<S, i256>(values, len, ctx).await,
503-
target_type => {
504-
vortex_bail!("cannot export DecimalArray as Arrow decimal value type {target_type}")
505-
}
506-
}
507-
}
508-
509-
/// Launches the CUDA kernel that casts from Vortex storage type `S` to Arrow output type `D`.
510-
///
511-
/// The caller must ensure this is not a narrowing cast.
512-
async fn decimal_cast<S, D>(
513-
values: BufferHandle,
514-
len: usize,
515-
ctx: &mut CudaExecutionCtx,
516-
) -> VortexResult<BufferHandle>
517-
where
518-
S: NativeDecimalType + DeviceRepr,
519-
D: NativeDecimalType + DeviceRepr,
520-
{
521-
if len == 0 {
522-
return ctx
523-
.ensure_on_device(BufferHandle::new_host(
524-
Buffer::<D>::empty().into_byte_buffer(),
525-
))
526-
.await;
527-
}
528-
529-
let mut output_buffer = ctx.device_alloc::<D>(len)?;
530-
531-
let values_view = values.cuda_view::<S>()?;
532-
let len_u64 = len as u64;
533-
let cuda_function = ctx.load_function_with_suffixes(
534-
"decimal_cast",
535-
&[&S::DECIMAL_TYPE.to_string(), &D::DECIMAL_TYPE.to_string()],
536-
)?;
537-
538-
ctx.launch_kernel(&cuda_function, len, |args| {
539-
args.arg(&values_view).arg(&mut output_buffer).arg(&len_u64);
540-
})?;
541-
542-
Ok(BufferHandle::new_device(Arc::new(CudaDeviceBuffer::new(
543-
output_buffer,
544-
)?)))
479+
widen_decimal_values(values, values_type, target_type, len, ctx).await
545480
}
546481

547482
/// Export Vortex binary views as an Arrow Device array with `Utf8View`/`BinaryView` layout.

‎vortex-cuda/src/dynamic_dispatch/plan_builder.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,8 @@ fn is_dyn_dispatch_compatible(array: &ArrayRef) -> bool {
133133
_ => false,
134134
};
135135
}
136+
// TODO(connor): Add Widen fusion after fixing sign extension of signed narrow BitPacked and
137+
// FoR children in dynamic_dispatch.cu. Those paths currently zero-extend to the output width.
136138
(id == FoR.id() && array.as_::<FoR>().references().is_global())
137139
|| id == ZigZag.id()
138140
|| id == Primitive.id()

0 commit comments

Comments
 (0)