Skip to content

Commit 7c2df2c

Browse files
authored
feat(cuda): make GPU dispatch mode configurable (#7621)
Allow for explicitly choosing dyn dispatch or standalone kernels. We primarily do this to be able to test and bench the standalone CUDA kernels. In a follow up, I'll change the logic to prefer standalone kernels, if available for an array tree, over dyn dispatch. Signed-off-by: Alexander Droste <[email protected]>
1 parent 193d655 commit 7c2df2c

8 files changed

Lines changed: 69 additions & 0 deletions

File tree

‎vortex-cuda/benches/alp_cuda.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ use vortex::encodings::alp::ALPFloat;
3131
use vortex::encodings::alp::alp_encode;
3232
use vortex::error::VortexExpect;
3333
use vortex::session::VortexSession;
34+
use vortex_cuda::CudaDispatchMode;
3435
use vortex_cuda::CudaSession;
3536
use vortex_cuda::executor::CudaArrayExt;
3637
use vortex_cuda_macros::cuda_available;
@@ -109,6 +110,7 @@ where
109110

110111
let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())
111112
.vortex_expect("failed to create execution context")
113+
.with_dispatch_mode(CudaDispatchMode::StandaloneOnly)
112114
.with_launch_strategy(Arc::new(timed));
113115

114116
for _ in 0..iters {

‎vortex-cuda/benches/bitpacked_cuda.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ use vortex::encodings::fastlanes::BitPackedData;
3131
use vortex::encodings::fastlanes::unpack_iter::BitPacked;
3232
use vortex::error::VortexExpect;
3333
use vortex::session::VortexSession;
34+
use vortex_cuda::CudaDispatchMode;
3435
use vortex_cuda::CudaSession;
3536
use vortex_cuda::executor::CudaArrayExt;
3637
use vortex_cuda_macros::cuda_available;
@@ -127,6 +128,7 @@ where
127128

128129
let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())
129130
.vortex_expect("failed to create execution context")
131+
.with_dispatch_mode(CudaDispatchMode::StandaloneOnly)
130132
.with_launch_strategy(Arc::new(timed));
131133

132134
for _ in 0..iters {
@@ -172,6 +174,7 @@ where
172174

173175
let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())
174176
.vortex_expect("failed to create execution context")
177+
.with_dispatch_mode(CudaDispatchMode::StandaloneOnly)
175178
.with_launch_strategy(Arc::new(timed));
176179

177180
for _ in 0..iters {

‎vortex-cuda/benches/date_time_parts_cuda.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ use vortex::error::VortexExpect;
3030
use vortex::extension::datetime::TimeUnit;
3131
use vortex::extension::datetime::Timestamp;
3232
use vortex::session::VortexSession;
33+
use vortex_cuda::CudaDispatchMode;
3334
use vortex_cuda::CudaSession;
3435
use vortex_cuda::executor::CudaArrayExt;
3536
use vortex_cuda_macros::cuda_available;
@@ -68,6 +69,7 @@ fn benchmark_datetimeparts(c: &mut Criterion) {
6869

6970
let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())
7071
.vortex_expect("failed to create execution context")
72+
.with_dispatch_mode(CudaDispatchMode::StandaloneOnly)
7173
.with_launch_strategy(Arc::new(timed));
7274

7375
for _ in 0..iters {

‎vortex-cuda/benches/dict_cuda.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ use vortex::buffer::Buffer;
2626
use vortex::dtype::NativePType;
2727
use vortex::error::VortexExpect;
2828
use vortex::session::VortexSession;
29+
use vortex_cuda::CudaDispatchMode;
2930
use vortex_cuda::CudaSession;
3031
use vortex_cuda::executor::CudaArrayExt;
3132
use vortex_cuda_macros::cuda_available;
@@ -96,6 +97,7 @@ where
9697

9798
let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())
9899
.vortex_expect("failed to create execution context")
100+
.with_dispatch_mode(CudaDispatchMode::StandaloneOnly)
99101
.with_launch_strategy(Arc::new(timed));
100102

101103
for _ in 0..iters {

‎vortex-cuda/benches/for_cuda.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ use vortex::encodings::fastlanes::FoRArray;
3333
use vortex::error::VortexExpect;
3434
use vortex::scalar::Scalar;
3535
use vortex::session::VortexSession;
36+
use vortex_cuda::CudaDispatchMode;
3637
use vortex_cuda::CudaSession;
3738
use vortex_cuda::executor::CudaArrayExt;
3839
use vortex_cuda_macros::cuda_available;
@@ -91,6 +92,7 @@ where
9192

9293
let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())
9394
.vortex_expect("failed to create execution context")
95+
.with_dispatch_mode(CudaDispatchMode::StandaloneOnly)
9496
.with_launch_strategy(Arc::new(timed));
9597

9698
for _ in 0..iters {
@@ -129,6 +131,7 @@ where
129131

130132
let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())
131133
.vortex_expect("failed to create execution context")
134+
.with_dispatch_mode(CudaDispatchMode::StandaloneOnly)
132135
.with_launch_strategy(Arc::new(timed));
133136

134137
for _ in 0..iters {

‎vortex-cuda/src/executor.rs‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,25 @@ impl CudaKernelEvents {
5757
}
5858
}
5959

60+
/// Controls which GPU dispatch strategy is used when executing arrays.
61+
///
62+
/// By default, `execute_cuda` tries fused dynamic dispatch first,
63+
/// then falls back to standalone per-encoding kernels. Benchmarks and tests
64+
/// can force a specific strategy to get stable, isolated measurements.
65+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
66+
pub enum CudaDispatchMode {
67+
/// Automatically choose the best strategy (current default behavior).
68+
/// Tries fused → partially-fused → standalone → CPU fallback.
69+
#[default]
70+
Auto,
71+
/// Only use standalone per-encoding `CudaExecute` kernels.
72+
/// Skips the fused/partially-fused dynamic dispatch planner entirely.
73+
StandaloneOnly,
74+
/// Only use fused or partially-fused dynamic dispatch.
75+
/// Returns an error if the array is not dyn-dispatch-compatible.
76+
DynDispatchOnly,
77+
}
78+
6079
/// CUDA execution context.
6180
///
6281
/// Provides access to the CUDA context and stream for kernel execution.
@@ -66,6 +85,7 @@ pub struct CudaExecutionCtx {
6685
ctx: ExecutionCtx,
6786
cuda_session: CudaSession,
6887
strategy: Arc<dyn LaunchStrategy>,
88+
dispatch_mode: CudaDispatchMode,
6989
}
7090

7191
impl CudaExecutionCtx {
@@ -77,6 +97,7 @@ impl CudaExecutionCtx {
7797
ctx,
7898
cuda_session,
7999
strategy: Arc::new(DefaultLaunchStrategy),
100+
dispatch_mode: CudaDispatchMode::Auto,
80101
}
81102
}
82103

@@ -94,6 +115,15 @@ impl CudaExecutionCtx {
94115
self
95116
}
96117

118+
/// Set the dispatch mode for the execution context.
119+
///
120+
/// This controls whether `execute_cuda` uses fused dynamic dispatch,
121+
/// standalone per-encoding kernels, or the automatic (default) strategy.
122+
pub fn with_dispatch_mode(mut self, dispatch_mode: CudaDispatchMode) -> Self {
123+
self.dispatch_mode = dispatch_mode;
124+
self
125+
}
126+
97127
/// Perform an external kernel launch, with events created and logged via the configured
98128
/// [`LaunchStrategy`].
99129
///
@@ -276,6 +306,11 @@ impl CudaExecutionCtx {
276306
self.ctx.session()
277307
}
278308

309+
/// Returns the current dispatch mode.
310+
pub fn dispatch_mode(&self) -> CudaDispatchMode {
311+
self.dispatch_mode
312+
}
313+
279314
/// Returns a reference to the CUDA session.
280315
pub(crate) fn cuda_session(&self) -> &CudaSession {
281316
&self.cuda_session

‎vortex-cuda/src/hybrid_dispatch/mod.rs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,8 +70,21 @@ pub async fn try_gpu_dispatch(
7070
array: &ArrayRef,
7171
ctx: &mut CudaExecutionCtx,
7272
) -> VortexResult<Canonical> {
73+
use crate::executor::CudaDispatchMode;
74+
7375
trace!(encoding = %array.encoding_id(), dtype = ?array.dtype(), len = array.len(), "try GPU dispatch");
7476

77+
// Short-circuit: standalone-only skips the plan builder entirely.
78+
if ctx.dispatch_mode() == CudaDispatchMode::StandaloneOnly {
79+
trace!(encoding = %array.encoding_id(), "standalone-only dispatch");
80+
return ctx
81+
.cuda_session()
82+
.kernel(&array.encoding_id())
83+
.ok_or_else(|| vortex_err!("No CUDA kernel for encoding {:?}", array.encoding_id()))?
84+
.execute(array.clone(), ctx)
85+
.await;
86+
}
87+
7588
match DispatchPlan::new(array)? {
7689
DispatchPlan::Fused(plan) => {
7790
let materialized = plan.materialize(ctx).await?;
@@ -99,6 +112,14 @@ pub async fn try_gpu_dispatch(
99112
materialized.execute(array.len(), ctx)
100113
}
101114
DispatchPlan::Unfused => {
115+
// In dyn-dispatch-only mode, don't fall back to standalone kernels.
116+
if ctx.dispatch_mode() == CudaDispatchMode::DynDispatchOnly {
117+
return Err(vortex_err!(
118+
"Array with encoding {:?} is not dyn-dispatch-compatible",
119+
array.encoding_id()
120+
));
121+
}
122+
102123
// Unfused kernel dispatch fallback.
103124
ctx.cuda_session()
104125
.kernel(&array.encoding_id())

‎vortex-cuda/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ pub use canonical::CanonicalCudaExt;
2727
pub use device_buffer::CudaBufferExt;
2828
pub use device_buffer::CudaDeviceBuffer;
2929
pub use device_read_at::CopyDeviceReadAt;
30+
pub use executor::CudaDispatchMode;
3031
pub use executor::CudaExecutionCtx;
3132
pub use executor::CudaKernelEvents;
3233
use kernel::ALPExecutor;

0 commit comments

Comments
 (0)