Skip to content

Commit e0d17f6

Browse files
authored
fix(cuda): report missing driver library as an error (#10873)
## Summary Makes missing CUDA driver support a normal initialization error instead of a panic. Previously, CudaContext::new and device_count called raw driver functions that panic when the CUDA library cannot be loaded. Callers had to probe for the library first or catch the panic, despite these APIs returning Result. Check driver availability in result::init and return CUDA_ERROR_NOT_INITIALIZED when loading fails. Guard error-name and description lookup too, so formatting that error does not try to call the unavailable driver. With this handled in the driver wrapper, cuda_available no longer needs its own probe, and CudaSession::try_default no longer needs catch_unwind. The raw sys APIs retain their documented panic behavior. Driver tests also stop silently passing on arbitrary context or stream creation failures. They skip on NOT_INITIALIZED or NO_DEVICE and fail on other setup errors. ## Changes - fix(cuda): report a missing driver library as an error - refactor(cuda): share the driver error description lookup - test(cuda): fail driver tests whose CUDA setup fails - Name the shared availability helper is_cuda_driver_present --------- Signed-off-by: Alexander Droste <[email protected]>
1 parent cafee53 commit e0d17f6

5 files changed

Lines changed: 116 additions & 45 deletions

File tree

‎vortex-cuda/driver/src/result.rs‎

Lines changed: 39 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
//! Driver calls returning [`Result`]s, for callers that need the raw handles.
55
66
use std::ffi::CStr;
7+
use std::ffi::c_char;
78
use std::ffi::c_uint;
89
use std::ffi::c_void;
910
use std::fmt;
@@ -19,25 +20,40 @@ pub struct DriverError(pub CUresult);
1920
impl DriverError {
2021
/// The driver's name for the error, such as `CUDA_ERROR_INVALID_VALUE`.
2122
pub fn error_name(&self) -> Result<&'static CStr, DriverError> {
22-
let mut name = std::ptr::null();
23-
// SAFETY: The driver writes a pointer to a static string.
24-
unsafe { sys::cuGetErrorName(self.0, &raw mut name) }.result()?;
25-
// SAFETY: Successful calls return a NUL-terminated static string.
26-
Ok(unsafe { CStr::from_ptr(name) })
23+
self.describe(sys::cuGetErrorName)
2724
}
2825

2926
/// The driver's description of the error.
3027
pub fn error_string(&self) -> Result<&'static CStr, DriverError> {
31-
let mut description = std::ptr::null();
28+
self.describe(sys::cuGetErrorString)
29+
}
30+
31+
/// Returns the static error name or description written by `describe`.
32+
fn describe(
33+
&self,
34+
describe: unsafe fn(CUresult, *mut *const c_char) -> CUresult,
35+
) -> Result<&'static CStr, DriverError> {
36+
// Errors are formatted even when the driver library is missing.
37+
if !is_cuda_driver_present() {
38+
return Err(*self);
39+
}
40+
let mut text = std::ptr::null();
3241
// SAFETY: The driver writes a pointer to a static string.
33-
unsafe { sys::cuGetErrorString(self.0, &raw mut description) }.result()?;
42+
unsafe { describe(self.0, &raw mut text) }.result()?;
3443
// SAFETY: Successful calls return a NUL-terminated static string.
35-
Ok(unsafe { CStr::from_ptr(description) })
44+
Ok(unsafe { CStr::from_ptr(text) })
3645
}
3746
}
3847

3948
impl fmt::Debug for DriverError {
4049
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
50+
if let Some(reason) = sys::driver_load_error() {
51+
return write!(
52+
f,
53+
"DriverError({}, CUDA driver unavailable: {reason})",
54+
self.0.0
55+
);
56+
}
4157
match self.error_name() {
4258
Ok(name) => write!(f, "DriverError({}, {})", self.0.0, name.to_string_lossy()),
4359
Err(_) => write!(f, "DriverError({})", self.0.0),
@@ -47,6 +63,9 @@ impl fmt::Debug for DriverError {
4763

4864
impl fmt::Display for DriverError {
4965
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
66+
if let Some(reason) = sys::driver_load_error() {
67+
return write!(f, "CUDA driver unavailable: {reason}");
68+
}
5069
match (self.error_name(), self.error_string()) {
5170
(Ok(name), Ok(description)) => write!(
5271
f,
@@ -74,11 +93,23 @@ impl CUresult {
7493
}
7594

7695
/// Initializes the driver; repeated calls are cheap.
96+
///
97+
/// Fails with `CUDA_ERROR_NOT_INITIALIZED` if the driver library cannot be loaded, where the raw
98+
/// [`sys`] functions would panic, so that the safe API reports a host without CUDA as an error.
7799
pub fn init() -> Result<(), DriverError> {
100+
if !is_cuda_driver_present() {
101+
return Err(DriverError(CUresult::CUDA_ERROR_NOT_INITIALIZED));
102+
}
78103
// SAFETY: cuInit has no preconditions.
79104
unsafe { sys::cuInit(0) }.result()
80105
}
81106

107+
/// Whether the driver library is loaded, which the raw [`sys`] functions require.
108+
fn is_cuda_driver_present() -> bool {
109+
// SAFETY: Loads the driver library, which any driver call would load as well.
110+
unsafe { sys::is_culib_present() }
111+
}
112+
82113
/// Allocates `bytes` of page-locked host memory with the given `cuMemHostAlloc` flags.
83114
///
84115
/// # Safety

‎vortex-cuda/driver/src/sys.rs‎

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ use std::ffi::c_char;
1919
use std::ffi::c_int;
2020
use std::ffi::c_uint;
2121
use std::ffi::c_void;
22+
use std::fmt;
2223
use std::sync::OnceLock;
2324

2425
use libloading::Library;
@@ -294,15 +295,40 @@ driver_api! {
294295
fn cuGetErrorString(error: CUresult, description: *mut *const c_char);
295296
}
296297

298+
/// Distinguishes an unavailable library from an incompatible library that opened successfully.
299+
#[derive(Debug)]
300+
pub(crate) enum DriverLoadError {
301+
LibraryUnavailable,
302+
MissingSymbol(String),
303+
}
304+
305+
impl fmt::Display for DriverLoadError {
306+
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
307+
match self {
308+
Self::LibraryUnavailable => {
309+
write!(
310+
f,
311+
"none of [{}] could be loaded",
312+
DRIVER_LIBRARIES.join(", ")
313+
)
314+
}
315+
Self::MissingSymbol(reason) => f.write_str(reason),
316+
}
317+
}
318+
}
319+
320+
/// The cached library-loading or symbol-resolution error, loading the driver on first use.
321+
pub(crate) fn driver_load_error() -> Option<&'static DriverLoadError> {
322+
driver().err()
323+
}
324+
297325
/// The process-wide driver entry points, loaded on first use.
298-
fn driver() -> Result<&'static Driver, &'static str> {
299-
static DRIVER: OnceLock<Result<Driver, String>> = OnceLock::new();
326+
fn driver() -> Result<&'static Driver, &'static DriverLoadError> {
327+
static DRIVER: OnceLock<Result<Driver, DriverLoadError>> = OnceLock::new();
300328
DRIVER
301329
.get_or_init(|| {
302-
let library = open_library()
303-
.ok_or_else(|| format!("none of {DRIVER_LIBRARIES:?} could be loaded"))?;
304-
Driver::load(library)
330+
let library = open_library().ok_or(DriverLoadError::LibraryUnavailable)?;
331+
Driver::load(library).map_err(DriverLoadError::MissingSymbol)
305332
})
306333
.as_ref()
307-
.map_err(String::as_str)
308334
}

‎vortex-cuda/driver/src/tests.rs‎

Lines changed: 39 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,18 +13,51 @@ use crate::HostSlice;
1313
use crate::SyncOnDrop;
1414
use crate::sys;
1515

16-
/// A context and two streams, or `None` without a CUDA device.
16+
/// A context and two streams, or `None` without a CUDA driver or device.
17+
///
18+
/// Any other failure fails the test, rather than passing it without running it.
1719
fn streams() -> Option<(Arc<CudaContext>, Arc<CudaStream>, Arc<CudaStream>)> {
18-
// SAFETY: Only probes for the driver library.
19-
if !unsafe { sys::is_culib_present() } {
20+
if !driver_available(sys::driver_load_error()) {
2021
return None;
2122
}
22-
let context = CudaContext::new(0).ok()?;
23-
let first = context.new_stream().ok()?;
24-
let second = context.new_stream().ok()?;
23+
let context = match CudaContext::new(0) {
24+
Ok(context) => context,
25+
Err(DriverError(sys::CUresult::CUDA_ERROR_NO_DEVICE)) => return None,
26+
Err(error) => panic!("CUDA initialization failed: {error}"),
27+
};
28+
let first = context.new_stream().expect("stream");
29+
let second = context.new_stream().expect("stream");
2530
Some((context, first, second))
2631
}
2732

33+
fn driver_available(error: Option<&sys::DriverLoadError>) -> bool {
34+
match error {
35+
None => true,
36+
Some(sys::DriverLoadError::LibraryUnavailable) => false,
37+
Some(error) => panic!("CUDA driver loading failed: {error}"),
38+
}
39+
}
40+
41+
#[test]
42+
fn missing_driver_skips_tests() {
43+
assert!(!driver_available(Some(
44+
&sys::DriverLoadError::LibraryUnavailable
45+
)));
46+
}
47+
48+
#[test]
49+
fn loaded_driver_runs_tests() {
50+
assert!(driver_available(None));
51+
}
52+
53+
#[test]
54+
#[should_panic(expected = "CUDA driver loading failed: cuMemAllocAsync: undefined symbol")]
55+
fn missing_driver_symbol_fails_tests() {
56+
driver_available(Some(&sys::DriverLoadError::MissingSymbol(
57+
"cuMemAllocAsync: undefined symbol".to_owned(),
58+
)));
59+
}
60+
2861
#[test]
2962
fn host_device_round_trip() -> Result<(), DriverError> {
3063
let Some((context, stream, _)) = streams() else {

‎vortex-cuda/src/lib.rs‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
66
use tracing::info;
77
use vortex_cuda_driver::CudaContext;
8-
use vortex_cuda_driver::sys;
98

109
pub mod arrow;
1110
mod canonical;
@@ -103,15 +102,8 @@ use crate::kernel::SequenceExecutor;
103102
use crate::kernel::SliceExecutor;
104103

105104
/// Checks if a CUDA driver and at least one CUDA device are available.
106-
///
107-
/// The driver crate loads `libcuda` lazily and panics if the driver library is absent, so we
108-
/// first probe for it with `is_culib_present`. Creating the context then fails gracefully with
109-
/// `Err`, rather than panicking, when the driver is present but no usable device is.
110105
pub fn cuda_available() -> bool {
111-
// SAFETY: `is_culib_present` only tries to dlopen the driver library to test for its
112-
// presence; it upholds no further invariants.
113-
let driver_present = unsafe { sys::is_culib_present() };
114-
driver_present && CudaContext::new(0).is_ok()
106+
CudaContext::new(0).is_ok()
115107
}
116108

117109
/// Registers CUDA kernels.

‎vortex-cuda/src/session.rs‎

Lines changed: 5 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,6 @@
33

44
use std::any::Any;
55
use std::fmt::Debug;
6-
use std::panic::AssertUnwindSafe;
7-
use std::panic::catch_unwind;
86
use std::sync::Arc;
97

108
use vortex::array::ArrayId;
@@ -131,20 +129,11 @@ impl CudaSession {
131129
/// Unlike [`Default::default`], this returns an error instead of panicking when CUDA cannot be
132130
/// initialized.
133131
pub fn try_default() -> VortexResult<Self> {
134-
// The driver crate panics rather than returning an error when the CUDA driver library
135-
// cannot be loaded, so catch any unwind here to uphold this constructor's no-panic contract.
136-
match catch_unwind(AssertUnwindSafe(|| -> VortexResult<Self> {
137-
let context = CudaContext::new(0)
138-
.map_err(|err| vortex_err!("failed to initialize CUDA device 0: {err}"))?;
139-
let this = Self::new(context);
140-
initialize_cuda(&this);
141-
Ok(this)
142-
})) {
143-
Ok(result) => result,
144-
Err(_) => Err(vortex_err!(
145-
"failed to initialize CUDA: the driver library is unavailable"
146-
)),
147-
}
132+
let context = CudaContext::new(0)
133+
.map_err(|err| vortex_err!("failed to initialize CUDA device 0: {err}"))?;
134+
let this = Self::new(context);
135+
initialize_cuda(&this);
136+
Ok(this)
148137
}
149138

150139
/// Creates a new CUDA execution context.

0 commit comments

Comments
 (0)