Skip to content

Commit 65935f1

Browse files
authored
Fix crash when registering/unregistering callbacks concurrently under free-threaded Python (NVIDIA#3308)
* Lock the callback bookkeeping map in cuda.bindings Under free-threaded Python, concurrent register/unregister of callbacks (cuLogs*, *AsyncNotification) corrupts the shared m_global._allocated std::map. Guard it with a cython.pymutex. driver.pyx and runtime.pyx are regenerated from cybind, not hand-edited. Closes NVIDIA#3282 * add a test case to increase probability triggering contention
1 parent 1cda28d commit 65935f1

3 files changed

Lines changed: 79 additions & 11 deletions

File tree

‎cuda_bindings_13/cuda/bindings/driver.pyx‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# SPDX-License-Identifier: Apache-2.0
33

44
# This code was automatically generated with version 13.4.1. Do not modify it directly.
5-
# CYTHON-BINDINGS-GENERATED-DO-NOT-MODIFY-THIS-FILE: format=1; content-sha256=348b02f0a76a3a271fbb11535f02d660d811d3c21a877e46bfc070bbabf43e76
5+
# CYTHON-BINDINGS-GENERATED-DO-NOT-MODIFY-THIS-FILE: format=1; content-sha256=8932ee959b6a2f1e69316f813cb8e8492c8e112094812dc1310c49970f00779a
66
from typing import Any, Optional
77
import cython
88
import ctypes
@@ -30899,7 +30899,7 @@ def cuDeviceRegisterAsyncNotification(device, callbackFunc, userData):
3089930899
if err != cydriver.CUDA_SUCCESS:
3090030900
free(cbData)
3090130901
else:
30902-
m_global._allocated[int(callback)] = cbData
30902+
m_global._insert(int(callback), cbData)
3090330903
_helper_input_void_ptr_free(&cyuserDataHelper)
3090430904
if err != cydriver.CUDA_SUCCESS:
3090530905
return (_CUresult(err), None)
@@ -30948,8 +30948,7 @@ def cuDeviceUnregisterAsyncNotification(device, callback):
3094830948
with nogil:
3094930949
err = cydriver.cuDeviceUnregisterAsyncNotification(cydevice, cycallback)
3095030950
if err == cydriver.CUDA_SUCCESS:
30951-
free(m_global._allocated[pcallback])
30952-
m_global._allocated.erase(<void_ptr>pcallback)
30951+
free(m_global._take(pcallback))
3095330952
return (_CUresult(err),)
3095430953

3095530954
@cython.embedsignature(True)
@@ -55743,7 +55742,7 @@ def cuLogsRegisterCallback(callbackFunc, userData):
5574355742
if err != cydriver.CUDA_SUCCESS:
5574455743
free(cbData)
5574555744
else:
55746-
m_global._allocated[int(callback_out)] = cbData
55745+
m_global._insert(int(callback_out), cbData)
5574755746
_helper_input_void_ptr_free(&cyuserDataHelper)
5574855747
if err != cydriver.CUDA_SUCCESS:
5574955748
return (_CUresult(err), None)
@@ -55774,8 +55773,7 @@ def cuLogsUnregisterCallback(callback):
5577455773
with nogil:
5577555774
err = cydriver.cuLogsUnregisterCallback(cycallback)
5577655775
if err == cydriver.CUDA_SUCCESS:
55777-
free(m_global._allocated[pcallback])
55778-
m_global._allocated.erase(<void_ptr>pcallback)
55776+
free(m_global._take(pcallback))
5577955777
return (_CUresult(err),)
5578055778

5578155779
@cython.embedsignature(True)
@@ -57375,6 +57373,19 @@ def cuGraphicsVDPAURegisterOutputSurface(vdpSurface, unsigned int flags):
5737557373

5737657374
cdef class cudaBindingsDriverGlobal:
5737757375
cdef map[void_ptr, void*] _allocated
57376+
cdef cython.pymutex _lock
57377+
57378+
cdef void _insert(self, void_ptr key, void* value) noexcept:
57379+
with self._lock:
57380+
self._allocated[key] = value
57381+
57382+
cdef void* _take(self, void_ptr key) noexcept:
57383+
cdef void* value = NULL
57384+
with self._lock:
57385+
if self._allocated.count(key) != 0:
57386+
value = self._allocated[key]
57387+
self._allocated.erase(key)
57388+
return value
5737857389

5737957390
def __dealloc__(self):
5738057391
for item in self._allocated:

‎cuda_bindings_13/cuda/bindings/runtime.pyx‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# SPDX-License-Identifier: Apache-2.0
33

44
# This code was automatically generated with version 13.4.1. Do not modify it directly.
5-
# CYTHON-BINDINGS-GENERATED-DO-NOT-MODIFY-THIS-FILE: format=1; content-sha256=ea95eb12334cbd4c1bf3fc8e890fb6f9493063b12faea853bb3d8c6288934182
5+
# CYTHON-BINDINGS-GENERATED-DO-NOT-MODIFY-THIS-FILE: format=1; content-sha256=4cc5d08de16bc7cac781ba6e469ffe16b0d8e30beea5ec6f5e579f0fae24e834
66
from typing import Any, Optional
77
import cython
88
import ctypes
@@ -21228,7 +21228,7 @@ def cudaDeviceRegisterAsyncNotification(int device, callbackFunc, userData):
2122821228
if err != cyruntime.cudaSuccess:
2122921229
free(cbData)
2123021230
else:
21231-
m_global._allocated[int(callback)] = cbData
21231+
m_global._insert(int(callback), cbData)
2123221232
_helper_input_void_ptr_free(&cyuserDataHelper)
2123321233

2123421234
if err != cyruntime.cudaSuccess:
@@ -21270,8 +21270,7 @@ def cudaDeviceUnregisterAsyncNotification(int device, callback):
2127021270
with nogil:
2127121271
err = cyruntime.cudaDeviceUnregisterAsyncNotification(device, cycallback)
2127221272
if err == cyruntime.cudaSuccess:
21273-
free(m_global._allocated[pcallback])
21274-
m_global._allocated.erase(<void_ptr>pcallback)
21273+
free(m_global._take(pcallback))
2127521274
return (_cudaError_t(err),)
2127621275

2127721276
@cython.embedsignature(True)
@@ -40262,6 +40261,19 @@ def getLocalRuntimeVersion():
4026240261

4026340262
cdef class cudaBindingsRuntimeGlobal:
4026440263
cdef map[void_ptr, void*] _allocated
40264+
cdef cython.pymutex _lock
40265+
40266+
cdef void _insert(self, void_ptr key, void* value) noexcept:
40267+
with self._lock:
40268+
self._allocated[key] = value
40269+
40270+
cdef void* _take(self, void_ptr key) noexcept:
40271+
cdef void* value = NULL
40272+
with self._lock:
40273+
if self._allocated.count(key) != 0:
40274+
value = self._allocated[key]
40275+
self._allocated.erase(key)
40276+
return value
4026540277

4026640278
def __dealloc__(self):
4026740279
for item in self._allocated:
Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
import threading
5+
6+
import pytest
7+
8+
from cuda.bindings import driver
9+
from cuda_python_test_helpers import driver_version_less_than
10+
11+
NUM_THREADS = 16
12+
NUM_ITERATIONS = 500
13+
14+
15+
@pytest.mark.agent_authored(model="claude-sonnet-5-5")
16+
@pytest.mark.skipif(
17+
driver_version_less_than(12090),
18+
reason="cuLogs* functions were introduced in CUDA 12.9",
19+
)
20+
@pytest.mark.thread_unsafe(reason="starts its own threads and needs them to overlap")
21+
def test_culogs_register_unregister_from_many_threads():
22+
(err,) = driver.cuInit(0)
23+
assert err == driver.CUresult.CUDA_SUCCESS
24+
25+
barrier = threading.Barrier(NUM_THREADS)
26+
failures = []
27+
28+
def worker():
29+
try:
30+
barrier.wait()
31+
for _ in range(NUM_ITERATIONS):
32+
err, handle = driver.cuLogsRegisterCallback(0, None)
33+
assert err == driver.CUresult.CUDA_SUCCESS
34+
(err,) = driver.cuLogsUnregisterCallback(handle)
35+
assert err == driver.CUresult.CUDA_SUCCESS
36+
except BaseException as e:
37+
failures.append(e)
38+
39+
threads = [threading.Thread(target=worker) for _ in range(NUM_THREADS)]
40+
for t in threads:
41+
t.start()
42+
for t in threads:
43+
t.join()
44+
45+
assert not failures, failures

0 commit comments

Comments
 (0)