Skip to content

Commit 2224499

Browse files
Andy-Jostclaude
andauthored
test(cuda.bindings): reuse worker-thread contexts under pytest-run-parallel (#3122)
* test(cuda.bindings): reuse worker-thread contexts under pytest-run-parallel The pytest-run-parallel plugin in the bindings conftest made each worker thread create and destroy a CUDA context around every test. The driver serializes context creation, so with four threads every test paid about 0.75 s. On free-threaded rows that run the bindings tests, the step took 36 to 39 minutes and the jobs hit the 60-minute limit. Keep one context per running thread, but reuse it. A pool hands each worker an idle context or creates one; the worker pushes it, runs the test, synchronizes, and pops it, so the previous current context is restored. A context that reports an error at the sync is destroyed, not reused. The plugin destroys the pooled contexts at each module boundary, so a context lives no longer than the module-scoped ctx fixture does in a run without threads. Co-Authored-By: Claude Opus 5.5 <[email protected]> * test(cuda.bindings): capture thread-locally in tests run on parallel threads With pooled contexts the worker threads of a test start together; before, serialized context creation kept them apart. Three tests begin a stream capture in global mode, and in that mode a call such as cuGraphInstantiate in one thread fails while another thread is capturing. The host-function graph tests failed this way on the free-threaded rows. Capture in thread-local mode instead; what the tests check does not depend on it. Drop the synchronize at the end of each pooled test for the same reason: it could break a global capture running in another thread. Pooled contexts now behave like the module-scoped ctx fixture, which does not synchronize between tests either. Co-Authored-By: Claude Opus 5.5 <[email protected]> --------- Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 4ac0f82 commit 2224499

3 files changed

Lines changed: 84 additions & 16 deletions

File tree

‎cuda_bindings/tests/conftest.py‎

Lines changed: 72 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import inspect
77
import pathlib
88
import sys
9+
import threading
910
from contextlib import contextmanager
1011

1112
import pytest
@@ -37,19 +38,70 @@ def pytest_configure(config):
3738
config.pluginmanager.register(_CudaBindingsParallelPlugin(), name="_cuda_bindings_parallel_plugin")
3839

3940

40-
@contextmanager
41-
def _thread_context():
42-
# Context setting up `device` and `ctx` for individual threads on
43-
# pytest-run-parallel
44-
err, device = cuda.cuDeviceGet(0)
45-
assert err == cuda.CUresult.CUDA_SUCCESS
46-
err, ctx = cuda.cuCtxCreate(None, 0, device)
47-
assert err == cuda.CUresult.CUDA_SUCCESS
48-
try:
49-
yield device, ctx
50-
finally:
51-
(err,) = cuda.cuCtxDestroy(ctx)
41+
class _ThreadContextPool:
42+
"""CUDA contexts for pytest-run-parallel worker threads, reused across tests.
43+
44+
pytest-run-parallel starts new threads for every test, and each thread
45+
needs a current context. Creating and destroying a context per thread per
46+
test serializes in the driver: with four threads it cost about 0.75 s per
47+
test and made the free-threaded bindings run more than ten times slower.
48+
49+
The pool gives each running thread a context of its own, as before, and
50+
keeps it for the next test. The plugin empties the pool at the end of each
51+
test module, so a context lives no longer than the module-scoped ``ctx``
52+
fixture does in a run without threads.
53+
54+
The worker threads of a test now run at the same time. Do not add a
55+
synchronizing call between tests here: while another thread captures a
56+
stream in global mode, such a call fails and breaks that capture.
57+
"""
58+
59+
def __init__(self):
60+
self._lock = threading.Lock()
61+
self._idle = []
62+
63+
@staticmethod
64+
def _create():
65+
err, device = cuda.cuDeviceGet(0)
66+
assert err == cuda.CUresult.CUDA_SUCCESS
67+
# cuCtxCreate makes the new context current; pop it so that context()
68+
# can push and pop it like a reused one.
69+
err, ctx = cuda.cuCtxCreate(None, 0, device)
70+
assert err == cuda.CUresult.CUDA_SUCCESS
71+
err, _ = cuda.cuCtxPopCurrent()
72+
assert err == cuda.CUresult.CUDA_SUCCESS
73+
return device, ctx
74+
75+
@contextmanager
76+
def context(self):
77+
with self._lock:
78+
entry = self._idle.pop() if self._idle else None
79+
if entry is None:
80+
entry = self._create()
81+
device, ctx = entry
82+
(err,) = cuda.cuCtxPushCurrent(ctx)
5283
assert err == cuda.CUresult.CUDA_SUCCESS
84+
try:
85+
yield device, ctx
86+
finally:
87+
# Pop rather than unset, so that the context that was current
88+
# before the test (on the main thread, for tests that are not run
89+
# in parallel) is current again.
90+
err, _ = cuda.cuCtxPopCurrent()
91+
assert err == cuda.CUresult.CUDA_SUCCESS
92+
with self._lock:
93+
self._idle.append(entry)
94+
95+
def clear(self):
96+
"""Destroy the idle contexts. Call only when no test is running."""
97+
with self._lock:
98+
entries, self._idle = self._idle, []
99+
for _, ctx in entries:
100+
(err,) = cuda.cuCtxDestroy(ctx)
101+
assert err == cuda.CUresult.CUDA_SUCCESS
102+
103+
104+
_thread_contexts = _ThreadContextPool()
53105

54106

55107
def _wrap_worker_cuda_test(func):
@@ -63,7 +115,7 @@ def _wrap_worker_cuda_test(func):
63115
@functools.wraps(func)
64116
def wrapper(*args, **kwargs):
65117
kwargs = dict(kwargs) # copy before mutating
66-
with _thread_context() as (device, ctx):
118+
with _thread_contexts.context() as (device, ctx):
67119
# device is None when reusing an existing context (defensive path);
68120
# keep whatever the fixture provided in kwargs as-is.
69121
if wants_device and device is not None:
@@ -103,6 +155,13 @@ def pytest_collection_modifyitems(self, config, items):
103155
if _item_needs_thread_ctx(item):
104156
item.obj = _wrap_worker_cuda_test(item.obj)
105157

158+
@pytest.hookimpl(trylast=True)
159+
def pytest_runtest_teardown(self, item, nextitem):
160+
# The worker threads of this test have joined. Destroy the pooled
161+
# contexts when the next test is in another module, or there is none.
162+
if nextitem is None or getattr(nextitem, "module", None) is not getattr(item, "module", None):
163+
_thread_contexts.clear()
164+
106165

107166
@pytest.fixture(scope="module")
108167
def cuda_driver():

‎cuda_bindings/tests/test_cuda.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -706,7 +706,10 @@ def host_fn(userData):
706706
err, stream = cuda.cuStreamCreate(0)
707707
assert err == cuda.CUresult.CUDA_SUCCESS
708708
try:
709-
(err,) = cuda.cuStreamBeginCapture(stream, cuda.CUstreamCaptureMode.CU_STREAM_CAPTURE_MODE_GLOBAL)
709+
# Thread-local mode: pytest-run-parallel runs this test in several threads
710+
# at once, and in global mode a call such as cuGraphInstantiate in one
711+
# thread fails while another thread is capturing.
712+
(err,) = cuda.cuStreamBeginCapture(stream, cuda.CUstreamCaptureMode.CU_STREAM_CAPTURE_MODE_THREAD_LOCAL)
710713
assert err == cuda.CUresult.CUDA_SUCCESS
711714
(err,) = cuda.cuLaunchHostFunc(stream, callback, ctypes.addressof(counter))
712715
assert err == cuda.CUresult.CUDA_SUCCESS

‎cuda_bindings/tests/test_cudart.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -616,7 +616,10 @@ def test_cudart_cudaStreamGetCaptureInfo():
616616
assert status == cudart.cudaStreamCaptureStatus.cudaStreamCaptureStatusNone
617617

618618
# start capture
619-
(err,) = cudart.cudaStreamBeginCapture(stream, cudart.cudaStreamCaptureMode.cudaStreamCaptureModeGlobal)
619+
# Thread-local mode: pytest-run-parallel runs this test in several threads
620+
# at once, and in global mode a call such as cudaGraphInstantiate in one
621+
# thread fails while another thread is capturing.
622+
(err,) = cudart.cudaStreamBeginCapture(stream, cudart.cudaStreamCaptureMode.cudaStreamCaptureModeThreadLocal)
620623
assertSuccess(err)
621624

622625
# validate that stream is capturing now
@@ -1469,7 +1472,10 @@ def host_fn(userData):
14691472
err, stream = cudart.cudaStreamCreate()
14701473
assertSuccess(err)
14711474
try:
1472-
(err,) = cudart.cudaStreamBeginCapture(stream, cudart.cudaStreamCaptureMode.cudaStreamCaptureModeGlobal)
1475+
# Thread-local mode: pytest-run-parallel runs this test in several threads
1476+
# at once, and in global mode a call such as cudaGraphInstantiate in one
1477+
# thread fails while another thread is capturing.
1478+
(err,) = cudart.cudaStreamBeginCapture(stream, cudart.cudaStreamCaptureMode.cudaStreamCaptureModeThreadLocal)
14731479
assertSuccess(err)
14741480
(err,) = cudart.cudaLaunchHostFunc(stream, callback, ctypes.addressof(counter))
14751481
assertSuccess(err)

0 commit comments

Comments
 (0)