Skip to content

Commit db28018

Browse files
authored
cuda.core: allow updating Buffer deallocation streams (NVIDIA#2602)
* cuda.core: allow updating Buffer deallocation streams * docs(cuda.core): add deallocation stream transfer example Show how to move a Buffer between streams with an event so its eventual deallocation remains correctly ordered. * test(cuda.core): use shared memory resource instrumentation Keep the deallocation-stream tests aligned with the centralized test helper merged in NVIDIA#2624.
1 parent 0eb23c4 commit db28018

7 files changed

Lines changed: 232 additions & 4 deletions

File tree

‎cuda_core/cuda/core/_memory/_buffer.pyi‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,41 @@ class Buffer:
127127
stream : :obj:`~_stream.Stream` | :obj:`~graph.GraphBuilder`, optional
128128
The stream object to use for asynchronous deallocation. If None,
129129
the deallocation stream stored in the handle is used.
130+
131+
See Also
132+
--------
133+
set_deallocation_stream
134+
Change the deallocation stream without closing the buffer.
135+
"""
136+
137+
def set_deallocation_stream(self, stream: Stream | GraphBuilder) -> None:
138+
"""Change the stream that orders this buffer's eventual deallocation.
139+
140+
The buffer remains open and usable. A later :meth:`close` without a
141+
stream, garbage collection, or release of the final retained device
142+
pointer handle uses the replacement stream.
143+
144+
This method does not synchronize streams or establish dependencies.
145+
The caller must ensure that allocation and all accesses are ordered
146+
before the deallocation on ``stream``.
147+
148+
Parameters
149+
----------
150+
stream : :obj:`~_stream.Stream` | :obj:`~graph.GraphBuilder`
151+
The stream to use for eventual asynchronous deallocation.
152+
153+
Raises
154+
------
155+
RuntimeError
156+
If the buffer is already closed, or if a default-stream token
157+
cannot be bound because no CUDA context is current.
158+
TypeError
159+
If ``stream`` is ``None`` or is not an accepted stream object.
160+
161+
Notes
162+
-----
163+
Synchronizing concurrent mutation and destruction of the same buffer
164+
is the caller's responsibility.
130165
"""
131166

132167
def __enter__(self):

‎cuda_core/cuda/core/_memory/_buffer.pyx‎

Lines changed: 46 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -252,7 +252,7 @@ cdef class Buffer:
252252
# The parent process's stream is not portable across processes, so the
253253
# pickle path cannot thread an explicit stream through. Seed the
254254
# imported buffer's deallocation with the current context's default
255-
# stream; the receiver can override via buffer.close(stream).
255+
# stream; the receiver can override it before or during close.
256256
return Buffer.from_ipc_descriptor(mr, ipc_descriptor, stream=default_stream())
257257

258258
def __reduce__(self) -> tuple[object, ...]:
@@ -347,9 +347,45 @@ cdef class Buffer:
347347
stream : :obj:`~_stream.Stream` | :obj:`~graph.GraphBuilder`, optional
348348
The stream object to use for asynchronous deallocation. If None,
349349
the deallocation stream stored in the handle is used.
350+
351+
See Also
352+
--------
353+
set_deallocation_stream
354+
Change the deallocation stream without closing the buffer.
350355
"""
351356
Buffer_close(self, stream)
352357

358+
def set_deallocation_stream(self, stream: Stream | GraphBuilder) -> None:
359+
"""Change the stream that orders this buffer's eventual deallocation.
360+
361+
The buffer remains open and usable. A later :meth:`close` without a
362+
stream, garbage collection, or release of the final retained device
363+
pointer handle uses the replacement stream.
364+
365+
This method does not synchronize streams or establish dependencies.
366+
The caller must ensure that allocation and all accesses are ordered
367+
before the deallocation on ``stream``.
368+
369+
Parameters
370+
----------
371+
stream : :obj:`~_stream.Stream` | :obj:`~graph.GraphBuilder`
372+
The stream to use for eventual asynchronous deallocation.
373+
374+
Raises
375+
------
376+
RuntimeError
377+
If the buffer is already closed, or if a default-stream token
378+
cannot be bound because no CUDA context is current.
379+
TypeError
380+
If ``stream`` is ``None`` or is not an accepted stream object.
381+
382+
Notes
383+
-----
384+
Synchronizing concurrent mutation and destruction of the same buffer
385+
is the caller's responsibility.
386+
"""
387+
Buffer_set_deallocation_stream(self, stream)
388+
353389
def __enter__(self):
354390
return self
355391

@@ -707,15 +743,21 @@ cdef tuple Buffer_coerce_batch(object buffers, str what, str single_hint):
707743
return tuple(out)
708744

709745

746+
cdef inline void Buffer_set_deallocation_stream(Buffer self, object stream):
747+
"""Validate and replace a live buffer's deallocation recipe."""
748+
if not self._h_ptr:
749+
raise RuntimeError("Cannot set the deallocation stream on a closed Buffer")
750+
cdef Stream s = Stream_accept(stream)
751+
_apply_deallocation_stream(self._h_ptr, s._h_stream)
752+
753+
710754
cdef inline void Buffer_close(Buffer self, object stream):
711755
"""Close a buffer, freeing its memory."""
712-
cdef Stream s
713756
if not self._h_ptr:
714757
return
715758
# Update deallocation stream if provided
716759
if stream is not None:
717-
s = Stream_accept(stream)
718-
_apply_deallocation_stream(self._h_ptr, s._h_stream)
760+
Buffer_set_deallocation_stream(self, stream)
719761
# Reset handle - RAII deleter will free the memory (and release owner ref in C++)
720762
self._h_ptr.reset()
721763
self._size = 0

‎cuda_core/docs/source/api.rst‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,14 @@ Memory management
7777
ManagedMemoryResourceOptions
7878
VirtualMemoryResourceOptions
7979

80+
A :class:`Buffer` records the stream that will order its eventual deallocation.
81+
Use :meth:`Buffer.set_deallocation_stream` to replace that stream without
82+
closing the buffer. Changing the recorded stream does not synchronize streams;
83+
the caller must order allocation and every access before the deallocation,
84+
using events or other CUDA synchronization mechanisms as needed. See
85+
:cuda-core-example:`buffer_deallocation_stream.py <buffer_deallocation_stream.py>`
86+
for a complete example.
87+
8088

8189
CUDA compilation toolchain
8290
--------------------------

‎cuda_core/docs/source/examples.rst‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,13 @@ Linking and graphs
3939
- :cuda-core-example:`cuda_graphs.py <cuda_graphs.py>`
4040
captures and replays a multi-kernel CUDA graph to reduce launch overhead.
4141

42+
Memory management
43+
-----------------
44+
45+
- :cuda-core-example:`buffer_deallocation_stream.py <buffer_deallocation_stream.py>`
46+
transfers a buffer between streams and safely changes the stream that orders
47+
its deallocation.
48+
4249
Interoperability and memory access
4350
----------------------------------
4451

‎cuda_core/docs/source/release/1.2.0-notes.rst‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@ Fixes and enhancements
3535
free recipe can pin that context.
3636
(`#2497 <https://github.com/NVIDIA/cuda-python/issues/2497>`__)
3737

38+
- Added :meth:`Buffer.set_deallocation_stream` to change the stream that orders
39+
a buffer's eventual deallocation without closing the buffer.
40+
(`#2600 <https://github.com/NVIDIA/cuda-python/issues/2600>`__)
41+
3842
- Explicit calls to ``deallocate()`` on pool-backed memory resources and
3943
:class:`GraphMemoryResource` now propagate errors from the underlying CUDA
4044
free operation. Previously, these errors could be suppressed. Automatic
Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
#
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
# ################################################################################
6+
#
7+
# This example transfers a buffer from a producer stream to a consumer stream.
8+
# An event orders the consumer after the producer. The buffer then records the
9+
# consumer stream for its eventual deallocation.
10+
#
11+
# ################################################################################
12+
13+
# /// script
14+
# dependencies = ["cuda_bindings", "cuda_core"]
15+
# ///
16+
17+
import ctypes
18+
19+
from cuda.core import Device, LegacyPinnedMemoryResource
20+
21+
22+
def produce_data(device, stream, size, value):
23+
"""Allocate and fill a buffer on the producer stream."""
24+
buffer = device.allocate(size, stream=stream)
25+
buffer.fill(value, stream=stream)
26+
ready = stream.record()
27+
return buffer, ready
28+
29+
30+
def consume_data(buffer, ready, output, stream):
31+
"""Submit consumer work and transfer the deallocation stream."""
32+
stream.wait(ready)
33+
buffer.set_deallocation_stream(stream)
34+
buffer.copy_to(output, stream=stream)
35+
36+
37+
def main():
38+
device = Device()
39+
device.set_current()
40+
producer_stream = device.create_stream()
41+
consumer_stream = device.create_stream()
42+
pinned_mr = LegacyPinnedMemoryResource()
43+
44+
size = 4096
45+
value = 42
46+
buffer = None
47+
ready = None
48+
output = None
49+
50+
try:
51+
output = pinned_mr.allocate(size)
52+
buffer, ready = produce_data(device, producer_stream, size, value)
53+
consume_data(buffer, ready, output, consumer_stream)
54+
55+
# No stream argument is needed. The buffer now records consumer_stream.
56+
# The free operation runs after the copy on that stream.
57+
buffer.close()
58+
buffer = None
59+
consumer_stream.sync()
60+
61+
result = ctypes.string_at(int(output.handle), output.size)
62+
assert result == bytes([value]) * size
63+
print("Buffer deallocation stream transfer completed.")
64+
finally:
65+
if buffer is not None:
66+
buffer.close()
67+
if output is not None:
68+
output.close()
69+
if ready is not None:
70+
ready.close()
71+
consumer_stream.close()
72+
producer_stream.close()
73+
74+
75+
if __name__ == "__main__":
76+
main()

‎cuda_core/tests/test_memory.py‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -516,6 +516,62 @@ def test_mr_deallocate_receives_stream():
516516
assert telemetry["deallocations"][-1]["stream"].handle == stream.handle
517517

518518

519+
@pytest.mark.agent_authored(model="gpt-5.6")
520+
@pytest.mark.parametrize(
521+
("configuration", "destruction"),
522+
[
523+
("initialization", "close"),
524+
("initialization", "gc"),
525+
("setter", "close"),
526+
("setter", "gc"),
527+
("close", "close"),
528+
],
529+
)
530+
def test_buffer_deallocation_stream_configuration_paths(configuration, destruction):
531+
"""Creation, mutation, and close overrides use the requested stream."""
532+
import gc
533+
534+
device = Device()
535+
device.set_current()
536+
initial_stream = device.create_stream()
537+
target_stream = device.create_stream()
538+
CapturingMR, telemetry = make_instrumented_memory_resource(record_streams=True)
539+
mr = CapturingMR(device)
540+
541+
stream = target_stream if configuration == "initialization" else initial_stream
542+
buf = Buffer.from_handle(1, 1024, mr=mr, stream=stream)
543+
if configuration == "setter":
544+
handle = buf.handle
545+
buf.set_deallocation_stream(target_stream)
546+
assert buf.handle == handle
547+
assert buf.size == 1024
548+
549+
if destruction == "close":
550+
buf.close(stream=target_stream if configuration == "close" else None)
551+
else:
552+
del buf
553+
gc.collect()
554+
555+
assert len(telemetry["deallocations"]) == 1
556+
assert telemetry["deallocations"][0]["stream"].handle == target_stream.handle
557+
558+
559+
@pytest.mark.agent_authored(model="gpt-5.6")
560+
def test_set_deallocation_stream_rejects_none_and_closed_buffer():
561+
device = Device()
562+
device.set_current()
563+
stream = device.create_stream()
564+
mr = StubMemoryResource(device)
565+
buf = Buffer.from_handle(1, 1024, mr=mr, stream=stream)
566+
567+
with pytest.raises(TypeError, match="stream is required"):
568+
buf.set_deallocation_stream(None)
569+
570+
buf.close()
571+
with pytest.raises(RuntimeError, match="closed Buffer"):
572+
buf.set_deallocation_stream(stream)
573+
574+
519575
@pytest.mark.parametrize("buffer_type", [Buffer, ManagedBuffer])
520576
def test_from_handle_mr_records_default_stream(buffer_type):
521577
"""When a Buffer/ManagedBuffer is minted via :meth:`from_handle` with ``mr``

0 commit comments

Comments
 (0)