Skip to content

Commit 818f5d2

Browse files
rparolinclaude
andcommitted
feat(cuda.core): unified 1..N managed_memory.prefetch with cydriver
Rewrite prefetch() with the unified single-or-batched signature targeted by issue NVIDIA#1333: - prefetch(targets, location, *, options=None, stream) - targets accepts a single Buffer or a sequence of Buffers - location accepts a Location dataclass, Device, int (-1 = host), or a sequence broadcasting to per-buffer locations - length mismatch raises ValueError; empty targets raises ValueError - options is reserved for future per-call flags and must be None - stream moved to the end, kept keyword-only Internals: switch from Python-level driver.cuMemPrefetchAsync to Cython-level cydriver.cuMemPrefetchAsync via cimport cydriver, with HANDLE_RETURN. Replace the runtime _V2_BINDINGS check with compile-time IF CUDA_CORE_BUILD_MAJOR >= 13 / ELSE per the codebase precedent in _managed_memory_resource.pyx, _memory_pool.pyx, _tensor_map.pyx. N>1 dispatches to cydriver.cuMemPrefetchBatchAsync (CUDA 13 only); on CUDA 12 builds, batched prefetch raises NotImplementedError. Single-range prefetch continues to work on both CUDA 12 and 13 builds. The location_type= keyword is removed; callers express location kind via the Location dataclass added in 20d036e. The advise() and discard_prefetch() functions still use the legacy _normalize_managed_location helper and Python-level driver calls; they will be migrated in their own tasks. Also drops test_managed_memory_prefetch_uses_legacy_bindings_signature, which monkeypatched the Python-level driver.cuMemPrefetchAsync — no longer applicable since the prefetch path uses cydriver. The corresponding advise legacy-bindings test stays for now (advise still uses Python driver). Closes Andy-Jost's review comment that the existing API is "non-Pythonic" by making it Pythonic in a different direction (typed Location dataclass) while preserving the free-function shape pending Leo's tie-break on ManagedBuffer subclass. Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
1 parent dc46535 commit 818f5d2

2 files changed

Lines changed: 284 additions & 73 deletions

File tree

‎cuda_core/cuda/core/_memory/_managed_memory_ops.pyx‎

Lines changed: 166 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,19 @@
44

55
from __future__ import annotations
66

7+
from cpython.mem cimport PyMem_Free, PyMem_Malloc
8+
from libc.stdint cimport uintptr_t
9+
10+
from cuda.bindings cimport cydriver
711
from cuda.core._memory._buffer cimport Buffer, _init_mem_attrs
12+
from cuda.core._resource_handles cimport as_cu
813
from cuda.core._stream cimport Stream, Stream_accept
14+
from cuda.core._utils.cuda_utils cimport HANDLE_RETURN
915

1016
from cuda.core._utils.cuda_utils import driver, handle_return
1117
from cuda.core._utils.version import binding_version
1218
from cuda.core._device import Device
19+
from cuda.core._memory._managed_location import Location, _coerce_location
1320

1421

1522
cdef tuple _VALID_MANAGED_LOCATION_TYPES = (
@@ -228,6 +235,74 @@ cdef void _require_managed_buffer(Buffer self, str what):
228235
raise ValueError(f"{what} requires a managed-memory allocation")
229236

230237

238+
# Coerce ``targets`` (single Buffer or sequence) to a tuple[Buffer, ...].
239+
cdef tuple _coerce_buffer_targets(object targets, str what):
240+
cdef list out
241+
if isinstance(targets, Buffer):
242+
return (<Buffer>targets,)
243+
if isinstance(targets, (list, tuple)):
244+
if not targets:
245+
raise ValueError(f"{what}: empty targets sequence")
246+
out = []
247+
for t in targets:
248+
if not isinstance(t, Buffer):
249+
raise TypeError(
250+
f"{what}: each target must be a Buffer, got {type(t).__name__}"
251+
)
252+
out.append(t)
253+
return tuple(out)
254+
raise TypeError(
255+
f"{what}: targets must be a Buffer or sequence of Buffer, "
256+
f"got {type(targets).__name__}"
257+
)
258+
259+
260+
# Broadcast a single location across ``n`` targets, or coerce a length-N
261+
# sequence elementwise.
262+
cdef tuple _broadcast_locations(object location, Py_ssize_t n, bint allow_none, str what):
263+
cdef object coerced
264+
if isinstance(location, (list, tuple)):
265+
if len(location) != n:
266+
raise ValueError(
267+
f"{what}: location length {len(location)} does not match "
268+
f"targets length {n}"
269+
)
270+
return tuple(_coerce_location(loc, allow_none=allow_none) for loc in location)
271+
coerced = _coerce_location(location, allow_none=allow_none)
272+
return tuple([coerced] * n)
273+
274+
275+
IF CUDA_CORE_BUILD_MAJOR >= 13:
276+
# Convert a Location dataclass to a cydriver.CUmemLocation struct.
277+
cdef inline cydriver.CUmemLocation _to_cumemlocation(object loc):
278+
cdef cydriver.CUmemLocation out
279+
cdef str kind = loc.kind
280+
if kind == "device":
281+
out.type = cydriver.CUmemLocationType.CU_MEM_LOCATION_TYPE_DEVICE
282+
out.id = <int>loc.id
283+
elif kind == "host":
284+
out.type = cydriver.CUmemLocationType.CU_MEM_LOCATION_TYPE_HOST
285+
out.id = 0
286+
elif kind == "host_numa":
287+
out.type = cydriver.CUmemLocationType.CU_MEM_LOCATION_TYPE_HOST_NUMA
288+
out.id = <int>loc.id
289+
else: # host_numa_current
290+
out.type = cydriver.CUmemLocationType.CU_MEM_LOCATION_TYPE_HOST_NUMA_CURRENT
291+
out.id = 0
292+
return out
293+
ELSE:
294+
# CUDA 12 cuMemPrefetchAsync takes a device ordinal (-1 = host).
295+
cdef inline int _to_legacy_device(object loc) except? -2:
296+
cdef str kind = loc.kind
297+
if kind == "device":
298+
return <int>loc.id
299+
if kind == "host":
300+
return -1
301+
raise RuntimeError(
302+
f"location_type={kind!r} requires a CUDA 13 build of cuda.core"
303+
)
304+
305+
231306
cdef void _require_managed_discard_prefetch_support(str what):
232307
global _DISCARD_PREFETCH_SUPPORTED
233308
if _DISCARD_PREFETCH_SUPPORTED < 0:
@@ -293,59 +368,106 @@ def advise(
293368

294369

295370
def prefetch(
296-
target: Buffer,
297-
location: Device | int | None = None,
371+
targets,
372+
location=None,
298373
*,
299-
stream: Stream | GraphBuilder,
300-
location_type: str | None = None,
374+
options=None,
375+
stream,
301376
):
302-
"""Prefetch a managed-memory allocation range to a target location.
377+
"""Prefetch one or more managed-memory ranges to a target location.
303378
304379
Parameters
305380
----------
306-
target : :class:`Buffer`
307-
Managed allocation to operate on.
308-
location : :obj:`~_device.Device` | int | None, optional
309-
Target location. When ``location_type`` is ``None``, values are
310-
interpreted as a device ordinal, ``-1`` for host, or ``None``.
311-
A location is required for prefetch.
312-
stream : :obj:`~_stream.Stream` | :obj:`~_graph.GraphBuilder`
313-
Keyword argument specifying the stream for the asynchronous prefetch.
314-
location_type : str | None, optional
315-
Explicit location kind. Supported values are ``"device"``, ``"host"``,
316-
``"host_numa"``, and ``"host_numa_current"``.
381+
targets : :class:`Buffer` | Sequence[:class:`Buffer`]
382+
One or more managed allocations to operate on.
383+
location : :class:`Location` | :obj:`~_device.Device` | int | Sequence[...]
384+
Target location(s). A single location applies to all targets; a
385+
sequence must match ``len(targets)``. ``Device`` and ``int`` values
386+
are coerced to :class:`Location` (``-1`` maps to host).
387+
options : None
388+
Reserved for future per-call flags. Must be ``None``.
389+
stream : :class:`~_stream.Stream` | :class:`~graph.GraphBuilder`
390+
Stream for the asynchronous prefetch (keyword-only).
391+
392+
Raises
393+
------
394+
NotImplementedError
395+
If ``len(targets) > 1`` on a CUDA 12 build of ``cuda.core``.
317396
"""
318-
if not isinstance(target, Buffer):
319-
raise TypeError(f"prefetch target must be a Buffer, got {type(target).__name__}")
320-
cdef Buffer buf = <Buffer>target
321-
_require_managed_buffer(buf, "prefetch")
397+
if options is not None:
398+
raise TypeError(
399+
f"prefetch options must be None (reserved); got {type(options).__name__}"
400+
)
401+
cdef tuple bufs = _coerce_buffer_targets(targets, "prefetch")
402+
cdef Py_ssize_t n = len(bufs)
403+
cdef tuple locs = _broadcast_locations(location, n, False, "prefetch")
322404
cdef Stream s = Stream_accept(stream)
323-
cdef object ptr = buf.handle
324-
cdef size_t nbytes = buf._size
325405

326-
location = _normalize_managed_location(
327-
location,
328-
location_type,
329-
"prefetch",
330-
)
331-
if _managed_location_uses_v2_bindings():
332-
handle_return(
333-
driver.cuMemPrefetchAsync(
334-
ptr,
335-
nbytes,
336-
location,
337-
_MANAGED_OPERATION_FLAGS,
338-
s.handle,
339-
)
340-
)
406+
cdef Buffer buf
407+
for buf in bufs:
408+
_require_managed_buffer(buf, "prefetch")
409+
410+
if n == 1:
411+
_do_single_prefetch(<Buffer>bufs[0], locs[0], s)
341412
else:
342-
handle_return(
343-
driver.cuMemPrefetchAsync(
344-
ptr,
345-
nbytes,
346-
_managed_location_to_legacy_device(location, "prefetch"),
347-
s.handle,
348-
)
413+
_do_batch_prefetch(bufs, locs, s)
414+
415+
416+
cdef void _do_single_prefetch(Buffer buf, object loc, Stream s):
417+
cdef cydriver.CUdeviceptr cu_ptr = as_cu(buf._h_ptr)
418+
cdef size_t nbytes = buf._size
419+
cdef cydriver.CUstream hstream = as_cu(s._h_stream)
420+
IF CUDA_CORE_BUILD_MAJOR >= 13:
421+
cdef cydriver.CUmemLocation cu_loc = _to_cumemlocation(loc)
422+
with nogil:
423+
HANDLE_RETURN(cydriver.cuMemPrefetchAsync(cu_ptr, nbytes, cu_loc, 0, hstream))
424+
ELSE:
425+
cdef int dev_int = _to_legacy_device(loc)
426+
with nogil:
427+
HANDLE_RETURN(cydriver.cuMemPrefetchAsync(cu_ptr, nbytes, dev_int, hstream))
428+
429+
430+
cdef void _do_batch_prefetch(tuple bufs, tuple locs, Stream s):
431+
IF CUDA_CORE_BUILD_MAJOR >= 13:
432+
cdef Py_ssize_t n = len(bufs)
433+
cdef cydriver.CUstream hstream = as_cu(s._h_stream)
434+
cdef cydriver.CUdeviceptr* ptrs = <cydriver.CUdeviceptr*>PyMem_Malloc(
435+
n * sizeof(cydriver.CUdeviceptr)
436+
)
437+
cdef size_t* sizes = <size_t*>PyMem_Malloc(n * sizeof(size_t))
438+
cdef cydriver.CUmemLocation* loc_arr = <cydriver.CUmemLocation*>PyMem_Malloc(
439+
n * sizeof(cydriver.CUmemLocation)
440+
)
441+
cdef size_t* loc_indices = <size_t*>PyMem_Malloc(n * sizeof(size_t))
442+
if not (ptrs and sizes and loc_arr and loc_indices):
443+
PyMem_Free(ptrs)
444+
PyMem_Free(sizes)
445+
PyMem_Free(loc_arr)
446+
PyMem_Free(loc_indices)
447+
raise MemoryError()
448+
cdef Buffer buf
449+
cdef Py_ssize_t i
450+
try:
451+
for i in range(n):
452+
buf = <Buffer>bufs[i]
453+
ptrs[i] = as_cu(buf._h_ptr)
454+
sizes[i] = buf._size
455+
loc_arr[i] = _to_cumemlocation(locs[i])
456+
loc_indices[i] = <size_t>i
457+
with nogil:
458+
HANDLE_RETURN(cydriver.cuMemPrefetchBatchAsync(
459+
ptrs, sizes, <size_t>n,
460+
loc_arr, loc_indices, <size_t>n,
461+
0, hstream,
462+
))
463+
finally:
464+
PyMem_Free(ptrs)
465+
PyMem_Free(sizes)
466+
PyMem_Free(loc_arr)
467+
PyMem_Free(loc_indices)
468+
ELSE:
469+
raise NotImplementedError(
470+
"batched prefetch requires a CUDA 13 build of cuda.core"
349471
)
350472

351473

0 commit comments

Comments
 (0)