|
4 | 4 |
|
5 | 5 | from __future__ import annotations |
6 | 6 |
|
| 7 | +from cpython.mem cimport PyMem_Free, PyMem_Malloc |
| 8 | +from libc.stdint cimport uintptr_t |
| 9 | + |
| 10 | +from cuda.bindings cimport cydriver |
7 | 11 | from cuda.core._memory._buffer cimport Buffer, _init_mem_attrs |
| 12 | +from cuda.core._resource_handles cimport as_cu |
8 | 13 | from cuda.core._stream cimport Stream, Stream_accept |
| 14 | +from cuda.core._utils.cuda_utils cimport HANDLE_RETURN |
9 | 15 |
|
10 | 16 | from cuda.core._utils.cuda_utils import driver, handle_return |
11 | 17 | from cuda.core._utils.version import binding_version |
12 | 18 | from cuda.core._device import Device |
| 19 | +from cuda.core._memory._managed_location import Location, _coerce_location |
13 | 20 |
|
14 | 21 |
|
15 | 22 | cdef tuple _VALID_MANAGED_LOCATION_TYPES = ( |
@@ -228,6 +235,74 @@ cdef void _require_managed_buffer(Buffer self, str what): |
228 | 235 | raise ValueError(f"{what} requires a managed-memory allocation") |
229 | 236 |
|
230 | 237 |
|
| 238 | +# Coerce ``targets`` (single Buffer or sequence) to a tuple[Buffer, ...]. |
| 239 | +cdef tuple _coerce_buffer_targets(object targets, str what): |
| 240 | + cdef list out |
| 241 | + if isinstance(targets, Buffer): |
| 242 | + return (<Buffer>targets,) |
| 243 | + if isinstance(targets, (list, tuple)): |
| 244 | + if not targets: |
| 245 | + raise ValueError(f"{what}: empty targets sequence") |
| 246 | + out = [] |
| 247 | + for t in targets: |
| 248 | + if not isinstance(t, Buffer): |
| 249 | + raise TypeError( |
| 250 | + f"{what}: each target must be a Buffer, got {type(t).__name__}" |
| 251 | + ) |
| 252 | + out.append(t) |
| 253 | + return tuple(out) |
| 254 | + raise TypeError( |
| 255 | + f"{what}: targets must be a Buffer or sequence of Buffer, " |
| 256 | + f"got {type(targets).__name__}" |
| 257 | + ) |
| 258 | + |
| 259 | + |
| 260 | +# Broadcast a single location across ``n`` targets, or coerce a length-N |
| 261 | +# sequence elementwise. |
| 262 | +cdef tuple _broadcast_locations(object location, Py_ssize_t n, bint allow_none, str what): |
| 263 | + cdef object coerced |
| 264 | + if isinstance(location, (list, tuple)): |
| 265 | + if len(location) != n: |
| 266 | + raise ValueError( |
| 267 | + f"{what}: location length {len(location)} does not match " |
| 268 | + f"targets length {n}" |
| 269 | + ) |
| 270 | + return tuple(_coerce_location(loc, allow_none=allow_none) for loc in location) |
| 271 | + coerced = _coerce_location(location, allow_none=allow_none) |
| 272 | + return tuple([coerced] * n) |
| 273 | + |
| 274 | + |
| 275 | +IF CUDA_CORE_BUILD_MAJOR >= 13: |
| 276 | + # Convert a Location dataclass to a cydriver.CUmemLocation struct. |
| 277 | + cdef inline cydriver.CUmemLocation _to_cumemlocation(object loc): |
| 278 | + cdef cydriver.CUmemLocation out |
| 279 | + cdef str kind = loc.kind |
| 280 | + if kind == "device": |
| 281 | + out.type = cydriver.CUmemLocationType.CU_MEM_LOCATION_TYPE_DEVICE |
| 282 | + out.id = <int>loc.id |
| 283 | + elif kind == "host": |
| 284 | + out.type = cydriver.CUmemLocationType.CU_MEM_LOCATION_TYPE_HOST |
| 285 | + out.id = 0 |
| 286 | + elif kind == "host_numa": |
| 287 | + out.type = cydriver.CUmemLocationType.CU_MEM_LOCATION_TYPE_HOST_NUMA |
| 288 | + out.id = <int>loc.id |
| 289 | + else: # host_numa_current |
| 290 | + out.type = cydriver.CUmemLocationType.CU_MEM_LOCATION_TYPE_HOST_NUMA_CURRENT |
| 291 | + out.id = 0 |
| 292 | + return out |
| 293 | +ELSE: |
| 294 | + # CUDA 12 cuMemPrefetchAsync takes a device ordinal (-1 = host). |
| 295 | + cdef inline int _to_legacy_device(object loc) except? -2: |
| 296 | + cdef str kind = loc.kind |
| 297 | + if kind == "device": |
| 298 | + return <int>loc.id |
| 299 | + if kind == "host": |
| 300 | + return -1 |
| 301 | + raise RuntimeError( |
| 302 | + f"location_type={kind!r} requires a CUDA 13 build of cuda.core" |
| 303 | + ) |
| 304 | + |
| 305 | + |
231 | 306 | cdef void _require_managed_discard_prefetch_support(str what): |
232 | 307 | global _DISCARD_PREFETCH_SUPPORTED |
233 | 308 | if _DISCARD_PREFETCH_SUPPORTED < 0: |
@@ -293,59 +368,106 @@ def advise( |
293 | 368 |
|
294 | 369 |
|
295 | 370 | def prefetch( |
296 | | - target: Buffer, |
297 | | - location: Device | int | None = None, |
| 371 | + targets, |
| 372 | + location=None, |
298 | 373 | *, |
299 | | - stream: Stream | GraphBuilder, |
300 | | - location_type: str | None = None, |
| 374 | + options=None, |
| 375 | + stream, |
301 | 376 | ): |
302 | | - """Prefetch a managed-memory allocation range to a target location. |
| 377 | + """Prefetch one or more managed-memory ranges to a target location. |
303 | 378 |
|
304 | 379 | Parameters |
305 | 380 | ---------- |
306 | | - target : :class:`Buffer` |
307 | | - Managed allocation to operate on. |
308 | | - location : :obj:`~_device.Device` | int | None, optional |
309 | | - Target location. When ``location_type`` is ``None``, values are |
310 | | - interpreted as a device ordinal, ``-1`` for host, or ``None``. |
311 | | - A location is required for prefetch. |
312 | | - stream : :obj:`~_stream.Stream` | :obj:`~_graph.GraphBuilder` |
313 | | - Keyword argument specifying the stream for the asynchronous prefetch. |
314 | | - location_type : str | None, optional |
315 | | - Explicit location kind. Supported values are ``"device"``, ``"host"``, |
316 | | - ``"host_numa"``, and ``"host_numa_current"``. |
| 381 | + targets : :class:`Buffer` | Sequence[:class:`Buffer`] |
| 382 | + One or more managed allocations to operate on. |
| 383 | + location : :class:`Location` | :obj:`~_device.Device` | int | Sequence[...] |
| 384 | + Target location(s). A single location applies to all targets; a |
| 385 | + sequence must match ``len(targets)``. ``Device`` and ``int`` values |
| 386 | + are coerced to :class:`Location` (``-1`` maps to host). |
| 387 | + options : None |
| 388 | + Reserved for future per-call flags. Must be ``None``. |
| 389 | + stream : :class:`~_stream.Stream` | :class:`~graph.GraphBuilder` |
| 390 | + Stream for the asynchronous prefetch (keyword-only). |
| 391 | +
|
| 392 | + Raises |
| 393 | + ------ |
| 394 | + NotImplementedError |
| 395 | + If ``len(targets) > 1`` on a CUDA 12 build of ``cuda.core``. |
317 | 396 | """ |
318 | | - if not isinstance(target, Buffer): |
319 | | - raise TypeError(f"prefetch target must be a Buffer, got {type(target).__name__}") |
320 | | - cdef Buffer buf = <Buffer>target |
321 | | - _require_managed_buffer(buf, "prefetch") |
| 397 | + if options is not None: |
| 398 | + raise TypeError( |
| 399 | + f"prefetch options must be None (reserved); got {type(options).__name__}" |
| 400 | + ) |
| 401 | + cdef tuple bufs = _coerce_buffer_targets(targets, "prefetch") |
| 402 | + cdef Py_ssize_t n = len(bufs) |
| 403 | + cdef tuple locs = _broadcast_locations(location, n, False, "prefetch") |
322 | 404 | cdef Stream s = Stream_accept(stream) |
323 | | - cdef object ptr = buf.handle |
324 | | - cdef size_t nbytes = buf._size |
325 | 405 |
|
326 | | - location = _normalize_managed_location( |
327 | | - location, |
328 | | - location_type, |
329 | | - "prefetch", |
330 | | - ) |
331 | | - if _managed_location_uses_v2_bindings(): |
332 | | - handle_return( |
333 | | - driver.cuMemPrefetchAsync( |
334 | | - ptr, |
335 | | - nbytes, |
336 | | - location, |
337 | | - _MANAGED_OPERATION_FLAGS, |
338 | | - s.handle, |
339 | | - ) |
340 | | - ) |
| 406 | + cdef Buffer buf |
| 407 | + for buf in bufs: |
| 408 | + _require_managed_buffer(buf, "prefetch") |
| 409 | + |
| 410 | + if n == 1: |
| 411 | + _do_single_prefetch(<Buffer>bufs[0], locs[0], s) |
341 | 412 | else: |
342 | | - handle_return( |
343 | | - driver.cuMemPrefetchAsync( |
344 | | - ptr, |
345 | | - nbytes, |
346 | | - _managed_location_to_legacy_device(location, "prefetch"), |
347 | | - s.handle, |
348 | | - ) |
| 413 | + _do_batch_prefetch(bufs, locs, s) |
| 414 | + |
| 415 | + |
| 416 | +cdef void _do_single_prefetch(Buffer buf, object loc, Stream s): |
| 417 | + cdef cydriver.CUdeviceptr cu_ptr = as_cu(buf._h_ptr) |
| 418 | + cdef size_t nbytes = buf._size |
| 419 | + cdef cydriver.CUstream hstream = as_cu(s._h_stream) |
| 420 | + IF CUDA_CORE_BUILD_MAJOR >= 13: |
| 421 | + cdef cydriver.CUmemLocation cu_loc = _to_cumemlocation(loc) |
| 422 | + with nogil: |
| 423 | + HANDLE_RETURN(cydriver.cuMemPrefetchAsync(cu_ptr, nbytes, cu_loc, 0, hstream)) |
| 424 | + ELSE: |
| 425 | + cdef int dev_int = _to_legacy_device(loc) |
| 426 | + with nogil: |
| 427 | + HANDLE_RETURN(cydriver.cuMemPrefetchAsync(cu_ptr, nbytes, dev_int, hstream)) |
| 428 | + |
| 429 | + |
| 430 | +cdef void _do_batch_prefetch(tuple bufs, tuple locs, Stream s): |
| 431 | + IF CUDA_CORE_BUILD_MAJOR >= 13: |
| 432 | + cdef Py_ssize_t n = len(bufs) |
| 433 | + cdef cydriver.CUstream hstream = as_cu(s._h_stream) |
| 434 | + cdef cydriver.CUdeviceptr* ptrs = <cydriver.CUdeviceptr*>PyMem_Malloc( |
| 435 | + n * sizeof(cydriver.CUdeviceptr) |
| 436 | + ) |
| 437 | + cdef size_t* sizes = <size_t*>PyMem_Malloc(n * sizeof(size_t)) |
| 438 | + cdef cydriver.CUmemLocation* loc_arr = <cydriver.CUmemLocation*>PyMem_Malloc( |
| 439 | + n * sizeof(cydriver.CUmemLocation) |
| 440 | + ) |
| 441 | + cdef size_t* loc_indices = <size_t*>PyMem_Malloc(n * sizeof(size_t)) |
| 442 | + if not (ptrs and sizes and loc_arr and loc_indices): |
| 443 | + PyMem_Free(ptrs) |
| 444 | + PyMem_Free(sizes) |
| 445 | + PyMem_Free(loc_arr) |
| 446 | + PyMem_Free(loc_indices) |
| 447 | + raise MemoryError() |
| 448 | + cdef Buffer buf |
| 449 | + cdef Py_ssize_t i |
| 450 | + try: |
| 451 | + for i in range(n): |
| 452 | + buf = <Buffer>bufs[i] |
| 453 | + ptrs[i] = as_cu(buf._h_ptr) |
| 454 | + sizes[i] = buf._size |
| 455 | + loc_arr[i] = _to_cumemlocation(locs[i]) |
| 456 | + loc_indices[i] = <size_t>i |
| 457 | + with nogil: |
| 458 | + HANDLE_RETURN(cydriver.cuMemPrefetchBatchAsync( |
| 459 | + ptrs, sizes, <size_t>n, |
| 460 | + loc_arr, loc_indices, <size_t>n, |
| 461 | + 0, hstream, |
| 462 | + )) |
| 463 | + finally: |
| 464 | + PyMem_Free(ptrs) |
| 465 | + PyMem_Free(sizes) |
| 466 | + PyMem_Free(loc_arr) |
| 467 | + PyMem_Free(loc_indices) |
| 468 | + ELSE: |
| 469 | + raise NotImplementedError( |
| 470 | + "batched prefetch requires a CUDA 13 build of cuda.core" |
349 | 471 | ) |
350 | 472 |
|
351 | 473 |
|
|
0 commit comments