Skip to content

Commit 991ea99

Browse files
juenglinjuenglin
committed
test(bindings): cap memory pools missed in #3140
#3140 capped the memory pools created by test_cuda_memPool_attr and test_cudart_MemPool_attr but missed three more sites with the same uncapped-pool (maxSize 0) pattern: * cuda_bindings_13/tests/test_cuda.py: test_ffi_coverage_new_mempool_attributes (v2 driver API, no max_size set). * cuda_bindings_12/tests/test_cuda.py: test_cuda_memPool_attr (no POOL_SIZE constant or maxSize) and test_cuda_graphMem_attr (no xfail_on_graph_mempool_oom fixture; cuda_bindings_12 was added after #3140). * cuda_bindings_12/tests/test_cudart.py: test_cudart_MemPool_attr (same). A pool with maxSize 0 reserves virtual address space of about twice the device memory; on riscv64 Sv39 (39-bit VA space) this fails with CUDA_ERROR_OUT_OF_MEMORY on GPUs with enough device memory.
1 parent 3cc4edb commit 991ea99

3 files changed

Lines changed: 40 additions & 1 deletion

File tree

‎cuda_bindings_12/tests/test_cuda.py‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,12 @@ def supportsManagedMemory():
3232
return err == cudart.cudaError_t.cudaSuccess and isSupported
3333

3434

35+
# Cap for memory pools created by tests. A pool created with maxSize 0 reserves
36+
# virtual address space of about twice the device memory, and that reservation
37+
# can fail on systems with a limited virtual address space.
38+
POOL_SIZE = 2 * 1024 * 1024 # 2 MiB
39+
40+
3541
def supportsCudaAPI(name):
3642
return name in dir(cuda)
3743

@@ -333,6 +339,7 @@ def test_cuda_memPool_attr():
333339
poolProps.allocType = cuda.CUmemAllocationType.CU_MEM_ALLOCATION_TYPE_PINNED
334340
poolProps.location.id = 0
335341
poolProps.location.type = cuda.CUmemLocationType.CU_MEM_LOCATION_TYPE_DEVICE
342+
poolProps.maxSize = POOL_SIZE
336343

337344
attr_list = [None] * 8
338345
err, pool = cuda.cuMemPoolCreate(poolProps)
@@ -539,9 +546,27 @@ def test_cuda_mem_range_attr():
539546
assert err == cuda.CUresult.CUDA_SUCCESS
540547

541548

549+
@pytest.fixture
550+
def xfail_on_graph_mempool_oom():
551+
"""Return a check that xfails the calling test if the graph memory pool cannot be created.
552+
553+
The graph memory pool reserves virtual address space of about twice the
554+
device memory, and unlike pools from cuMemPoolCreate it cannot be capped.
555+
On systems with a limited virtual address space this reservation can fail
556+
with CUDA_ERROR_OUT_OF_MEMORY, especially when another uncapped pool
557+
already exists in the process.
558+
"""
559+
560+
def check(err):
561+
if err == cuda.CUresult.CUDA_ERROR_OUT_OF_MEMORY:
562+
pytest.xfail("graph memory pool creation failed with CUDA_ERROR_OUT_OF_MEMORY")
563+
564+
return check
565+
566+
542567
@pytest.mark.skipif(driverVersionLessThan(11040) or not supportsMemoryPool(), reason="Mempool for graphs not supported")
543568
@pytest.mark.thread_unsafe(reason="used high memory can be higher if threaded.")
544-
def test_cuda_graphMem_attr():
569+
def test_cuda_graphMem_attr(xfail_on_graph_mempool_oom):
545570
(err,) = cuda.cuInit(0)
546571
assert err == cuda.CUresult.CUDA_SUCCESS
547572
err, device = cuda.cuDeviceGet(0)
@@ -564,6 +589,12 @@ def test_cuda_graphMem_attr():
564589
params.bytesize = allocSize
565590

566591
err, allocNode = cuda.cuGraphAddMemAllocNode(graph, None, 0, params)
592+
if err == cuda.CUresult.CUDA_ERROR_OUT_OF_MEMORY:
593+
(destroy_err,) = cuda.cuGraphDestroy(graph)
594+
assert destroy_err == cuda.CUresult.CUDA_SUCCESS
595+
(destroy_err,) = cuda.cuStreamDestroy(stream)
596+
assert destroy_err == cuda.CUresult.CUDA_SUCCESS
597+
xfail_on_graph_mempool_oom(err)
567598
assert err == cuda.CUresult.CUDA_SUCCESS
568599
err, freeNode = cuda.cuGraphAddMemFreeNode(graph, [allocNode], 1, params.dptr)
569600
assert err == cuda.CUresult.CUDA_SUCCESS

‎cuda_bindings_12/tests/test_cudart.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,12 @@ def supportsMemoryPool():
3131
return isSuccess(err) and isSupported
3232

3333

34+
# Cap for memory pools created by tests. A pool created with maxSize 0 reserves
35+
# virtual address space of about twice the device memory, and that reservation
36+
# can fail on systems with a limited virtual address space.
37+
POOL_SIZE = 2 * 1024 * 1024 # 2 MiB
38+
39+
3440
def supportsSparseTexturesDeviceFilter():
3541
err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrSparseCudaArraySupported, 0)
3642
return isSuccess(err) and isSupported
@@ -425,6 +431,7 @@ def test_cudart_MemPool_attr():
425431
poolProps.allocType = cudart.cudaMemAllocationType.cudaMemAllocationTypePinned
426432
poolProps.location.id = 0
427433
poolProps.location.type = cudart.cudaMemLocationType.cudaMemLocationTypeDevice
434+
poolProps.maxSize = POOL_SIZE
428435

429436
attr_list = [None] * 8
430437
err, pool = cudart.cudaMemPoolCreate(poolProps)

‎cuda_bindings_13/tests/test_cuda.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -603,6 +603,7 @@ def test_ffi_coverage_new_mempool_attributes():
603603
pool_props.alloc_type = driver.MemAllocationType.PINNED
604604
pool_props.location.id = 0
605605
pool_props.location.type = driver.MemLocationType.DEVICE
606+
pool_props.max_size = POOL_SIZE
606607
pool = driver.mem_pool_create(pool_props)
607608
try:
608609
for attr in [

0 commit comments

Comments
 (0)