Skip to content

Commit 52a834f

Browse files
committed
test: xfail graph mempool allocation OOM
Handle graph allocation OOMs with the shared mempool xfail helper so graph tests report the known platform condition instead of cascading failures. xref: #2084 (comment) section 2
1 parent 93e86ec commit 52a834f

7 files changed

Lines changed: 172 additions & 111 deletions

‎cuda_core/tests/conftest.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import os
66
import pathlib
77
import sys
8+
from contextlib import contextmanager
89
from importlib.metadata import PackageNotFoundError, distribution
910

1011
import pytest
@@ -104,6 +105,15 @@ def create_pinned_memory_resource_or_xfail(*args, xfail_device=None, **kwargs):
104105
raise
105106

106107

108+
@contextmanager
109+
def xfail_on_graph_mempool_oom(device=0):
110+
try:
111+
yield
112+
except CUDAError as e:
113+
xfail_if_mempool_oom(e, "cuGraphAddMemAllocNode", device)
114+
raise
115+
116+
107117
def _device_id_from_resource_options(device, args, kwargs):
108118
if device is not None:
109119
return device

‎cuda_core/tests/graph/test_graph_definition.py‎

Lines changed: 66 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
from helpers.graph_kernels import compile_common_kernels
1111
from helpers.misc import try_create_condition
1212

13+
from conftest import xfail_on_graph_mempool_oom
1314
from cuda.core import Device, LaunchConfig
1415
from cuda.core.graph import (
1516
AllocNode,
@@ -201,13 +202,15 @@ def _build_disconnected():
201202
def graph_spec(request, init_cuda):
202203
if request.param is not _build_empty:
203204
_skip_if_no_mempool()
204-
return request.param()
205+
with xfail_on_graph_mempool_oom():
206+
return request.param()
205207

206208

207209
@pytest.fixture(params=_NONEMPTY_BUILDERS)
208210
def nonempty_graph_spec(request, init_cuda):
209211
_skip_if_no_mempool()
210-
return request.param()
212+
with xfail_on_graph_mempool_oom():
213+
return request.param()
211214

212215

213216
# =============================================================================
@@ -562,7 +565,8 @@ def node_spec(request, init_cuda):
562565
if spec.needs_mempool:
563566
_skip_if_no_mempool()
564567
g = GraphDefinition()
565-
node, expected_attrs = spec.builder(g)
568+
with xfail_on_graph_mempool_oom():
569+
node, expected_attrs = spec.builder(g)
566570
return spec, g, node, expected_attrs
567571

568572

@@ -803,18 +807,20 @@ def test_alloc_zero_size_fails(sample_graphdef):
803807
def test_free_creates_dependency(sample_graphdef):
804808
"""Free node depends on its predecessor."""
805809
_skip_if_no_mempool()
806-
alloc = sample_graphdef.allocate(ALLOC_SIZE)
807-
free = alloc.deallocate(alloc.dptr)
810+
with xfail_on_graph_mempool_oom():
811+
alloc = sample_graphdef.allocate(ALLOC_SIZE)
812+
free = alloc.deallocate(alloc.dptr)
808813
assert alloc in free.pred
809814

810815

811816
def test_alloc_free_chain(sample_graphdef):
812817
"""Alloc and free can be chained."""
813818
_skip_if_no_mempool()
814-
a1 = sample_graphdef.allocate(ALLOC_SIZE)
815-
a2 = a1.allocate(ALLOC_SIZE)
816-
f2 = a2.deallocate(a2.dptr)
817-
f1 = f2.deallocate(a1.dptr)
819+
with xfail_on_graph_mempool_oom():
820+
a1 = sample_graphdef.allocate(ALLOC_SIZE)
821+
a2 = a1.allocate(ALLOC_SIZE)
822+
f2 = a2.deallocate(a2.dptr)
823+
f1 = f2.deallocate(a1.dptr)
818824
assert a1 in a2.pred
819825
assert a2 in f2.pred
820826
assert f2 in f1.pred
@@ -842,15 +848,17 @@ def test_alloc_device_option(sample_graphdef, device_spec):
842848
"""Device can be specified as int or Device object."""
843849
_skip_if_no_mempool()
844850
device = Device()
845-
node = sample_graphdef.allocate(ALLOC_SIZE, device=device_spec(device))
851+
with xfail_on_graph_mempool_oom(device):
852+
node = sample_graphdef.allocate(ALLOC_SIZE, device=device_spec(device))
846853
assert node.dptr != 0
847854

848855

849856
def test_alloc_peer_access(mempool_device_x2):
850857
"""AllocNode.peer_access reflects requested peers."""
851858
d0, d1 = mempool_device_x2
852859
g = GraphDefinition()
853-
node = g.allocate(ALLOC_SIZE, device=d0.device_id, peer_access=[d1.device_id])
860+
with xfail_on_graph_mempool_oom(d0):
861+
node = g.allocate(ALLOC_SIZE, device=d0.device_id, peer_access=[d1.device_id])
854862
assert d1.device_id in node.peer_access
855863

856864

@@ -863,8 +871,9 @@ def test_alloc_peer_access(mempool_device_x2):
863871
def test_join_merges_branches(sample_graphdef, num_branches):
864872
"""join() with multiple branches creates correct dependencies."""
865873
_skip_if_no_mempool()
866-
branches = [sample_graphdef.allocate(ALLOC_SIZE) for _ in range(num_branches)]
867-
joined = sample_graphdef.join(*branches)
874+
with xfail_on_graph_mempool_oom():
875+
branches = [sample_graphdef.allocate(ALLOC_SIZE) for _ in range(num_branches)]
876+
joined = sample_graphdef.join(*branches)
868877
assert isinstance(joined, EmptyNode)
869878
assert set(joined.pred) == set(branches)
870879

@@ -956,8 +965,9 @@ def test_instantiate_empty_graph(sample_graphdef, inst_kwargs):
956965
def test_instantiate_with_nodes(sample_graphdef, inst_kwargs):
957966
"""Graph with nodes can be instantiated."""
958967
_skip_if_no_mempool()
959-
sample_graphdef.allocate(ALLOC_SIZE)
960-
sample_graphdef.allocate(ALLOC_SIZE)
968+
with xfail_on_graph_mempool_oom():
969+
sample_graphdef.allocate(ALLOC_SIZE)
970+
sample_graphdef.allocate(ALLOC_SIZE)
961971
graph = _instantiate(sample_graphdef, inst_kwargs)
962972
assert graph is not None
963973

@@ -997,8 +1007,9 @@ def test_instantiate_and_execute_kernel(sample_graphdef, inst_kwargs):
9971007
def test_instantiate_and_execute_alloc_free(sample_graphdef, inst_kwargs):
9981008
"""Graph with alloc/free can be executed."""
9991009
_skip_if_no_mempool()
1000-
alloc = sample_graphdef.allocate(ALLOC_SIZE)
1001-
alloc.deallocate(alloc.dptr)
1010+
with xfail_on_graph_mempool_oom():
1011+
alloc = sample_graphdef.allocate(ALLOC_SIZE)
1012+
alloc.deallocate(alloc.dptr)
10021013

10031014
stream = Device().create_stream()
10041015
graph = _instantiate_and_upload(sample_graphdef, inst_kwargs, stream)
@@ -1010,9 +1021,10 @@ def test_instantiate_and_execute_alloc_free(sample_graphdef, inst_kwargs):
10101021
def test_instantiate_and_execute_memset(sample_graphdef, inst_kwargs):
10111022
"""Graph with alloc/memset/free can be executed."""
10121023
_skip_if_no_mempool()
1013-
alloc = sample_graphdef.allocate(ALLOC_SIZE)
1014-
ms = alloc.memset(alloc.dptr, 0xAB, ALLOC_SIZE)
1015-
ms.deallocate(alloc.dptr)
1024+
with xfail_on_graph_mempool_oom():
1025+
alloc = sample_graphdef.allocate(ALLOC_SIZE)
1026+
ms = alloc.memset(alloc.dptr, 0xAB, ALLOC_SIZE)
1027+
ms.deallocate(alloc.dptr)
10161028

10171029
stream = Device().create_stream()
10181030
graph = _instantiate_and_upload(sample_graphdef, inst_kwargs, stream)
@@ -1026,12 +1038,13 @@ def test_instantiate_and_execute_memcpy(sample_graphdef, inst_kwargs):
10261038
_skip_if_no_mempool()
10271039
import ctypes
10281040

1029-
src_alloc = sample_graphdef.allocate(ALLOC_SIZE)
1030-
dst_alloc = sample_graphdef.allocate(ALLOC_SIZE)
1031-
dep = sample_graphdef.join(src_alloc, dst_alloc)
1032-
ms = dep.memset(src_alloc.dptr, 0xAB, ALLOC_SIZE)
1033-
cp = ms.memcpy(dst_alloc.dptr, src_alloc.dptr, ALLOC_SIZE)
1034-
cp.deallocate(src_alloc.dptr)
1041+
with xfail_on_graph_mempool_oom():
1042+
src_alloc = sample_graphdef.allocate(ALLOC_SIZE)
1043+
dst_alloc = sample_graphdef.allocate(ALLOC_SIZE)
1044+
dep = sample_graphdef.join(src_alloc, dst_alloc)
1045+
ms = dep.memset(src_alloc.dptr, 0xAB, ALLOC_SIZE)
1046+
cp = ms.memcpy(dst_alloc.dptr, src_alloc.dptr, ALLOC_SIZE)
1047+
cp.deallocate(src_alloc.dptr)
10351048

10361049
stream = Device().create_stream()
10371050
graph = _instantiate_and_upload(sample_graphdef, inst_kwargs, stream)
@@ -1166,11 +1179,12 @@ def test_instantiate_and_execute_if_then(sample_graphdef):
11661179
set_handle = mod.get_kernel("set_handle")
11671180
add_one = mod.get_kernel("add_one")
11681181

1169-
alloc = sample_graphdef.allocate(ctypes.sizeof(ctypes.c_int))
1170-
ms = alloc.memset(alloc.dptr, 0, ctypes.sizeof(ctypes.c_int))
1171-
setter = ms.launch(LaunchConfig(grid=1, block=1), set_handle, condition, 1)
1172-
if_node = setter.if_then(condition)
1173-
if_node.then.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr)
1182+
with xfail_on_graph_mempool_oom():
1183+
alloc = sample_graphdef.allocate(ctypes.sizeof(ctypes.c_int))
1184+
ms = alloc.memset(alloc.dptr, 0, ctypes.sizeof(ctypes.c_int))
1185+
setter = ms.launch(LaunchConfig(grid=1, block=1), set_handle, condition, 1)
1186+
if_node = setter.if_then(condition)
1187+
if_node.then.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr)
11741188

11751189
graph = sample_graphdef.instantiate()
11761190
stream = Device().create_stream()
@@ -1198,13 +1212,14 @@ def test_instantiate_and_execute_if_else(sample_graphdef):
11981212
set_handle = mod.get_kernel("set_handle")
11991213
add_one = mod.get_kernel("add_one")
12001214

1201-
alloc = sample_graphdef.allocate(ctypes.sizeof(ctypes.c_int))
1202-
ms = alloc.memset(alloc.dptr, 0, ctypes.sizeof(ctypes.c_int))
1203-
setter = ms.launch(LaunchConfig(grid=1, block=1), set_handle, condition, 0)
1204-
ie_node = setter.if_else(condition)
1205-
ie_node.then.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr)
1206-
n1 = ie_node.else_.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr)
1207-
n1.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr)
1215+
with xfail_on_graph_mempool_oom():
1216+
alloc = sample_graphdef.allocate(ctypes.sizeof(ctypes.c_int))
1217+
ms = alloc.memset(alloc.dptr, 0, ctypes.sizeof(ctypes.c_int))
1218+
setter = ms.launch(LaunchConfig(grid=1, block=1), set_handle, condition, 0)
1219+
ie_node = setter.if_else(condition)
1220+
ie_node.then.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr)
1221+
n1 = ie_node.else_.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr)
1222+
n1.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr)
12081223

12091224
graph = sample_graphdef.instantiate()
12101225
stream = Device().create_stream()
@@ -1232,12 +1247,13 @@ def test_instantiate_and_execute_switch(sample_graphdef):
12321247
set_handle = mod.get_kernel("set_handle")
12331248
add_one = mod.get_kernel("add_one")
12341249

1235-
alloc = sample_graphdef.allocate(ctypes.sizeof(ctypes.c_int))
1236-
ms = alloc.memset(alloc.dptr, 0, ctypes.sizeof(ctypes.c_int))
1237-
setter = ms.launch(LaunchConfig(grid=1, block=1), set_handle, condition, 2)
1238-
sw_node = setter.switch(condition, 4)
1239-
for branch in sw_node.branches:
1240-
branch.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr)
1250+
with xfail_on_graph_mempool_oom():
1251+
alloc = sample_graphdef.allocate(ctypes.sizeof(ctypes.c_int))
1252+
ms = alloc.memset(alloc.dptr, 0, ctypes.sizeof(ctypes.c_int))
1253+
setter = ms.launch(LaunchConfig(grid=1, block=1), set_handle, condition, 2)
1254+
sw_node = setter.switch(condition, 4)
1255+
for branch in sw_node.branches:
1256+
branch.launch(LaunchConfig(grid=1, block=1), add_one, alloc.dptr)
12411257

12421258
graph = sample_graphdef.instantiate()
12431259
stream = Device().create_stream()
@@ -1272,7 +1288,8 @@ def test_conditional_node_type_preserved_by_nodes(sample_graphdef):
12721288
def test_debug_dot_print_creates_file(sample_graphdef, dot_file):
12731289
"""debug_dot_print writes a DOT file."""
12741290
_skip_if_no_mempool()
1275-
sample_graphdef.allocate(ALLOC_SIZE)
1291+
with xfail_on_graph_mempool_oom():
1292+
sample_graphdef.allocate(ALLOC_SIZE)
12761293
sample_graphdef.debug_dot_print(str(dot_file))
12771294
assert dot_file.exists()
12781295
content = dot_file.read_text()
@@ -1282,7 +1299,8 @@ def test_debug_dot_print_creates_file(sample_graphdef, dot_file):
12821299
def test_debug_dot_print_with_options(sample_graphdef, dot_file):
12831300
"""debug_dot_print accepts GraphDebugPrintOptions."""
12841301
_skip_if_no_mempool()
1285-
sample_graphdef.allocate(ALLOC_SIZE)
1302+
with xfail_on_graph_mempool_oom():
1303+
sample_graphdef.allocate(ALLOC_SIZE)
12861304
options = GraphDebugPrintOptions(verbose=True, handles=True)
12871305
sample_graphdef.debug_dot_print(str(dot_file), options)
12881306
assert dot_file.exists()
@@ -1291,6 +1309,7 @@ def test_debug_dot_print_with_options(sample_graphdef, dot_file):
12911309
def test_debug_dot_print_invalid_options(sample_graphdef, dot_file):
12921310
"""debug_dot_print rejects invalid options type."""
12931311
_skip_if_no_mempool()
1294-
sample_graphdef.allocate(ALLOC_SIZE)
1312+
with xfail_on_graph_mempool_oom():
1313+
sample_graphdef.allocate(ALLOC_SIZE)
12951314
with pytest.raises(TypeError, match="options must be a GraphDebugPrintOptions"):
12961315
sample_graphdef.debug_dot_print(str(dot_file), "invalid")

‎cuda_core/tests/graph/test_graph_definition_errors.py‎

Lines changed: 24 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
from helpers.graph_kernels import compile_common_kernels
1010
from helpers.misc import try_create_condition
1111

12+
from conftest import xfail_on_graph_mempool_oom
1213
from cuda.core import Device, LaunchConfig
1314
from cuda.core._utils.cuda_utils import CUDAError
1415
from cuda.core.graph import (
@@ -69,7 +70,8 @@ def test_memset_invalid_value_size(init_cuda):
6970
"""memset with 3-byte value (not 1, 2, or 4) raises ValueError."""
7071
_skip_if_no_mempool()
7172
g = GraphDefinition()
72-
alloc = g.allocate(1024)
73+
with xfail_on_graph_mempool_oom():
74+
alloc = g.allocate(1024)
7375
with pytest.raises(ValueError):
7476
alloc.memset(alloc.dptr, b"\x01\x02\x03", 100)
7577

@@ -113,8 +115,9 @@ def test_join_single_predecessor(init_cuda):
113115
"""node.join() with no extra args creates a single-dep empty node."""
114116
_skip_if_no_mempool()
115117
g = GraphDefinition()
116-
a = g.allocate(1024)
117-
joined = a.join()
118+
with xfail_on_graph_mempool_oom():
119+
a = g.allocate(1024)
120+
joined = a.join()
118121
assert isinstance(joined, EmptyNode)
119122
assert set(joined.pred) == {a}
120123

@@ -136,7 +139,8 @@ def test_unmatched_alloc_succeeds(init_cuda):
136139
"""Alloc without corresponding free is valid (graph-scoped lifetime)."""
137140
_skip_if_no_mempool()
138141
g = GraphDefinition()
139-
g.allocate(1024)
142+
with xfail_on_graph_mempool_oom():
143+
g.allocate(1024)
140144
graph = g.instantiate()
141145
stream = Device().create_stream()
142146
graph.launch(stream)
@@ -174,10 +178,11 @@ def test_while_loop_zero_iterations(init_cuda):
174178

175179
g = GraphDefinition()
176180
condition = g.create_condition(default_value=0)
177-
alloc = g.allocate(SIZEOF_INT)
178-
ms = alloc.memset(alloc.dptr, 0, SIZEOF_INT)
179-
loop = ms.while_loop(condition)
180-
loop.body.launch(cfg, add_one, alloc.dptr)
181+
with xfail_on_graph_mempool_oom():
182+
alloc = g.allocate(SIZEOF_INT)
183+
ms = alloc.memset(alloc.dptr, 0, SIZEOF_INT)
184+
loop = ms.while_loop(condition)
185+
loop.body.launch(cfg, add_one, alloc.dptr)
181186

182187
graph = g.instantiate()
183188
stream = Device().create_stream()
@@ -202,10 +207,11 @@ def test_if_then_false_skips_body(init_cuda):
202207

203208
g = GraphDefinition()
204209
condition = g.create_condition(default_value=0)
205-
alloc = g.allocate(SIZEOF_INT)
206-
ms = alloc.memset(alloc.dptr, 0, SIZEOF_INT)
207-
if_node = ms.if_then(condition)
208-
if_node.then.launch(cfg, add_one, alloc.dptr)
210+
with xfail_on_graph_mempool_oom():
211+
alloc = g.allocate(SIZEOF_INT)
212+
ms = alloc.memset(alloc.dptr, 0, SIZEOF_INT)
213+
if_node = ms.if_then(condition)
214+
if_node.then.launch(cfg, add_one, alloc.dptr)
209215

210216
graph = g.instantiate()
211217
stream = Device().create_stream()
@@ -230,11 +236,12 @@ def test_switch_oob_skips_all_branches(init_cuda):
230236

231237
g = GraphDefinition()
232238
condition = g.create_condition(default_value=99)
233-
alloc = g.allocate(SIZEOF_INT)
234-
ms = alloc.memset(alloc.dptr, 0, SIZEOF_INT)
235-
sw = ms.switch(condition, 3)
236-
for branch in sw.branches:
237-
branch.launch(cfg, add_one, alloc.dptr)
239+
with xfail_on_graph_mempool_oom():
240+
alloc = g.allocate(SIZEOF_INT)
241+
ms = alloc.memset(alloc.dptr, 0, SIZEOF_INT)
242+
sw = ms.switch(condition, 3)
243+
for branch in sw.branches:
244+
branch.launch(cfg, add_one, alloc.dptr)
238245

239246
graph = g.instantiate()
240247
stream = Device().create_stream()

‎cuda_core/tests/graph/test_graph_definition_integration.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import numpy as np
99
import pytest
1010

11+
from conftest import xfail_on_graph_mempool_oom
1112
from cuda.core import Device, EventOptions, LaunchConfig, Program, ProgramOptions
1213
from cuda.core._utils.cuda_utils import driver, handle_return
1314
from cuda.core.graph import GraphDefinition
@@ -204,7 +205,8 @@ def test_heat_diffusion(init_cuda):
204205
host_ptr = handle_return(driver.cuMemAllocHost(_HEAT_N * SIZEOF_FLOAT))
205206

206207
try:
207-
_run_heat_graph(dev, k_heat, k_countdown, host_ptr)
208+
with xfail_on_graph_mempool_oom(dev):
209+
_run_heat_graph(dev, k_heat, k_countdown, host_ptr)
208210
finally:
209211
handle_return(driver.cuMemFreeHost(host_ptr))
210212

@@ -314,7 +316,8 @@ def test_bisection_root(init_cuda):
314316
host_ptr = handle_return(driver.cuMemAllocHost(SIZEOF_FLOAT))
315317

316318
try:
317-
_run_bisection_graph(dev, k_eval, k_hi, k_lo, k_cd, k_check, k_newton, host_ptr)
319+
with xfail_on_graph_mempool_oom(dev):
320+
_run_bisection_graph(dev, k_eval, k_hi, k_lo, k_cd, k_check, k_newton, host_ptr)
318321
finally:
319322
handle_return(driver.cuMemFreeHost(host_ptr))
320323

@@ -416,7 +419,8 @@ def test_switch_dispatch(init_cuda, mode, expected):
416419
host_ptr = handle_return(driver.cuMemAllocHost(SIZEOF_INT))
417420

418421
try:
419-
_run_switch_graph(dev, mode, k_negate, k_double, k_square, host_ptr)
422+
with xfail_on_graph_mempool_oom(dev):
423+
_run_switch_graph(dev, mode, k_negate, k_double, k_square, host_ptr)
420424

421425
result = ctypes.c_int.from_address(host_ptr).value
422426
assert result == expected

0 commit comments

Comments
 (0)