Skip to content

Commit 08a42d7

Browse files
authored
test(cuda.core): handle default managed pool OOM on Windows MCDM (follow-up to #3004) (#3055)
test(cuda.core): handle default managed pool OOM on Windows MCDM (follow-up to #3004) On some Windows MCDM machines the default managed pool lookup fails with CUDA_ERROR_OUT_OF_MEMORY (VA reservation, nvbugs5815123) instead of CUDA_ERROR_NOT_SUPPORTED, which #3004 exposed by removing the OOM xfail. - create_managed_memory_resource_or_skip() skips the no-options path when concurrent managed access is unavailable, before the lookup. - test_default_pool_error_without_concurrent_access xfails on Windows CUDA_ERROR_OUT_OF_MEMORY.
1 parent c7e6f09 commit 08a42d7

3 files changed

Lines changed: 58 additions & 3 deletions

File tree

‎cuda_core/tests/helpers/memory.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
import pytest
77

8-
from cuda.core import ManagedMemoryResource, PinnedMemoryResource
8+
from cuda.core import Device, ManagedMemoryResource, PinnedMemoryResource
99
from cuda.core._utils.cuda_utils import CUDAError
1010

1111

@@ -33,6 +33,13 @@ def skip_if_managed_memory_unsupported(device):
3333

3434
def create_managed_memory_resource_or_skip(*args, xfail_device=None, **kwargs):
3535
# Keep the established "skip" helper name for call-site readability.
36+
if not args and kwargs.get("options") is None and not Device().properties.concurrent_managed_access:
37+
# Without options this looks up the device's default managed pool, which
38+
# supports managed allocations only with concurrent managed access. Check
39+
# the property instead of the lookup's error: the driver returns
40+
# CUDA_ERROR_NOT_SUPPORTED on some machines and CUDA_ERROR_OUT_OF_MEMORY
41+
# on others. Dedicated pools (options given) work without it.
42+
pytest.skip("Device does not support concurrent managed memory access")
3643
try:
3744
return ManagedMemoryResource(*args, **kwargs)
3845
except CUDAError as e:

‎cuda_core/tests/test_helpers.py‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
from helpers.buffers import PatternGen, compare_equal_buffers, make_scratch_buffer, thread_unsafe_on_windows
1313
from helpers.latch import LatchKernel
1414
from helpers.logging import TimestampedLogger
15+
from helpers.memory import create_managed_memory_resource_or_skip
1516
from helpers.oom_diagnostics import (
1617
DEFAULT_FILENAME,
1718
OOM_MARKER,
@@ -467,3 +468,40 @@ def test_gl_context_not_on_nvidia_gpu_reason_rejects_other_vendors(monkeypatch,
467468
reason = gl_context_not_on_nvidia_gpu_reason()
468469
assert reason is not None
469470
assert "not on an NVIDIA GPU" in reason
471+
472+
473+
def _patch_managed_mr_factory(monkeypatch, concurrent_managed_access):
474+
# Stand in for the helper's Device and ManagedMemoryResource so the skip
475+
# decision does not depend on the local device.
476+
device = types.SimpleNamespace(
477+
properties=types.SimpleNamespace(concurrent_managed_access=concurrent_managed_access)
478+
)
479+
monkeypatch.setattr("helpers.memory.Device", lambda: device)
480+
factory = Mock(name="ManagedMemoryResource")
481+
monkeypatch.setattr("helpers.memory.ManagedMemoryResource", factory)
482+
return factory
483+
484+
485+
@pytest.mark.thread_unsafe(reason="patches module globals of helpers.memory")
486+
@pytest.mark.agent_authored(model="claude-opus-5.5")
487+
def test_managed_mr_helper_skips_default_pool_without_concurrent_access(monkeypatch):
488+
factory = _patch_managed_mr_factory(monkeypatch, concurrent_managed_access=False)
489+
with pytest.raises(pytest.skip.Exception, match="concurrent managed memory access"):
490+
create_managed_memory_resource_or_skip()
491+
factory.assert_not_called()
492+
493+
494+
@pytest.mark.thread_unsafe(reason="patches module globals of helpers.memory")
495+
@pytest.mark.parametrize(
496+
("concurrent_managed_access", "args", "kwargs"),
497+
[
498+
pytest.param(False, ("options",), {}, id="positional-options"),
499+
pytest.param(False, (), {"options": "options"}, id="keyword-options"),
500+
pytest.param(True, (), {}, id="default-pool-with-concurrent-access"),
501+
],
502+
)
503+
@pytest.mark.agent_authored(model="claude-opus-5.5")
504+
def test_managed_mr_helper_constructs_when_pool_is_usable(monkeypatch, concurrent_managed_access, args, kwargs):
505+
factory = _patch_managed_mr_factory(monkeypatch, concurrent_managed_access)
506+
assert create_managed_memory_resource_or_skip(*args, **kwargs) is factory.return_value
507+
factory.assert_called_once_with(*args, **kwargs)

‎cuda_core/tests/test_managed_memory_warning.py‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
These tests only run on affected platforms (concurrent_managed_access is False).
99
"""
1010

11+
import sys
1112
import warnings
1213

1314
import pytest
@@ -16,6 +17,7 @@
1617
import cuda.bindings
1718
from cuda.core import Device, ManagedMemoryResource, ManagedMemoryResourceOptions
1819
from cuda.core._memory._managed_memory_resource import reset_concurrent_access_warning
20+
from cuda.core._utils.cuda_utils import CUDAError
1921

2022
_cuda_major = int(cuda.bindings.__version__.split(".")[0])
2123

@@ -51,8 +53,16 @@ def device_without_concurrent_managed_access(init_cuda):
5153
@requires_cuda_13
5254
def test_default_pool_error_without_concurrent_access(device_without_concurrent_managed_access):
5355
"""ManagedMemoryResource() raises RuntimeError when the default pool doesn't support managed."""
54-
with pytest.raises(RuntimeError, match="does not support managed allocations"):
55-
ManagedMemoryResource()
56+
try:
57+
with pytest.raises(RuntimeError, match="does not support managed allocations"):
58+
ManagedMemoryResource()
59+
except CUDAError as e:
60+
# nvbugs5815123: on some Windows machines the driver fails to reserve VA
61+
# for the default managed pool lookup and returns CUDA_ERROR_OUT_OF_MEMORY
62+
# instead of CUDA_ERROR_NOT_SUPPORTED, so the RuntimeError is never raised.
63+
if sys.platform == "win32" and "CUDA_ERROR_OUT_OF_MEMORY" in str(e):
64+
pytest.xfail("default managed pool lookup ran out of VA on Windows (nvbugs5815123)")
65+
raise
5666

5767

5868
@requires_cuda_13

0 commit comments

Comments
 (0)