Skip to content

Commit 540d408

Browse files
committed
test(cuda_core): reserve the driver's two pool address windows at session start
The driver's default memory pool and graph memory pool each permanently reserve roughly twice the installed device memory of virtual address space, and neither can be capped or released. On a large-memory GPU with a bounded per-process address space, whether the second reservation still finds a contiguous range depends on how fragmented the space has become by the time some test needs it, which is what makes the failures in NVIDIA#2381 intermittent. Measurements and documentation references: NVIDIA#2381 (comment) Take both reservations up front in a session-scoped autouse fixture so they land back to back in a nearly empty address space. This does not reduce the footprint; it makes the outcome deterministic. If either is refused the session aborts with an explanation, since every later test needing that pool would fail the same way and a cascade of identical OOM errors says nothing about the cause. Available address space is reported before and after via cuMemAddressReserve rather than an OS query, so the measurement behaves the same on Windows and Linux. CUDA_CORE_TEST_SKIP_EARLY_RESERVATION=1 disables the fixture; CUDA_CORE_TEST_FORCE_RESERVATION_FAILURE=1 exercises the abort path.
1 parent be1e752 commit 540d408

3 files changed

Lines changed: 471 additions & 0 deletions

File tree

‎cuda_core/tests/conftest.py‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131

3232
from cuda_python_test_helpers.marks import skipif_need_cuda_headers # noqa: F401 (re-exported for tests)
3333
from cuda_python_test_helpers.mempool import xfail_if_mempool_oom
34+
from helpers import va_reservation
3435
from helpers.constants import POOL_SIZE
3536

3637
import cuda.core
@@ -55,6 +56,53 @@ def pytest_configure(config):
5556
config.pluginmanager.register(_CudaCoreParallelPlugin(), name="_cuda_core_parallel_plugin")
5657

5758

59+
_reservation_report = None
60+
61+
62+
@pytest.fixture(scope="session", autouse=True)
63+
def reserve_driver_pools(request, session_setup):
64+
"""Take the driver's two large address-space reservations before anything else.
65+
66+
Session-scoped and autouse so both reservations land back to back in a
67+
nearly empty address space, ahead of any test that could fragment it.
68+
Depends on session_setup for cuInit. See helpers/va_reservation.py for why
69+
this matters, and issue #2381.
70+
71+
Aborts the session with an explanation if either reservation is refused:
72+
every later test that needs the pool would fail the same way, and the
73+
resulting cascade of identical OOM errors says nothing about the cause.
74+
"""
75+
global _reservation_report
76+
77+
if int(os.environ.get("CUDA_CORE_TEST_SKIP_EARLY_RESERVATION", 0)) != 0:
78+
yield
79+
return
80+
81+
with _init_cuda_context() as device:
82+
_reservation_report = va_reservation.reserve_driver_pools(device)
83+
84+
terminal_reporter = request.config.pluginmanager.get_plugin("terminalreporter")
85+
if terminal_reporter is not None:
86+
# Written here rather than only in the summary so the numbers survive a
87+
# session that dies partway through.
88+
terminal_reporter.write_sep("=", "cuda_core address space reservation")
89+
for line in _reservation_report.lines():
90+
terminal_reporter.write_line(line)
91+
92+
if _reservation_report.failed:
93+
pytest.exit(va_reservation.build_failure_message(_reservation_report), returncode=1)
94+
95+
yield
96+
97+
98+
def pytest_terminal_summary(terminalreporter):
99+
if _reservation_report is None or _reservation_report.failed:
100+
return
101+
terminalreporter.write_sep("=", "cuda_core address space reservation")
102+
for line in _reservation_report.lines():
103+
terminalreporter.write_line(line)
104+
105+
58106
@contextmanager
59107
def _init_cuda_context():
60108
# TODO: rename this to e.g. init_context
Lines changed: 319 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,319 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
"""Force the driver's two large address-space reservations up front (issue #2381).
5+
6+
The driver keeps two pools per device that it never gives back, and each one
7+
reserves a virtual address window of roughly twice the installed device memory
8+
when it is first touched:
9+
10+
- the **default device mempool**, reserved by ``cuDeviceGetMemPool``
11+
- the **graph memory pool**, reserved by ``cuGraphAddMemAllocNode`` at node
12+
creation time
13+
14+
Neither is capped by ``max_size`` and neither is released by
15+
``cuDeviceGraphMemTrim``, ``cuGraphDestroy``, or anything else short of process
16+
exit. On a large-memory GPU with a bounded per-process address space -- 357 GiB
17+
per pool on a 179 GiB device -- the two together consume most of the budget, and
18+
whether the *second* one finds a contiguous range depends on how fragmented the
19+
space has become. That is what makes the full-suite failures intermittent, and
20+
why they always begin at the first test to need whichever pool came second.
21+
22+
Taking both reservations at session start, back to back into a nearly empty
23+
address space, removes test order and accumulated fragmentation from the
24+
question. It does not reduce the footprint; it makes the outcome deterministic.
25+
26+
Measurement here goes through ``cuMemAddressReserve``, not the OS, so it behaves
27+
the same on Windows and Linux.
28+
"""
29+
30+
from __future__ import annotations
31+
32+
import os
33+
import time
34+
35+
from cuda.bindings import driver
36+
37+
MIB = 1024 * 1024
38+
GIB = 1024 * MIB
39+
40+
# cuMemAddressReserve wants a power-of-two alignment and a size that is a
41+
# multiple of it. 2 MiB is the granularity the driver uses for pools.
42+
VA_ALIGNMENT = 2 * MIB
43+
# Above any plausible per-process budget, so the descending probe below always
44+
# starts from a size that fails.
45+
MAX_PROBE_BYTES = 1 << 46
46+
47+
# Each driver-managed pool reserves about this multiple of installed device
48+
# memory. Used to express remaining headroom in units of "one more pool".
49+
POOL_RESERVATION_MULTIPLE = 2
50+
51+
52+
def align_up(size: int) -> int:
53+
"""Round to a multiple of the reservation alignment.
54+
55+
Device memory sizes are not generally a multiple of it, and
56+
cuMemAddressReserve rejects a size that is not with
57+
CUDA_ERROR_INVALID_VALUE -- which would otherwise read as "no address space
58+
left" at every size probed.
59+
"""
60+
return ((size + VA_ALIGNMENT - 1) // VA_ALIGNMENT) * VA_ALIGNMENT
61+
62+
63+
def _reserve_and_release(size: int) -> bool:
64+
"""True if the driver still grants a contiguous reservation of ``size``.
65+
66+
Reserving costs address space but no memory, so this reads what the address
67+
space can satisfy without perturbing it. Raises if the release fails, since
68+
a leaked reservation would corrupt every later measurement.
69+
"""
70+
err, ptr = driver.cuMemAddressReserve(align_up(size), VA_ALIGNMENT, 0, 0)
71+
if err != driver.CUresult.CUDA_SUCCESS:
72+
return False
73+
(err,) = driver.cuMemAddressFree(ptr, align_up(size))
74+
if err != driver.CUresult.CUDA_SUCCESS:
75+
raise RuntimeError(f"cuMemAddressFree({size:#x}) -> {err!r}; address space measurement is unreliable")
76+
return True
77+
78+
79+
def largest_reservable(reserve=None, max_bytes: int = MAX_PROBE_BYTES, refine_steps: int = 4) -> int:
80+
"""Largest contiguous reservation the driver still grants.
81+
82+
Halves down from ``max_bytes`` rather than doubling up from the granularity,
83+
because a *refused* reservation allocates nothing and returns immediately
84+
while releasing a granted one is expensive -- hundreds of milliseconds for a
85+
large range, and seconds on some configurations. Descending pays that cost
86+
once instead of once per rung.
87+
88+
Halving alone only resolves to a power of two, which is too coarse to show
89+
what the reservations cost when the budget is much larger than they are, so
90+
``refine_steps`` bisections then narrow the answer. Each bisection risks one
91+
more expensive release, hence the small default.
92+
93+
``reserve`` is injectable so the search can be tested without a GPU.
94+
"""
95+
reserve = _reserve_and_release if reserve is None else reserve
96+
97+
size = max_bytes
98+
while size >= VA_ALIGNMENT and not reserve(size):
99+
size //= 2
100+
if size < VA_ALIGNMENT:
101+
return 0
102+
if size == max_bytes:
103+
return size # nothing was refused, so there is no bracket to narrow
104+
105+
low, high = size, size * 2 # high was refused on the way down
106+
for _ in range(refine_steps):
107+
middle = ((low + high) // 2 // VA_ALIGNMENT) * VA_ALIGNMENT
108+
if middle <= low or middle >= high:
109+
break
110+
if reserve(middle):
111+
low = middle
112+
else:
113+
high = middle
114+
return low
115+
116+
117+
def vmm_supported(device_id: int = 0) -> bool:
118+
"""True if this device exposes the virtual memory management APIs."""
119+
err, dev = driver.cuDeviceGet(device_id)
120+
if err != driver.CUresult.CUDA_SUCCESS:
121+
return False
122+
attribute = driver.CUdevice_attribute.CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED
123+
err, supported = driver.cuDeviceGetAttribute(attribute, dev)
124+
return err == driver.CUresult.CUDA_SUCCESS and bool(supported)
125+
126+
127+
def format_bytes(value: int | None) -> str:
128+
if value is None:
129+
return "unknown"
130+
return f"{value / GIB:.2f} GiB"
131+
132+
133+
class Reservation:
134+
"""One driver-managed pool that has to be materialized."""
135+
136+
def __init__(self, name: str, detail: str, materialize):
137+
self.name = name
138+
self.detail = detail
139+
self._materialize = materialize
140+
self.error: str | None = None
141+
142+
def run(self) -> bool:
143+
"""Materialize the pool. Returns True on success, recording any error."""
144+
try:
145+
self._materialize()
146+
except Exception as exc: # surfaced to the user by build_failure_message
147+
self.error = f"{type(exc).__name__}: {exc}"
148+
return False
149+
return True
150+
151+
@property
152+
def succeeded(self) -> bool:
153+
return self.error is None
154+
155+
156+
def _materialize_default_mempool(device):
157+
"""Touch the device's default memory pool.
158+
159+
cuDeviceGetMemPool is the call that makes the reservation; the allocation
160+
only proves the pool is usable afterwards.
161+
"""
162+
buffer = device.memory_resource.allocate(8, stream=device.default_stream)
163+
device.default_stream.sync()
164+
buffer.close(device.default_stream)
165+
device.default_stream.sync()
166+
167+
168+
def _materialize_graph_mempool(device):
169+
"""Add one graph memory-allocation node, which reserves the graph pool.
170+
171+
The graph is discarded immediately: the address reservation it triggers
172+
outlives it, which is the whole point.
173+
"""
174+
from cuda.core.graph import GraphDefinition
175+
176+
definition = GraphDefinition()
177+
definition.allocate(1024)
178+
del definition
179+
180+
181+
def _forced_failure():
182+
"""Stand-in for a refused reservation, so the abort path can be exercised.
183+
184+
Set CUDA_CORE_TEST_FORCE_RESERVATION_FAILURE=1 to check what this suite
185+
reports on a machine whose address space is too small, without needing one.
186+
"""
187+
raise RuntimeError("CUDA_ERROR_OUT_OF_MEMORY: simulated refusal (CUDA_CORE_TEST_FORCE_RESERVATION_FAILURE is set)")
188+
189+
190+
def reservations_for(device) -> list[Reservation]:
191+
if os.environ.get("CUDA_CORE_TEST_FORCE_RESERVATION_FAILURE", "0") not in ("0", ""):
192+
return [
193+
Reservation("default device mempool", "cuDeviceGetMemPool", _forced_failure),
194+
Reservation("graph memory pool", "cuGraphAddMemAllocNode", _forced_failure),
195+
]
196+
return [
197+
Reservation(
198+
"default device mempool",
199+
"cuDeviceGetMemPool",
200+
lambda: _materialize_default_mempool(device),
201+
),
202+
Reservation(
203+
"graph memory pool",
204+
"cuGraphAddMemAllocNode",
205+
lambda: _materialize_graph_mempool(device),
206+
),
207+
]
208+
209+
210+
class ReservationReport:
211+
"""What the early reservations cost, for the terminal."""
212+
213+
def __init__(self, device_name, device_memory, before, after, reservations, measured, seconds=0.0):
214+
self.device_name = device_name
215+
self.device_memory = device_memory
216+
self.before = before
217+
self.after = after
218+
self.reservations = reservations
219+
self.measured = measured
220+
self.seconds = seconds
221+
222+
@property
223+
def failed(self) -> list[Reservation]:
224+
return [item for item in self.reservations if not item.succeeded]
225+
226+
@property
227+
def pool_reservation_bytes(self) -> int | None:
228+
if self.device_memory is None:
229+
return None
230+
return align_up(POOL_RESERVATION_MULTIPLE * self.device_memory)
231+
232+
def lines(self) -> list[str]:
233+
out = [f"device 0: {self.device_name} ({format_bytes(self.device_memory)} device memory)"]
234+
if self.measured:
235+
out.append(f"largest reservable range before: {format_bytes(self.before)}")
236+
else:
237+
out.append("largest reservable range: not measured (no virtual memory management support)")
238+
239+
for item in self.reservations:
240+
status = "reserved" if item.succeeded else f"FAILED: {item.error}"
241+
out.append(f" {item.name:<24} {item.detail:<24} {status}")
242+
243+
if self.measured:
244+
# A drop here is a *lower* bound on what was taken: the driver may
245+
# carve its reservations out of a region other than the largest
246+
# hole, in which case the largest hole does not move at all.
247+
change = None if self.before is None or self.after is None else self.before - self.after
248+
if change is None:
249+
note = ""
250+
elif change > 0:
251+
note = f" (largest hole shrank by {format_bytes(change)})"
252+
else:
253+
note = " (largest hole unchanged)"
254+
out.append(f"largest reservable range after: {format_bytes(self.after)}{note}")
255+
pool_bytes = self.pool_reservation_bytes
256+
if pool_bytes and self.after is not None:
257+
out.append(
258+
f"remaining headroom: {self.after // pool_bytes} more pool-sized "
259+
f"({format_bytes(pool_bytes)}) reservations [{self.seconds:.1f}s measuring]"
260+
)
261+
return out
262+
263+
264+
def build_failure_message(report: ReservationReport) -> str:
265+
"""Explain, for a human, why this machine cannot run the suite."""
266+
pool_bytes = report.pool_reservation_bytes
267+
failed = ", ".join(item.name for item in report.failed)
268+
lines = [
269+
"",
270+
"cuda_core tests cannot run on this machine: the CUDA driver could not reserve",
271+
f"address space for {failed}.",
272+
"",
273+
f" device 0 {report.device_name}",
274+
f" installed device memory {format_bytes(report.device_memory)}",
275+
f" needed per driver-managed pool {format_bytes(pool_bytes)} of *virtual address space*",
276+
f" largest range still available {format_bytes(report.after if report.measured else None)}",
277+
"",
278+
]
279+
for item in report.failed:
280+
lines.append(f" {item.name} ({item.detail}): {item.error}")
281+
lines += [
282+
"",
283+
"The driver keeps two pools per device -- the default memory pool and the graph",
284+
"memory pool -- and reserves roughly twice the installed device memory of address",
285+
"space for each one the first time it is used. Neither reservation can be capped,",
286+
"and neither is released before the process exits. On a large-memory GPU with a",
287+
"bounded per-process address space the two together can exceed the budget, and no",
288+
"amount of freeing device memory helps: the exhausted resource is address space,",
289+
"not memory. Expect cuMemGetInfo to report plenty free while this fails.",
290+
"",
291+
"Options: run on a device with less memory, run the graph tests in a separate",
292+
"process from the rest of the suite so the two reservations never coexist, or on",
293+
"Windows check whether the driver model (WDDM/MCDM/TCC) bounds the address space",
294+
"more tightly than expected. See issue #2381.",
295+
"",
296+
]
297+
return "\n".join(lines)
298+
299+
300+
def reserve_driver_pools(device, measure: bool = True) -> ReservationReport:
301+
"""Materialize both driver-managed pools, measuring address space around them."""
302+
device_memory = None
303+
err, _free, total = driver.cuMemGetInfo()
304+
if err == driver.CUresult.CUDA_SUCCESS:
305+
device_memory = int(total)
306+
307+
measured = measure and vmm_supported(device.device_id)
308+
started = time.perf_counter()
309+
before = largest_reservable() if measured else None
310+
elapsed = time.perf_counter() - started
311+
312+
reservations = reservations_for(device)
313+
for item in reservations:
314+
item.run()
315+
316+
started = time.perf_counter()
317+
after = largest_reservable() if measured else None
318+
elapsed += time.perf_counter() - started
319+
return ReservationReport(device.name, device_memory, before, after, reservations, measured, elapsed)

0 commit comments

Comments
 (0)