|
| 1 | +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. |
| 2 | +# SPDX-License-Identifier: Apache-2.0 |
| 3 | + |
| 4 | +"""Force the driver's two large address-space reservations up front (issue #2381). |
| 5 | +
|
| 6 | +The driver keeps two pools per device that it never gives back, and each one |
| 7 | +reserves a virtual address window of roughly twice the installed device memory |
| 8 | +when it is first touched: |
| 9 | +
|
| 10 | +- the **default device mempool**, reserved by ``cuDeviceGetMemPool`` |
| 11 | +- the **graph memory pool**, reserved by ``cuGraphAddMemAllocNode`` at node |
| 12 | + creation time |
| 13 | +
|
| 14 | +Neither is capped by ``max_size`` and neither is released by |
| 15 | +``cuDeviceGraphMemTrim``, ``cuGraphDestroy``, or anything else short of process |
| 16 | +exit. On a large-memory GPU with a bounded per-process address space -- 357 GiB |
| 17 | +per pool on a 179 GiB device -- the two together consume most of the budget, and |
| 18 | +whether the *second* one finds a contiguous range depends on how fragmented the |
| 19 | +space has become. That is what makes the full-suite failures intermittent, and |
| 20 | +why they always begin at the first test to need whichever pool came second. |
| 21 | +
|
| 22 | +Taking both reservations at session start, back to back into a nearly empty |
| 23 | +address space, removes test order and accumulated fragmentation from the |
| 24 | +question. It does not reduce the footprint; it makes the outcome deterministic. |
| 25 | +
|
| 26 | +Measurement here goes through ``cuMemAddressReserve``, not the OS, so it behaves |
| 27 | +the same on Windows and Linux. |
| 28 | +""" |
| 29 | + |
| 30 | +from __future__ import annotations |
| 31 | + |
| 32 | +import os |
| 33 | +import time |
| 34 | + |
| 35 | +from cuda.bindings import driver |
| 36 | + |
| 37 | +MIB = 1024 * 1024 |
| 38 | +GIB = 1024 * MIB |
| 39 | + |
| 40 | +# cuMemAddressReserve wants a power-of-two alignment and a size that is a |
| 41 | +# multiple of it. 2 MiB is the granularity the driver uses for pools. |
| 42 | +VA_ALIGNMENT = 2 * MIB |
| 43 | +# Above any plausible per-process budget, so the descending probe below always |
| 44 | +# starts from a size that fails. |
| 45 | +MAX_PROBE_BYTES = 1 << 46 |
| 46 | + |
| 47 | +# Each driver-managed pool reserves about this multiple of installed device |
| 48 | +# memory. Used to express remaining headroom in units of "one more pool". |
| 49 | +POOL_RESERVATION_MULTIPLE = 2 |
| 50 | + |
| 51 | + |
| 52 | +def align_up(size: int) -> int: |
| 53 | + """Round to a multiple of the reservation alignment. |
| 54 | +
|
| 55 | + Device memory sizes are not generally a multiple of it, and |
| 56 | + cuMemAddressReserve rejects a size that is not with |
| 57 | + CUDA_ERROR_INVALID_VALUE -- which would otherwise read as "no address space |
| 58 | + left" at every size probed. |
| 59 | + """ |
| 60 | + return ((size + VA_ALIGNMENT - 1) // VA_ALIGNMENT) * VA_ALIGNMENT |
| 61 | + |
| 62 | + |
| 63 | +def _reserve_and_release(size: int) -> bool: |
| 64 | + """True if the driver still grants a contiguous reservation of ``size``. |
| 65 | +
|
| 66 | + Reserving costs address space but no memory, so this reads what the address |
| 67 | + space can satisfy without perturbing it. Raises if the release fails, since |
| 68 | + a leaked reservation would corrupt every later measurement. |
| 69 | + """ |
| 70 | + err, ptr = driver.cuMemAddressReserve(align_up(size), VA_ALIGNMENT, 0, 0) |
| 71 | + if err != driver.CUresult.CUDA_SUCCESS: |
| 72 | + return False |
| 73 | + (err,) = driver.cuMemAddressFree(ptr, align_up(size)) |
| 74 | + if err != driver.CUresult.CUDA_SUCCESS: |
| 75 | + raise RuntimeError(f"cuMemAddressFree({size:#x}) -> {err!r}; address space measurement is unreliable") |
| 76 | + return True |
| 77 | + |
| 78 | + |
| 79 | +def largest_reservable(reserve=None, max_bytes: int = MAX_PROBE_BYTES, refine_steps: int = 4) -> int: |
| 80 | + """Largest contiguous reservation the driver still grants. |
| 81 | +
|
| 82 | + Halves down from ``max_bytes`` rather than doubling up from the granularity, |
| 83 | + because a *refused* reservation allocates nothing and returns immediately |
| 84 | + while releasing a granted one is expensive -- hundreds of milliseconds for a |
| 85 | + large range, and seconds on some configurations. Descending pays that cost |
| 86 | + once instead of once per rung. |
| 87 | +
|
| 88 | + Halving alone only resolves to a power of two, which is too coarse to show |
| 89 | + what the reservations cost when the budget is much larger than they are, so |
| 90 | + ``refine_steps`` bisections then narrow the answer. Each bisection risks one |
| 91 | + more expensive release, hence the small default. |
| 92 | +
|
| 93 | + ``reserve`` is injectable so the search can be tested without a GPU. |
| 94 | + """ |
| 95 | + reserve = _reserve_and_release if reserve is None else reserve |
| 96 | + |
| 97 | + size = max_bytes |
| 98 | + while size >= VA_ALIGNMENT and not reserve(size): |
| 99 | + size //= 2 |
| 100 | + if size < VA_ALIGNMENT: |
| 101 | + return 0 |
| 102 | + if size == max_bytes: |
| 103 | + return size # nothing was refused, so there is no bracket to narrow |
| 104 | + |
| 105 | + low, high = size, size * 2 # high was refused on the way down |
| 106 | + for _ in range(refine_steps): |
| 107 | + middle = ((low + high) // 2 // VA_ALIGNMENT) * VA_ALIGNMENT |
| 108 | + if middle <= low or middle >= high: |
| 109 | + break |
| 110 | + if reserve(middle): |
| 111 | + low = middle |
| 112 | + else: |
| 113 | + high = middle |
| 114 | + return low |
| 115 | + |
| 116 | + |
| 117 | +def vmm_supported(device_id: int = 0) -> bool: |
| 118 | + """True if this device exposes the virtual memory management APIs.""" |
| 119 | + err, dev = driver.cuDeviceGet(device_id) |
| 120 | + if err != driver.CUresult.CUDA_SUCCESS: |
| 121 | + return False |
| 122 | + attribute = driver.CUdevice_attribute.CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED |
| 123 | + err, supported = driver.cuDeviceGetAttribute(attribute, dev) |
| 124 | + return err == driver.CUresult.CUDA_SUCCESS and bool(supported) |
| 125 | + |
| 126 | + |
| 127 | +def format_bytes(value: int | None) -> str: |
| 128 | + if value is None: |
| 129 | + return "unknown" |
| 130 | + return f"{value / GIB:.2f} GiB" |
| 131 | + |
| 132 | + |
| 133 | +class Reservation: |
| 134 | + """One driver-managed pool that has to be materialized.""" |
| 135 | + |
| 136 | + def __init__(self, name: str, detail: str, materialize): |
| 137 | + self.name = name |
| 138 | + self.detail = detail |
| 139 | + self._materialize = materialize |
| 140 | + self.error: str | None = None |
| 141 | + |
| 142 | + def run(self) -> bool: |
| 143 | + """Materialize the pool. Returns True on success, recording any error.""" |
| 144 | + try: |
| 145 | + self._materialize() |
| 146 | + except Exception as exc: # surfaced to the user by build_failure_message |
| 147 | + self.error = f"{type(exc).__name__}: {exc}" |
| 148 | + return False |
| 149 | + return True |
| 150 | + |
| 151 | + @property |
| 152 | + def succeeded(self) -> bool: |
| 153 | + return self.error is None |
| 154 | + |
| 155 | + |
| 156 | +def _materialize_default_mempool(device): |
| 157 | + """Touch the device's default memory pool. |
| 158 | +
|
| 159 | + cuDeviceGetMemPool is the call that makes the reservation; the allocation |
| 160 | + only proves the pool is usable afterwards. |
| 161 | + """ |
| 162 | + buffer = device.memory_resource.allocate(8, stream=device.default_stream) |
| 163 | + device.default_stream.sync() |
| 164 | + buffer.close(device.default_stream) |
| 165 | + device.default_stream.sync() |
| 166 | + |
| 167 | + |
| 168 | +def _materialize_graph_mempool(device): |
| 169 | + """Add one graph memory-allocation node, which reserves the graph pool. |
| 170 | +
|
| 171 | + The graph is discarded immediately: the address reservation it triggers |
| 172 | + outlives it, which is the whole point. |
| 173 | + """ |
| 174 | + from cuda.core.graph import GraphDefinition |
| 175 | + |
| 176 | + definition = GraphDefinition() |
| 177 | + definition.allocate(1024) |
| 178 | + del definition |
| 179 | + |
| 180 | + |
| 181 | +def _forced_failure(): |
| 182 | + """Stand-in for a refused reservation, so the abort path can be exercised. |
| 183 | +
|
| 184 | + Set CUDA_CORE_TEST_FORCE_RESERVATION_FAILURE=1 to check what this suite |
| 185 | + reports on a machine whose address space is too small, without needing one. |
| 186 | + """ |
| 187 | + raise RuntimeError("CUDA_ERROR_OUT_OF_MEMORY: simulated refusal (CUDA_CORE_TEST_FORCE_RESERVATION_FAILURE is set)") |
| 188 | + |
| 189 | + |
| 190 | +def reservations_for(device) -> list[Reservation]: |
| 191 | + if os.environ.get("CUDA_CORE_TEST_FORCE_RESERVATION_FAILURE", "0") not in ("0", ""): |
| 192 | + return [ |
| 193 | + Reservation("default device mempool", "cuDeviceGetMemPool", _forced_failure), |
| 194 | + Reservation("graph memory pool", "cuGraphAddMemAllocNode", _forced_failure), |
| 195 | + ] |
| 196 | + return [ |
| 197 | + Reservation( |
| 198 | + "default device mempool", |
| 199 | + "cuDeviceGetMemPool", |
| 200 | + lambda: _materialize_default_mempool(device), |
| 201 | + ), |
| 202 | + Reservation( |
| 203 | + "graph memory pool", |
| 204 | + "cuGraphAddMemAllocNode", |
| 205 | + lambda: _materialize_graph_mempool(device), |
| 206 | + ), |
| 207 | + ] |
| 208 | + |
| 209 | + |
| 210 | +class ReservationReport: |
| 211 | + """What the early reservations cost, for the terminal.""" |
| 212 | + |
| 213 | + def __init__(self, device_name, device_memory, before, after, reservations, measured, seconds=0.0): |
| 214 | + self.device_name = device_name |
| 215 | + self.device_memory = device_memory |
| 216 | + self.before = before |
| 217 | + self.after = after |
| 218 | + self.reservations = reservations |
| 219 | + self.measured = measured |
| 220 | + self.seconds = seconds |
| 221 | + |
| 222 | + @property |
| 223 | + def failed(self) -> list[Reservation]: |
| 224 | + return [item for item in self.reservations if not item.succeeded] |
| 225 | + |
| 226 | + @property |
| 227 | + def pool_reservation_bytes(self) -> int | None: |
| 228 | + if self.device_memory is None: |
| 229 | + return None |
| 230 | + return align_up(POOL_RESERVATION_MULTIPLE * self.device_memory) |
| 231 | + |
| 232 | + def lines(self) -> list[str]: |
| 233 | + out = [f"device 0: {self.device_name} ({format_bytes(self.device_memory)} device memory)"] |
| 234 | + if self.measured: |
| 235 | + out.append(f"largest reservable range before: {format_bytes(self.before)}") |
| 236 | + else: |
| 237 | + out.append("largest reservable range: not measured (no virtual memory management support)") |
| 238 | + |
| 239 | + for item in self.reservations: |
| 240 | + status = "reserved" if item.succeeded else f"FAILED: {item.error}" |
| 241 | + out.append(f" {item.name:<24} {item.detail:<24} {status}") |
| 242 | + |
| 243 | + if self.measured: |
| 244 | + # A drop here is a *lower* bound on what was taken: the driver may |
| 245 | + # carve its reservations out of a region other than the largest |
| 246 | + # hole, in which case the largest hole does not move at all. |
| 247 | + change = None if self.before is None or self.after is None else self.before - self.after |
| 248 | + if change is None: |
| 249 | + note = "" |
| 250 | + elif change > 0: |
| 251 | + note = f" (largest hole shrank by {format_bytes(change)})" |
| 252 | + else: |
| 253 | + note = " (largest hole unchanged)" |
| 254 | + out.append(f"largest reservable range after: {format_bytes(self.after)}{note}") |
| 255 | + pool_bytes = self.pool_reservation_bytes |
| 256 | + if pool_bytes and self.after is not None: |
| 257 | + out.append( |
| 258 | + f"remaining headroom: {self.after // pool_bytes} more pool-sized " |
| 259 | + f"({format_bytes(pool_bytes)}) reservations [{self.seconds:.1f}s measuring]" |
| 260 | + ) |
| 261 | + return out |
| 262 | + |
| 263 | + |
| 264 | +def build_failure_message(report: ReservationReport) -> str: |
| 265 | + """Explain, for a human, why this machine cannot run the suite.""" |
| 266 | + pool_bytes = report.pool_reservation_bytes |
| 267 | + failed = ", ".join(item.name for item in report.failed) |
| 268 | + lines = [ |
| 269 | + "", |
| 270 | + "cuda_core tests cannot run on this machine: the CUDA driver could not reserve", |
| 271 | + f"address space for {failed}.", |
| 272 | + "", |
| 273 | + f" device 0 {report.device_name}", |
| 274 | + f" installed device memory {format_bytes(report.device_memory)}", |
| 275 | + f" needed per driver-managed pool {format_bytes(pool_bytes)} of *virtual address space*", |
| 276 | + f" largest range still available {format_bytes(report.after if report.measured else None)}", |
| 277 | + "", |
| 278 | + ] |
| 279 | + for item in report.failed: |
| 280 | + lines.append(f" {item.name} ({item.detail}): {item.error}") |
| 281 | + lines += [ |
| 282 | + "", |
| 283 | + "The driver keeps two pools per device -- the default memory pool and the graph", |
| 284 | + "memory pool -- and reserves roughly twice the installed device memory of address", |
| 285 | + "space for each one the first time it is used. Neither reservation can be capped,", |
| 286 | + "and neither is released before the process exits. On a large-memory GPU with a", |
| 287 | + "bounded per-process address space the two together can exceed the budget, and no", |
| 288 | + "amount of freeing device memory helps: the exhausted resource is address space,", |
| 289 | + "not memory. Expect cuMemGetInfo to report plenty free while this fails.", |
| 290 | + "", |
| 291 | + "Options: run on a device with less memory, run the graph tests in a separate", |
| 292 | + "process from the rest of the suite so the two reservations never coexist, or on", |
| 293 | + "Windows check whether the driver model (WDDM/MCDM/TCC) bounds the address space", |
| 294 | + "more tightly than expected. See issue #2381.", |
| 295 | + "", |
| 296 | + ] |
| 297 | + return "\n".join(lines) |
| 298 | + |
| 299 | + |
| 300 | +def reserve_driver_pools(device, measure: bool = True) -> ReservationReport: |
| 301 | + """Materialize both driver-managed pools, measuring address space around them.""" |
| 302 | + device_memory = None |
| 303 | + err, _free, total = driver.cuMemGetInfo() |
| 304 | + if err == driver.CUresult.CUDA_SUCCESS: |
| 305 | + device_memory = int(total) |
| 306 | + |
| 307 | + measured = measure and vmm_supported(device.device_id) |
| 308 | + started = time.perf_counter() |
| 309 | + before = largest_reservable() if measured else None |
| 310 | + elapsed = time.perf_counter() - started |
| 311 | + |
| 312 | + reservations = reservations_for(device) |
| 313 | + for item in reservations: |
| 314 | + item.run() |
| 315 | + |
| 316 | + started = time.perf_counter() |
| 317 | + after = largest_reservable() if measured else None |
| 318 | + elapsed += time.perf_counter() - started |
| 319 | + return ReservationReport(device.name, device_memory, before, after, reservations, measured, elapsed) |
0 commit comments