Skip to content

Commit 66cbcaf

Browse files
Andy-Jostclaude
andcommitted
test(cuda.core): close leftover owned pools and run the teardown gc.collect() only for pool-owning tests
The `init_cuda` teardown added by #2484 runs `gc.collect()` after every test. In a single measurement run on one L40S GPU (not a CI runner), that collect costs 59 ms median per teardown and the `cuCtxSynchronize()` after it costs 0.03 ms. Removing the collect ran the suite at 0.63 of baseline with every test passing on a 46 GB GPU. That run was on Linux; the OOM scenario of #2381 was a Windows-side VA budget, so this PR keeps the collect where a pool can be left behind and must be judged on its Windows MCDM and WDDM CI rows. Co-Authored-By: Claude Fable 5.1 <[email protected]>
1 parent 5078fe6 commit 66cbcaf

20 files changed

Lines changed: 552 additions & 44 deletions

‎ci/tools/check_mempool_hygiene.py‎

Lines changed: 231 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,34 @@
11
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
22
# SPDX-License-Identifier: Apache-2.0
33

4-
"""Check that tests do not create uncapped CUDA memory pools.
4+
"""Check the memory-pool hygiene of the cuda_core tests.
55
6-
A pool created without ``max_size`` reserves an address-space window sized from
7-
installed device memory rather than from what the test allocates, and the whole
8-
cuda_core suite shares one process. Enough of those reservations exhaust the
9-
address space, after which the rest of the session fails with
10-
CUDA_ERROR_OUT_OF_MEMORY on a device with free physical memory.
6+
A memory resource constructed with an options object creates a pool that the
7+
test owns; one constructed without options wraps the device's current pool and
8+
owns nothing. The whole cuda_core suite shares one process, and an owned pool
9+
keeps its address-space reservation until the pool is destroyed and the
10+
stream-ordered frees of its allocations retire. Three rules keep that bounded:
1111
12-
See cuda_core/tests/AGENTS.md for the rule this enforces.
12+
1. max_size: a Device or Pinned pool sets ``max_size``. A pool without it
13+
reserves an address-space window sized from installed device memory rather
14+
than from what the test allocates, and enough of those reservations exhaust
15+
the address space, after which the rest of the session fails with
16+
CUDA_ERROR_OUT_OF_MEMORY on a device with free physical memory. Opt out with
17+
``# uncapped-pool-ok: <reason>``.
18+
2. close: a Device or Pinned pool is bound to a name and closed in the function
19+
that creates it, by ``.close()`` on that name or by a ``for`` loop that
20+
closes every element of the list the name holds. Memory resources are not
21+
context managers. Opt out with ``# unclosed-pool-ok: <reason>`` when the
22+
close happens elsewhere.
23+
3. owns_pool: a test that creates a pool of any kind, directly or through a
24+
function in the same module, carries ``@pytest.mark.owns_pool`` on the
25+
function, on its class, or in the module's ``pytestmark``; a fixture that
26+
creates a pool calls ``request.node.add_marker("owns_pool")``. The marker
27+
makes the ``init_cuda`` teardown run ``gc.collect()`` before it drains the
28+
context. Calls are followed by bare name and through ``self``; a pool created
29+
through a function passed in as a parameter is not seen.
30+
31+
See cuda_core/tests/AGENTS.md for the rules this enforces.
1332
"""
1433

1534
from __future__ import annotations
@@ -27,7 +46,24 @@
2746
CAPPABLE_OPTIONS = frozenset({"DeviceMemoryResourceOptions", "PinnedMemoryResourceOptions"})
2847
CAPPABLE_RESOURCES = frozenset({"DeviceMemoryResource", "PinnedMemoryResource"})
2948

30-
OPT_OUT_MARKER = "uncapped-pool-ok"
49+
# Every options class. Passing one to a resource creates an owned pool.
50+
POOL_OPTIONS = CAPPABLE_OPTIONS | {"ManagedMemoryResourceOptions", "VirtualMemoryResourceOptions"}
51+
# Resource constructors and the number of leading positional arguments that are
52+
# not options (the device). More positional arguments, or any keyword, pass options.
53+
POOL_RESOURCES = {
54+
"DeviceMemoryResource": 1,
55+
"PinnedMemoryResource": 0,
56+
"ManagedMemoryResource": 0,
57+
"VirtualMemoryResource": 1,
58+
}
59+
# Factories in cuda_core/tests/helpers/memory.py; they create a pool when given options.
60+
POOL_HELPERS = frozenset({"create_managed_memory_resource_or_skip", "create_pinned_memory_resource_or_xfail"})
61+
HELPER_NON_OPTION_KEYWORDS = frozenset({"xfail_device"})
62+
CAPPABLE_HELPERS = frozenset({"create_pinned_memory_resource_or_xfail"})
63+
64+
UNCAPPED_MARKER = "uncapped-pool-ok"
65+
UNCLOSED_MARKER = "unclosed-pool-ok"
66+
OWNS_POOL = "owns_pool"
3167

3268

3369
def _callee_name(node: ast.Call) -> str:
@@ -39,6 +75,17 @@ def _callee_name(node: ast.Call) -> str:
3975
return ""
4076

4177

78+
def _decorator_name(node: ast.AST) -> str:
79+
"""The last name of a decorator or mark expression: ``pytest.mark.owns_pool`` -> ``owns_pool``."""
80+
if isinstance(node, ast.Call):
81+
node = node.func
82+
if isinstance(node, ast.Attribute):
83+
return node.attr
84+
if isinstance(node, ast.Name):
85+
return node.id
86+
return ""
87+
88+
4289
def _is_capped(node: ast.Call) -> bool:
4390
# ``**kwargs`` (arg is None) may carry max_size; do not guess.
4491
return any(kw.arg is None or kw.arg == "max_size" for kw in node.keywords)
@@ -53,21 +100,40 @@ def _dict_is_capped(node: ast.Dict) -> bool:
53100
return False
54101

55102

56-
def _opted_out(lines: list[str], node: ast.AST) -> bool:
57-
"""True if the call, or the line above it, carries the opt-out marker."""
103+
def _passes_options(node: ast.Call) -> bool:
104+
"""True if a resource constructor or pool helper call passes options, so it creates a pool."""
105+
name = _callee_name(node)
106+
if name in POOL_RESOURCES:
107+
return len(node.args) > POOL_RESOURCES[name] or bool(node.keywords)
108+
if name in POOL_HELPERS:
109+
return bool(node.args) or any(kw.arg not in HELPER_NON_OPTION_KEYWORDS for kw in node.keywords)
110+
return False
111+
112+
113+
def _creates_pool(node: ast.Call) -> bool:
114+
"""True if the call creates an owned pool of any kind (rule 3)."""
115+
return _callee_name(node) in POOL_OPTIONS or _passes_options(node)
116+
117+
118+
def _creates_cappable_pool(node: ast.Call) -> bool:
119+
"""True if the call creates a Device or Pinned pool (rule 2)."""
120+
return _callee_name(node) in CAPPABLE_RESOURCES | CAPPABLE_HELPERS and _passes_options(node)
121+
122+
123+
def _opted_out(lines: list[str], node: ast.AST, markers: tuple[str, ...]) -> bool:
124+
"""True if the call, or the line above it, carries one of the opt-out markers."""
58125
start = max(node.lineno - 2, 0) # -1 for 0-based, -1 more for a preceding comment
59126
end = getattr(node, "end_lineno", node.lineno)
60-
return any(OPT_OUT_MARKER in line for line in lines[start:end])
127+
return any(marker in line for marker in markers for line in lines[start:end])
61128

62129

63-
def violations_in(path: Path) -> list[str]:
64-
"""Return one message per uncapped pool construction in ``path``."""
65-
source = path.read_text(encoding="utf-8")
66-
lines = source.splitlines()
130+
def _calls(node: ast.AST) -> list[ast.Call]:
131+
return [n for n in ast.walk(node) if isinstance(n, ast.Call)]
132+
133+
134+
def _uncapped_pools(tree: ast.Module, lines: list[str], path: Path) -> list[str]:
67135
found = []
68-
for node in ast.walk(ast.parse(source, filename=str(path))):
69-
if not isinstance(node, ast.Call):
70-
continue
136+
for node in _calls(tree):
71137
name = _callee_name(node)
72138
if name in CAPPABLE_OPTIONS:
73139
uncapped = not _is_capped(node)
@@ -77,13 +143,152 @@ def violations_in(path: Path) -> list[str]:
77143
uncapped = any(not _dict_is_capped(d) for d in dicts)
78144
else:
79145
continue
80-
if uncapped and not _opted_out(lines, node):
146+
if uncapped and not _opted_out(lines, node, (UNCAPPED_MARKER,)):
81147
found.append(f"{path.as_posix()}:{node.lineno}: {name} without max_size")
82148
return found
83149

84150

151+
def _names_closed_in(node: ast.AST) -> set[str]:
152+
"""Names ``x`` with an ``x.close(...)`` call inside ``node``."""
153+
return {
154+
call.func.value.id
155+
for call in _calls(node)
156+
if isinstance(call.func, ast.Attribute) and call.func.attr == "close" and isinstance(call.func.value, ast.Name)
157+
}
158+
159+
160+
def _unclosed_pools(func: ast.FunctionDef, lines: list[str], path: Path) -> list[str]:
161+
closed_names = _names_closed_in(func)
162+
for node in ast.walk(func):
163+
# ``for mr in mrs: mr.close()`` closes every pool bound to ``mrs``.
164+
if (
165+
isinstance(node, ast.For)
166+
and isinstance(node.target, ast.Name)
167+
and isinstance(node.iter, ast.Name)
168+
and node.target.id in _names_closed_in(node)
169+
):
170+
closed_names.add(node.iter.id)
171+
bound_to = {}
172+
for node in ast.walk(func):
173+
if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
174+
# Every call in the value, so a pool built inside a comprehension is bound to the list's name.
175+
for call in _calls(node.value):
176+
bound_to[id(call)] = node.targets[0].id
177+
found = []
178+
for call in _calls(func):
179+
if not _creates_cappable_pool(call) or bound_to.get(id(call)) in closed_names:
180+
continue
181+
# A call marked uncapped-pool-ok raises before any pool exists, so there is nothing to close.
182+
if _opted_out(lines, call, (UNCAPPED_MARKER, UNCLOSED_MARKER)):
183+
continue
184+
found.append(f"{path.as_posix()}:{call.lineno}: {_callee_name(call)} in {func.name} is not closed")
185+
return found
186+
187+
188+
def _functions(tree: ast.Module):
189+
"""Yield ``(function, enclosing classes)`` for module-level functions and methods."""
190+
191+
def visit(body, classes):
192+
for node in body:
193+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
194+
yield node, classes
195+
elif isinstance(node, ast.ClassDef):
196+
yield from visit(node.body, [*classes, node])
197+
198+
yield from visit(tree.body, [])
199+
200+
201+
def _callee_names(func: ast.FunctionDef) -> set[str]:
202+
"""Names of the functions ``func`` calls by bare name or through ``self``/``cls``."""
203+
names = set()
204+
for call in _calls(func):
205+
target = call.func
206+
if isinstance(target, ast.Name):
207+
names.add(target.id)
208+
elif (
209+
isinstance(target, ast.Attribute)
210+
and isinstance(target.value, ast.Name)
211+
and target.value.id in ("self", "cls")
212+
):
213+
names.add(target.attr)
214+
return names
215+
216+
217+
def _pool_creators(functions) -> set[int]:
218+
"""Ids of the functions that create a pool, directly or through another function in the module."""
219+
creators = {id(func) for func, _ in functions if any(_creates_pool(call) for call in _calls(func))}
220+
callees = {id(func): _callee_names(func) for func, _ in functions}
221+
changed = True
222+
while changed:
223+
creator_names = {func.name for func, _ in functions if id(func) in creators}
224+
changed = False
225+
for func, _ in functions:
226+
if id(func) not in creators and callees[id(func)] & creator_names:
227+
creators.add(id(func))
228+
changed = True
229+
return creators
230+
231+
232+
def _module_marks(tree: ast.Module) -> set[str]:
233+
marks = set()
234+
for node in tree.body:
235+
if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "pytestmark" for t in node.targets):
236+
values = node.value.elts if isinstance(node.value, (ast.List, ast.Tuple)) else [node.value]
237+
marks.update(_decorator_name(value) for value in values)
238+
return marks
239+
240+
241+
def _has_decorator(node: ast.AST, name: str) -> bool:
242+
return any(_decorator_name(dec) == name for dec in node.decorator_list)
243+
244+
245+
def _adds_owns_pool_marker(func: ast.FunctionDef) -> bool:
246+
return any(
247+
isinstance(call.func, ast.Attribute)
248+
and call.func.attr == "add_marker"
249+
and call.args
250+
and isinstance(call.args[0], ast.Constant)
251+
and call.args[0].value == OWNS_POOL
252+
for call in _calls(func)
253+
)
254+
255+
256+
def _unmarked_pool_owners(tree: ast.Module, functions, path: Path) -> list[str]:
257+
creators = _pool_creators(functions)
258+
module_marked = OWNS_POOL in _module_marks(tree)
259+
found = []
260+
for func, classes in functions:
261+
if id(func) not in creators:
262+
continue
263+
qualname = ".".join([*(cls.name for cls in classes), func.name])
264+
marked = module_marked or any(_has_decorator(node, OWNS_POOL) for node in (func, *classes))
265+
if _has_decorator(func, "fixture") and not _adds_owns_pool_marker(func):
266+
found.append(
267+
f"{path.as_posix()}:{func.lineno}: fixture {qualname} creates a memory pool "
268+
f'but does not call request.node.add_marker("{OWNS_POOL}")'
269+
)
270+
elif func.name.startswith("test_") and not marked:
271+
found.append(
272+
f"{path.as_posix()}:{func.lineno}: {qualname} creates a memory pool but is not marked {OWNS_POOL}"
273+
)
274+
return found
275+
276+
277+
def violations_in(path: Path) -> list[str]:
278+
"""Return one message per hygiene violation in ``path``."""
279+
source = path.read_text(encoding="utf-8")
280+
lines = source.splitlines()
281+
tree = ast.parse(source, filename=str(path))
282+
functions = list(_functions(tree))
283+
found = _uncapped_pools(tree, lines, path)
284+
for func, _ in functions:
285+
found.extend(_unclosed_pools(func, lines, path))
286+
found.extend(_unmarked_pool_owners(tree, functions, path))
287+
return found
288+
289+
85290
def main(argv: list[str] | None = None) -> int:
86-
parser = argparse.ArgumentParser(description=__doc__)
291+
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
87292
parser.add_argument(
88293
"paths",
89294
nargs="*",
@@ -97,12 +302,15 @@ def main(argv: list[str] | None = None) -> int:
97302
if not violations:
98303
return 0
99304

100-
print("error: memory pools created by tests must set max_size:", file=sys.stderr)
305+
print("error: memory-pool hygiene violations in tests:", file=sys.stderr)
101306
for violation in violations:
102307
print(f" - {violation}", file=sys.stderr)
103308
print(
104-
f"Use the suite-wide POOL_SIZE from cuda_core/tests/helpers/constants.py, or annotate a\n"
105-
f"deliberate exception with a '# {OPT_OUT_MARKER}: <reason>' comment.\n"
309+
f"Pools created by tests must set max_size (POOL_SIZE from cuda_core/tests/helpers/constants.py)\n"
310+
f"and must be closed in the function that creates them (.close() on the name, or a loop over the list).\n"
311+
f"Annotate a deliberate exception with a '# {UNCAPPED_MARKER}: <reason>' or\n"
312+
f"'# {UNCLOSED_MARKER}: <reason>' comment. A test that creates a pool carries\n"
313+
f'@pytest.mark.{OWNS_POOL}; a fixture that creates one calls request.node.add_marker("{OWNS_POOL}").\n'
106314
f"See cuda_core/tests/AGENTS.md.",
107315
file=sys.stderr,
108316
)

0 commit comments

Comments
 (0)