11# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
22# SPDX-License-Identifier: Apache-2.0
33
4- """Check that tests do not create uncapped CUDA memory pools .
4+ """Check the memory-pool hygiene of the cuda_core tests .
55
6- A pool created without ``max_size`` reserves an address-space window sized from
7- installed device memory rather than from what the test allocates, and the whole
8- cuda_core suite shares one process. Enough of those reservations exhaust the
9- address space, after which the rest of the session fails with
10- CUDA_ERROR_OUT_OF_MEMORY on a device with free physical memory.
6+ A memory resource constructed with an options object creates a pool that the
7+ test owns; one constructed without options wraps the device's current pool and
8+ owns nothing. The whole cuda_core suite shares one process, and an owned pool
9+ keeps its address- space reservation until the pool is destroyed and the
10+ stream-ordered frees of its allocations retire. Three rules keep that bounded:
1111
12- See cuda_core/tests/AGENTS.md for the rule this enforces.
12+ 1. max_size: a Device or Pinned pool sets ``max_size``. A pool without it
13+ reserves an address-space window sized from installed device memory rather
14+ than from what the test allocates, and enough of those reservations exhaust
15+ the address space, after which the rest of the session fails with
16+ CUDA_ERROR_OUT_OF_MEMORY on a device with free physical memory. Opt out with
17+ ``# uncapped-pool-ok: <reason>``.
18+ 2. close: a Device or Pinned pool is bound to a name and closed in the function
19+ that creates it, by ``.close()`` on that name or by a ``for`` loop that
20+ closes every element of the list the name holds. Memory resources are not
21+ context managers. Opt out with ``# unclosed-pool-ok: <reason>`` when the
22+ close happens elsewhere.
23+ 3. owns_pool: a test that creates a pool of any kind, directly or through a
24+ function in the same module, carries ``@pytest.mark.owns_pool`` on the
25+ function, on its class, or in the module's ``pytestmark``; a fixture that
26+ creates a pool calls ``request.node.add_marker("owns_pool")``. The marker
27+ makes the ``init_cuda`` teardown run ``gc.collect()`` before it drains the
28+ context. Calls are followed by bare name and through ``self``; a pool created
29+ through a function passed in as a parameter is not seen.
30+
31+ See cuda_core/tests/AGENTS.md for the rules this enforces.
1332"""
1433
1534from __future__ import annotations
2746CAPPABLE_OPTIONS = frozenset ({"DeviceMemoryResourceOptions" , "PinnedMemoryResourceOptions" })
2847CAPPABLE_RESOURCES = frozenset ({"DeviceMemoryResource" , "PinnedMemoryResource" })
2948
30- OPT_OUT_MARKER = "uncapped-pool-ok"
49+ # Every options class. Passing one to a resource creates an owned pool.
50+ POOL_OPTIONS = CAPPABLE_OPTIONS | {"ManagedMemoryResourceOptions" , "VirtualMemoryResourceOptions" }
51+ # Resource constructors and the number of leading positional arguments that are
52+ # not options (the device). More positional arguments, or any keyword, pass options.
53+ POOL_RESOURCES = {
54+ "DeviceMemoryResource" : 1 ,
55+ "PinnedMemoryResource" : 0 ,
56+ "ManagedMemoryResource" : 0 ,
57+ "VirtualMemoryResource" : 1 ,
58+ }
59+ # Factories in cuda_core/tests/helpers/memory.py; they create a pool when given options.
60+ POOL_HELPERS = frozenset ({"create_managed_memory_resource_or_skip" , "create_pinned_memory_resource_or_xfail" })
61+ HELPER_NON_OPTION_KEYWORDS = frozenset ({"xfail_device" })
62+ CAPPABLE_HELPERS = frozenset ({"create_pinned_memory_resource_or_xfail" })
63+
64+ UNCAPPED_MARKER = "uncapped-pool-ok"
65+ UNCLOSED_MARKER = "unclosed-pool-ok"
66+ OWNS_POOL = "owns_pool"
3167
3268
3369def _callee_name (node : ast .Call ) -> str :
@@ -39,6 +75,17 @@ def _callee_name(node: ast.Call) -> str:
3975 return ""
4076
4177
78+ def _decorator_name (node : ast .AST ) -> str :
79+ """The last name of a decorator or mark expression: ``pytest.mark.owns_pool`` -> ``owns_pool``."""
80+ if isinstance (node , ast .Call ):
81+ node = node .func
82+ if isinstance (node , ast .Attribute ):
83+ return node .attr
84+ if isinstance (node , ast .Name ):
85+ return node .id
86+ return ""
87+
88+
4289def _is_capped (node : ast .Call ) -> bool :
4390 # ``**kwargs`` (arg is None) may carry max_size; do not guess.
4491 return any (kw .arg is None or kw .arg == "max_size" for kw in node .keywords )
@@ -53,21 +100,40 @@ def _dict_is_capped(node: ast.Dict) -> bool:
53100 return False
54101
55102
56- def _opted_out (lines : list [str ], node : ast .AST ) -> bool :
57- """True if the call, or the line above it, carries the opt-out marker."""
103+ def _passes_options (node : ast .Call ) -> bool :
104+ """True if a resource constructor or pool helper call passes options, so it creates a pool."""
105+ name = _callee_name (node )
106+ if name in POOL_RESOURCES :
107+ return len (node .args ) > POOL_RESOURCES [name ] or bool (node .keywords )
108+ if name in POOL_HELPERS :
109+ return bool (node .args ) or any (kw .arg not in HELPER_NON_OPTION_KEYWORDS for kw in node .keywords )
110+ return False
111+
112+
113+ def _creates_pool (node : ast .Call ) -> bool :
114+ """True if the call creates an owned pool of any kind (rule 3)."""
115+ return _callee_name (node ) in POOL_OPTIONS or _passes_options (node )
116+
117+
118+ def _creates_cappable_pool (node : ast .Call ) -> bool :
119+ """True if the call creates a Device or Pinned pool (rule 2)."""
120+ return _callee_name (node ) in CAPPABLE_RESOURCES | CAPPABLE_HELPERS and _passes_options (node )
121+
122+
123+ def _opted_out (lines : list [str ], node : ast .AST , markers : tuple [str , ...]) -> bool :
124+ """True if the call, or the line above it, carries one of the opt-out markers."""
58125 start = max (node .lineno - 2 , 0 ) # -1 for 0-based, -1 more for a preceding comment
59126 end = getattr (node , "end_lineno" , node .lineno )
60- return any (OPT_OUT_MARKER in line for line in lines [start :end ])
127+ return any (marker in line for marker in markers for line in lines [start :end ])
61128
62129
63- def violations_in (path : Path ) -> list [str ]:
64- """Return one message per uncapped pool construction in ``path``."""
65- source = path .read_text (encoding = "utf-8" )
66- lines = source .splitlines ()
130+ def _calls (node : ast .AST ) -> list [ast .Call ]:
131+ return [n for n in ast .walk (node ) if isinstance (n , ast .Call )]
132+
133+
134+ def _uncapped_pools (tree : ast .Module , lines : list [str ], path : Path ) -> list [str ]:
67135 found = []
68- for node in ast .walk (ast .parse (source , filename = str (path ))):
69- if not isinstance (node , ast .Call ):
70- continue
136+ for node in _calls (tree ):
71137 name = _callee_name (node )
72138 if name in CAPPABLE_OPTIONS :
73139 uncapped = not _is_capped (node )
@@ -77,13 +143,152 @@ def violations_in(path: Path) -> list[str]:
77143 uncapped = any (not _dict_is_capped (d ) for d in dicts )
78144 else :
79145 continue
80- if uncapped and not _opted_out (lines , node ):
146+ if uncapped and not _opted_out (lines , node , ( UNCAPPED_MARKER ,) ):
81147 found .append (f"{ path .as_posix ()} :{ node .lineno } : { name } without max_size" )
82148 return found
83149
84150
151+ def _names_closed_in (node : ast .AST ) -> set [str ]:
152+ """Names ``x`` with an ``x.close(...)`` call inside ``node``."""
153+ return {
154+ call .func .value .id
155+ for call in _calls (node )
156+ if isinstance (call .func , ast .Attribute ) and call .func .attr == "close" and isinstance (call .func .value , ast .Name )
157+ }
158+
159+
160+ def _unclosed_pools (func : ast .FunctionDef , lines : list [str ], path : Path ) -> list [str ]:
161+ closed_names = _names_closed_in (func )
162+ for node in ast .walk (func ):
163+ # ``for mr in mrs: mr.close()`` closes every pool bound to ``mrs``.
164+ if (
165+ isinstance (node , ast .For )
166+ and isinstance (node .target , ast .Name )
167+ and isinstance (node .iter , ast .Name )
168+ and node .target .id in _names_closed_in (node )
169+ ):
170+ closed_names .add (node .iter .id )
171+ bound_to = {}
172+ for node in ast .walk (func ):
173+ if isinstance (node , ast .Assign ) and len (node .targets ) == 1 and isinstance (node .targets [0 ], ast .Name ):
174+ # Every call in the value, so a pool built inside a comprehension is bound to the list's name.
175+ for call in _calls (node .value ):
176+ bound_to [id (call )] = node .targets [0 ].id
177+ found = []
178+ for call in _calls (func ):
179+ if not _creates_cappable_pool (call ) or bound_to .get (id (call )) in closed_names :
180+ continue
181+ # A call marked uncapped-pool-ok raises before any pool exists, so there is nothing to close.
182+ if _opted_out (lines , call , (UNCAPPED_MARKER , UNCLOSED_MARKER )):
183+ continue
184+ found .append (f"{ path .as_posix ()} :{ call .lineno } : { _callee_name (call )} in { func .name } is not closed" )
185+ return found
186+
187+
188+ def _functions (tree : ast .Module ):
189+ """Yield ``(function, enclosing classes)`` for module-level functions and methods."""
190+
191+ def visit (body , classes ):
192+ for node in body :
193+ if isinstance (node , (ast .FunctionDef , ast .AsyncFunctionDef )):
194+ yield node , classes
195+ elif isinstance (node , ast .ClassDef ):
196+ yield from visit (node .body , [* classes , node ])
197+
198+ yield from visit (tree .body , [])
199+
200+
201+ def _callee_names (func : ast .FunctionDef ) -> set [str ]:
202+ """Names of the functions ``func`` calls by bare name or through ``self``/``cls``."""
203+ names = set ()
204+ for call in _calls (func ):
205+ target = call .func
206+ if isinstance (target , ast .Name ):
207+ names .add (target .id )
208+ elif (
209+ isinstance (target , ast .Attribute )
210+ and isinstance (target .value , ast .Name )
211+ and target .value .id in ("self" , "cls" )
212+ ):
213+ names .add (target .attr )
214+ return names
215+
216+
217+ def _pool_creators (functions ) -> set [int ]:
218+ """Ids of the functions that create a pool, directly or through another function in the module."""
219+ creators = {id (func ) for func , _ in functions if any (_creates_pool (call ) for call in _calls (func ))}
220+ callees = {id (func ): _callee_names (func ) for func , _ in functions }
221+ changed = True
222+ while changed :
223+ creator_names = {func .name for func , _ in functions if id (func ) in creators }
224+ changed = False
225+ for func , _ in functions :
226+ if id (func ) not in creators and callees [id (func )] & creator_names :
227+ creators .add (id (func ))
228+ changed = True
229+ return creators
230+
231+
232+ def _module_marks (tree : ast .Module ) -> set [str ]:
233+ marks = set ()
234+ for node in tree .body :
235+ if isinstance (node , ast .Assign ) and any (isinstance (t , ast .Name ) and t .id == "pytestmark" for t in node .targets ):
236+ values = node .value .elts if isinstance (node .value , (ast .List , ast .Tuple )) else [node .value ]
237+ marks .update (_decorator_name (value ) for value in values )
238+ return marks
239+
240+
241+ def _has_decorator (node : ast .AST , name : str ) -> bool :
242+ return any (_decorator_name (dec ) == name for dec in node .decorator_list )
243+
244+
245+ def _adds_owns_pool_marker (func : ast .FunctionDef ) -> bool :
246+ return any (
247+ isinstance (call .func , ast .Attribute )
248+ and call .func .attr == "add_marker"
249+ and call .args
250+ and isinstance (call .args [0 ], ast .Constant )
251+ and call .args [0 ].value == OWNS_POOL
252+ for call in _calls (func )
253+ )
254+
255+
256+ def _unmarked_pool_owners (tree : ast .Module , functions , path : Path ) -> list [str ]:
257+ creators = _pool_creators (functions )
258+ module_marked = OWNS_POOL in _module_marks (tree )
259+ found = []
260+ for func , classes in functions :
261+ if id (func ) not in creators :
262+ continue
263+ qualname = "." .join ([* (cls .name for cls in classes ), func .name ])
264+ marked = module_marked or any (_has_decorator (node , OWNS_POOL ) for node in (func , * classes ))
265+ if _has_decorator (func , "fixture" ) and not _adds_owns_pool_marker (func ):
266+ found .append (
267+ f"{ path .as_posix ()} :{ func .lineno } : fixture { qualname } creates a memory pool "
268+ f'but does not call request.node.add_marker("{ OWNS_POOL } ")'
269+ )
270+ elif func .name .startswith ("test_" ) and not marked :
271+ found .append (
272+ f"{ path .as_posix ()} :{ func .lineno } : { qualname } creates a memory pool but is not marked { OWNS_POOL } "
273+ )
274+ return found
275+
276+
277+ def violations_in (path : Path ) -> list [str ]:
278+ """Return one message per hygiene violation in ``path``."""
279+ source = path .read_text (encoding = "utf-8" )
280+ lines = source .splitlines ()
281+ tree = ast .parse (source , filename = str (path ))
282+ functions = list (_functions (tree ))
283+ found = _uncapped_pools (tree , lines , path )
284+ for func , _ in functions :
285+ found .extend (_unclosed_pools (func , lines , path ))
286+ found .extend (_unmarked_pool_owners (tree , functions , path ))
287+ return found
288+
289+
85290def main (argv : list [str ] | None = None ) -> int :
86- parser = argparse .ArgumentParser (description = __doc__ )
291+ parser = argparse .ArgumentParser (description = __doc__ , formatter_class = argparse . RawDescriptionHelpFormatter )
87292 parser .add_argument (
88293 "paths" ,
89294 nargs = "*" ,
@@ -97,12 +302,15 @@ def main(argv: list[str] | None = None) -> int:
97302 if not violations :
98303 return 0
99304
100- print ("error: memory pools created by tests must set max_size :" , file = sys .stderr )
305+ print ("error: memory-pool hygiene violations in tests:" , file = sys .stderr )
101306 for violation in violations :
102307 print (f" - { violation } " , file = sys .stderr )
103308 print (
104- f"Use the suite-wide POOL_SIZE from cuda_core/tests/helpers/constants.py, or annotate a\n "
105- f"deliberate exception with a '# { OPT_OUT_MARKER } : <reason>' comment.\n "
309+ f"Pools created by tests must set max_size (POOL_SIZE from cuda_core/tests/helpers/constants.py)\n "
310+ f"and must be closed in the function that creates them (.close() on the name, or a loop over the list).\n "
311+ f"Annotate a deliberate exception with a '# { UNCAPPED_MARKER } : <reason>' or\n "
312+ f"'# { UNCLOSED_MARKER } : <reason>' comment. A test that creates a pool carries\n "
313+ f'@pytest.mark.{ OWNS_POOL } ; a fixture that creates one calls request.node.add_marker("{ OWNS_POOL } ").\n '
106314 f"See cuda_core/tests/AGENTS.md." ,
107315 file = sys .stderr ,
108316 )
0 commit comments