4848 PinnedMemoryResourceOptions ,
4949 _device ,
5050)
51- from cuda .core ._memory ._buffer import _live_memory_resource_count
51+ from cuda .core ._memory ._buffer import _live_owned_mempool_count , _live_va_reservation_count
5252from cuda .core ._utils .cuda_utils import handle_return
5353
5454
@@ -131,38 +131,30 @@ def _item_uses_init_cuda(item):
131131 return "init_cuda" in getattr (item , "fixturenames" , ())
132132
133133
134- # init_cuda collects at teardown only if the process-wide live MemoryResource
135- # count differs from its value at setup. That is sound only if no older
136- # resource is released during a test (it could cancel a new leak), so tests
137- # that construct resources must use init_cuda, and the leftover check below
138- # reports resources that outlive a test without it.
134+ # init_cuda collects at teardown only if the process-wide count of live *owned*
135+ # driver resources (owned memory pools and VA reservations) differs from its
136+ # value at setup. The counts come from the C++ RAII layer, so a borrowed pool
137+ # reference (the device default pool) and MemoryResource subclasses that hold no
138+ # pool or reservation do not move them. That makes the gate sound only if no
139+ # older owned resource is released during a test (it could cancel a new leak),
140+ # so tests that construct owned resources must use init_cuda, and the leftover
141+ # check below reports owned resources that outlive a test without it.
139142
140- # Live count after the last init_cuda test; None until one has run.
141- _mr_expected_live = None
142- # Live count at the start of the current item, before its fixtures set up.
143- _mr_live_at_item_start = 0
144- _mr_last_init_cuda_nodeid = None
145143
144+ def _live_resource_counts ():
145+ """(owned memory pools, VA reservations) live process-wide, as a tuple.
146146
147- def _device_default_memory_resource_count ():
148- """Number of main-thread Devices that currently cache a default MR.
149-
150- Reaches into Device thread-local state (``_device._tls``) on purpose; update
151- this and :func:`_release_device_default_memory_resources` if that moves.
147+ Only *owned* driver resources are counted (see the C++ accessors), so this
148+ excludes borrowed pool references and non-owning MemoryResource subclasses.
152149 """
153- return sum (d ._memory_resource is not None for d in getattr (_device ._tls , "devices" , ()))
154-
150+ return (_live_owned_mempool_count (), _live_va_reservation_count ())
155151
156- def _release_device_default_memory_resources ():
157- """Drop each main-thread Device's lazily created default memory resource.
158152
159- Otherwise they are still counted when init_cuda compares counts at
160- teardown and force a collect in every test that allocates through the
161- device default. The setter rejects None,
162- hence the private access.
163- """
164- for d in getattr (_device ._tls , "devices" , ()):
165- d ._memory_resource = None
153+ # Counts after the last init_cuda test; None until one has run.
154+ _mr_expected_live = None
155+ # Counts at the start of the current item, before its fixtures set up.
156+ _mr_live_at_item_start = (0 , 0 )
157+ _mr_last_init_cuda_nodeid = None
166158
167159
168160@pytest .hookimpl (wrapper = True )
@@ -171,17 +163,17 @@ def pytest_runtest_protocol(item, nextitem):
171163 # not the session start, because the count legitimately rises over a run
172164 # (interned GraphMemoryResource, pytest-retained class instances).
173165 global _mr_expected_live , _mr_live_at_item_start , _mr_last_init_cuda_nodeid
174- _mr_live_at_item_start = _live_memory_resource_count ()
166+ _mr_live_at_item_start = _live_resource_counts ()
175167 try :
176168 return (yield )
177169 finally :
178- live = _live_memory_resource_count ()
170+ live = _live_resource_counts ()
179171 if _item_uses_init_cuda (item ):
180172 _mr_expected_live = live
181173 _mr_last_init_cuda_nodeid = item .nodeid
182174 elif _mr_expected_live is not None :
183- # Only lower it : a released resource must not hide a later leak.
184- _mr_expected_live = min (_mr_expected_live , live )
175+ # Only lower each component : a released resource must not hide a later leak.
176+ _mr_expected_live = tuple ( min (a , b ) for a , b in zip ( _mr_expected_live , live ) )
185177
186178
187179class _CudaCoreParallelPlugin :
@@ -243,25 +235,27 @@ def init_cuda(request):
243235 )
244236
245237 # Leftover check: growth since the previous init_cuda test means something
246- # in between left a resource alive. Collect first to sweep a cyclic
247- # leftover, and before the count at setup is recorded so it is reflected there.
248- if _mr_expected_live is not None and _mr_live_at_item_start > _mr_expected_live :
238+ # in between left an owned resource alive. Collect first to sweep a cyclic
239+ # leftover, and before the counts at setup are recorded so they are reflected there.
240+ if _mr_expected_live is not None and any ( now > exp for now , exp in zip ( _mr_live_at_item_start , _mr_expected_live )) :
249241 gc .collect ()
250- held = _live_memory_resource_count () - _mr_expected_live
242+ now_counts = _live_resource_counts ()
243+ held = tuple (max (n - e , 0 ) for n , e in zip (now_counts , _mr_expected_live ))
244+ excess = tuple (s - e for s , e in zip (_mr_live_at_item_start , _mr_expected_live ))
251245 seed = getattr (request .config .option , "randomly_seed" , None )
252246 seed_msg = f" (pytest-randomly seed: { seed } )" if seed is not None else ""
253247 warnings .warn (
254- f"{ _mr_live_at_item_start - _mr_expected_live } memory resource(s) were left alive by "
255- f"tests that ran since { _mr_last_init_cuda_nodeid } ({ max ( held , 0 ) } still alive after a "
248+ f"{ excess } (owned_pools, va_reservations) memory resource(s) were left alive by "
249+ f"tests that ran since { _mr_last_init_cuda_nodeid } ({ held } still alive after a "
256250 f"collect). A test or fixture that constructs a memory resource must request "
257251 f"init_cuda.{ seed_msg } " ,
258252 UnreleasedMemoryResourceWarning ,
259253 stacklevel = 2 ,
260254 )
261255
262- # Live count at setup, excluding cached Device default resources (they are
263- # released at teardown before the count is compared) .
264- live_at_setup = _live_memory_resource_count () - _device_default_memory_resource_count ()
256+ # Counts at setup. The device default pool is a borrowed reference and is
257+ # not counted, so no cached-default bookkeeping is needed .
258+ live_at_setup = _live_resource_counts ()
265259 try :
266260 yield device
267261 finally :
@@ -270,13 +264,12 @@ def init_cuda(request):
270264 # torn-down fixtures are not counted below. Revisit if pytest changes
271265 # how it retains fixture values.
272266 request .node .funcargs .clear ()
273- # Only a resource in a reference cycle outlives the test frame, so
274- # collect only if the count differs from its value at setup. A lower
267+ # Only an owned resource in a reference cycle outlives the test frame, so
268+ # collect only if the counts differ from their values at setup. A lower
275269 # count means an older resource was released, which could mask a leak
276270 # in this test: collect in that case too.
277- _release_device_default_memory_resources ()
278- live_after_release = _live_memory_resource_count ()
279- if live_after_release != live_at_setup :
271+ live_after = _live_resource_counts ()
272+ if live_after != live_at_setup :
280273 gc .collect ()
281274 # Force any pool/allocation whose only remaining reference was a local
282275 # in this test's frame to actually get destroyed now, then drain the
@@ -288,12 +281,13 @@ def init_cuda(request):
288281 driver .cuCtxSynchronize ()
289282 _ = _device_unset_current ()
290283 # After the cleanup, so an escalated warning cannot skip it.
291- if live_after_release < live_at_setup :
284+ if any (now < snap for now , snap in zip (live_after , live_at_setup )):
285+ released = tuple (s - n for s , n in zip (live_at_setup , live_after ))
292286 warnings .warn (
293- f"{ request .node .nodeid } released { live_at_setup - live_after_release } memory resource(s) "
294- f"that existed before it started (count { live_after_release } , snapshot { live_at_setup } ). "
295- " A leak in this test could have been masked by that release. Find what held the "
296- "released resource." ,
287+ f"{ request .node .nodeid } released { released } (owned_pools, va_reservations) memory "
288+ f"resource(s) that existed before it started (counts { live_after } , snapshot "
289+ f" { live_at_setup } ). A leak in this test could have been masked by that release. "
290+ "Find what held the released resource." ,
297291 BaselineMemoryResourceReleasedWarning ,
298292 stacklevel = 2 ,
299293 )
0 commit comments