Skip to content

Commit 2ff8747

Browse files
Andy-Jostclaude
andauthored
test(cuda.core): keep the cycle collector out of the latch window in test_error_timing_incomplete (NVIDIA#3310)
* test(cuda.core): keep the cycle collector out of the latch window in test_error_timing_incomplete The test failed with "DID NOT RAISE" after about 60 s, the latch timeout, on one CI row. The likely cause is a finalizer that the cycle collector runs while the latch holds the stream: a free of memory synchronizes the context and waits for the latch. Collect before the launch and disable the collector until the latch is released. Create the end event before the launch. Use a 10 s latch and fail with the blocked time when the record call stalls. Closes NVIDIA#3296. Co-Authored-By: Claude Opus 5.5 <[email protected]> * test(cuda.core): time the whole latch window; request init_cuda Start the blocked-time check before the latch launch, so that a stall in the launch or the record reports the time. Re-enable the collector in its own finally block. Request init_cuda, as the test builds a pinned memory resource through the latch. Co-Authored-By: Claude Opus 5.5 <[email protected]> --------- Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 6681b39 commit 2ff8747

1 file changed

Lines changed: 38 additions & 10 deletions

File tree

‎cuda_core/tests/test_event.py‎

Lines changed: 38 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@
22
# SPDX-License-Identifier: Apache-2.0
33

44

5+
import gc
56
import math
7+
import time
68

79
import pytest
810
from helpers.latch import LatchKernel
@@ -15,6 +17,10 @@
1517
EventOptions,
1618
)
1719

20+
# Short enough that a stalled host call fails fast; long enough that the
21+
# latch cannot time out on its own during the few calls it guards.
22+
_LATCH_TIMEOUT_SEC = 10
23+
1824

1925
def test_event_init_disabled():
2026
with pytest.raises(RuntimeError, match=r"^Event objects cannot be instantiated directly\."):
@@ -120,22 +126,44 @@ def test_error_timing_recorded():
120126

121127
@pytest.mark.skipif(Device().compute_capability.major < 7, reason="__nanosleep is only available starting Volta (sm70)")
122128
@pytest.mark.thread_unsafe(reason="requires a barrier wait to avoid overlapping pinned latch allocations")
123-
def test_error_timing_incomplete():
124-
device = Device()
125-
device.set_current()
126-
latch = LatchKernel(device)
129+
@pytest.mark.agent_authored(model="claude-opus-5-5")
130+
def test_error_timing_incomplete(init_cuda):
131+
device = init_cuda
132+
latch = LatchKernel(device, timeout_sec=_LATCH_TIMEOUT_SEC)
127133
enabled = EventOptions(timing_enabled=True)
128134
stream = device.create_stream()
129135

130136
event1 = stream.record(options=enabled)
131-
latch.launch(stream)
132-
event3 = stream.record(options=enabled)
137+
# Create event3 before the latch holds the stream, so that no event
138+
# creation runs while the stream is held.
139+
event3 = device.create_event(options=enabled)
133140

134-
# event3 will never complete because the latch has not been released
135-
with pytest.raises(RuntimeError, match="^One or both events have not completed."):
136-
event3 - event1
141+
# A finalizer that frees memory synchronizes the context. If the cycle
142+
# collector runs one while the latch holds the stream, the host blocks
143+
# until the latch times out, and event3 completes (#3296). Collect now
144+
# and keep the collector off until the latch is released.
145+
gc.collect()
146+
gc_enabled = gc.isenabled()
147+
gc.disable()
148+
try:
149+
start = time.monotonic()
150+
latch.launch(stream)
151+
stream.record(event3)
152+
elapsed = time.monotonic() - start
153+
assert elapsed < _LATCH_TIMEOUT_SEC / 2, (
154+
f"a host call blocked for {elapsed:.1f} s while the latch held the stream "
155+
f"(latch timeout {_LATCH_TIMEOUT_SEC} s)"
156+
)
157+
# event3 cannot complete while the latch holds the stream
158+
with pytest.raises(RuntimeError, match="^One or both events have not completed."):
159+
event3 - event1
160+
finally:
161+
try:
162+
latch.release()
163+
finally:
164+
if gc_enabled:
165+
gc.enable()
137166

138-
latch.release()
139167
event3.sync()
140168
event3 - event1 # this should work
141169

0 commit comments

Comments
 (0)