44
55# Real GPU tests for cuda.core.checkpoint — no mocks.
66#
7- # Lifecycle tests exercise lightweight state/lock operations in-process and
8- # mutating checkpoint / restore cycles through an isolated coordinator/target
9- # process pair.
7+ # Driver-backed lifecycle tests run through an isolated coordinator/target
8+ # process pair so hangs can be timed out without wedging the pytest process.
109#
1110# Migration tests attempt GPU UUID remapping following the pattern from
1211# NVIDIA/cuda-checkpoint r580-migration-api.c. They require ≥2 GPUs of
13- # the same chip type and a driver that supports migration; the tests skip
14- # gracefully when the hardware or driver cannot satisfy this.
12+ # the same chip type, an unmasked CUDA device view, and a driver that supports
13+ # migration; the tests skip gracefully when the hardware or driver cannot
14+ # satisfy this.
1515
1616import os
1717import signal
@@ -37,26 +37,18 @@ def _checkpoint_available():
3737
3838
3939needs_checkpoint = pytest .mark .skipif (
40- sys .platform != "linux" or os . environ . get ( "CI" ) is not None or not _checkpoint_available (),
41- reason = "CUDA checkpoint API requires Linux, a supported driver/bindings, and a non-CI environment " ,
40+ sys .platform != "linux" or not _checkpoint_available (),
41+ reason = "CUDA checkpoint API requires Linux and a supported driver/bindings" ,
4242)
4343
4444
4545# -- Helpers ---------------------------------------------------------------
4646
4747
48- def _run_or_skip_unsupported (func , * args , ** kwargs ):
49- try :
50- return func (* args , ** kwargs )
51- except RuntimeError as exc :
52- if "CUDA checkpointing is not supported" in str (exc ):
53- pytest .skip (str (exc ))
54- raise
55-
56-
5748_SCENARIO_SKIP_EXIT_CODE = 77
5849
5950_SCENARIO_COMMON = r"""
51+ import os
6052import subprocess
6153import sys
6254from contextlib import suppress
@@ -214,36 +206,6 @@ def _run_checkpoint_scenario_or_skip(body: str, *, timeout: int = 90) -> None:
214206 )
215207
216208
217- # -- Fixtures --------------------------------------------------------------
218-
219-
220- @pytest .fixture
221- def self_process (init_cuda ):
222- """checkpoint.Process wrapping os.getpid(), with safety unlock on teardown.
223-
224- Records the initial device so tests that call ``set_current()`` on a
225- different device (e.g. migration tests) are side-effect free.
226- """
227- original_device = init_cuda
228- proc = checkpoint .Process (os .getpid ())
229- yield proc
230- # Ensure the process is not left locked if the test fails mid-lifecycle.
231- try :
232- st = proc .state
233- except Exception :
234- st = None
235- if st == "checkpointed" :
236- with suppress (Exception ):
237- proc .restore ()
238- with suppress (Exception ):
239- proc .unlock ()
240- elif st == "locked" :
241- with suppress (Exception ):
242- proc .unlock ()
243- # Restore the original device so init_cuda's teardown pops the right context.
244- original_device .set_current ()
245-
246-
247209# -- Input validation (no GPU / driver needed) -----------------------------
248210
249211
@@ -265,36 +227,87 @@ def test_public_symbols(self):
265227 assert checkpoint .__all__ == ["Process" ]
266228 assert not hasattr (checkpoint , "ProcessStateT" )
267229
230+ def test_pid_is_read_only (self ):
231+ proc = checkpoint .Process (1 )
232+ assert proc .pid == 1
233+ with pytest .raises (AttributeError ):
234+ proc .pid = 2
235+
268236
269237# -- Lifecycle (single GPU, real driver) -----------------------------------
270238
271239
272240@needs_checkpoint
273241class TestCheckpointLifecycle :
274- def test_initial_state_is_running (self , self_process ):
275- assert self_process .state == "running"
242+ def test_initial_state_is_running (self ):
243+ _run_checkpoint_scenario_or_skip (
244+ """
245+ target, _ = start_target()
246+ proc = checkpoint.Process(target.pid)
247+ try:
248+ assert proc.state == "running"
249+ finally:
250+ stop_target(target)
251+ """
252+ )
276253
277- def test_restore_thread_id_is_positive (self , self_process ):
278- tid = self_process .restore_thread_id
279- assert isinstance (tid , int )
280- assert tid > 0
254+ def test_restore_thread_id_is_positive (self ):
255+ _run_checkpoint_scenario_or_skip (
256+ """
257+ target, _ = start_target()
258+ proc = checkpoint.Process(target.pid)
259+ try:
260+ tid = proc.restore_thread_id
261+ assert isinstance(tid, int)
262+ assert tid > 0
263+ finally:
264+ stop_target(target)
265+ """
266+ )
281267
282- def test_lock_unlock (self , self_process ):
283- _run_or_skip_unsupported (self_process .lock )
284- assert self_process .state == "locked"
285- self_process .unlock ()
286- assert self_process .state == "running"
268+ def test_lock_unlock (self ):
269+ _run_checkpoint_scenario_or_skip (
270+ """
271+ target, _ = start_target()
272+ proc = checkpoint.Process(target.pid)
273+ try:
274+ run_or_skip_unsupported(proc.lock)
275+ assert proc.state == "locked"
276+ proc.unlock()
277+ assert proc.state == "running"
278+ finally:
279+ stop_target(target)
280+ """
281+ )
287282
288- def test_lock_default_timeout (self , self_process ):
283+ def test_lock_default_timeout (self ):
289284 """lock() with the default timeout_ms=0 (no timeout)."""
290- _run_or_skip_unsupported (self_process .lock )
291- assert self_process .state == "locked"
292- self_process .unlock ()
285+ _run_checkpoint_scenario_or_skip (
286+ """
287+ target, _ = start_target()
288+ proc = checkpoint.Process(target.pid)
289+ try:
290+ run_or_skip_unsupported(proc.lock)
291+ assert proc.state == "locked"
292+ proc.unlock()
293+ finally:
294+ stop_target(target)
295+ """
296+ )
293297
294- def test_lock_with_timeout (self , self_process ):
295- _run_or_skip_unsupported (self_process .lock , timeout_ms = 5000 )
296- assert self_process .state == "locked"
297- self_process .unlock ()
298+ def test_lock_with_timeout (self ):
299+ _run_checkpoint_scenario_or_skip (
300+ """
301+ target, _ = start_target()
302+ proc = checkpoint.Process(target.pid)
303+ try:
304+ run_or_skip_unsupported(proc.lock, timeout_ms=5000)
305+ assert proc.state == "locked"
306+ proc.unlock()
307+ finally:
308+ stop_target(target)
309+ """
310+ )
298311
299312 def test_full_cycle_no_migration (self ):
300313 """lock -> checkpoint -> restore -> unlock, verify state at each step."""
@@ -345,6 +358,11 @@ def test_rotation_migrates_context(self):
345358 _run_checkpoint_scenario_or_skip (
346359 """
347360 devices = Device.get_all_devices()
361+ if "CUDA_VISIBLE_DEVICES" in os.environ:
362+ skip(
363+ "GPU migration tests require an unmasked CUDA device view because "
364+ "the checkpoint mapping must cover every GPU visible to the kernel-mode driver"
365+ )
348366 if len(devices) < 2:
349367 skip("GPU migration tests require at least 2 GPUs")
350368 if find_same_chip_pair(devices) is None:
@@ -378,6 +396,11 @@ def test_swap_identical_gpus(self):
378396 _run_checkpoint_scenario_or_skip (
379397 """
380398 devices = Device.get_all_devices()
399+ if "CUDA_VISIBLE_DEVICES" in os.environ:
400+ skip(
401+ "GPU migration tests require an unmasked CUDA device view because "
402+ "the checkpoint mapping must cover every GPU visible to the kernel-mode driver"
403+ )
381404 pair = find_same_chip_pair(devices)
382405 if pair is None:
383406 skip("No two GPUs of the same chip type found")
0 commit comments