Skip to content

Commit 376acc7

Browse files
committed
Address checkpoint review follow-ups
1 parent 8f798f4 commit 376acc7

3 files changed

Lines changed: 111 additions & 79 deletions

File tree

‎cuda_core/cuda/core/checkpoint.py‎

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -50,18 +50,25 @@ class Process:
5050
Process ID of the CUDA process.
5151
"""
5252

53-
__slots__ = ("pid",)
53+
__slots__ = ("_pid",)
5454

5555
def __init__(self, pid: int):
56-
self.pid = _check_pid(pid)
56+
self._pid = _check_pid(pid)
57+
58+
@property
59+
def pid(self) -> int:
60+
"""
61+
Process ID of the CUDA process.
62+
"""
63+
return self._pid
5764

5865
@property
5966
def state(self) -> _ProcessStateT:
6067
"""
6168
CUDA checkpoint state for this process.
6269
"""
6370
driver = _get_driver()
64-
state = _call_driver(driver, driver.cuCheckpointProcessGetState, self.pid)
71+
state = _call_driver(driver, driver.cuCheckpointProcessGetState, self._pid)
6572
state_names = _get_process_state_names(driver)
6673
try:
6774
return state_names[state]
@@ -75,7 +82,7 @@ def restore_thread_id(self) -> int:
7582
CUDA restore thread ID for this process.
7683
"""
7784
driver = _get_driver()
78-
return _call_driver(driver, driver.cuCheckpointProcessGetRestoreThreadId, self.pid)
85+
return _call_driver(driver, driver.cuCheckpointProcessGetRestoreThreadId, self._pid)
7986

8087
def lock(self, timeout_ms: int = 0) -> None:
8188
"""
@@ -89,14 +96,14 @@ def lock(self, timeout_ms: int = 0) -> None:
8996
driver = _get_driver()
9097
args = driver.CUcheckpointLockArgs()
9198
args.timeoutMs = _check_timeout_ms(timeout_ms)
92-
_call_driver(driver, driver.cuCheckpointProcessLock, self.pid, args)
99+
_call_driver(driver, driver.cuCheckpointProcessLock, self._pid, args)
93100

94101
def checkpoint(self) -> None:
95102
"""
96103
Checkpoint the GPU memory contents of this locked process.
97104
"""
98105
driver = _get_driver()
99-
_call_driver(driver, driver.cuCheckpointProcessCheckpoint, self.pid, None)
106+
_call_driver(driver, driver.cuCheckpointProcessCheckpoint, self._pid, None)
100107

101108
def restore(self, gpu_mapping: _Mapping[_Any, _Any] | None = None) -> None:
102109
"""
@@ -107,18 +114,20 @@ def restore(self, gpu_mapping: _Mapping[_Any, _Any] | None = None) -> None:
107114
gpu_mapping : mapping, optional
108115
GPU UUID remapping from each checkpointed GPU UUID to the GPU UUID
109116
to restore onto. For migration workflows, provide mappings for
110-
every CUDA-visible GPU.
117+
every GPU visible to the kernel-mode driver. User-space masking
118+
such as ``CUDA_VISIBLE_DEVICES`` does not reduce this mapping
119+
requirement.
111120
"""
112121
driver = _get_driver()
113122
args = _make_restore_args(driver, gpu_mapping)
114-
_call_driver(driver, driver.cuCheckpointProcessRestore, self.pid, args)
123+
_call_driver(driver, driver.cuCheckpointProcessRestore, self._pid, args)
115124

116125
def unlock(self) -> None:
117126
"""
118127
Unlock this locked process so it can resume CUDA API calls.
119128
"""
120129
driver = _get_driver()
121-
_call_driver(driver, driver.cuCheckpointProcessUnlock, self.pid, None)
130+
_call_driver(driver, driver.cuCheckpointProcessUnlock, self._pid, None)
122131

123132

124133
def _get_driver():
@@ -169,10 +178,7 @@ def _call_driver(driver, func, *args):
169178
"Upgrade to a driver version with CUDA checkpoint API support."
170179
) from e
171180
raise
172-
return _handle_return(driver, result)
173-
174181

175-
def _handle_return(driver, result):
176182
err = result[0]
177183
not_supported_errors = (
178184
getattr(driver.CUresult, "CUDA_ERROR_NOT_FOUND", None),

‎cuda_core/docs/source/api.rst‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -214,8 +214,11 @@ this:
214214
``"checkpointed"``, or ``"failed"``. Restore may optionally remap GPUs by
215215
passing ``gpu_mapping`` from each checkpointed GPU UUID to the GPU UUID that
216216
should be used during restore. For migration workflows, provide mappings for
217-
every CUDA-visible GPU. The mapping may use ``CUuuid`` objects or the UUID
218-
strings returned by :attr:`Device.uuid`. A successful restore returns the
217+
every GPU visible to the NVIDIA kernel-mode driver at checkpoint time.
218+
User-space masking such as ``CUDA_VISIBLE_DEVICES`` does not reduce this
219+
mapping requirement, so applications that rely on user-space GPU masking may
220+
not be valid migration targets. The mapping may use ``CUuuid`` objects or the
221+
UUID strings returned by :attr:`Device.uuid`. A successful restore returns the
219222
process to the locked state; call ``Process.unlock`` after restore to allow
220223
CUDA API calls to resume.
221224

‎cuda_core/tests/test_checkpoint.py‎

Lines changed: 88 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,14 @@
44

55
# Real GPU tests for cuda.core.checkpoint — no mocks.
66
#
7-
# Lifecycle tests exercise lightweight state/lock operations in-process and
8-
# mutating checkpoint / restore cycles through an isolated coordinator/target
9-
# process pair.
7+
# Driver-backed lifecycle tests run through an isolated coordinator/target
8+
# process pair so hangs can be timed out without wedging the pytest process.
109
#
1110
# Migration tests attempt GPU UUID remapping following the pattern from
1211
# NVIDIA/cuda-checkpoint r580-migration-api.c. They require ≥2 GPUs of
13-
# the same chip type and a driver that supports migration; the tests skip
14-
# gracefully when the hardware or driver cannot satisfy this.
12+
# the same chip type, an unmasked CUDA device view, and a driver that supports
13+
# migration; the tests skip gracefully when the hardware or driver cannot
14+
# satisfy this.
1515

1616
import os
1717
import signal
@@ -37,26 +37,18 @@ def _checkpoint_available():
3737

3838

3939
needs_checkpoint = pytest.mark.skipif(
40-
sys.platform != "linux" or os.environ.get("CI") is not None or not _checkpoint_available(),
41-
reason="CUDA checkpoint API requires Linux, a supported driver/bindings, and a non-CI environment",
40+
sys.platform != "linux" or not _checkpoint_available(),
41+
reason="CUDA checkpoint API requires Linux and a supported driver/bindings",
4242
)
4343

4444

4545
# -- Helpers ---------------------------------------------------------------
4646

4747

48-
def _run_or_skip_unsupported(func, *args, **kwargs):
49-
try:
50-
return func(*args, **kwargs)
51-
except RuntimeError as exc:
52-
if "CUDA checkpointing is not supported" in str(exc):
53-
pytest.skip(str(exc))
54-
raise
55-
56-
5748
_SCENARIO_SKIP_EXIT_CODE = 77
5849

5950
_SCENARIO_COMMON = r"""
51+
import os
6052
import subprocess
6153
import sys
6254
from contextlib import suppress
@@ -214,36 +206,6 @@ def _run_checkpoint_scenario_or_skip(body: str, *, timeout: int = 90) -> None:
214206
)
215207

216208

217-
# -- Fixtures --------------------------------------------------------------
218-
219-
220-
@pytest.fixture
221-
def self_process(init_cuda):
222-
"""checkpoint.Process wrapping os.getpid(), with safety unlock on teardown.
223-
224-
Records the initial device so tests that call ``set_current()`` on a
225-
different device (e.g. migration tests) are side-effect free.
226-
"""
227-
original_device = init_cuda
228-
proc = checkpoint.Process(os.getpid())
229-
yield proc
230-
# Ensure the process is not left locked if the test fails mid-lifecycle.
231-
try:
232-
st = proc.state
233-
except Exception:
234-
st = None
235-
if st == "checkpointed":
236-
with suppress(Exception):
237-
proc.restore()
238-
with suppress(Exception):
239-
proc.unlock()
240-
elif st == "locked":
241-
with suppress(Exception):
242-
proc.unlock()
243-
# Restore the original device so init_cuda's teardown pops the right context.
244-
original_device.set_current()
245-
246-
247209
# -- Input validation (no GPU / driver needed) -----------------------------
248210

249211

@@ -265,36 +227,87 @@ def test_public_symbols(self):
265227
assert checkpoint.__all__ == ["Process"]
266228
assert not hasattr(checkpoint, "ProcessStateT")
267229

230+
def test_pid_is_read_only(self):
231+
proc = checkpoint.Process(1)
232+
assert proc.pid == 1
233+
with pytest.raises(AttributeError):
234+
proc.pid = 2
235+
268236

269237
# -- Lifecycle (single GPU, real driver) -----------------------------------
270238

271239

272240
@needs_checkpoint
273241
class TestCheckpointLifecycle:
274-
def test_initial_state_is_running(self, self_process):
275-
assert self_process.state == "running"
242+
def test_initial_state_is_running(self):
243+
_run_checkpoint_scenario_or_skip(
244+
"""
245+
target, _ = start_target()
246+
proc = checkpoint.Process(target.pid)
247+
try:
248+
assert proc.state == "running"
249+
finally:
250+
stop_target(target)
251+
"""
252+
)
276253

277-
def test_restore_thread_id_is_positive(self, self_process):
278-
tid = self_process.restore_thread_id
279-
assert isinstance(tid, int)
280-
assert tid > 0
254+
def test_restore_thread_id_is_positive(self):
255+
_run_checkpoint_scenario_or_skip(
256+
"""
257+
target, _ = start_target()
258+
proc = checkpoint.Process(target.pid)
259+
try:
260+
tid = proc.restore_thread_id
261+
assert isinstance(tid, int)
262+
assert tid > 0
263+
finally:
264+
stop_target(target)
265+
"""
266+
)
281267

282-
def test_lock_unlock(self, self_process):
283-
_run_or_skip_unsupported(self_process.lock)
284-
assert self_process.state == "locked"
285-
self_process.unlock()
286-
assert self_process.state == "running"
268+
def test_lock_unlock(self):
269+
_run_checkpoint_scenario_or_skip(
270+
"""
271+
target, _ = start_target()
272+
proc = checkpoint.Process(target.pid)
273+
try:
274+
run_or_skip_unsupported(proc.lock)
275+
assert proc.state == "locked"
276+
proc.unlock()
277+
assert proc.state == "running"
278+
finally:
279+
stop_target(target)
280+
"""
281+
)
287282

288-
def test_lock_default_timeout(self, self_process):
283+
def test_lock_default_timeout(self):
289284
"""lock() with the default timeout_ms=0 (no timeout)."""
290-
_run_or_skip_unsupported(self_process.lock)
291-
assert self_process.state == "locked"
292-
self_process.unlock()
285+
_run_checkpoint_scenario_or_skip(
286+
"""
287+
target, _ = start_target()
288+
proc = checkpoint.Process(target.pid)
289+
try:
290+
run_or_skip_unsupported(proc.lock)
291+
assert proc.state == "locked"
292+
proc.unlock()
293+
finally:
294+
stop_target(target)
295+
"""
296+
)
293297

294-
def test_lock_with_timeout(self, self_process):
295-
_run_or_skip_unsupported(self_process.lock, timeout_ms=5000)
296-
assert self_process.state == "locked"
297-
self_process.unlock()
298+
def test_lock_with_timeout(self):
299+
_run_checkpoint_scenario_or_skip(
300+
"""
301+
target, _ = start_target()
302+
proc = checkpoint.Process(target.pid)
303+
try:
304+
run_or_skip_unsupported(proc.lock, timeout_ms=5000)
305+
assert proc.state == "locked"
306+
proc.unlock()
307+
finally:
308+
stop_target(target)
309+
"""
310+
)
298311

299312
def test_full_cycle_no_migration(self):
300313
"""lock -> checkpoint -> restore -> unlock, verify state at each step."""
@@ -345,6 +358,11 @@ def test_rotation_migrates_context(self):
345358
_run_checkpoint_scenario_or_skip(
346359
"""
347360
devices = Device.get_all_devices()
361+
if "CUDA_VISIBLE_DEVICES" in os.environ:
362+
skip(
363+
"GPU migration tests require an unmasked CUDA device view because "
364+
"the checkpoint mapping must cover every GPU visible to the kernel-mode driver"
365+
)
348366
if len(devices) < 2:
349367
skip("GPU migration tests require at least 2 GPUs")
350368
if find_same_chip_pair(devices) is None:
@@ -378,6 +396,11 @@ def test_swap_identical_gpus(self):
378396
_run_checkpoint_scenario_or_skip(
379397
"""
380398
devices = Device.get_all_devices()
399+
if "CUDA_VISIBLE_DEVICES" in os.environ:
400+
skip(
401+
"GPU migration tests require an unmasked CUDA device view because "
402+
"the checkpoint mapping must cover every GPU visible to the kernel-mode driver"
403+
)
381404
pair = find_same_chip_pair(devices)
382405
if pair is None:
383406
skip("No two GPUs of the same chip type found")

0 commit comments

Comments
 (0)