Skip to content

Commit a3f1573

Browse files
leofangleofang
committed
CI: add fast-feedback probe-driver-swap job (workflow_dispatch only)
Allocates one L4 GPU + privileged container, runs install_gpu_driver.sh with DRIVER=610.43.02, then drives nvmlDeviceSetPersistenceMode via raw ctypes -- the exact NVML call that cuda.core's test_persistence_mode_enabled exercises. Exits 1 on NVML_ERROR_UNKNOWN so the smoke test fails loudly when the install path leaves the daemon dead. Total runtime ~5 min vs ~30 min for the full test matrix. Triggered by workflow_dispatch only -- this is an opt-in debugging job, not regular PR or nightly traffic.
1 parent 701cf2f commit a3f1573

1 file changed

Lines changed: 72 additions & 0 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -413,6 +413,78 @@ jobs:
413413
with:
414414
is-release: ${{ github.ref_type == 'tag' }}
415415

416+
# Fast-feedback probe for changes to ci/tools/install_gpu_driver.sh.
417+
# Allocates one L4 GPU + container, runs the driver swap to a
418+
# hard-coded version, then drives nvmlDeviceSetPersistenceMode via
419+
# raw ctypes -- the *exact* NVML call that cuda.core's
420+
# test_persistence_mode_enabled exercises. Total runtime is ~5 min
421+
# vs. ~30 min for a full test matrix.
422+
#
423+
# `workflow_dispatch` only -- this is an opt-in debugging job, not
424+
# part of regular PR or nightly traffic.
425+
probe-driver-swap:
426+
name: Probe custom-DRIVER install
427+
if: ${{ github.event_name == 'workflow_dispatch' && github.repository_owner == 'nvidia' }}
428+
runs-on: "linux-amd64-gpu-l4-latest-1"
429+
timeout-minutes: 15
430+
defaults:
431+
run:
432+
shell: bash --noprofile --norc -xeuo pipefail {0}
433+
container:
434+
options: -u root --security-opt seccomp=unconfined --shm-size 16g --privileged --pid=host
435+
image: ubuntu:22.04
436+
env:
437+
NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
438+
steps:
439+
- name: Checkout
440+
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
441+
442+
- name: Install host deps
443+
run: |
444+
apt-get update -qq
445+
apt-get -y install --no-install-recommends util-linux python3
446+
447+
- name: Install GPU driver
448+
env:
449+
DRIVER: '610.43.02'
450+
GPU_TYPE: 'l4'
451+
run: ./ci/tools/install_gpu_driver.sh
452+
453+
- name: Show post-install host + container state
454+
run: |
455+
nvidia-smi --query-gpu=name,driver_version,persistence_mode --format=csv
456+
echo
457+
echo "=== /run/nvidia-persistenced ==="
458+
ls -la /run/nvidia-persistenced/ 2>&1 || echo "MISSING"
459+
echo
460+
echo "=== nvidia-persistenced process ==="
461+
pgrep -laf nvidia-persistenced || echo "(none)"
462+
463+
- name: Drive nvmlDeviceSetPersistenceMode via ctypes
464+
run: |
465+
python3 <<'PYEOF'
466+
import ctypes, sys
467+
NVML_SUCCESS, NVML_ERROR_NO_PERMISSION, NVML_ERROR_UNKNOWN = 0, 4, 999
468+
nvml = ctypes.CDLL("libnvidia-ml.so.1")
469+
assert nvml.nvmlInit_v2() == 0, "nvmlInit_v2 failed"
470+
h = ctypes.c_void_p()
471+
assert nvml.nvmlDeviceGetHandleByIndex_v2(0, ctypes.byref(h)) == 0
472+
m = ctypes.c_uint(99)
473+
nvml.nvmlDeviceGetPersistenceMode(h, ctypes.byref(m))
474+
print(f"current persistence_mode = {m.value} (1=ENABLED, 0=DISABLED)")
475+
ret = nvml.nvmlDeviceSetPersistenceMode(h, 0)
476+
print(f"SET DISABLED -> {ret} # 0=SUCCESS, 4=NO_PERMISSION, 999=UNKNOWN")
477+
if ret == NVML_ERROR_UNKNOWN:
478+
print("FAIL: NVML_ERROR_UNKNOWN -- daemon-down failure mode reproduced", file=sys.stderr)
479+
sys.exit(1)
480+
if ret != NVML_SUCCESS:
481+
print(f"FAIL: unexpected return code {ret}", file=sys.stderr)
482+
sys.exit(1)
483+
# restore
484+
nvml.nvmlDeviceSetPersistenceMode(h, m.value)
485+
print("OK")
486+
PYEOF
487+
416488
checks:
417489
name: Check job status
418490
if: always()

0 commit comments

Comments
 (0)