Skip to content

Commit 8d8a9ef

Browse files
committed
CI: drop install_gpu_driver.sh experiments that turned out non-load-bearing
- Revert `pkill -TERM nvidia-persistenced` to `systemctl stop`; pkill alone didn't prevent the host dir's inode from flipping, the re-bind of /run/nvidia-persistenced/ is what restores the container's view. - Drop `nvidia-smi -pm 1`; the test exercises NVML's set call, which succeeds once the daemon socket is reachable regardless of current Persistence-M state. - Trim `set -x` blocks and `pgrep`/`ls -la`/`stat` diagnostics that served their purpose during debugging. Keeps the load-bearing changes (nsenter bash -s, /usr/(bin|lib) refresh filter, exec nvidia-persistenced --user root, the /run/nvidia-persistenced re-bind, cp --preserve=mode) and brings the diff against Justin's nvgha-driver back down to the strict minimum.
1 parent 2b34f1f commit 8d8a9ef

1 file changed

Lines changed: 30 additions & 63 deletions

File tree

‎ci/tools/install_gpu_driver.sh‎

Lines changed: 30 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -89,19 +89,7 @@ in_container() {
8989
host_install() {
9090
apt-get -y install build-essential dkms "linux-headers-$(uname -r)" psmisc kmod
9191

92-
# Take down nvidia-persistenced *without* systemctl. The packaged
93-
# systemd unit declares `RuntimeDirectory=nvidia-persistenced`, which
94-
# makes systemd unlink /run/nvidia-persistenced/ on stop. The
95-
# container has /run/nvidia-persistenced/ bind-mounted from host, and
96-
# the bind mount points to the dir's inode at container-start time --
97-
# if systemd removes the dir and the new daemon recreates it under a
98-
# different inode, the container's bind mount goes stale and its
99-
# /run/nvidia-persistenced/socket loses its link to the live daemon
100-
# endpoint (the file shows up with link count 0 inside the container).
101-
# NVML state-changing calls in the container then return
102-
# NVML_ERROR_UNKNOWN. Sending SIGTERM directly keeps the inode alive.
103-
pkill -TERM nvidia-persistenced 2>/dev/null || true
104-
systemctl stop dcgm-exporter 2>/dev/null || true
92+
systemctl stop nvidia-persistenced dcgm-exporter 2>/dev/null || true
10593
# if-test instead of `fuser ... || true` so a kill failure surfaces
10694
# (fuser exits 1 when nothing holds the device, which is the happy path).
10795
if fuser /dev/nvidia* >/dev/null 2>&1; then
@@ -127,35 +115,25 @@ host_install() {
127115
--accept-license --ui=none --no-cc-version-check --kernel-module-type="$KMT" )
128116
modprobe nvidia nvidia_uvm nvidia_modeset
129117

130-
# Bring nvidia-persistenced back up. We stopped it above, and the
131-
# `--silent --no-questions` .run installer drops `/usr/bin/nvidia-
132-
# persistenced` but does not reliably reinstall a usable systemd
133-
# unit -- so a previous attempt at `systemctl start nvidia-
134-
# persistenced.service` was a no-op. Exec the daemon directly; it
135-
# self-daemonizes and creates `/run/nvidia-persistenced/socket`,
136-
# which NVML clients in the test container need for state-changing
137-
# calls like `nvmlDeviceSetPersistenceMode` -- without it those
138-
# calls return NVML_ERROR_UNKNOWN.
118+
# Bring nvidia-persistenced back up. NVML state-changing calls from
119+
# inside the test container (e.g. nvmlDeviceSetPersistenceMode, which
120+
# cuda.core's test_persistence_mode_enabled exercises) talk to the
121+
# daemon via /run/nvidia-persistenced/socket; without a live daemon
122+
# they return NVML_ERROR_UNKNOWN.
139123
#
140-
# `--user root`: the daemon's default user is `nvidia-persistenced`,
141-
# which our apt purge of `nvidia-compute-utils-*` deleted. Without
142-
# this flag the daemon's setuid(3) call fails post-fork and the
143-
# process exits silently (which leaves Persistence-M flipping back
144-
# to Off the moment we exit the start window).
124+
# systemctl can't start the unit (the `--silent --no-questions` .run
125+
# installer drops /usr/bin/nvidia-persistenced but no usable systemd
126+
# unit), so exec the binary directly -- it self-daemonizes.
145127
#
146-
# Same latent gap exists in nv-gha-runners/vm-images' `nvgha-driver`;
147-
# their CUDA-runtime validation workload doesn't issue an NVML SET
148-
# write so they haven't surfaced it yet.
149-
set -x
150-
/usr/bin/nvidia-persistenced --verbose --user root 2>&1 || true
151-
sleep 1
152-
# Diagnostics: confirm the daemon is alive + socket present.
153-
pgrep -laf nvidia-persistenced || echo "WARN: nvidia-persistenced not running"
154-
ls -la /run/nvidia-persistenced/ 2>&1 || echo "WARN: /run/nvidia-persistenced missing"
155-
# Set persistence mode explicitly so we match the runner image's
156-
# `Persistence-M: On` baseline regardless of how the daemon came up.
157-
nvidia-smi -pm 1 || true
158-
set +x
128+
# `--user root` because the default `nvidia-persistenced` user was
129+
# deleted along with `nvidia-compute-utils-*` in the purge above;
130+
# without this flag the daemon's post-fork setuid() fails silently
131+
# and the process exits.
132+
#
133+
# nv-gha-runners/vm-images' `nvgha-driver` has the same latent gap;
134+
# its CUDA-runtime validation workload never issues an NVML SET
135+
# write so it hasn't surfaced there.
136+
/usr/bin/nvidia-persistenced --verbose --user root || true
159137
}
160138

161139
# Replace the toolkit's bind-mounted nvidia libs/binaries inside this
@@ -200,21 +178,13 @@ refresh_container_libs() {
200178
[ -n "$src" ] || { echo "skip $tgt: no host source" >&2; continue; }
201179
fi
202180
umount "$tgt" 2>/dev/null || true
203-
# --preserve=mode keeps the SUID bit. /usr/bin/nvidia-modprobe ships
204-
# 4755 and NVML's state-changing calls (e.g.
205-
# nvmlDeviceSetPersistenceMode) go through it; a plain `cp` strips
206-
# SUID and the call then fails with NVML_ERROR_UNKNOWN. The runner
207-
# team's nvgha-driver has the same bug; we differ here.
181+
# --preserve=mode keeps the SUID bit so the refresh doesn't silently
182+
# de-privilege binaries like nvidia-modprobe that ship 4755 (the
183+
# runner-team's nvgha-driver uses plain `cp` and has the same gap).
208184
cp -f --preserve=mode --remove-destination "$src" "$tgt" \
209185
|| echo "WARN: refresh failed for $tgt (src=$src)" >&2
210186
done
211187
ldconfig
212-
213-
# Diagnostic: confirm SUID survived on nvidia-modprobe (the load-bearing
214-
# piece). One-liner so the next CI log proves the fix.
215-
if [ -e /usr/bin/nvidia-modprobe ]; then
216-
stat -c 'refresh: %n mode=%a uid=%u' /usr/bin/nvidia-modprobe >&2
217-
fi
218188
}
219189

220190
if [ -z "${_NVDRV_NSENTERED:-}" ] && in_container; then
@@ -228,22 +198,19 @@ if [ -z "${_NVDRV_NSENTERED:-}" ] && in_container; then
228198
|| { echo "::error::container needs 'options: --privileged --pid=host'" >&2; exit 1; }
229199
refresh_container_libs
230200

231-
# Re-bind /run/nvidia-persistenced from host. The container's original
232-
# bind mount of this dir was taken at container-start time and points
233-
# to the host's then-current inode. Even with `pkill` (instead of
234-
# systemctl) the host dir is recreated by the new daemon under a fresh
235-
# inode -- leaving the container's bind mount stranded on a deleted
236-
# inode (socket file shows up with link count 0). Re-do the bind mount
237-
# so in-container NVML clients see the live daemon endpoint. Needs
238-
# CAP_SYS_ADMIN, which we get from the --privileged --pid=host the
239-
# workflow adds for custom-DRIVER rows.
201+
# Re-bind /run/nvidia-persistenced from the host. The container's
202+
# original bind mount was pinned to the host's at-container-start
203+
# inode; the daemon stop+restart cycle recreates the dir under a
204+
# fresh inode, stranding the bind mount on the deleted one (socket
205+
# shows up with link count 0 inside the container, NVML SET calls
206+
# return NVML_ERROR_UNKNOWN). Re-bind from /proc/1/root so the
207+
# container picks up the live host dir. Needs CAP_SYS_ADMIN, which
208+
# the workflow grants via --privileged --pid=host on custom-DRIVER
209+
# rows.
240210
if [ -d /proc/1/root/run/nvidia-persistenced ]; then
241-
set -x
242211
umount /run/nvidia-persistenced 2>/dev/null || true
243212
mkdir -p /run/nvidia-persistenced
244213
mount --bind /proc/1/root/run/nvidia-persistenced /run/nvidia-persistenced
245-
ls -la /run/nvidia-persistenced/ >&2
246-
set +x
247214
fi
248215
else
249216
host_install

0 commit comments

Comments
 (0)