@@ -89,19 +89,7 @@ in_container() {
8989host_install () {
9090 apt-get -y install build-essential dkms " linux-headers-$( uname -r) " psmisc kmod
9191
92- # Take down nvidia-persistenced *without* systemctl. The packaged
93- # systemd unit declares `RuntimeDirectory=nvidia-persistenced`, which
94- # makes systemd unlink /run/nvidia-persistenced/ on stop. The
95- # container has /run/nvidia-persistenced/ bind-mounted from host, and
96- # the bind mount points to the dir's inode at container-start time --
97- # if systemd removes the dir and the new daemon recreates it under a
98- # different inode, the container's bind mount goes stale and its
99- # /run/nvidia-persistenced/socket loses its link to the live daemon
100- # endpoint (the file shows up with link count 0 inside the container).
101- # NVML state-changing calls in the container then return
102- # NVML_ERROR_UNKNOWN. Sending SIGTERM directly keeps the inode alive.
103- pkill -TERM nvidia-persistenced 2> /dev/null || true
104- systemctl stop dcgm-exporter 2> /dev/null || true
92+ systemctl stop nvidia-persistenced dcgm-exporter 2> /dev/null || true
10593 # if-test instead of `fuser ... || true` so a kill failure surfaces
10694 # (fuser exits 1 when nothing holds the device, which is the happy path).
10795 if fuser /dev/nvidia* > /dev/null 2>&1 ; then
@@ -127,35 +115,25 @@ host_install() {
127115 --accept-license --ui=none --no-cc-version-check --kernel-module-type=" $KMT " )
128116 modprobe nvidia nvidia_uvm nvidia_modeset
129117
130- # Bring nvidia-persistenced back up. We stopped it above, and the
131- # `--silent --no-questions` .run installer drops `/usr/bin/nvidia-
132- # persistenced` but does not reliably reinstall a usable systemd
133- # unit -- so a previous attempt at `systemctl start nvidia-
134- # persistenced.service` was a no-op. Exec the daemon directly; it
135- # self-daemonizes and creates `/run/nvidia-persistenced/socket`,
136- # which NVML clients in the test container need for state-changing
137- # calls like `nvmlDeviceSetPersistenceMode` -- without it those
138- # calls return NVML_ERROR_UNKNOWN.
118+ # Bring nvidia-persistenced back up. NVML state-changing calls from
119+ # inside the test container (e.g. nvmlDeviceSetPersistenceMode, which
120+ # cuda.core's test_persistence_mode_enabled exercises) talk to the
121+ # daemon via /run/nvidia-persistenced/socket; without a live daemon
122+ # they return NVML_ERROR_UNKNOWN.
139123 #
140- # `--user root`: the daemon's default user is `nvidia-persistenced`,
141- # which our apt purge of `nvidia-compute-utils-*` deleted. Without
142- # this flag the daemon's setuid(3) call fails post-fork and the
143- # process exits silently (which leaves Persistence-M flipping back
144- # to Off the moment we exit the start window).
124+ # systemctl can't start the unit (the `--silent --no-questions` .run
125+ # installer drops /usr/bin/nvidia-persistenced but no usable systemd
126+ # unit), so exec the binary directly -- it self-daemonizes.
145127 #
146- # Same latent gap exists in nv-gha-runners/vm-images' `nvgha-driver`;
147- # their CUDA-runtime validation workload doesn't issue an NVML SET
148- # write so they haven't surfaced it yet.
149- set -x
150- /usr/bin/nvidia-persistenced --verbose --user root 2>&1 || true
151- sleep 1
152- # Diagnostics: confirm the daemon is alive + socket present.
153- pgrep -laf nvidia-persistenced || echo " WARN: nvidia-persistenced not running"
154- ls -la /run/nvidia-persistenced/ 2>&1 || echo " WARN: /run/nvidia-persistenced missing"
155- # Set persistence mode explicitly so we match the runner image's
156- # `Persistence-M: On` baseline regardless of how the daemon came up.
157- nvidia-smi -pm 1 || true
158- set +x
128+ # `--user root` because the default `nvidia-persistenced` user was
129+ # deleted along with `nvidia-compute-utils-*` in the purge above;
130+ # without this flag the daemon's post-fork setuid() fails silently
131+ # and the process exits.
132+ #
133+ # nv-gha-runners/vm-images' `nvgha-driver` has the same latent gap;
134+ # its CUDA-runtime validation workload never issues an NVML SET
135+ # write so it hasn't surfaced there.
136+ /usr/bin/nvidia-persistenced --verbose --user root || true
159137}
160138
161139# Replace the toolkit's bind-mounted nvidia libs/binaries inside this
@@ -200,21 +178,13 @@ refresh_container_libs() {
200178 [ -n " $src " ] || { echo " skip $tgt : no host source" >&2 ; continue ; }
201179 fi
202180 umount " $tgt " 2> /dev/null || true
203- # --preserve=mode keeps the SUID bit. /usr/bin/nvidia-modprobe ships
204- # 4755 and NVML's state-changing calls (e.g.
205- # nvmlDeviceSetPersistenceMode) go through it; a plain `cp` strips
206- # SUID and the call then fails with NVML_ERROR_UNKNOWN. The runner
207- # team's nvgha-driver has the same bug; we differ here.
181+ # --preserve=mode keeps the SUID bit so the refresh doesn't silently
182+ # de-privilege binaries like nvidia-modprobe that ship 4755 (the
183+ # runner-team's nvgha-driver uses plain `cp` and has the same gap).
208184 cp -f --preserve=mode --remove-destination " $src " " $tgt " \
209185 || echo " WARN: refresh failed for $tgt (src=$src )" >&2
210186 done
211187 ldconfig
212-
213- # Diagnostic: confirm SUID survived on nvidia-modprobe (the load-bearing
214- # piece). One-liner so the next CI log proves the fix.
215- if [ -e /usr/bin/nvidia-modprobe ]; then
216- stat -c ' refresh: %n mode=%a uid=%u' /usr/bin/nvidia-modprobe >&2
217- fi
218188}
219189
220190if [ -z " ${_NVDRV_NSENTERED:- } " ] && in_container; then
@@ -228,22 +198,19 @@ if [ -z "${_NVDRV_NSENTERED:-}" ] && in_container; then
228198 || { echo " ::error::container needs 'options: --privileged --pid=host'" >&2 ; exit 1; }
229199 refresh_container_libs
230200
231- # Re-bind /run/nvidia-persistenced from host. The container's original
232- # bind mount of this dir was taken at container-start time and points
233- # to the host's then-current inode. Even with `pkill` (instead of
234- # systemctl) the host dir is recreated by the new daemon under a fresh
235- # inode -- leaving the container's bind mount stranded on a deleted
236- # inode (socket file shows up with link count 0 ). Re-do the bind mount
237- # so in- container NVML clients see the live daemon endpoint . Needs
238- # CAP_SYS_ADMIN, which we get from the --privileged --pid=host the
239- # workflow adds for custom-DRIVER rows.
201+ # Re-bind /run/nvidia-persistenced from the host. The container's
202+ # original bind mount was pinned to the host's at- container-start
203+ # inode; the daemon stop+restart cycle recreates the dir under a
204+ # fresh inode, stranding the bind mount on the deleted one (socket
205+ # shows up with link count 0 inside the container, NVML SET calls
206+ # return NVML_ERROR_UNKNOWN ). Re-bind from /proc/1/root so the
207+ # container picks up the live host dir . Needs CAP_SYS_ADMIN, which
208+ # the workflow grants via --privileged --pid=host on custom-DRIVER
209+ # rows.
240210 if [ -d /proc/1/root/run/nvidia-persistenced ]; then
241- set -x
242211 umount /run/nvidia-persistenced 2> /dev/null || true
243212 mkdir -p /run/nvidia-persistenced
244213 mount --bind /proc/1/root/run/nvidia-persistenced /run/nvidia-persistenced
245- ls -la /run/nvidia-persistenced/ >&2
246- set +x
247214 fi
248215else
249216 host_install
0 commit comments