Skip to content

Commit 903fbcc

Browse files
authored
CI: stop display container before switching driver modes (NVIDIA#3184)
* CI: stop display container before switching driver modes Closes NVIDIA#3183 Stop NVDisplay.ContainerLocalSystem before invoking nvidia-smi -fdm and wait for every NVDisplay.Container process to exit. After a driver install, the display container can crash-loop while the GPU is cycled. If its teardown overlaps PnP removal, nvlddmkm can remain loaded and nvidia-smi may never recover. Bound the process drain at 60 seconds and report remaining PIDs on failure. Preserve the existing mode-specific logging and device-cycle stabilization behavior. * Put the service stop under 60 second timer as well. * CI: prevent display container restart during mode switching Temporarily disable NVDisplay.ContainerLocalSystem while changing driver modes so queued recovery actions cannot restart it during the device cycle. Handle in-flight starts, restore the original startup type in a finally block, and report the requested driver mode by name. * Implement coderabbit suggestions
1 parent 06e3aa3 commit 903fbcc

1 file changed

Lines changed: 79 additions & 29 deletions

File tree

‎ci/tools/configure_driver_mode.ps1‎

Lines changed: 79 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -11,50 +11,100 @@
1111
# Inputs (env):
1212
# DRIVER_MODE One of WDDM, TCC, MCDM.
1313

14+
function Stop-DisplayContainer {
15+
param (
16+
[System.ServiceProcess.ServiceController]$Service
17+
)
18+
19+
# A crash-looping display-container process can race the subsequent device
20+
# cycle and prevent the NVIDIA kernel driver from unloading.
21+
Write-Output "Stopping NVDisplay.ContainerLocalSystem..."
22+
$deadline = (Get-Date).AddSeconds(60)
23+
24+
Write-Output "Waiting for NVDisplay.ContainerLocalSystem to stop and NVDisplay.Container processes to exit..."
25+
while ($true) {
26+
$Service.Refresh()
27+
# A recovery start may already be in flight when the service is
28+
# disabled. Stop it as soon as it reaches a stoppable state.
29+
if ($Service.Status -eq "Running" -or $Service.Status -eq "Paused") {
30+
Stop-Service -InputObject $Service -Force -NoWait -ErrorAction Stop
31+
$Service.Refresh()
32+
}
33+
$container_processes = @(Get-Process -Name "NVDisplay.Container" -ErrorAction SilentlyContinue)
34+
if ($Service.Status -eq "Stopped" -and $container_processes.Count -eq 0) {
35+
break
36+
}
37+
if ((Get-Date) -ge $deadline) {
38+
$process_ids = $container_processes.Id -join ", "
39+
if (-not $process_ids) { $process_ids = "none" }
40+
throw "Timed out after 60s waiting for the NVIDIA display container (service status: $($Service.Status); remaining NVDisplay.Container PIDs: $process_ids)"
41+
}
42+
Start-Sleep -Milliseconds 500
43+
}
44+
}
45+
1446
function Set-DriverMode {
1547

1648
# Map matrix DRIVER_MODE to nvidia-smi -fdm code.
1749
# This assumes we have the prior knowledge on which GPU can use which mode.
1850
$driver_mode = $env:DRIVER_MODE
1951
if ($driver_mode -eq "WDDM") {
20-
Write-Output "Setting driver mode to WDDM..."
21-
nvidia-smi -fdm 0
52+
$driver_mode_code = 0
2253
} elseif ($driver_mode -eq "TCC") {
23-
Write-Output "Setting driver mode to TCC..."
24-
nvidia-smi -fdm 1
54+
$driver_mode_code = 1
2555
} elseif ($driver_mode -eq "MCDM") {
26-
Write-Output "Setting driver mode to MCDM..."
27-
nvidia-smi -fdm 2
56+
$driver_mode_code = 2
2857
} else {
2958
Write-Output "Unknown driver mode: $driver_mode"
3059
exit 1
3160
}
3261

33-
# Only restart NVIDIA display adapters, not other display devices (e.g. QEMU VGA).
34-
$nvidia_devices = @(Get-PnpDevice -Class Display -FriendlyName "NVIDIA*")
35-
$gpu_count = $nvidia_devices.Count
36-
foreach ($device in $nvidia_devices) {
37-
Write-Output "Restarting device: $($device.FriendlyName) ($($device.InstanceId))"
38-
pnputil /disable-device "$($device.InstanceId)"
39-
pnputil /enable-device "$($device.InstanceId)"
40-
}
62+
$display_container_service = Get-Service -Name "NVDisplay.ContainerLocalSystem" -ErrorAction Stop
63+
$display_container_start_type = $display_container_service.StartType
64+
try {
65+
# A queued recovery restart cannot be canceled, so keep the service
66+
# disabled until the device cycle and NVML recovery are complete.
67+
Write-Output "Temporarily disabling NVDisplay.ContainerLocalSystem..."
68+
Set-Service -InputObject $display_container_service -StartupType Disabled -ErrorAction Stop
69+
Stop-DisplayContainer -Service $display_container_service
4170

42-
# Initial settle after the device cycle.
43-
Start-Sleep -Seconds 5
71+
Write-Output "Setting driver mode to $driver_mode..."
72+
nvidia-smi -fdm $driver_mode_code
73+
if ($LASTEXITCODE -ne 0) {
74+
throw "nvidia-smi -fdm $driver_mode_code ($driver_mode) failed with exit code $LASTEXITCODE"
75+
}
4476

45-
# Poll nvidia-smi for N consecutive successes (N == cycled GPUs)
46-
# so a mid-init "ok" flap doesn't fool the loop; bail after ~60s.
47-
Write-Output "Waiting for nvidia-smi/NVML to come back up after device cycle..."
48-
$deadline = (Get-Date).AddSeconds(60)
49-
$consecutive_ok = 0
50-
do {
51-
Start-Sleep -Seconds 3
52-
& nvidia-smi.exe 2>&1 | Out-Null
53-
if ($LASTEXITCODE -eq 0) { $consecutive_ok++ } else { $consecutive_ok = 0 }
54-
} while ($consecutive_ok -lt $gpu_count -and (Get-Date) -lt $deadline)
55-
if ($consecutive_ok -lt $gpu_count) {
56-
Write-Error "nvidia-smi did not return cleanly $gpu_count times in a row within 60s of the device cycle"
57-
exit 1
77+
# Only restart NVIDIA display adapters, not other display devices (e.g. QEMU VGA).
78+
$nvidia_devices = @(Get-PnpDevice -Class Display -FriendlyName "NVIDIA*")
79+
$gpu_count = $nvidia_devices.Count
80+
if ($gpu_count -eq 0) {
81+
throw "No NVIDIA display devices were found."
82+
}
83+
foreach ($device in $nvidia_devices) {
84+
Write-Output "Restarting device: $($device.FriendlyName) ($($device.InstanceId))"
85+
pnputil /disable-device "$($device.InstanceId)"
86+
pnputil /enable-device "$($device.InstanceId)"
87+
}
88+
89+
# Initial settle after the device cycle.
90+
Start-Sleep -Seconds 5
91+
92+
# Poll nvidia-smi for N consecutive successes (N == cycled GPUs)
93+
# so a mid-init "ok" flap doesn't fool the loop; bail after ~60s.
94+
Write-Output "Waiting for nvidia-smi/NVML to come back up after device cycle..."
95+
$deadline = (Get-Date).AddSeconds(60)
96+
$consecutive_ok = 0
97+
do {
98+
Start-Sleep -Seconds 3
99+
& nvidia-smi.exe 2>&1 | Out-Null
100+
if ($LASTEXITCODE -eq 0) { $consecutive_ok++ } else { $consecutive_ok = 0 }
101+
} while ($consecutive_ok -lt $gpu_count -and (Get-Date) -lt $deadline)
102+
if ($consecutive_ok -lt $gpu_count) {
103+
throw "nvidia-smi did not return cleanly $gpu_count times in a row within 60s of the device cycle"
104+
}
105+
} finally {
106+
Write-Output "Restoring NVDisplay.ContainerLocalSystem startup type to $display_container_start_type..."
107+
Set-Service -Name $display_container_service.Name -StartupType $display_container_start_type -ErrorAction Stop
58108
}
59109
}
60110

0 commit comments

Comments
 (0)