|
11 | 11 | # Inputs (env): |
12 | 12 | # DRIVER_MODE One of WDDM, TCC, MCDM. |
13 | 13 |
|
| 14 | +function Stop-DisplayContainer { |
| 15 | + param ( |
| 16 | + [System.ServiceProcess.ServiceController]$Service |
| 17 | + ) |
| 18 | + |
| 19 | + # A crash-looping display-container process can race the subsequent device |
| 20 | + # cycle and prevent the NVIDIA kernel driver from unloading. |
| 21 | + Write-Output "Stopping NVDisplay.ContainerLocalSystem..." |
| 22 | + $deadline = (Get-Date).AddSeconds(60) |
| 23 | + |
| 24 | + Write-Output "Waiting for NVDisplay.ContainerLocalSystem to stop and NVDisplay.Container processes to exit..." |
| 25 | + while ($true) { |
| 26 | + $Service.Refresh() |
| 27 | + # A recovery start may already be in flight when the service is |
| 28 | + # disabled. Stop it as soon as it reaches a stoppable state. |
| 29 | + if ($Service.Status -eq "Running" -or $Service.Status -eq "Paused") { |
| 30 | + Stop-Service -InputObject $Service -Force -NoWait -ErrorAction Stop |
| 31 | + $Service.Refresh() |
| 32 | + } |
| 33 | + $container_processes = @(Get-Process -Name "NVDisplay.Container" -ErrorAction SilentlyContinue) |
| 34 | + if ($Service.Status -eq "Stopped" -and $container_processes.Count -eq 0) { |
| 35 | + break |
| 36 | + } |
| 37 | + if ((Get-Date) -ge $deadline) { |
| 38 | + $process_ids = $container_processes.Id -join ", " |
| 39 | + if (-not $process_ids) { $process_ids = "none" } |
| 40 | + throw "Timed out after 60s waiting for the NVIDIA display container (service status: $($Service.Status); remaining NVDisplay.Container PIDs: $process_ids)" |
| 41 | + } |
| 42 | + Start-Sleep -Milliseconds 500 |
| 43 | + } |
| 44 | +} |
| 45 | + |
14 | 46 | function Set-DriverMode { |
15 | 47 |
|
16 | 48 | # Map matrix DRIVER_MODE to nvidia-smi -fdm code. |
17 | 49 | # This assumes we have the prior knowledge on which GPU can use which mode. |
18 | 50 | $driver_mode = $env:DRIVER_MODE |
19 | 51 | if ($driver_mode -eq "WDDM") { |
20 | | - Write-Output "Setting driver mode to WDDM..." |
21 | | - nvidia-smi -fdm 0 |
| 52 | + $driver_mode_code = 0 |
22 | 53 | } elseif ($driver_mode -eq "TCC") { |
23 | | - Write-Output "Setting driver mode to TCC..." |
24 | | - nvidia-smi -fdm 1 |
| 54 | + $driver_mode_code = 1 |
25 | 55 | } elseif ($driver_mode -eq "MCDM") { |
26 | | - Write-Output "Setting driver mode to MCDM..." |
27 | | - nvidia-smi -fdm 2 |
| 56 | + $driver_mode_code = 2 |
28 | 57 | } else { |
29 | 58 | Write-Output "Unknown driver mode: $driver_mode" |
30 | 59 | exit 1 |
31 | 60 | } |
32 | 61 |
|
33 | | - # Only restart NVIDIA display adapters, not other display devices (e.g. QEMU VGA). |
34 | | - $nvidia_devices = @(Get-PnpDevice -Class Display -FriendlyName "NVIDIA*") |
35 | | - $gpu_count = $nvidia_devices.Count |
36 | | - foreach ($device in $nvidia_devices) { |
37 | | - Write-Output "Restarting device: $($device.FriendlyName) ($($device.InstanceId))" |
38 | | - pnputil /disable-device "$($device.InstanceId)" |
39 | | - pnputil /enable-device "$($device.InstanceId)" |
40 | | - } |
| 62 | + $display_container_service = Get-Service -Name "NVDisplay.ContainerLocalSystem" -ErrorAction Stop |
| 63 | + $display_container_start_type = $display_container_service.StartType |
| 64 | + try { |
| 65 | + # A queued recovery restart cannot be canceled, so keep the service |
| 66 | + # disabled until the device cycle and NVML recovery are complete. |
| 67 | + Write-Output "Temporarily disabling NVDisplay.ContainerLocalSystem..." |
| 68 | + Set-Service -InputObject $display_container_service -StartupType Disabled -ErrorAction Stop |
| 69 | + Stop-DisplayContainer -Service $display_container_service |
41 | 70 |
|
42 | | - # Initial settle after the device cycle. |
43 | | - Start-Sleep -Seconds 5 |
| 71 | + Write-Output "Setting driver mode to $driver_mode..." |
| 72 | + nvidia-smi -fdm $driver_mode_code |
| 73 | + if ($LASTEXITCODE -ne 0) { |
| 74 | + throw "nvidia-smi -fdm $driver_mode_code ($driver_mode) failed with exit code $LASTEXITCODE" |
| 75 | + } |
44 | 76 |
|
45 | | - # Poll nvidia-smi for N consecutive successes (N == cycled GPUs) |
46 | | - # so a mid-init "ok" flap doesn't fool the loop; bail after ~60s. |
47 | | - Write-Output "Waiting for nvidia-smi/NVML to come back up after device cycle..." |
48 | | - $deadline = (Get-Date).AddSeconds(60) |
49 | | - $consecutive_ok = 0 |
50 | | - do { |
51 | | - Start-Sleep -Seconds 3 |
52 | | - & nvidia-smi.exe 2>&1 | Out-Null |
53 | | - if ($LASTEXITCODE -eq 0) { $consecutive_ok++ } else { $consecutive_ok = 0 } |
54 | | - } while ($consecutive_ok -lt $gpu_count -and (Get-Date) -lt $deadline) |
55 | | - if ($consecutive_ok -lt $gpu_count) { |
56 | | - Write-Error "nvidia-smi did not return cleanly $gpu_count times in a row within 60s of the device cycle" |
57 | | - exit 1 |
| 77 | + # Only restart NVIDIA display adapters, not other display devices (e.g. QEMU VGA). |
| 78 | + $nvidia_devices = @(Get-PnpDevice -Class Display -FriendlyName "NVIDIA*") |
| 79 | + $gpu_count = $nvidia_devices.Count |
| 80 | + if ($gpu_count -eq 0) { |
| 81 | + throw "No NVIDIA display devices were found." |
| 82 | + } |
| 83 | + foreach ($device in $nvidia_devices) { |
| 84 | + Write-Output "Restarting device: $($device.FriendlyName) ($($device.InstanceId))" |
| 85 | + pnputil /disable-device "$($device.InstanceId)" |
| 86 | + pnputil /enable-device "$($device.InstanceId)" |
| 87 | + } |
| 88 | + |
| 89 | + # Initial settle after the device cycle. |
| 90 | + Start-Sleep -Seconds 5 |
| 91 | + |
| 92 | + # Poll nvidia-smi for N consecutive successes (N == cycled GPUs) |
| 93 | + # so a mid-init "ok" flap doesn't fool the loop; bail after ~60s. |
| 94 | + Write-Output "Waiting for nvidia-smi/NVML to come back up after device cycle..." |
| 95 | + $deadline = (Get-Date).AddSeconds(60) |
| 96 | + $consecutive_ok = 0 |
| 97 | + do { |
| 98 | + Start-Sleep -Seconds 3 |
| 99 | + & nvidia-smi.exe 2>&1 | Out-Null |
| 100 | + if ($LASTEXITCODE -eq 0) { $consecutive_ok++ } else { $consecutive_ok = 0 } |
| 101 | + } while ($consecutive_ok -lt $gpu_count -and (Get-Date) -lt $deadline) |
| 102 | + if ($consecutive_ok -lt $gpu_count) { |
| 103 | + throw "nvidia-smi did not return cleanly $gpu_count times in a row within 60s of the device cycle" |
| 104 | + } |
| 105 | + } finally { |
| 106 | + Write-Output "Restoring NVDisplay.ContainerLocalSystem startup type to $display_container_start_type..." |
| 107 | + Set-Service -Name $display_container_service.Name -StartupType $display_container_start_type -ErrorAction Stop |
58 | 108 | } |
59 | 109 | } |
60 | 110 |
|
|
0 commit comments