fix: use Recreate strategy for GPU workloads to prevent rolling update deadlock (#196)

Defilan · web-flow · commit 2e4518106cd9 · 2026-03-02T08:53:28.000-08:00
Signed-off-by: Christopher Maher &lt;chris@mahercode.io&gt;
diff --git a/internal/controller/inferenceservice_controller.go b/internal/controller/inferenceservice_controller.go
@@ -839,6 +839,13 @@ func (r *InferenceServiceReconciler) constructDeployment(
 	}
 
 	if gpuCount > 0 {
+		// Use Recreate strategy for GPU workloads to prevent deadlock:
+		// RollingUpdate requires the new pod to be Ready before terminating the old,
+		// but the new pod cannot schedule if the old pod holds the only available GPU(s).
+		deployment.Spec.Strategy = appsv1.DeploymentStrategy{
+			Type: appsv1.RecreateDeploymentStrategyType,
+		}
+
 		tolerations := []corev1.Toleration{
 			{
 				Key:      "nvidia.com/gpu",
diff --git a/internal/controller/inferenceservice_controller_test.go b/internal/controller/inferenceservice_controller_test.go
@@ -255,6 +255,9 @@ var _ = Describe("Multi-GPU Deployment Construction", func() {
 			By("verifying GPU resource limits")
 			gpuLimit := container.Resources.Limits["nvidia.com/gpu"]
 			Expect(gpuLimit).To(Equal(resource.MustParse("2")))
+
+			By("verifying Recreate strategy is set to avoid GPU deadlock")
+			Expect(deployment.Spec.Strategy.Type).To(Equal(appsv1.RecreateDeploymentStrategyType))
 		})
 
 		It("should include multi-GPU args for 4 GPU model", func() {
@@ -579,6 +582,9 @@ var _ = Describe("Multi-GPU Deployment Construction", func() {
 				}
 			}
 			Expect(hasNvidiaToleration).To(BeTrue())
+
+			By("verifying Recreate strategy is set for GPU workloads")
+			Expect(deployment.Spec.Strategy.Type).To(Equal(appsv1.RecreateDeploymentStrategyType))
 		})
 
 		It("should apply custom node selector from InferenceService spec", func() {
@@ -1840,6 +1846,7 @@ var _ = Describe("constructDeployment additional cases", func() {
 		deployment := reconciler.constructDeployment(isvc, model, 1)
 		Expect(deployment.Spec.Template.Spec.Tolerations).To(BeEmpty())
 		Expect(deployment.Spec.Template.Spec.NodeSelector).To(BeEmpty())
+		Expect(deployment.Spec.Strategy.Type).To(Equal(appsv1.DeploymentStrategyType("")))
 	})
 
 	It("should use explicit GPU layers from Model spec", func() {

Original file line number	Diff line number	Diff line change
`@@ -839,6 +839,13 @@ func (r *InferenceServiceReconciler) constructDeployment(`
`839`	`839`	`}`
`840`	`840`
`841`	`841`	`if gpuCount > 0 {`
	`842`	`+ // Use Recreate strategy for GPU workloads to prevent deadlock:`
	`843`	`+ // RollingUpdate requires the new pod to be Ready before terminating the old,`
	`844`	`+ // but the new pod cannot schedule if the old pod holds the only available GPU(s).`
	`845`	`+ deployment.Spec.Strategy = appsv1.DeploymentStrategy{`
	`846`	`+ Type: appsv1.RecreateDeploymentStrategyType,`
	`847`	`+ }`
	`848`	`+`
`842`	`849`	`tolerations := []corev1.Toleration{`
`843`	`850`	`{`
`844`	`851`	`Key: "nvidia.com/gpu",`