Skip to content

Commit 956d0c3

Browse files
mnenciaarmru
andauthored
fix(major-upgrade): wait for pods to fully terminate before starting upgrade job (#10298)
During a major version upgrade, the upgrade job could be created while old instance pods were still gracefully terminating. This happened because `deleteAllPodsInMajorUpgradePreparation` treated pods with a `deletionTimestamp` as already gone, returning "all clear" to proceed with job creation. Track terminating pods/jobs separately and only signal readiness to proceed when all pods and jobs have been completely removed from the API server. Reduce the requeue interval from 30s to 10s since we are actively waiting for termination to complete. Closes #10297 Signed-off-by: Marco Nenciarini <[email protected]> Signed-off-by: Armando Ruocco <[email protected]> Co-authored-by: Armando Ruocco <[email protected]>
1 parent 1e3ac51 commit 956d0c3

2 files changed

Lines changed: 63 additions & 4 deletions

File tree

‎pkg/reconciler/majorupgrade/reconciler.go‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -135,33 +135,40 @@ func deleteAllPodsInMajorUpgradePreparation(
135135
jobs []batchv1.Job,
136136
) (*ctrl.Result, error) {
137137
foundSomethingToDelete := false
138+
waitingForDeletion := false
138139

139140
for _, pod := range instances {
140141
if pod.GetDeletionTimestamp() != nil {
142+
waitingForDeletion = true
141143
continue
142144
}
143145

144146
foundSomethingToDelete = true
145147
if err := c.Delete(ctx, &pod); err != nil {
146-
return nil, err
148+
if !apierrs.IsNotFound(err) {
149+
return nil, err
150+
}
147151
}
148152
}
149153

150154
for _, job := range jobs {
151155
if job.GetDeletionTimestamp() != nil {
156+
waitingForDeletion = true
152157
continue
153158
}
154159

155160
foundSomethingToDelete = true
156161
if err := c.Delete(ctx, &job, &client.DeleteOptions{
157162
PropagationPolicy: ptr.To(metav1.DeletePropagationForeground),
158163
}); err != nil {
159-
return nil, err
164+
if !apierrs.IsNotFound(err) {
165+
return nil, err
166+
}
160167
}
161168
}
162169

163-
if foundSomethingToDelete {
164-
return &ctrl.Result{RequeueAfter: 30 * time.Second}, nil
170+
if foundSomethingToDelete || waitingForDeletion {
171+
return &ctrl.Result{RequeueAfter: 10 * time.Second}, nil
165172
}
166173

167174
return nil, nil

‎pkg/reconciler/majorupgrade/reconciler_test.go‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,58 @@ var _ = Describe("Major upgrade rollback handling", func() {
189189
})
190190
})
191191

192+
var _ = Describe("deleteAllPodsInMajorUpgradePreparation", func() {
193+
It("deletes pods and requeues when pods have no deletion timestamp", func(ctx SpecContext) {
194+
pod := corev1.Pod{
195+
ObjectMeta: metav1.ObjectMeta{
196+
Name: "cluster-example-1",
197+
Namespace: "default",
198+
},
199+
}
200+
201+
fakeClient := fake.NewClientBuilder().
202+
WithScheme(schemeBuilder.BuildWithAllKnownScheme()).
203+
WithRuntimeObjects(&pod).
204+
Build()
205+
206+
result, err := deleteAllPodsInMajorUpgradePreparation(ctx, fakeClient, []corev1.Pod{pod}, nil)
207+
Expect(err).ToNot(HaveOccurred())
208+
Expect(result).ToNot(BeNil())
209+
Expect(*result).To(Equal(ctrl.Result{RequeueAfter: 10 * time.Second}))
210+
})
211+
212+
It("requeues when pods are still terminating", func(ctx SpecContext) {
213+
now := metav1.Now()
214+
terminatingPod := corev1.Pod{
215+
ObjectMeta: metav1.ObjectMeta{
216+
Name: "cluster-example-1",
217+
Namespace: "default",
218+
DeletionTimestamp: &now,
219+
Finalizers: []string{"test-finalizer"},
220+
},
221+
}
222+
223+
fakeClient := fake.NewClientBuilder().
224+
WithScheme(schemeBuilder.BuildWithAllKnownScheme()).
225+
Build()
226+
227+
result, err := deleteAllPodsInMajorUpgradePreparation(ctx, fakeClient, []corev1.Pod{terminatingPod}, nil)
228+
Expect(err).ToNot(HaveOccurred())
229+
Expect(result).ToNot(BeNil())
230+
Expect(*result).To(Equal(ctrl.Result{RequeueAfter: 10 * time.Second}))
231+
})
232+
233+
It("returns nil when no pods or jobs exist", func(ctx SpecContext) {
234+
fakeClient := fake.NewClientBuilder().
235+
WithScheme(schemeBuilder.BuildWithAllKnownScheme()).
236+
Build()
237+
238+
result, err := deleteAllPodsInMajorUpgradePreparation(ctx, fakeClient, nil, nil)
239+
Expect(err).ToNot(HaveOccurred())
240+
Expect(result).To(BeNil())
241+
})
242+
})
243+
192244
var _ = Describe("Major upgrade job definition", func() {
193245
It("sets BackoffLimit to 0", func() {
194246
cluster := &apiv1.Cluster{

0 commit comments

Comments
 (0)