Skip to content

Commit 09b91e5

Browse files
armrumnencia
andauthored
fix: allow unrecoverable annotation to work on non-ready pods (#9968)
Pods annotated with alpha.cnpg.io/unrecoverable=true were not being deleted when the postgres process was not running, because the reconciliation loop returned early waiting for pods to report their status before reaching the unrecoverable instance check. Move the reconcileUnrecoverableInstances call before the early return so that unrecoverable pods are handled regardless of their readiness state. Closes #9947 Signed-off-by: Armando Ruocco <[email protected]> Signed-off-by: Marco Nenciarini <[email protected]> Co-authored-by: Marco Nenciarini <[email protected]>
1 parent 3d6e698 commit 09b91e5

2 files changed

Lines changed: 78 additions & 5 deletions

File tree

‎internal/controller/cluster_controller.go‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1003,6 +1003,14 @@ func (r *ClusterReconciler) reconcilePods(
10031003
return r.createPrimaryInstance(ctx, cluster)
10041004
}
10051005

1006+
// Handle instances marked as unrecoverable before waiting for pods to be ready.
1007+
// This ensures pods annotated with alpha.cnpg.io/unrecoverable=true are
1008+
// deleted even when they can't report their status (e.g., postgres process
1009+
// not running, startup probe failing).
1010+
if res, err := r.reconcileUnrecoverableInstances(ctx, cluster, resources); !res.IsZero() || err != nil {
1011+
return res, err
1012+
}
1013+
10061014
// Stop acting here if there are non-ready Pods unless in maintenance reusing PVCs.
10071015
// The user have chosen to wait for the missing nodes to come up
10081016
if !(cluster.IsNodeMaintenanceWindowInProgress() && cluster.IsReusePVCEnabled()) &&
@@ -1023,11 +1031,6 @@ func (r *ClusterReconciler) reconcilePods(
10231031
return r.joinReplicaInstance(ctx, newNodeSerial, cluster)
10241032
}
10251033

1026-
// Are there nodes to be removed? Remove one of them
1027-
if res, err := r.reconcileUnrecoverableInstances(ctx, cluster, resources); !res.IsZero() || err != nil {
1028-
return res, err
1029-
}
1030-
10311034
// Should we scale down the cluster?
10321035
if cluster.Status.Instances > cluster.Spec.Instances {
10331036
if err := r.scaleDownCluster(ctx, cluster, resources); err != nil {

‎internal/controller/cluster_unrecoverable_test.go‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,4 +112,74 @@ var _ = Describe("Unrecoverable replicas", func() {
112112

113113
Expect(result).To(ConsistOf("cluster-example-3", "cluster-example-4"))
114114
})
115+
116+
It("Collects unrecoverable instances regardless of pod readiness status", func(ctx SpecContext) {
117+
makeNonReadyPod := func(name string, unrecoverable string) corev1.Pod {
118+
pod := corev1.Pod{
119+
ObjectMeta: metav1.ObjectMeta{
120+
Name: name,
121+
Annotations: map[string]string{
122+
utils.UnrecoverableInstanceAnnotationName: unrecoverable,
123+
},
124+
},
125+
Status: corev1.PodStatus{
126+
Phase: corev1.PodRunning,
127+
Conditions: []corev1.PodCondition{
128+
{
129+
Type: corev1.PodReady,
130+
Status: corev1.ConditionFalse,
131+
},
132+
},
133+
},
134+
}
135+
return pod
136+
}
137+
138+
cluster := &apiv1.Cluster{
139+
ObjectMeta: metav1.ObjectMeta{},
140+
Spec: apiv1.ClusterSpec{
141+
Instances: 2,
142+
},
143+
Status: apiv1.ClusterStatus{
144+
CurrentPrimary: "cluster-example-1",
145+
TargetPrimary: "cluster-example-1",
146+
},
147+
}
148+
149+
// Primary pod is ready
150+
primaryPod := corev1.Pod{
151+
ObjectMeta: metav1.ObjectMeta{
152+
Name: "cluster-example-1",
153+
Annotations: map[string]string{},
154+
},
155+
Status: corev1.PodStatus{
156+
Phase: corev1.PodRunning,
157+
Conditions: []corev1.PodCondition{
158+
{
159+
Type: corev1.PodReady,
160+
Status: corev1.ConditionTrue,
161+
},
162+
},
163+
},
164+
}
165+
166+
// Replica pod is not ready (e.g., postgres process not running, startup probe
167+
// failing) but annotated as unrecoverable by the user
168+
nonReadyUnrecoverablePod := makeNonReadyPod("cluster-example-2", "true")
169+
170+
result := collectNamesOfUnrecoverableInstances(
171+
ctx,
172+
cluster,
173+
&managedResources{
174+
instances: corev1.PodList{
175+
Items: []corev1.Pod{
176+
primaryPod,
177+
nonReadyUnrecoverablePod,
178+
},
179+
},
180+
},
181+
)
182+
183+
Expect(result).To(ConsistOf("cluster-example-2"))
184+
})
115185
})

0 commit comments

Comments
 (0)