Skip to content

Commit ff48dfa

Browse files
armrumnencia
andauthored
fix(hibernation): report WaitingForHealthy condition for non-healthy clusters (#10193)
When the hibernation annotation is set on a non-healthy cluster (e.g. CrashLoopBackOff), the operator correctly defers hibernation until the cluster recovers. However, this decision was previously silent — no condition was set, leaving `cnpg status` unable to report why hibernation had not started. This PR addresses the missing reporting aspect of the issue. Non-healthy clusters with the hibernation annotation now receive a `WaitingForHealthy` condition (`Status=False`), providing clear feedback through `cnpg status` where previously there was none. The underlying behavior — deferring hibernation until the cluster is healthy — remains unchanged. No pre-existing behavior is altered: - Healthy clusters requesting hibernation still proceed immediately - Already-ongoing hibernation (DeletingPods, WaitingPodsDeletion, Hibernated) still bypasses the healthy-phase guard - Clusters without the annotation remain unaffected Note: this does not solve the broader problem of hibernating a cluster that will never become healthy on its own (e.g. persistent CrashLoopBackOff). That would require a separate discussion about allowing forced hibernation when PVCs are already fully provisioned. Ref #10024 Signed-off-by: Armando Ruocco <[email protected]> Signed-off-by: Marco Nenciarini <[email protected]> Co-authored-by: Marco Nenciarini <[email protected]>
1 parent 6f88e7d commit ff48dfa

4 files changed

Lines changed: 89 additions & 8 deletions

File tree

‎pkg/reconciler/hibernation/reconciler.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,11 @@ func Reconcile(
5353
case HibernationConditionReasonWaitingPodsDeletion:
5454
return &ctrl.Result{RequeueAfter: 5 * time.Second}, nil
5555

56+
case HibernationConditionReasonWaitingForHealthy:
57+
// The cluster is not healthy yet, let the reconciliation loop
58+
// continue; hibernation will proceed once the cluster becomes healthy
59+
return nil, nil
60+
5661
default:
5762
return &ctrl.Result{}, nil
5863
}

‎pkg/reconciler/hibernation/reconciler_test.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,29 @@ var _ = Describe("Reconcile resources", func() {
133133
Expect(mock.deletedPods).To(BeEmpty())
134134
})
135135

136+
It("lets the reconciliation loop proceed when the cluster is waiting to be healthy", func(ctx SpecContext) {
137+
mock := &clientMock{}
138+
cluster := apiv1.Cluster{
139+
ObjectMeta: metav1.ObjectMeta{
140+
Annotations: map[string]string{
141+
utils.HibernationAnnotationName: HibernationOn,
142+
},
143+
},
144+
Status: apiv1.ClusterStatus{
145+
Conditions: []metav1.Condition{
146+
{
147+
Type: HibernationConditionType,
148+
Status: metav1.ConditionFalse,
149+
Reason: HibernationConditionReasonWaitingForHealthy,
150+
},
151+
},
152+
},
153+
}
154+
// A Reconcile with a nil return will allow cluster reconciliation to proceed
155+
Expect(Reconcile(ctx, mock, &cluster, nil)).To(BeNil())
156+
Expect(mock.deletedPods).To(BeEmpty())
157+
})
158+
136159
It("deletes the primary pod if available", func(ctx SpecContext) {
137160
mock := &clientMock{}
138161
cluster := apiv1.Cluster{

‎pkg/reconciler/hibernation/status.go‎

Lines changed: 31 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,11 @@ const (
6161
// hibernation condition that is used when the operator is waiting for a Pod
6262
// to be deleted
6363
HibernationConditionReasonWaitingPodsDeletion = "WaitingPodsDeletion"
64+
65+
// HibernationConditionReasonWaitingForHealthy is the value of the
66+
// hibernation condition that is used when the cluster is not healthy
67+
// and hibernation cannot proceed yet
68+
HibernationConditionReasonWaitingForHealthy = "WaitingForHealthy"
6469
)
6570

6671
// ErrInvalidHibernationValue is raised when the hibernation annotation has
@@ -85,11 +90,16 @@ func EnrichStatus(
8590
return
8691
}
8792

88-
// We proceed to hibernate the cluster only when it is ready.
89-
// Hibernating a non-ready cluster may be dangerous since the PVCs
90-
// won't be completely created.
91-
// We should stop the enrich status only when the cluster is unhealthy and the process hasn't already started
93+
// We defer hibernation when the cluster is unhealthy and the process
94+
// hasn't already started. Hibernating a non-ready cluster may be
95+
// dangerous since the PVCs won't be completely created.
9296
if cluster.Status.Phase != apiv1.PhaseHealthy && !isHibernationOngoing(cluster) {
97+
meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{
98+
Type: HibernationConditionType,
99+
Status: metav1.ConditionFalse,
100+
Reason: HibernationConditionReasonWaitingForHealthy,
101+
Message: "Waiting for the cluster to be healthy before proceeding with hibernation",
102+
})
93103
return
94104
}
95105

@@ -127,7 +137,22 @@ func isHibernationEnabled(cluster *apiv1.Cluster) bool {
127137
return cluster.Annotations[utils.HibernationAnnotationName] == HibernationOn
128138
}
129139

130-
// isHibernationOngoing check if the cluster is doing the hibernation process
140+
// isHibernationOngoing checks if the cluster has already started the
141+
// hibernation process (i.e. pod deletion has begun or completed).
142+
// It excludes informational states like WaitingForHealthy that should
143+
// not bypass the healthy-phase safety guard.
131144
func isHibernationOngoing(cluster *apiv1.Cluster) bool {
132-
return meta.FindStatusCondition(cluster.Status.Conditions, HibernationConditionType) != nil
145+
condition := meta.FindStatusCondition(cluster.Status.Conditions, HibernationConditionType)
146+
if condition == nil {
147+
return false
148+
}
149+
150+
switch condition.Reason {
151+
case HibernationConditionReasonDeletingPods,
152+
HibernationConditionReasonWaitingPodsDeletion,
153+
HibernationConditionReasonHibernated:
154+
return true
155+
default:
156+
return false
157+
}
133158
}

‎pkg/reconciler/hibernation/status_test.go‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ var _ = Describe("Status enrichment", func() {
120120
Expect(hibernationCondition.Reason).To(Equal(HibernationConditionReasonDeletingPods))
121121
})
122122

123-
It("doesn't enrich the status while the cluster is not ready", func(ctx SpecContext) {
123+
It("sets a WaitingForHealthy condition when the cluster is not ready", func(ctx SpecContext) {
124124
cluster := apiv1.Cluster{
125125
ObjectMeta: metav1.ObjectMeta{
126126
Annotations: map[string]string{
@@ -134,7 +134,35 @@ var _ = Describe("Status enrichment", func() {
134134

135135
EnrichStatus(ctx, &cluster, []corev1.Pod{{}})
136136
hibernationCondition := meta.FindStatusCondition(cluster.Status.Conditions, HibernationConditionType)
137-
Expect(hibernationCondition).To(BeNil())
137+
Expect(hibernationCondition).ToNot(BeNil())
138+
Expect(hibernationCondition.Status).To(Equal(metav1.ConditionFalse))
139+
Expect(hibernationCondition.Reason).To(Equal(HibernationConditionReasonWaitingForHealthy))
140+
})
141+
142+
It("transitions from WaitingForHealthy to DeletingPods when the cluster becomes healthy", func(ctx SpecContext) {
143+
cluster := apiv1.Cluster{
144+
ObjectMeta: metav1.ObjectMeta{
145+
Annotations: map[string]string{
146+
utils.HibernationAnnotationName: HibernationOn,
147+
},
148+
},
149+
Status: apiv1.ClusterStatus{
150+
Phase: apiv1.PhaseCreatingReplica,
151+
},
152+
}
153+
154+
// First call: cluster not healthy
155+
EnrichStatus(ctx, &cluster, []corev1.Pod{{}})
156+
hibernationCondition := meta.FindStatusCondition(cluster.Status.Conditions, HibernationConditionType)
157+
Expect(hibernationCondition).ToNot(BeNil())
158+
Expect(hibernationCondition.Reason).To(Equal(HibernationConditionReasonWaitingForHealthy))
159+
160+
// Cluster becomes healthy
161+
cluster.Status.Phase = apiv1.PhaseHealthy
162+
EnrichStatus(ctx, &cluster, []corev1.Pod{{}})
163+
hibernationCondition = meta.FindStatusCondition(cluster.Status.Conditions, HibernationConditionType)
164+
Expect(hibernationCondition).ToNot(BeNil())
165+
Expect(hibernationCondition.Reason).To(Equal(HibernationConditionReasonDeletingPods))
138166
})
139167

140168
It("waits for each Pod to be deleted gracefully", func(ctx SpecContext) {

0 commit comments

Comments
 (0)