Skip to content

Commit 96e445c

Browse files
leonardocearmrumnencia
authored
fix: avoid spurious failovers when the primary status endpoint fails transiently (#10445)
When the operator calls /pg/status on the primary during a reconciliation loop and the request fails, a failover could be triggered even if the primary pod was still Ready from the kubelet's perspective. Since a successful /pg/status implies a passing readiness probe but not vice versa, a failing /pg/status on an otherwise Ready pod is a strong signal of a transient condition (e.g. a network hiccup between the operator and the pod) rather than a genuine PostgreSQL health problem. The reconciliation loop now requeues instead of proceeding when the current primary equals the target primary, the pod is Ready, but /pg/status is failing, applying the same conservative approach already used when the kubelet has not yet refreshed a positive readiness probe. A follow-up PR (#10509) will surface the deferred failover as a cluster phase so the state is visible via `kubectl get cluster`. See #10403 Signed-off-by: Leonardo Cecchi <[email protected]> Signed-off-by: Armando Ruocco <[email protected]> Signed-off-by: Marco Nenciarini <[email protected]> Co-authored-by: Armando Ruocco <[email protected]> Co-authored-by: Marco Nenciarini <[email protected]>
1 parent 0083245 commit 96e445c

4 files changed

Lines changed: 261 additions & 31 deletions

File tree

‎internal/controller/cluster_controller.go‎

Lines changed: 69 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -505,37 +505,8 @@ func (r *ClusterReconciler) reconcile(ctx context.Context, cluster *apiv1.Cluste
505505
return ctrl.Result{}, err
506506
}
507507

508-
// The instance list is sorted and will present the primary as the first
509-
// element, followed by the replicas, the most updated coming first.
510-
// Pods that are not responding will be at the end of the list. We use
511-
// the information reported by the instance manager to sort the
512-
// instances. When we need to elect a new primary, we take the first item
513-
// on this list.
514-
//
515-
// Here we check the readiness status of the first Pod as we can't
516-
// promote an instance that is not ready from the Kubernetes
517-
// point-of-view: the services will not forward traffic to it even if
518-
// PostgreSQL is up and running.
519-
//
520-
// An instance can be up and running even if the readiness probe is
521-
// negative: this is going to happen, i.e., when an instance is
522-
// un-fenced, and the Kubelet still hasn't refreshed the status of the
523-
// readiness probe.
524-
if instancesStatus.Len() > 0 {
525-
mostAdvancedInstance := instancesStatus.Items[0]
526-
hasHTTPStatus := mostAdvancedInstance.HasHTTPStatus()
527-
isPodReady := mostAdvancedInstance.IsPodReady
528-
529-
if hasHTTPStatus && !isPodReady {
530-
// The readiness probe status from the Kubelet is not updated, so
531-
// we need to wait for it to be refreshed
532-
contextLogger.Info(
533-
"Waiting for the Kubelet to refresh the readiness probe",
534-
"mostAdvancedInstanceName", mostAdvancedInstance.Pod.Name,
535-
"hasHTTPStatus", hasHTTPStatus,
536-
"isPodReady", isPodReady)
537-
return ctrl.Result{RequeueAfter: 1 * time.Second}, nil
538-
}
508+
if res := r.evaluatePodReadinessGuards(ctx, cluster, instancesStatus); res != nil {
509+
return *res, nil
539510
}
540511

541512
// If the user has requested to hibernate the cluster, we do that before
@@ -598,6 +569,73 @@ func (r *ClusterReconciler) reconcile(ctx context.Context, cluster *apiv1.Cluste
598569
return setStatusPluginHook(ctx, r.Client, cnpgiClient.GetPluginClientFromContext(ctx), cluster)
599570
}
600571

572+
// evaluatePodReadinessGuards short-circuits the reconciliation loop with a
573+
// requeue when the readiness information reported by the instance managers is
574+
// not consistent with a promotion or failover being safe. We can only promote
575+
// an instance that is Ready from the Kubernetes point of view, otherwise the
576+
// read-write service will not route traffic to it even if PostgreSQL is up
577+
// and running.
578+
//
579+
// Two branches are checked in order:
580+
//
581+
// - Kubelet has not refreshed the readiness probe yet. The first element of
582+
// the (post-sort) instance list is reporting a healthy /pg/status but the
583+
// kubelet has not yet flipped the readiness probe to True (typical for a
584+
// short window after un-fencing an instance). Waiting here prevents
585+
// electing a primary that Kubernetes will refuse to route traffic to.
586+
//
587+
// - Primary pod is Ready but its /pg/status endpoint is failing. A failing
588+
// /pg/status on an otherwise Ready pod usually indicates an
589+
// operator-to-pod connectivity issue rather than a genuine PostgreSQL
590+
// health problem. The operator trusts the kubelet's readiness probe as
591+
// the source of truth and defers its own failover decision until
592+
// Kubernetes agrees the primary is not Ready.
593+
//
594+
// Without this branch the failover-election path would promote the
595+
// healthiest replica: PostgresqlStatusList.Less pushes items with an
596+
// Error to the tail, so the erroring primary ends up behind a healthy
597+
// replica at Items[0] and gets supplanted on a mere network hiccup.
598+
//
599+
// Returns nil when the caller can proceed with the rest of the reconciliation.
600+
func (r *ClusterReconciler) evaluatePodReadinessGuards(
601+
ctx context.Context,
602+
cluster *apiv1.Cluster,
603+
instancesStatus postgres.PostgresqlStatusList,
604+
) *ctrl.Result {
605+
if instancesStatus.Len() == 0 {
606+
return nil
607+
}
608+
609+
contextLogger := log.FromContext(ctx)
610+
611+
firstInstance := instancesStatus.Items[0]
612+
hasHTTPStatus := firstInstance.HasHTTPStatus()
613+
isPodReady := firstInstance.IsPodReady
614+
615+
if hasHTTPStatus && !isPodReady {
616+
// The readiness probe status from the kubelet has not been refreshed
617+
// yet, so we wait rather than electing a primary that Kubernetes will
618+
// refuse to route traffic to.
619+
contextLogger.Info(
620+
"Waiting for the Kubelet to refresh the readiness probe",
621+
"instanceName", firstInstance.Pod.Name,
622+
"hasHTTPStatus", hasHTTPStatus,
623+
"isPodReady", isPodReady)
624+
return &ctrl.Result{RequeueAfter: 10 * time.Second}
625+
}
626+
627+
if cluster.Status.CurrentPrimary != "" &&
628+
cluster.Status.CurrentPrimary == cluster.Status.TargetPrimary &&
629+
instancesStatus.IsPodReadyAndNotReporting(cluster.Status.CurrentPrimary) {
630+
contextLogger.Warning(
631+
"Primary pod is ready but status endpoint is failing, requeueing",
632+
"primaryPodName", cluster.Status.CurrentPrimary)
633+
return &ctrl.Result{RequeueAfter: 10 * time.Second}
634+
}
635+
636+
return nil
637+
}
638+
601639
func (r *ClusterReconciler) ensureNoFailoverOnFullDisk(
602640
ctx context.Context,
603641
cluster *apiv1.Cluster,

‎internal/controller/cluster_controller_test.go‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,14 @@ SPDX-License-Identifier: Apache-2.0
2020
package controller
2121

2222
import (
23+
"fmt"
24+
"sort"
2325
"time"
2426

2527
cnpgTypes "github.com/cloudnative-pg/machinery/pkg/types"
2628
batchv1 "k8s.io/api/batch/v1"
2729
corev1 "k8s.io/api/core/v1"
30+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2831
"k8s.io/apimachinery/pkg/types"
2932
"sigs.k8s.io/controller-runtime/pkg/reconcile"
3033

@@ -367,3 +370,134 @@ var _ = Describe("isNodeUnschedulableOrBeingDrained", func() {
367370
Entry("node has an unknown taint", nodeWithUnknownTaint, false),
368371
)
369372
})
373+
374+
var _ = Describe("evaluatePodReadinessGuards", func() {
375+
const (
376+
primaryName = "cluster-1"
377+
replicaName = "cluster-2"
378+
newPrimaryName = "cluster-3"
379+
)
380+
381+
var (
382+
env *testingEnvironment
383+
cluster *apiv1.Cluster
384+
)
385+
386+
BeforeEach(func() {
387+
env = buildTestEnvironment()
388+
namespace := newFakeNamespace(env.client)
389+
cluster = newFakeCNPGCluster(env.client, namespace)
390+
})
391+
392+
errStatusFailing := fmt.Errorf("status endpoint failing")
393+
394+
readyReportingReplica := postgres.PostgresqlStatus{
395+
Pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: replicaName}},
396+
IsPodReady: true,
397+
}
398+
readyReportingPrimary := postgres.PostgresqlStatus{
399+
Pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: primaryName}},
400+
IsPodReady: true,
401+
}
402+
readyErroringPrimary := postgres.PostgresqlStatus{
403+
Pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: primaryName}},
404+
IsPodReady: true,
405+
Error: errStatusFailing,
406+
}
407+
unreadyErroringPrimary := postgres.PostgresqlStatus{
408+
Pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: primaryName}},
409+
IsPodReady: false,
410+
Error: errStatusFailing,
411+
}
412+
readyErroringReplica := postgres.PostgresqlStatus{
413+
Pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: replicaName}},
414+
IsPodReady: true,
415+
Error: errStatusFailing,
416+
}
417+
// kubeletStaleReporting is the "kubelet has not refreshed the probe yet"
418+
// shape: instance manager reports success (Error==nil, so HasHTTPStatus
419+
// returns true) but the pod is not yet Ready from the kubelet.
420+
kubeletStaleReporting := postgres.PostgresqlStatus{
421+
Pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: primaryName}},
422+
IsPodReady: false,
423+
}
424+
425+
DescribeTable(
426+
"guards behaviour",
427+
func(ctx SpecContext, currentPrimary, targetPrimary string, items []postgres.PostgresqlStatus, requeue bool) {
428+
cluster.Status.CurrentPrimary = currentPrimary
429+
cluster.Status.TargetPrimary = targetPrimary
430+
431+
result := env.clusterReconciler.evaluatePodReadinessGuards(
432+
ctx, cluster, postgres.PostgresqlStatusList{Items: items},
433+
)
434+
435+
if requeue {
436+
Expect(result).NotTo(BeNil())
437+
Expect(result.RequeueAfter).To(Equal(10 * time.Second))
438+
} else {
439+
Expect(result).To(BeNil())
440+
}
441+
},
442+
Entry("happy path: primary Ready and reporting, no guard fires",
443+
primaryName, primaryName,
444+
[]postgres.PostgresqlStatus{readyReportingPrimary, readyReportingReplica}, false),
445+
Entry("kubelet has not refreshed the readiness probe yet",
446+
primaryName, primaryName,
447+
[]postgres.PostgresqlStatus{kubeletStaleReporting, readyReportingReplica}, true),
448+
Entry("transient /pg/status failure on Ready primary requeues",
449+
primaryName, primaryName,
450+
[]postgres.PostgresqlStatus{readyReportingReplica, readyErroringPrimary}, true),
451+
Entry("guard is scoped to steady-state (CurrentPrimary == TargetPrimary)",
452+
primaryName, newPrimaryName,
453+
[]postgres.PostgresqlStatus{readyReportingReplica, readyErroringPrimary}, false),
454+
Entry("bootstrap: no primary elected yet",
455+
"", "",
456+
[]postgres.PostgresqlStatus{}, false),
457+
Entry("primary absent from status list (defensive)",
458+
primaryName, primaryName,
459+
[]postgres.PostgresqlStatus{readyReportingReplica}, false),
460+
Entry("current primary set but target primary cleared: guard does not fire",
461+
primaryName, "",
462+
[]postgres.PostgresqlStatus{readyReportingPrimary, readyErroringReplica}, false),
463+
Entry("erroring replica with reporting primary does not fire the guard",
464+
primaryName, primaryName,
465+
[]postgres.PostgresqlStatus{readyReportingPrimary, readyErroringReplica}, false),
466+
Entry("primary unready and not reporting (genuine failover case)",
467+
primaryName, primaryName,
468+
[]postgres.PostgresqlStatus{readyReportingReplica, unreadyErroringPrimary}, false),
469+
Entry("empty status list",
470+
primaryName, primaryName,
471+
[]postgres.PostgresqlStatus{}, false),
472+
)
473+
474+
It("fires after the production sort pushes the erroring primary to the tail", func(ctx SpecContext) {
475+
// Build the list in "primary first" order as the instance manager
476+
// would populate it, then rely on PostgresqlStatusList.Less to push
477+
// the erroring primary to the tail. This locks the guard's
478+
// rationale against a regression in the sort.
479+
statusList := postgres.PostgresqlStatusList{
480+
Items: []postgres.PostgresqlStatus{
481+
{
482+
Pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: primaryName}},
483+
IsPodReady: true,
484+
Error: errStatusFailing,
485+
},
486+
{
487+
Pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: replicaName}},
488+
IsPodReady: true,
489+
},
490+
},
491+
}
492+
sort.Sort(&statusList)
493+
Expect(statusList.Items[0].Pod.Name).To(Equal(replicaName),
494+
"sort must push the erroring primary to the tail")
495+
496+
cluster.Status.CurrentPrimary = primaryName
497+
cluster.Status.TargetPrimary = primaryName
498+
499+
result := env.clusterReconciler.evaluatePodReadinessGuards(ctx, cluster, statusList)
500+
Expect(result).NotTo(BeNil())
501+
Expect(result.RequeueAfter).To(Equal(10 * time.Second))
502+
})
503+
})

‎pkg/postgres/status.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -352,6 +352,24 @@ func (list PostgresqlStatusList) IsPodReporting(podname string) bool {
352352
return false
353353
}
354354

355+
// IsPodReadyAndNotReporting returns true if the pod with the given name is
356+
// marked as ready by the kubelet but is not successfully reporting its status
357+
// via the /pg/status endpoint. This indicates a likely transient failure
358+
// (e.g. a network hiccup between the operator and the pod) rather than a
359+
// genuine PostgreSQL health problem.
360+
//
361+
// "Not reporting" is the negation of IsPodReporting: the /pg/status call
362+
// against the pod returned an error.
363+
func (list PostgresqlStatusList) IsPodReadyAndNotReporting(podname string) bool {
364+
for _, item := range list.Items {
365+
if item.Pod.Name == podname {
366+
return item.IsPodReady && item.Error != nil
367+
}
368+
}
369+
370+
return false
371+
}
372+
355373
// IsComplete checks the PostgreSQL status list for Pods which
356374
// contain errors. Returns true if everything is green and
357375
// false otherwise

‎pkg/postgres/status_test.go‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,46 @@ var _ = Describe("PostgreSQL status real", func() {
239239
})
240240
})
241241

242+
var _ = Describe("IsPodReadyAndNotReporting", func() {
243+
errStatusEndpointFailing := fmt.Errorf("status endpoint failing")
244+
245+
podList := PostgresqlStatusList{
246+
Items: []PostgresqlStatus{
247+
{
248+
Pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "primary"}},
249+
IsPodReady: true,
250+
Error: errStatusEndpointFailing,
251+
},
252+
{
253+
Pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "replica-reporting"}},
254+
IsPodReady: true,
255+
Error: nil,
256+
},
257+
{
258+
Pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "replica-not-ready"}},
259+
IsPodReady: false,
260+
Error: errStatusEndpointFailing,
261+
},
262+
},
263+
}
264+
265+
It("returns true when the pod is ready but the status endpoint is failing", func() {
266+
Expect(podList.IsPodReadyAndNotReporting("primary")).To(BeTrue())
267+
})
268+
269+
It("returns false when the pod is ready and reporting", func() {
270+
Expect(podList.IsPodReadyAndNotReporting("replica-reporting")).To(BeFalse())
271+
})
272+
273+
It("returns false when the pod is not ready and not reporting", func() {
274+
Expect(podList.IsPodReadyAndNotReporting("replica-not-ready")).To(BeFalse())
275+
})
276+
277+
It("returns false when the pod is not in the list", func() {
278+
Expect(podList.IsPodReadyAndNotReporting("unknown")).To(BeFalse())
279+
})
280+
})
281+
242282
var _ = Describe("Configuration report", func() {
243283
DescribeTable(
244284
"Configuration report",

0 commit comments

Comments
 (0)