Skip to content

Commit aa830e9

Browse files
armrumnencia
authored andcommitted
feat: switch primary-status-unreachable signal from Phase to Event
The prior commit on this branch (f1204b22c) surfaced the /pg/status unreachability guard as a new v1 Phase, PhaseWaitingForPrimaryStatus. On reflection, Event is a better fit for this observation: - The signal is transient (a reconcile-loop observation, not a lifecycle state). Phase is single-slot and would flap between Healthy and Waiting on each 10s requeue, polluting the one-line `kubectl get cluster` view rather than informing it. - Phase permanently extends the v1 API enum; Events do not. - The Kubernetes event recorder natively deduplicates by (involvedObject, reason, message), so a sustained outage rolls up into one logical event with a growing count and moving lastTimestamp. This commit drops PhaseWaitingForPrimaryStatus and its RegisterPhase call, and emits a Warning PrimaryStatusUnreachable event from evaluatePodReadinessGuards in its place. The event message is kept stable (no %v of the error) to preserve dedup; the underlying error is logged at the guard site via contextLogger.Warning with an 'err' structured field, and the event message points users there. IsPodReadyAndNotReporting now returns (bool, error) so the guard's log line can carry the instance manager's error without a second iteration over the status list. The helper has a single caller, so the signature change is zero blast radius. The (ctrl.Result, error) signature of evaluatePodReadinessGuards introduced in f1204b22c is reverted to a single ctrl.Result return: without the RegisterPhase call, the function has no error source, and golangci-lint's unparam rightly flags result 1 as always nil. Deliberately out of scope (follow-ups): - No max-duration escape when operator-to-primary networking is persistently broken while the kubelet readiness probe keeps passing. /readyz and /pg/status are served by different handlers in the instance manager and can diverge. - No new Prometheus metric for time since last successful /pg/status. - No event emission on the kubelet-stale-readiness branch (expected sub-second, event noise not worth the added signal). Depends on #10445. Signed-off-by: Armando Ruocco <[email protected]>
1 parent 18cfb8f commit aa830e9

5 files changed

Lines changed: 97 additions & 43 deletions

File tree

‎api/v1/cluster_types.go‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -700,12 +700,6 @@ const (
700700
// PhaseWaitingForInstancesToBeActive is a waiting phase that is triggered when an instance pod is not active
701701
PhaseWaitingForInstancesToBeActive = "Waiting for the instances to become active"
702702

703-
// PhaseWaitingForPrimaryStatus is set when the current primary pod is
704-
// Ready from the kubelet's perspective but the operator is failing to
705-
// collect its status via the /pg/status endpoint. Failover is deferred
706-
// until Kubernetes marks the primary pod as not Ready.
707-
PhaseWaitingForPrimaryStatus = "Waiting to re-establish contact with the primary instance"
708-
709703
// PhaseOnlineUpgrading for when the instance manager is being upgraded in place
710704
PhaseOnlineUpgrading = "Online upgrade in progress"
711705

‎internal/controller/cluster_controller.go‎

Lines changed: 29 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -505,8 +505,8 @@ func (r *ClusterReconciler) reconcile(ctx context.Context, cluster *apiv1.Cluste
505505
return ctrl.Result{}, err
506506
}
507507

508-
if res, err := r.evaluatePodReadinessGuards(ctx, cluster, instancesStatus); err != nil || !res.IsZero() {
509-
return res, err
508+
if res := r.evaluatePodReadinessGuards(ctx, cluster, instancesStatus); !res.IsZero() {
509+
return res, nil
510510
}
511511

512512
// If the user has requested to hibernate the cluster, we do that before
@@ -589,7 +589,11 @@ func (r *ClusterReconciler) reconcile(ctx context.Context, cluster *apiv1.Cluste
589589
// operator-to-pod connectivity issue rather than a genuine PostgreSQL
590590
// health problem. The operator trusts the kubelet's readiness probe as
591591
// the source of truth and defers its own failover decision until
592-
// Kubernetes agrees the primary is not Ready.
592+
// Kubernetes agrees the primary is not Ready. A PrimaryStatusUnreachable
593+
// Warning event is emitted so the deferral is visible via
594+
// `kubectl describe cluster` and the events API; the Kubernetes event
595+
// recorder dedupes by reason + message, so a sustained outage rolls up
596+
// into one logical event with a growing count.
593597
//
594598
// Without this branch the failover-election path would promote the
595599
// healthiest replica: PostgresqlStatusList.Less pushes items with an
@@ -602,9 +606,9 @@ func (r *ClusterReconciler) evaluatePodReadinessGuards(
602606
ctx context.Context,
603607
cluster *apiv1.Cluster,
604608
instancesStatus postgres.PostgresqlStatusList,
605-
) (ctrl.Result, error) {
609+
) ctrl.Result {
606610
if instancesStatus.Len() == 0 {
607-
return ctrl.Result{}, nil
611+
return ctrl.Result{}
608612
}
609613

610614
contextLogger := log.FromContext(ctx)
@@ -622,28 +626,32 @@ func (r *ClusterReconciler) evaluatePodReadinessGuards(
622626
"instanceName", firstInstance.Pod.Name,
623627
"hasHTTPStatus", hasHTTPStatus,
624628
"isPodReady", isPodReady)
625-
return ctrl.Result{RequeueAfter: 10 * time.Second}, nil
629+
return ctrl.Result{RequeueAfter: 10 * time.Second}
626630
}
627631

628632
if cluster.Status.CurrentPrimary != "" &&
629-
cluster.Status.CurrentPrimary == cluster.Status.TargetPrimary &&
630-
instancesStatus.IsPodReadyAndNotReporting(cluster.Status.CurrentPrimary) {
631-
contextLogger.Warning(
632-
"Primary pod is ready but status endpoint is failing, requeueing",
633-
"instanceName", cluster.Status.CurrentPrimary)
634-
if err := r.RegisterPhase(
635-
ctx,
636-
cluster,
637-
apiv1.PhaseWaitingForPrimaryStatus,
638-
"The primary pod is Ready but the operator cannot reach its /pg/status endpoint. "+
639-
"Failover is deferred until Kubernetes marks the primary as not Ready.",
640-
); err != nil {
641-
return ctrl.Result{}, err
633+
cluster.Status.CurrentPrimary == cluster.Status.TargetPrimary {
634+
if unreachable, statusErr := instancesStatus.IsPodReadyAndNotReporting(
635+
cluster.Status.CurrentPrimary,
636+
); unreachable {
637+
contextLogger.Warning(
638+
"Primary pod is Ready but /pg/status is failing, deferring failover",
639+
"instanceName", cluster.Status.CurrentPrimary,
640+
"err", statusErr)
641+
r.Recorder.Eventf(
642+
cluster,
643+
"Warning",
644+
"PrimaryStatusUnreachable",
645+
"Primary pod %s is Ready but the operator cannot reach its /pg/status endpoint; "+
646+
"failover deferred until Kubernetes marks the primary as not Ready. "+
647+
"See operator logs for the underlying error.",
648+
cluster.Status.CurrentPrimary,
649+
)
650+
return ctrl.Result{RequeueAfter: 10 * time.Second}
642651
}
643-
return ctrl.Result{RequeueAfter: 10 * time.Second}, nil
644652
}
645653

646-
return ctrl.Result{}, nil
654+
return ctrl.Result{}
647655
}
648656

649657
func (r *ClusterReconciler) ensureNoFailoverOnFullDisk(

‎internal/controller/cluster_controller_test.go‎

Lines changed: 44 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
corev1 "k8s.io/api/core/v1"
3030
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3131
"k8s.io/apimachinery/pkg/types"
32+
"k8s.io/client-go/tools/record"
3233
"sigs.k8s.io/controller-runtime/pkg/reconcile"
3334

3435
apiv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1"
@@ -428,10 +429,9 @@ var _ = Describe("evaluatePodReadinessGuards", func() {
428429
cluster.Status.CurrentPrimary = currentPrimary
429430
cluster.Status.TargetPrimary = targetPrimary
430431

431-
result, err := env.clusterReconciler.evaluatePodReadinessGuards(
432+
result := env.clusterReconciler.evaluatePodReadinessGuards(
432433
ctx, cluster, postgres.PostgresqlStatusList{Items: items},
433434
)
434-
Expect(err).NotTo(HaveOccurred())
435435

436436
if requeue {
437437
Expect(result.RequeueAfter).To(Equal(10 * time.Second))
@@ -496,8 +496,48 @@ var _ = Describe("evaluatePodReadinessGuards", func() {
496496
cluster.Status.CurrentPrimary = primaryName
497497
cluster.Status.TargetPrimary = primaryName
498498

499-
result, err := env.clusterReconciler.evaluatePodReadinessGuards(ctx, cluster, statusList)
500-
Expect(err).NotTo(HaveOccurred())
499+
result := env.clusterReconciler.evaluatePodReadinessGuards(ctx, cluster, statusList)
501500
Expect(result.RequeueAfter).To(Equal(10 * time.Second))
502501
})
502+
503+
It("emits a Warning event when the primary is Ready but /pg/status is failing", func(ctx SpecContext) {
504+
cluster.Status.CurrentPrimary = primaryName
505+
cluster.Status.TargetPrimary = primaryName
506+
507+
result := env.clusterReconciler.evaluatePodReadinessGuards(
508+
ctx, cluster,
509+
postgres.PostgresqlStatusList{Items: []postgres.PostgresqlStatus{
510+
readyReportingReplica, readyErroringPrimary,
511+
}},
512+
)
513+
Expect(result.RequeueAfter).To(Equal(10 * time.Second))
514+
515+
fakeRecorder, ok := env.clusterReconciler.Recorder.(*record.FakeRecorder)
516+
Expect(ok).To(BeTrue(), "test environment must wire a FakeRecorder")
517+
518+
var recorded string
519+
Eventually(fakeRecorder.Events, "1s").Should(Receive(&recorded))
520+
Expect(recorded).To(HavePrefix("Warning PrimaryStatusUnreachable"))
521+
Expect(recorded).To(ContainSubstring(primaryName))
522+
Expect(recorded).To(ContainSubstring("See operator logs"))
523+
})
524+
525+
It("does not emit an event on the kubelet-stale branch", func(ctx SpecContext) {
526+
cluster.Status.CurrentPrimary = primaryName
527+
cluster.Status.TargetPrimary = primaryName
528+
529+
result := env.clusterReconciler.evaluatePodReadinessGuards(
530+
ctx, cluster,
531+
postgres.PostgresqlStatusList{Items: []postgres.PostgresqlStatus{
532+
kubeletStaleReporting, readyReportingReplica,
533+
}},
534+
)
535+
Expect(result.RequeueAfter).To(Equal(10 * time.Second))
536+
537+
fakeRecorder, ok := env.clusterReconciler.Recorder.(*record.FakeRecorder)
538+
Expect(ok).To(BeTrue())
539+
540+
Consistently(fakeRecorder.Events, "100ms").ShouldNot(Receive(),
541+
"kubelet-stale branch must stay event-less to avoid noise")
542+
})
503543
})

‎pkg/postgres/status.go‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -356,18 +356,22 @@ func (list PostgresqlStatusList) IsPodReporting(podname string) bool {
356356
// marked as ready by the kubelet but is not successfully reporting its status
357357
// via the /pg/status endpoint. This indicates a likely transient failure
358358
// (e.g. a network hiccup between the operator and the pod) rather than a
359-
// genuine PostgreSQL health problem.
359+
// genuine PostgreSQL health problem. The second return value is the
360+
// underlying /pg/status error, meaningful only when the boolean is true.
360361
//
361362
// "Not reporting" is the negation of IsPodReporting: the /pg/status call
362363
// against the pod returned an error.
363-
func (list PostgresqlStatusList) IsPodReadyAndNotReporting(podname string) bool {
364+
func (list PostgresqlStatusList) IsPodReadyAndNotReporting(podname string) (bool, error) {
364365
for _, item := range list.Items {
365366
if item.Pod.Name == podname {
366-
return item.IsPodReady && item.Error != nil
367+
if item.IsPodReady && item.Error != nil {
368+
return true, item.Error
369+
}
370+
return false, nil
367371
}
368372
}
369373

370-
return false
374+
return false, nil
371375
}
372376

373377
// IsComplete checks the PostgreSQL status list for Pods which

‎pkg/postgres/status_test.go‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -262,20 +262,28 @@ var _ = Describe("IsPodReadyAndNotReporting", func() {
262262
},
263263
}
264264

265-
It("returns true when the pod is ready but the status endpoint is failing", func() {
266-
Expect(podList.IsPodReadyAndNotReporting("primary")).To(BeTrue())
265+
It("returns true and the underlying error when the pod is ready but the status endpoint is failing", func() {
266+
ok, err := podList.IsPodReadyAndNotReporting("primary")
267+
Expect(ok).To(BeTrue())
268+
Expect(err).To(MatchError(errStatusEndpointFailing))
267269
})
268270

269-
It("returns false when the pod is ready and reporting", func() {
270-
Expect(podList.IsPodReadyAndNotReporting("replica-reporting")).To(BeFalse())
271+
It("returns false and a nil error when the pod is ready and reporting", func() {
272+
ok, err := podList.IsPodReadyAndNotReporting("replica-reporting")
273+
Expect(ok).To(BeFalse())
274+
Expect(err).ToNot(HaveOccurred())
271275
})
272276

273-
It("returns false when the pod is not ready and not reporting", func() {
274-
Expect(podList.IsPodReadyAndNotReporting("replica-not-ready")).To(BeFalse())
277+
It("returns false and a nil error when the pod is not ready and not reporting", func() {
278+
ok, err := podList.IsPodReadyAndNotReporting("replica-not-ready")
279+
Expect(ok).To(BeFalse())
280+
Expect(err).ToNot(HaveOccurred())
275281
})
276282

277-
It("returns false when the pod is not in the list", func() {
278-
Expect(podList.IsPodReadyAndNotReporting("unknown")).To(BeFalse())
283+
It("returns false and a nil error when the pod is not in the list", func() {
284+
ok, err := podList.IsPodReadyAndNotReporting("unknown")
285+
Expect(ok).To(BeFalse())
286+
Expect(err).ToNot(HaveOccurred())
279287
})
280288
})
281289

0 commit comments

Comments
 (0)