Skip to content

Commit e494009

Browse files
armruleonardocemnencia
authored
fix: report failed instance creation jobs instead of waiting forever (#11035)
When bootstrapping, restoring, or adding a replica, an instance creation Job can exhaust its backoff limit and end in a BackoffLimitExceeded failure. When that happened the cluster stayed in a transient creation phase (for example "Setting up primary") forever: the operator counted the failed Job as still running, so reconciliation kept waiting on it and never reported the failure. A Job is treated as running until it reports one successful completion. A Job that has permanently failed never reaches that state, so it looked the same as one still in progress, and the running-job guard returned early on every reconciliation without updating the phase. This change detects the terminal JobFailed condition and moves the cluster into the existing unrecoverable phase, with a reason that names the failed Job and points to its logs. Detection keys off the JobFailed condition, not the failed-pod count, so a Job that is still retrying within its backoff limit is left alone. The check runs after the major version upgrade reconciliation, which owns its own Job lifecycle, so upgrade Jobs are not affected. The failed Job and its Pods are left in place so their logs stay available. There is no single recipe to recover: the cause has to be found in the logs and fixed before the instance can be created again. Unit tests cover the new Job failure detection, and a controller test verifies that a Job past its backoff limit moves the cluster to the unrecoverable phase. Closes #11042 Refs #10733 Signed-off-by: Armando Ruocco <[email protected]> Signed-off-by: Leonardo Cecchi <[email protected]> Signed-off-by: Marco Nenciarini <[email protected]> Co-authored-by: Leonardo Cecchi <[email protected]> Co-authored-by: Marco Nenciarini <[email protected]>
1 parent b00a63d commit e494009

5 files changed

Lines changed: 127 additions & 0 deletions

File tree

‎internal/controller/cluster_controller.go‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
"errors"
2727
"fmt"
2828
"reflect"
29+
"strings"
2930
"time"
3031

3132
"github.com/cloudnative-pg/machinery/pkg/log"
@@ -872,6 +873,24 @@ func (r *ClusterReconciler) reconcileResources(
872873
return *result, err
873874
}
874875

876+
// A Job that creates an instance (bootstrap, recovery or replica creation)
877+
// is counted as "running" until it succeeds, so a Job that has exhausted its
878+
// backoff limit would otherwise keep the cluster waiting forever. Surface the
879+
// failure in the phase instead. The cause is in the job logs and has to be
880+
// investigated: there is no predefined recipe.
881+
if failedJobs := resources.failedJobNames(); len(failedJobs) > 0 {
882+
contextLogger.Warning("An instance creation job has failed", "failedJobs", failedJobs)
883+
884+
reason := fmt.Sprintf("Instance creation failed for the following jobs: %s. "+
885+
"Check the job logs to investigate the cause of the failure.",
886+
strings.Join(failedJobs, ", "))
887+
888+
if err := r.RegisterPhase(ctx, cluster, apiv1.PhaseUnrecoverable, reason); err != nil {
889+
return ctrl.Result{}, err
890+
}
891+
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
892+
}
893+
875894
runningJobs := resources.runningJobNames()
876895

877896
// Act on Pods and PVCs only if there is nothing that is currently being created or deleted

‎internal/controller/cluster_controller_test.go‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -795,3 +795,48 @@ var _ = Describe("getPluginsNeededForReconcile", func() {
795795
To(Equal([]string{"plugin-shared"}))
796796
})
797797
})
798+
799+
var _ = Describe("reconcileResources surfaces a permanently failed instance creation job", func() {
800+
var env *testingEnvironment
801+
var namespace string
802+
803+
BeforeEach(func() {
804+
env = buildTestEnvironment()
805+
namespace = newFakeNamespace(env.client)
806+
})
807+
808+
// recoveryJob returns a recovery Job for the cluster with the given status,
809+
// mimicking the bootstrap/recovery Job the operator creates.
810+
recoveryJob := func(cluster *apiv1.Cluster, status batchv1.JobStatus) batchv1.Job {
811+
return batchv1.Job{
812+
ObjectMeta: metav1.ObjectMeta{
813+
Name: cluster.Name + "-1-full-recovery",
814+
Namespace: namespace,
815+
},
816+
Status: status,
817+
}
818+
}
819+
820+
It("reports the unrecoverable phase when a job reaches its backoff limit",
821+
func(ctx SpecContext) {
822+
cluster := newFakeCNPGCluster(env.client, namespace)
823+
failedJob := recoveryJob(cluster, batchv1.JobStatus{
824+
Failed: 7,
825+
Conditions: []batchv1.JobCondition{
826+
{Type: batchv1.JobFailed, Status: corev1.ConditionTrue, Reason: "BackoffLimitExceeded"},
827+
},
828+
})
829+
830+
resources := &managedResources{jobs: batchv1.JobList{Items: []batchv1.Job{failedJob}}}
831+
832+
res, err := env.clusterReconciler.reconcileResources(ctx, cluster, resources, postgres.PostgresqlStatusList{})
833+
Expect(err).ToNot(HaveOccurred())
834+
Expect(res.RequeueAfter).To(BeNumerically(">", 0))
835+
836+
var updated apiv1.Cluster
837+
Expect(env.client.Get(ctx, types.NamespacedName{Name: cluster.Name, Namespace: namespace}, &updated)).
838+
To(Succeed())
839+
Expect(updated.Status.Phase).To(Equal(apiv1.PhaseUnrecoverable))
840+
Expect(updated.Status.PhaseReason).To(ContainSubstring(failedJob.Name))
841+
})
842+
})

‎internal/controller/cluster_status.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,18 @@ func (resources *managedResources) runningJobNames() []string {
6969
return result
7070
}
7171

72+
// failedJobNames returns the names of the jobs that have permanently failed,
73+
// i.e. they have exhausted their backoff limit
74+
func (resources *managedResources) failedJobNames() []string {
75+
result := make([]string, 0, len(resources.jobs.Items))
76+
for _, job := range resources.jobs.Items {
77+
if utils.JobHasFailed(job) {
78+
result = append(result, job.Name)
79+
}
80+
}
81+
return result
82+
}
83+
7284
// Check if every managed Pod is active and will be schedules
7385
func (resources *managedResources) inactiveInstanceNames() []string {
7486
result := make([]string, 0, len(resources.instances.Items))

‎pkg/utils/job_conditions.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ package utils
2121

2222
import (
2323
batchv1 "k8s.io/api/batch/v1"
24+
corev1 "k8s.io/api/core/v1"
2425
)
2526

2627
// JobHasOneCompletion Completion check if a certain job is complete
@@ -32,6 +33,21 @@ func JobHasOneCompletion(job batchv1.Job) bool {
3233
return job.Status.Succeeded == requestedCompletions
3334
}
3435

36+
// JobHasFailed checks whether a Job has permanently failed, i.e. it has
37+
// exhausted its backoff limit. It keys off the terminal JobFailed condition
38+
// rather than Status.Failed because the latter is non-zero while the Job is
39+
// still retrying within its backoff limit.
40+
func JobHasFailed(job batchv1.Job) bool {
41+
// job.Status.Conditions is []batchv1.JobCondition, not []metav1.Condition,
42+
// so meta.FindStatusCondition cannot be used here.
43+
for _, condition := range job.Status.Conditions {
44+
if condition.Type == batchv1.JobFailed && condition.Status == corev1.ConditionTrue {
45+
return true
46+
}
47+
}
48+
return false
49+
}
50+
3551
// FilterJobsWithOneCompletion returns jobs that have one completion
3652
func FilterJobsWithOneCompletion(jobList []batchv1.Job) []batchv1.Job {
3753
var result []batchv1.Job

‎pkg/utils/job_conditions_test.go‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ package utils
2121

2222
import (
2323
batchv1 "k8s.io/api/batch/v1"
24+
corev1 "k8s.io/api/core/v1"
2425

2526
. "github.com/onsi/ginkgo/v2"
2627
. "github.com/onsi/gomega"
@@ -43,4 +44,38 @@ var _ = Describe("Job conditions", func() {
4344
Expect(JobHasOneCompletion(nonCompleteJob)).To(BeFalse())
4445
Expect(JobHasOneCompletion(completeJob)).To(BeTrue())
4546
})
47+
48+
DescribeTable("detects if a job has permanently failed",
49+
func(job batchv1.Job, expected bool) {
50+
Expect(JobHasFailed(job)).To(Equal(expected))
51+
},
52+
Entry("a job that has not started yet", batchv1.Job{}, false),
53+
Entry("a running job with one failed pod still within its backoff limit",
54+
batchv1.Job{Status: batchv1.JobStatus{Failed: 1}}, false),
55+
Entry("a successfully completed job",
56+
batchv1.Job{Status: batchv1.JobStatus{
57+
Conditions: []batchv1.JobCondition{
58+
{Type: batchv1.JobComplete, Status: corev1.ConditionTrue},
59+
},
60+
}}, false),
61+
Entry("a job that reached its backoff limit",
62+
batchv1.Job{Status: batchv1.JobStatus{
63+
Failed: 7,
64+
Conditions: []batchv1.JobCondition{
65+
{Type: batchv1.JobFailed, Status: corev1.ConditionTrue, Reason: "BackoffLimitExceeded"},
66+
},
67+
}}, true),
68+
Entry("a job whose failure condition is not active",
69+
batchv1.Job{Status: batchv1.JobStatus{
70+
Conditions: []batchv1.JobCondition{
71+
{Type: batchv1.JobFailed, Status: corev1.ConditionFalse},
72+
},
73+
}}, false),
74+
Entry("a job that is past its backoff limit but whose pods are still terminating",
75+
batchv1.Job{Status: batchv1.JobStatus{
76+
Conditions: []batchv1.JobCondition{
77+
{Type: batchv1.JobFailureTarget, Status: corev1.ConditionTrue, Reason: "BackoffLimitExceeded"},
78+
},
79+
}}, false),
80+
)
4681
})

0 commit comments

Comments
 (0)