Skip to content

Commit 6645e21

Browse files
scooby87myasnikovdaniil
authored andcommitted
fix(postgres): fail a backup job as soon as its cnpg Backup fails
A failed cnpg.io/Backup kept its BackupJob Running for 30 minutes as a "transient" failure, on the premise that CloudNativePG may move a Backup out of the failed phase. It does not: its backup controller returns early on any Backup already failed or completed, so the wait only delayed the verdict and hid it behind a misleading reason. A failed Backup now fails the BackupJob at once with the error CNPG recorded, pointing to the plugin-barman-cloud sidecar log when that error is only the plugin's exit status. A Backup in any other phase keeps the BackupJob waiting as before, now with a WaitingForCNPGBackup condition naming the Backup and its phase instead of no condition at all. It is still not capped: CloudNativePG keeps a healthy Backup pending while another backup of the same Cluster runs or while it restarts one, and a fixed deadline would fail those. Assisted-by: LLM Signed-off-by: Alexey Artamonov <[email protected]> (cherry picked from commit fe6968b) Signed-off-by: Myasnikov Daniil <[email protected]>
1 parent 95bb0ea commit 6645e21

2 files changed

Lines changed: 186 additions & 38 deletions

File tree

‎internal/backupcontroller/cnpgstrategy_controller.go‎

Lines changed: 36 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -175,13 +175,11 @@ const (
175175
// against an unreachable target reproduces the same FATAL.
176176
cnpgRecoveryMaxInspectPods = 5
177177

178-
// Cap on the wall-clock time a BackupJob can spend observing a
179-
// cnpg.io/Backup stuck in phase=failed before the driver gives up and
180-
// marks the BackupJob Failed. CNPG can transition out of "failed" on
181-
// internal retries (e.g. transient instance-manager restart), so we
182-
// don't fail on the first observation. But without this cap, a
183-
// permanently-broken Backup would pin the BackupJob in Running forever,
184-
// blocking later runs and wedging the Plan-controller queue.
178+
// Cap on the wall-clock time a BackupJob waits for the barman-cloud
179+
// plugin rollout before the cnpg.io/Backup exists. A Backup that exists
180+
// is not capped: CloudNativePG keeps a healthy one pending while another
181+
// backup of the same Cluster runs, or while it restarts one on another
182+
// instance, so no fixed deadline tells those apart from a stuck one.
185183
cnpgDefaultBackupDeadline = 30 * time.Minute
186184

187185
cnpgClusterHealthyPhase = "Cluster in healthy state"
@@ -453,44 +451,46 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
453451
return ctrl.Result{}, nil
454452

455453
case cnpgBackupPhaseFailed:
456-
// CNPG can transition the same Backup CR back out of "failed" when it
457-
// retries internally (e.g., after a transient instance-manager
458-
// restart), so we don't fail the BackupJob on the first observation.
459-
// But without a cap a permanently-broken Backup would pin the
460-
// BackupJob in Running forever; once StartedAt + cnpgDefaultBackupDeadline
461-
// has elapsed and the cnpg.io/Backup is still failed, we give up.
462-
if cnpgBackupDeadlineExceeded(j.Status.StartedAt) {
463-
final := message
464-
if final == "" {
465-
final = fmt.Sprintf(
466-
"cnpg.io Backup remained in phase=failed past %s deadline",
467-
cnpgDefaultBackupDeadline)
468-
}
469-
return r.markBackupJobFailed(ctx, j, final)
470-
}
471-
if message == "" {
472-
message = "cnpg.io Backup reported phase=failed; awaiting recovery or retry"
473-
}
474-
apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
454+
// CloudNativePG does not reconcile a Backup out of a terminal phase,
455+
// so waiting on a failed one cannot change the outcome.
456+
return r.markBackupJobFailed(ctx, j, cnpgBackupFailedMessage(cnpgBackup, message))
457+
458+
default:
459+
waiting := fmt.Sprintf("waiting for cnpg.io/Backup %s/%s, phase %q",
460+
cnpgBackup.Namespace, cnpgBackup.Name, phase)
461+
if apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
475462
Type: "Ready",
476463
Status: metav1.ConditionFalse,
477-
Reason: "CNPGBackupTransientFailure",
478-
Message: message,
479-
})
480-
if err := r.Status().Update(ctx, j); err != nil {
481-
return ctrl.Result{}, err
464+
Reason: "WaitingForCNPGBackup",
465+
Message: waiting,
466+
}) {
467+
if err := r.Status().Update(ctx, j); err != nil {
468+
return ctrl.Result{}, err
469+
}
482470
}
483471
return ctrl.Result{RequeueAfter: cnpgPollInterval}, nil
472+
}
473+
}
484474

485-
default:
486-
return ctrl.Result{RequeueAfter: cnpgPollInterval}, nil
475+
// cnpgBackupFailedMessage carries the error CloudNativePG recorded. When the
476+
// plugin run itself failed that is only its exit status, and the storage
477+
// error behind it is logged by the plugin-barman-cloud sidecar.
478+
func cnpgBackupFailedMessage(b *cnpgtypes.Backup, errText string) string {
479+
if errText == "" {
480+
errText = "no error recorded"
481+
}
482+
message := fmt.Sprintf("cnpg.io/Backup %s/%s failed: %s", b.Namespace, b.Name, errText)
483+
if strings.Contains(errText, "exit status") {
484+
message += "; the cause is in the log of the plugin-barman-cloud sidecar of the instance that ran it"
487485
}
486+
return message
488487
}
489488

490489
// cnpgBackupDeadlineExceeded reports whether enough wall-clock time has
491-
// elapsed since the BackupJob started that we should stop polling a stuck
492-
// cnpg.io/Backup and fail the run. Returns false when StartedAt is nil so
493-
// the very first reconcile (which sets StartedAt) does not trip the gate.
490+
// elapsed since the BackupJob started that we should stop waiting for the
491+
// barman-cloud plugin rollout and fail the run. Returns false when
492+
// StartedAt is nil so the very first reconcile (which sets StartedAt) does
493+
// not trip the gate.
494494
func cnpgBackupDeadlineExceeded(startedAt *metav1.Time) bool {
495495
if startedAt == nil {
496496
return false

‎internal/backupcontroller/cnpgstrategy_controller_test.go‎

Lines changed: 150 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2332,6 +2332,154 @@ func TestReconcileCNPG_StartedAtReturnsEarlyToAvoidStaleResourceVersion(t *testi
23322332
}
23332333
}
23342334

2335+
// CloudNativePG never reconciles a Backup out of a terminal phase (the
2336+
// backup controller returns early on failed and completed), so a failed
2337+
// Backup fails the BackupJob at once. Any other phase keeps the BackupJob
2338+
// waiting past the backup deadline: CNPG keeps a healthy Backup pending
2339+
// while another backup of the Cluster runs or while it restarts one.
2340+
func TestReconcileCNPG_BackupPhaseOutcome(t *testing.T) {
2341+
apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup
2342+
strategyGroup := strategyv1alpha1.GroupVersion.Group
2343+
const ns, appName = "tenant", "pg"
2344+
clusterName := cnpgClusterNameForApp(appName)
2345+
expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute))
2346+
2347+
cases := []struct {
2348+
name string
2349+
phase string
2350+
errText string
2351+
startedAt time.Time
2352+
wantPhase backupsv1alpha1.BackupJobPhase
2353+
wantMessage []string
2354+
notInMessage []string
2355+
}{
2356+
{name: "failed fails at once", phase: cnpgBackupPhaseFailed, errText: "rpc error: code = Unknown desc = exit status 1", startedAt: time.Now(),
2357+
wantPhase: backupsv1alpha1.BackupJobPhaseFailed, wantMessage: []string{"bj-cnpg", "exit status 1", "plugin-barman-cloud"}},
2358+
{name: "failed without an error text fails at once", phase: cnpgBackupPhaseFailed, startedAt: time.Now(),
2359+
wantPhase: backupsv1alpha1.BackupJobPhaseFailed, wantMessage: []string{"bj-cnpg", "no error recorded"}, notInMessage: []string{"plugin-barman-cloud"}},
2360+
{name: "failed before the plugin ran points nowhere else", phase: cnpgBackupPhaseFailed, errText: "cannot backup a hibernated cluster", startedAt: time.Now(),
2361+
wantPhase: backupsv1alpha1.BackupJobPhaseFailed, wantMessage: []string{"bj-cnpg", "hibernated"}, notInMessage: []string{"plugin-barman-cloud"}},
2362+
{name: "pending keeps waiting", phase: "pending", startedAt: time.Now(),
2363+
wantPhase: backupsv1alpha1.BackupJobPhaseRunning, wantMessage: []string{"bj-cnpg", "pending"}},
2364+
{name: "pending is not capped by the backup deadline", phase: "pending", startedAt: expired,
2365+
wantPhase: backupsv1alpha1.BackupJobPhaseRunning, wantMessage: []string{"bj-cnpg", "pending"}},
2366+
{name: "no phase yet keeps waiting", phase: "", startedAt: expired,
2367+
wantPhase: backupsv1alpha1.BackupJobPhaseRunning, wantMessage: []string{"bj-cnpg"}},
2368+
{name: "started keeps running", phase: "started", startedAt: expired,
2369+
wantPhase: backupsv1alpha1.BackupJobPhaseRunning, wantMessage: []string{"bj-cnpg", "started"}},
2370+
{name: "walArchivingFailing keeps waiting", phase: "walArchivingFailing", startedAt: expired,
2371+
wantPhase: backupsv1alpha1.BackupJobPhaseRunning, wantMessage: []string{"bj-cnpg", "walArchivingFailing"}},
2372+
{name: "completed succeeds", phase: cnpgBackupPhaseComplete, startedAt: expired,
2373+
wantPhase: backupsv1alpha1.BackupJobPhaseSucceeded},
2374+
}
2375+
for _, tc := range cases {
2376+
t.Run(tc.name, func(t *testing.T) {
2377+
startedAt := metav1.NewTime(tc.startedAt)
2378+
job := &backupsv1alpha1.BackupJob{
2379+
ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bj"},
2380+
Spec: backupsv1alpha1.BackupJobSpec{
2381+
ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: appName},
2382+
},
2383+
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt},
2384+
}
2385+
strategy := &strategyv1alpha1.CNPG{
2386+
ObjectMeta: metav1.ObjectMeta{Name: "cnpg-strategy"},
2387+
Spec: strategyv1alpha1.CNPGSpec{
2388+
Template: strategyv1alpha1.CNPGTemplate{
2389+
BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/"},
2390+
},
2391+
},
2392+
}
2393+
cluster := &cnpgtypes.Cluster{
2394+
ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: clusterName, UID: "cluster-uid"},
2395+
}
2396+
cnpgBackup := &cnpgtypes.Backup{
2397+
ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bj-cnpg", Labels: map[string]string{
2398+
backupsv1alpha1.OwningJobNameLabel: job.Name,
2399+
backupsv1alpha1.OwningJobNamespaceLabel: job.Namespace,
2400+
}},
2401+
Spec: cnpgtypes.BackupSpec{Cluster: cnpgtypes.ClusterReference{Name: clusterName}},
2402+
Status: cnpgtypes.BackupStatus{Phase: tc.phase, Error: tc.errText},
2403+
}
2404+
c := newCNPGStrategyTestClient(t, job, strategy, newPostgresApp(appName, ns), cluster, cnpgBackup)
2405+
r := &BackupJobReconciler{Client: c, Interface: cnpgDynamicFor(t, cluster)}
2406+
resolved := &ResolvedBackupConfig{
2407+
StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: strategy.Name},
2408+
}
2409+
2410+
j := &backupsv1alpha1.BackupJob{}
2411+
if err := c.Get(context.Background(), client.ObjectKeyFromObject(job), j); err != nil {
2412+
t.Fatalf("get BackupJob: %v", err)
2413+
}
2414+
res, err := r.reconcileCNPG(context.Background(), j, resolved)
2415+
if err != nil {
2416+
t.Fatalf("reconcileCNPG: %v", err)
2417+
}
2418+
if j.Status.Phase != tc.wantPhase {
2419+
t.Fatalf("BackupJob phase = %q (%s), want %q", j.Status.Phase, j.Status.Message, tc.wantPhase)
2420+
}
2421+
2422+
message := j.Status.Message
2423+
if tc.wantPhase == backupsv1alpha1.BackupJobPhaseRunning {
2424+
if res.RequeueAfter == 0 {
2425+
t.Errorf("expected a requeue while the cnpg.io/Backup is in progress, got %+v", res)
2426+
}
2427+
persisted := &backupsv1alpha1.BackupJob{}
2428+
if err := c.Get(context.Background(), client.ObjectKeyFromObject(job), persisted); err != nil {
2429+
t.Fatalf("get BackupJob after reconcile: %v", err)
2430+
}
2431+
ready := apimeta.FindStatusCondition(persisted.Status.Conditions, "Ready")
2432+
if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "WaitingForCNPGBackup" {
2433+
t.Fatalf("expected Ready=False/WaitingForCNPGBackup while waiting, got %+v", ready)
2434+
}
2435+
message = ready.Message
2436+
2437+
// A pass that observes the same phase again must not write the
2438+
// BackupJob status: the driver polls every few seconds.
2439+
statusWrites := 0
2440+
r.Client = interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
2441+
SubResourceUpdate: func(ctx context.Context, cl client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption) error {
2442+
statusWrites++
2443+
return cl.SubResource(sub).Update(ctx, obj, opts...)
2444+
},
2445+
SubResourcePatch: func(ctx context.Context, cl client.Client, sub string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error {
2446+
statusWrites++
2447+
return cl.SubResource(sub).Patch(ctx, obj, patch, opts...)
2448+
},
2449+
})
2450+
if _, err := r.reconcileCNPG(context.Background(), persisted, resolved); err != nil {
2451+
t.Fatalf("second reconcileCNPG: %v", err)
2452+
}
2453+
if statusWrites != 0 {
2454+
t.Errorf("an unchanged phase wrote the BackupJob status %d time(s)", statusWrites)
2455+
}
2456+
}
2457+
for _, want := range tc.wantMessage {
2458+
if !strings.Contains(message, want) {
2459+
t.Errorf("message %q does not mention %q", message, want)
2460+
}
2461+
}
2462+
for _, unwanted := range tc.notInMessage {
2463+
if strings.Contains(message, unwanted) {
2464+
t.Errorf("message %q mentions %q", message, unwanted)
2465+
}
2466+
}
2467+
2468+
artifacts := &backupsv1alpha1.BackupList{}
2469+
if err := c.List(context.Background(), artifacts, client.InNamespace(ns)); err != nil {
2470+
t.Fatalf("list Backups: %v", err)
2471+
}
2472+
wantArtifacts := 0
2473+
if tc.wantPhase == backupsv1alpha1.BackupJobPhaseSucceeded {
2474+
wantArtifacts = 1
2475+
}
2476+
if len(artifacts.Items) != wantArtifacts {
2477+
t.Errorf("got %d Backup artifacts, want %d", len(artifacts.Items), wantArtifacts)
2478+
}
2479+
})
2480+
}
2481+
}
2482+
23352483
// cnpgRolloutFixture drives reconcileCNPG against one Postgres app whose
23362484
// Cluster and instance pods a test swaps between passes, standing in for
23372485
// CNPG rolling the instances.
@@ -2516,8 +2664,8 @@ func TestReconcileCNPG_WaitsForPluginRolloutBeforeBackup(t *testing.T) {
25162664
if !apimeta.IsStatusConditionTrue(j.Status.Conditions, backupCondPluginRolledOut) {
25172665
t.Errorf("%s condition = %+v, want True", backupCondPluginRolledOut, apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut))
25182666
}
2519-
if cond := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); cond != nil {
2520-
t.Errorf("Ready condition = %+v left over from the wait while the backup runs", cond)
2667+
if cond := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); cond == nil || cond.Reason != "WaitingForCNPGBackup" {
2668+
t.Errorf("Ready condition = %+v, want the wait on the cnpg.io/Backup rather than one left over from the rollout", cond)
25212669
}
25222670
}
25232671

0 commit comments

Comments
 (0)