Skip to content

Commit bb52267

Browse files
committed
fix(postgres): give a held backup job its plugin rollout deadline back
A job that latched the barman-cloud plugin rollout and then waited for another backup of the application, or for a restore of it, had that wait charged to the rollout deadline. Released after more than the deadline, it failed on the first instance that was not ready yet with "rollout did not finish", although no rollout of its own had been running while it waited. The hold is now recorded in a DestinationWriteHeld condition, and its release time restarts the rollout deadline when it is later than the rollout's start. The instance pods are still measured against the rollout's start: one that rolled while the job was held carries the plugin and counts. Assisted-by: LLM Signed-off-by: Alexey Artamonov <[email protected]>
1 parent 0d3eff4 commit bb52267

2 files changed

Lines changed: 205 additions & 2 deletions

File tree

‎internal/backupcontroller/cnpgstrategy_controller.go‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,8 @@ const (
176176
cnpgRecoveryMaxInspectPods = 5
177177

178178
// Cap on the wall-clock time a BackupJob waits for the barman-cloud
179-
// plugin rollout, counted from the PluginRolledOut condition, or retries
179+
// plugin rollout, counted from the PluginRolledOut condition or from the
180+
// release recorded in DestinationWriteHeld, whichever is later, or retries
180181
// writing the WAL archive destination, counted from the first failure
181182
// since the last successful write or the last wait for another backup or
182183
// a restore, before the cnpg.io/Backup exists. A Backup that exists is
@@ -209,6 +210,13 @@ const (
209210
// app, or for a restore of it, before it tried.
210211
backupCondDestinationApplied = "DestinationApplied"
211212

213+
// Condition Type recorded while the job is held off writing the WAL
214+
// archive destination, by another backup of the app in flight or by a
215+
// restore of it, and set False when released. The release time restarts
216+
// the plugin rollout deadline, which that wait must not spend; the
217+
// instance pods still have to postdate the rollout's own start.
218+
backupCondDestinationWriteHeld = "DestinationWriteHeld"
219+
212220
cnpgPodRoleLabel = "cnpg.io/podRole"
213221
cnpgPodRoleInstance = "instance"
214222
)
@@ -349,6 +357,7 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
349357
}
350358
if restore != "" {
351359
apimeta.RemoveStatusCondition(&j.Status.Conditions, backupCondDestinationApplied)
360+
apimeta.SetStatusCondition(&j.Status.Conditions, cnpgDestinationWriteHeld(ConditionReasonRestoreInProgress))
352361
return r.holdForRestore(ctx, j, restore)
353362
}
354363
holder, err := r.otherCNPGBackupInFlight(ctx, j, clusterName)
@@ -357,9 +366,18 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
357366
}
358367
if holder != "" {
359368
apimeta.RemoveStatusCondition(&j.Status.Conditions, backupCondDestinationApplied)
369+
apimeta.SetStatusCondition(&j.Status.Conditions, cnpgDestinationWriteHeld("WaitingForOtherBackup"))
360370
return r.waitCNPGBackupJob(ctx, j, "WaitingForOtherBackup", fmt.Sprintf(
361371
"waiting for BackupJob %s of the same application to finish its backup, which writes to the shared WAL archive destination", holder))
362372
}
373+
if apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld) != nil {
374+
apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
375+
Type: backupCondDestinationWriteHeld,
376+
Status: metav1.ConditionFalse,
377+
Reason: "Released",
378+
Message: "nothing holds the WAL archive destination write any more",
379+
})
380+
}
363381
}
364382

365383
attemptedAt := metav1.Now()
@@ -466,7 +484,12 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
466484
return ctrl.Result{}, err
467485
}
468486
if pending != "" {
469-
if cnpgBackupDeadlineExceeded(&rollout.LastTransitionTime) {
487+
deadlineFrom := rollout.LastTransitionTime
488+
if held := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld); held != nil &&
489+
held.LastTransitionTime.After(deadlineFrom.Time) {
490+
deadlineFrom = held.LastTransitionTime
491+
}
492+
if cnpgBackupDeadlineExceeded(&deadlineFrom) {
470493
return r.markBackupJobFailed(ctx, j, fmt.Sprintf(
471494
"barman-cloud plugin rollout did not finish within %s: %s", cnpgDefaultBackupDeadline, pending))
472495
}
@@ -933,6 +956,15 @@ func (r *BackupJobReconciler) otherCNPGBackupInFlight(ctx context.Context, j *ba
933956
return holders[0], nil
934957
}
935958

959+
func cnpgDestinationWriteHeld(reason string) metav1.Condition {
960+
return metav1.Condition{
961+
Type: backupCondDestinationWriteHeld,
962+
Status: metav1.ConditionTrue,
963+
Reason: reason,
964+
Message: "held off writing the WAL archive destination",
965+
}
966+
}
967+
936968
// waitCNPGBackupJob requeues the BackupJob and records why it waits in its
937969
// Ready condition, writing the status only when the conditions changed.
938970
func (r *BackupJobReconciler) waitCNPGBackupJob(ctx context.Context, j *backupsv1alpha1.BackupJob, reason, message string) (ctrl.Result, error) {

‎internal/backupcontroller/cnpgstrategy_controller_test.go‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3447,6 +3447,41 @@ func TestReconcileCNPG_AttachedPluginDoesNotDelayBackup(t *testing.T) {
34473447
if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut); cond != nil {
34483448
t.Errorf("%s condition = %+v on a Cluster the driver did not roll", backupCondPluginRolledOut, cond)
34493449
}
3450+
if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld); cond != nil {
3451+
t.Errorf("%s condition = %+v on a job nothing held", backupCondDestinationWriteHeld, cond)
3452+
}
3453+
}
3454+
3455+
// A release older than the rollout, from a wait before the plugin was
3456+
// attached, does not shorten the rollout that began after it.
3457+
func TestReconcileCNPG_EarlierReleaseDoesNotShortenTheRollout(t *testing.T) {
3458+
expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute))
3459+
j := newRolloutBackupJob(expired.Add(-time.Hour))
3460+
j.Status.Conditions = []metav1.Condition{
3461+
{
3462+
Type: backupCondDestinationWriteHeld,
3463+
Status: metav1.ConditionFalse,
3464+
Reason: "Released",
3465+
LastTransitionTime: metav1.NewTime(expired),
3466+
},
3467+
{
3468+
Type: backupCondPluginRolledOut,
3469+
Status: metav1.ConditionFalse,
3470+
Reason: "WaitingForPluginRollout",
3471+
LastTransitionTime: metav1.NewTime(time.Now().Add(-time.Minute)),
3472+
},
3473+
}
3474+
f := newCNPGRolloutFixture(t, j,
3475+
newRolloutCluster("Waiting for user action", buildBarmanPlugin(rolloutCluster, rolloutCluster)),
3476+
newRolloutInstancePod(expired.Add(-time.Hour)))
3477+
3478+
got, backups := f.pass()
3479+
if got.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
3480+
t.Fatalf("a rollout that began a minute ago failed on an older release: %s", got.Status.Message)
3481+
}
3482+
if len(backups) != 0 {
3483+
t.Fatalf("a cnpg.io/Backup was created against an unfinished rollout: %+v", backups)
3484+
}
34503485
}
34513486

34523487
// TestReconcileCNPG_PluginRolloutWaitIsBounded: a rollout that never ends
@@ -3476,6 +3511,142 @@ func TestReconcileCNPG_PluginRolloutWaitIsBounded(t *testing.T) {
34763511
}
34773512
}
34783513

3514+
// A job whose plugin rollout started, and which then waited for another
3515+
// backup of the app or for a restore of it, gets the rollout deadline back
3516+
// from the moment it was released: that wait is not a rollout of its own.
3517+
// The instance pods still count as rolled from the rollout's own start.
3518+
func TestReconcileCNPG_HeldJobRolloutDeadlineCountsFromRelease(t *testing.T) {
3519+
type hold struct {
3520+
name string
3521+
seed func(f *cnpgRolloutFixture) (release func())
3522+
waiting string
3523+
}
3524+
holds := []hold{
3525+
{
3526+
name: "another backup",
3527+
waiting: "WaitingForOtherBackup",
3528+
seed: func(f *cnpgRolloutFixture) func() {
3529+
f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning)
3530+
f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running")
3531+
return func() {
3532+
held := &cnpgtypes.Backup{}
3533+
if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj-other-cnpg"}, held); err != nil {
3534+
f.t.Fatalf("get holder Backup: %v", err)
3535+
}
3536+
held.Status.Phase = cnpgBackupPhaseComplete
3537+
if err := f.c.Update(context.Background(), held); err != nil {
3538+
f.t.Fatalf("complete holder Backup: %v", err)
3539+
}
3540+
}
3541+
},
3542+
},
3543+
{
3544+
name: "a restore",
3545+
waiting: ConditionReasonRestoreInProgress,
3546+
seed: func(f *cnpgRolloutFixture) func() {
3547+
restore := f.seedRestoreOfApp()
3548+
return func() {
3549+
restore.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded
3550+
if err := f.c.Status().Update(context.Background(), restore); err != nil {
3551+
f.t.Fatalf("finish the restore: %v", err)
3552+
}
3553+
}
3554+
},
3555+
},
3556+
}
3557+
stampedAt := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)).Truncate(time.Second)
3558+
newHeldRollout := func(t *testing.T, h hold, cluster *cnpgtypes.Cluster, pod *corev1.Pod) *cnpgRolloutFixture {
3559+
t.Helper()
3560+
j := newRolloutBackupJob(stampedAt.Add(-time.Hour))
3561+
j.Status.Conditions = []metav1.Condition{{
3562+
Type: backupCondPluginRolledOut,
3563+
Status: metav1.ConditionFalse,
3564+
Reason: "WaitingForPluginRollout",
3565+
LastTransitionTime: metav1.NewTime(stampedAt),
3566+
}}
3567+
f := newCNPGRolloutFixture(t, j, cluster, pod)
3568+
release := h.seed(f)
3569+
got, backups := f.pass()
3570+
for _, b := range backups {
3571+
if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" {
3572+
t.Fatalf("a held job created cnpg.io/Backup %s", b.Name)
3573+
}
3574+
}
3575+
if got.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
3576+
t.Fatalf("a held job did not wait: %s", got.Status.Message)
3577+
}
3578+
if ready := apimeta.FindStatusCondition(got.Status.Conditions, "Ready"); ready == nil || ready.Reason != h.waiting {
3579+
t.Fatalf("Ready = %+v, want %s", ready, h.waiting)
3580+
}
3581+
release()
3582+
return f
3583+
}
3584+
3585+
for _, h := range holds {
3586+
t.Run(h.name+": the rollout is still waited for after the release", func(t *testing.T) {
3587+
f := newHeldRollout(t, h,
3588+
newRolloutCluster("Waiting for user action", buildBarmanPlugin(rolloutCluster, rolloutCluster)),
3589+
newRolloutInstancePod(stampedAt.Add(-time.Hour)))
3590+
3591+
releasedAt := time.Now().Truncate(time.Second)
3592+
j, backups := f.pass()
3593+
if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
3594+
t.Fatalf("time held by %s was charged to the rollout: %s", h.name, j.Status.Message)
3595+
}
3596+
for _, b := range backups {
3597+
if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" {
3598+
t.Fatalf("cnpg.io/Backup %s was created against an unfinished rollout", b.Name)
3599+
}
3600+
}
3601+
if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForPluginRollout" {
3602+
t.Fatalf("Ready = %+v, want WaitingForPluginRollout", ready)
3603+
}
3604+
released := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld)
3605+
if released == nil || released.Status != metav1.ConditionFalse || released.LastTransitionTime.Time.Before(releasedAt) {
3606+
t.Fatalf("%s = %+v, want False since the release at %s", backupCondDestinationWriteHeld, released, releasedAt)
3607+
}
3608+
if rollout := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut); rollout == nil || !rollout.LastTransitionTime.Time.Equal(stampedAt) {
3609+
t.Errorf("%s = %+v, want its stamp kept at %s", backupCondPluginRolledOut, rollout, stampedAt)
3610+
}
3611+
3612+
released.LastTransitionTime = metav1.NewTime(stampedAt)
3613+
if err := f.c.Status().Update(context.Background(), j); err != nil {
3614+
t.Fatalf("age the release: %v", err)
3615+
}
3616+
j, _ = f.pass()
3617+
if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed || !strings.Contains(j.Status.Message, "rollout did not finish") {
3618+
t.Fatalf("phase = %q (%s), want Failed once the rollout outlived the deadline after the release", j.Status.Phase, j.Status.Message)
3619+
}
3620+
})
3621+
3622+
t.Run(h.name+": an instance that rolled while the job was held counts", func(t *testing.T) {
3623+
f := newHeldRollout(t, h,
3624+
newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)),
3625+
newRolloutInstancePod(stampedAt.Add(time.Minute)))
3626+
3627+
var j *backupsv1alpha1.BackupJob
3628+
own := 0
3629+
for range 3 {
3630+
var backups []cnpgtypes.Backup
3631+
j, backups = f.pass()
3632+
own = 0
3633+
for _, b := range backups {
3634+
if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" {
3635+
own++
3636+
}
3637+
}
3638+
if own != 0 || j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
3639+
break
3640+
}
3641+
}
3642+
if own != 1 {
3643+
t.Fatalf("got %d cnpg.io/Backups of the job against an instance that rolled after the rollout began; Ready = %+v, phase %q (%s)",
3644+
own, apimeta.FindStatusCondition(j.Status.Conditions, "Ready"), j.Status.Phase, j.Status.Message)
3645+
}
3646+
})
3647+
}
3648+
}
3649+
34793650
// TestReconcileCNPGRestore_MissingStrategyFailsClosedAfterDeadline covers the
34803651
// restore-path blocker: a RestoreJob whose referenced CNPG Strategy CR never
34813652
// appears must NOT requeue forever. requeueRestoreStrategyNotReady gates

0 commit comments

Comments
 (0)