Skip to content

Commit 9c33f17

Browse files
committed
fix(postgres): restart the plugin rollout whenever a backup job attaches it
Only the first attach after the latch restarted the rollout. In the platform flow the chart renders no spec.plugins, so an in-place restore recreates the Cluster without the plugin, and a backup job held through that restore attaches it again once released. The rollout condition then still carried its earlier start, the restored instances were newer than it and ready, and the Backup was created in the same pass, before CloudNativePG had rolled anything; such a Backup fails with "requested plugin is not available" or "instance manager was restarted". Every write of the job that finds the plugin missing now starts the rollout over. A plugin attached by another writer is still timed from the latch. Assisted-by: LLM Signed-off-by: Alexey Artamonov <[email protected]>
1 parent bb52267 commit 9c33f17

2 files changed

Lines changed: 57 additions & 13 deletions

File tree

‎internal/backupcontroller/cnpgstrategy_controller.go‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -175,12 +175,13 @@ const (
175175
// against an unreachable target reproduces the same FATAL.
176176
cnpgRecoveryMaxInspectPods = 5
177177

178-
// Cap on the wall-clock time a BackupJob waits for the barman-cloud
179-
// plugin rollout, counted from the PluginRolledOut condition or from the
180-
// release recorded in DestinationWriteHeld, whichever is later, or retries
181-
// writing the WAL archive destination, counted from the first failure
182-
// since the last successful write or the last wait for another backup or
183-
// a restore, before the cnpg.io/Backup exists. A Backup that exists is
178+
// Cap on the wall-clock time a BackupJob spends before its
179+
// cnpg.io/Backup exists on each of two waits. The plugin rollout is
180+
// counted from the PluginRolledOut condition, or from the release
181+
// recorded in DestinationWriteHeld when that is later. Retrying the WAL
182+
// archive destination write is counted from the first failure since the
183+
// last successful write or the last wait for another backup or a
184+
// restore. A Backup that exists is
184185
// not capped: CloudNativePG keeps a healthy one pending while another
185186
// backup of the same Cluster runs, or while it restarts one on another
186187
// instance, so no fixed deadline tells those apart from a stuck one.
@@ -200,8 +201,9 @@ const (
200201
backupCondPluginRolledOut = "PluginRolledOut"
201202

202203
// Reason of the PluginRolledOut latch recorded before the write that
203-
// attaches the plugin. The write that attaches it replaces the latch with
204-
// its own time, which retries can put long after the latch.
204+
// attaches the plugin. Every write of this job that attaches it replaces
205+
// the condition with its own time, which retries can put long after the
206+
// latch.
205207
cnpgReasonAttachingPlugin = "AttachingPlugin"
206208

207209
// Condition Type recorded while writing the WAL archive destination keeps
@@ -454,9 +456,10 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
454456
}
455457
return ctrl.Result{RequeueAfter: cnpgPollInterval}, nil
456458
}
457-
if rollout.Reason == cnpgReasonAttachingPlugin && attachment.missing {
458-
// Instances recreated between the latch and this write run
459-
// without the plugin, and the rollout deadline starts here. A
459+
if attachment.missing {
460+
// This write attached the plugin, so its rollout starts here:
461+
// instances recreated since the latch, or by a restore that
462+
// re-rendered the Cluster without the plugin, run without it. A
460463
// plugin someone else attached since the latch keeps it: the
461464
// rollout that attach started may already be over.
462465
message := fmt.Sprintf("attached the barman-cloud plugin to cnpg.io/Cluster %s/%s, waiting for its instances to roll", j.Namespace, clusterName)

‎internal/backupcontroller/cnpgstrategy_controller_test.go‎

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2830,6 +2830,11 @@ func TestReconcileCNPG_PluginRolloutDeadlineCountsFromAttach(t *testing.T) {
28302830
if cond := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); cond == nil || cond.Reason != "WaitingForPluginRollout" {
28312831
t.Fatalf("Ready = %+v, want WaitingForPluginRollout", cond)
28322832
}
2833+
attached := &cnpgtypes.Cluster{}
2834+
if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: rolloutCluster}, attached); err != nil {
2835+
t.Fatalf("get Cluster: %v", err)
2836+
}
2837+
f.observeCNPG(newRolloutCluster(cnpgClusterHealthyPhase, attached.Spec.Plugins...))
28332838

28342839
rollout := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut)
28352840
rollout.LastTransitionTime = metav1.NewTime(expired)
@@ -2919,8 +2924,9 @@ func TestReconcileCNPG_PluginRolloutCountsFromTheSuccessfulAttach(t *testing.T)
29192924
}
29202925

29212926
// A plugin attached by someone else after the latch, another backup job or
2922-
// a restore, started the rollout before this job's write, which only finds
2923-
// it there: instances that rolled since the latch carry the plugin.
2927+
// the chart re-rendering a restored Cluster outside the platform flow,
2928+
// started the rollout before this job's write, which only finds it there:
2929+
// instances that rolled since the latch carry the plugin.
29242930
func TestReconcileCNPG_PluginAttachedByAnotherWriterKeepsTheLatch(t *testing.T) {
29252931
oldPod := newRolloutInstancePod(time.Now().Add(-time.Hour).Truncate(time.Second))
29262932
f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)),
@@ -3452,6 +3458,41 @@ func TestReconcileCNPG_AttachedPluginDoesNotDelayBackup(t *testing.T) {
34523458
}
34533459
}
34543460

3461+
// An in-place restore of the platform flow recreates the Cluster without the
3462+
// barman-cloud plugin, so a job held through it attaches the plugin again
3463+
// once released. That attach starts a rollout of its own, which the
3464+
// instances restored after the job's first rollout began have not done.
3465+
func TestReconcileCNPG_ReattachAfterRestoreRestartsTheRollout(t *testing.T) {
3466+
stampedAt := time.Now().Add(-10 * time.Minute).Truncate(time.Second)
3467+
j := newRolloutBackupJob(stampedAt.Add(-time.Hour))
3468+
j.Status.Conditions = []metav1.Condition{{
3469+
Type: backupCondPluginRolledOut,
3470+
Status: metav1.ConditionFalse,
3471+
Reason: "WaitingForPluginRollout",
3472+
LastTransitionTime: metav1.NewTime(stampedAt),
3473+
}}
3474+
f := newCNPGRolloutFixture(t, j, newRolloutCluster(cnpgClusterHealthyPhase),
3475+
newRolloutInstancePod(time.Now().Add(-2*time.Minute)))
3476+
restore := f.seedRestoreOfApp()
3477+
if got, _ := f.pass(); apimeta.FindStatusCondition(got.Status.Conditions, "Ready").Reason != ConditionReasonRestoreInProgress {
3478+
t.Fatalf("Ready = %+v, want the job held by the restore", apimeta.FindStatusCondition(got.Status.Conditions, "Ready"))
3479+
}
3480+
restore.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded
3481+
if err := f.c.Status().Update(context.Background(), restore); err != nil {
3482+
t.Fatalf("finish the restore: %v", err)
3483+
}
3484+
3485+
attachAt := time.Now().Truncate(time.Second)
3486+
got, backups := f.pass()
3487+
if len(backups) != 0 {
3488+
t.Fatalf("cnpg.io/Backup created in the pass that attached the plugin to the restored Cluster: %+v", backups)
3489+
}
3490+
rollout := apimeta.FindStatusCondition(got.Status.Conditions, backupCondPluginRolledOut)
3491+
if rollout == nil || rollout.Status != metav1.ConditionFalse || rollout.LastTransitionTime.Time.Before(attachAt) {
3492+
t.Fatalf("%s = %+v, want False since the attach at %s", backupCondPluginRolledOut, rollout, attachAt)
3493+
}
3494+
}
3495+
34553496
// A release older than the rollout, from a wait before the plugin was
34563497
// attached, does not shorten the rollout that began after it.
34573498
func TestReconcileCNPG_EarlierReleaseDoesNotShortenTheRollout(t *testing.T) {

0 commit comments

Comments
 (0)