Skip to content

Commit 048b350

Browse files
committed
fix(postgres): hold a queued backup job while its application is restored
The restore gate in Reconcile looks only at a BackupJob that has not started, and the CNPG driver stamps StartedAt on its first pass. A job that then waits behind another backup of the application, for as long as that backup runs, writes the WAL archive destination and the Cluster plugin once released without looking for a restore that began in the meantime. A write landing while the restore re-renders the Cluster can put the restored Cluster back on the source's WAL prefix, where recovery fails with "Expected empty archive". Every pass that is about to write the destination now looks for a restore of the application and holds the job the way the gate does. A job whose Backup already runs writes nothing and is not held. The hold resets the retry budget of the destination write, as the wait for another backup does, so time spent behind a restore does not fail the job on the first error after it. Signed-off-by: Alexey Artamonov <[email protected]> Assisted-by: LLM
1 parent bbb9ce7 commit 048b350

3 files changed

Lines changed: 187 additions & 7 deletions

File tree

‎internal/backupcontroller/backupjob_restore_lock.go‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -130,13 +130,14 @@ func ptrString(s *string) string {
130130
return *s
131131
}
132132

133-
// holdForRestore keeps a BackupJob that has not started yet waiting while a
134-
// restore writes into its application. It stays non-terminal, so a scheduled
135-
// run still happens once the restore ends, only later than its slot.
133+
// holdForRestore keeps a BackupJob waiting while a restore writes into its
134+
// application: one that has not started yet, or a CNPG one about to write
135+
// its WAL archive destination. It stays non-terminal, so a scheduled run
136+
// still happens once the restore ends, only later than its slot.
136137
//
137138
// A failed status write is returned so the manager retries it with backoff.
138-
// The job is held either way, since nothing past the gate ran, but a hold
139-
// nobody can see reads as a run stuck for no reason.
139+
// The job is held either way, since it writes nothing past the hold, but a
140+
// hold nobody can see reads as a run stuck for no reason.
140141
func (r *BackupJobReconciler) holdForRestore(ctx context.Context, j *backupsv1alpha1.BackupJob, restore string) (ctrl.Result, error) {
141142
if j.Status.Phase == backupsv1alpha1.BackupJobPhaseEmpty {
142143
j.Status.Phase = backupsv1alpha1.BackupJobPhasePending

‎internal/backupcontroller/cnpgstrategy_controller.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -334,6 +334,17 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
334334
// Backup exists and only while no other job's Backup is in flight.
335335
writeDestination := existingBackup == nil
336336
if writeDestination {
337+
// The gate in Reconcile sees only a job that has not started, and
338+
// this one may have waited in the queue for as long as a restore
339+
// takes to begin.
340+
restore, err := activeRestoreTargeting(ctx, r.Client, j.Namespace, j.Spec.ApplicationRef)
341+
if err != nil {
342+
return ctrl.Result{}, err
343+
}
344+
if restore != "" {
345+
apimeta.RemoveStatusCondition(&j.Status.Conditions, backupCondDestinationApplied)
346+
return r.holdForRestore(ctx, j, restore)
347+
}
337348
holder, err := r.otherCNPGBackupInFlight(ctx, j, clusterName)
338349
if err != nil {
339350
return ctrl.Result{}, err

‎internal/backupcontroller/cnpgstrategy_controller_test.go‎

Lines changed: 170 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"k8s.io/client-go/kubernetes/scheme"
2323
clienttesting "k8s.io/client-go/testing"
2424
"k8s.io/client-go/tools/record"
25+
ctrl "sigs.k8s.io/controller-runtime"
2526
"sigs.k8s.io/controller-runtime/pkg/client"
2627
clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
2728
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
@@ -2852,6 +2853,163 @@ func TestReconcileCNPG_QueueResetsDestinationRetryBudget(t *testing.T) {
28522853
}
28532854
}
28542855

2856+
// seedRestoreOfApp starts a RestoreJob that writes into the fixture's app.
2857+
func (f *cnpgRolloutFixture) seedRestoreOfApp() *backupsv1alpha1.RestoreJob {
2858+
f.t.Helper()
2859+
appGroup := backupsv1alpha1.DefaultApplicationAPIGroup
2860+
strategyGroup := strategyv1alpha1.GroupVersion.Group
2861+
backup := &backupsv1alpha1.Backup{
2862+
ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: "app-backup"},
2863+
Spec: backupsv1alpha1.BackupSpec{
2864+
ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &appGroup, Kind: postgresAppKind, Name: rolloutApp},
2865+
StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "cnpg-strategy"},
2866+
},
2867+
}
2868+
restore := &backupsv1alpha1.RestoreJob{
2869+
ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: "app-restore"},
2870+
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}},
2871+
}
2872+
for _, obj := range []client.Object{backup, restore} {
2873+
if err := f.c.Create(context.Background(), obj); err != nil {
2874+
f.t.Fatalf("seed %T: %v", obj, err)
2875+
}
2876+
}
2877+
return restore
2878+
}
2879+
2880+
// A job past its start writes the WAL archive destination only while no
2881+
// restore rewrites its application. The restore gate in Reconcile looks only
2882+
// at a job that has not started, and a job released from the queue, or one
2883+
// still before its first write, has started long ago: a write landing during
2884+
// the restore can put the restored Cluster back on the source's WAL prefix.
2885+
func TestReconcileCNPG_HoldsTheDestinationWriteWhileTheAppIsRestored(t *testing.T) {
2886+
for _, queued := range []bool{true, false} {
2887+
t.Run(fmt.Sprintf("released from the queue=%t", queued), func(t *testing.T) {
2888+
f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/")
2889+
if queued {
2890+
f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning)
2891+
f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running")
2892+
if j, _ := f.pass(); apimeta.FindStatusCondition(j.Status.Conditions, "Ready").Reason != "WaitingForOtherBackup" {
2893+
t.Fatalf("Ready = %+v, want the job queued", apimeta.FindStatusCondition(j.Status.Conditions, "Ready"))
2894+
}
2895+
}
2896+
restore := f.seedRestoreOfApp()
2897+
if queued {
2898+
held := &cnpgtypes.Backup{}
2899+
if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj-other-cnpg"}, held); err != nil {
2900+
t.Fatalf("get holder Backup: %v", err)
2901+
}
2902+
held.Status.Phase = cnpgBackupPhaseComplete
2903+
if err := f.c.Update(context.Background(), held); err != nil {
2904+
t.Fatalf("complete holder Backup: %v", err)
2905+
}
2906+
holder := &backupsv1alpha1.BackupJob{}
2907+
if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj-other"}, holder); err != nil {
2908+
t.Fatalf("get holder BackupJob: %v", err)
2909+
}
2910+
holder.Status.Phase = backupsv1alpha1.BackupJobPhaseSucceeded
2911+
if err := f.c.Status().Update(context.Background(), holder); err != nil {
2912+
t.Fatalf("finish holder BackupJob: %v", err)
2913+
}
2914+
}
2915+
2916+
j, backups, res := f.passResult()
2917+
if got := f.liveDestination(); got != "s3://other/" {
2918+
t.Errorf("destination written during a restore of the app: %q", got)
2919+
}
2920+
for _, b := range backups {
2921+
if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" {
2922+
t.Errorf("a cnpg.io/Backup was created during a restore of the app: %s", b.Name)
2923+
}
2924+
}
2925+
if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != ConditionReasonRestoreInProgress {
2926+
t.Errorf("Ready = %+v, want %s", ready, ConditionReasonRestoreInProgress)
2927+
}
2928+
if res.RequeueAfter == 0 {
2929+
t.Errorf("a held job is not requeued: %+v", res)
2930+
}
2931+
2932+
restore.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded
2933+
if err := f.c.Status().Update(context.Background(), restore); err != nil {
2934+
t.Fatalf("finish the restore: %v", err)
2935+
}
2936+
f.pass()
2937+
if got := f.liveDestination(); got != "s3://bucket/" {
2938+
t.Errorf("once the restore finished, destination = %q, want the job's own s3://bucket/", got)
2939+
}
2940+
})
2941+
}
2942+
}
2943+
2944+
// A job whose Backup already runs writes nothing, so a restore does not hold
2945+
// it: holding would strand a backup that is already moving data.
2946+
func TestReconcileCNPG_RestoreDoesNotHoldARunningBackup(t *testing.T) {
2947+
f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/")
2948+
f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running")
2949+
f.seedRestoreOfApp()
2950+
2951+
j, _ := f.pass()
2952+
if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForCNPGBackup" {
2953+
t.Errorf("Ready = %+v, want the running backup still followed", ready)
2954+
}
2955+
}
2956+
2957+
// An error looking for a restore is not taken as "no restore".
2958+
func TestReconcileCNPG_RestoreLookupErrorIsReturned(t *testing.T) {
2959+
f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/")
2960+
f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
2961+
List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error {
2962+
if _, ok := list.(*backupsv1alpha1.RestoreJobList); ok {
2963+
return apierrors.NewServiceUnavailable("cache not synced")
2964+
}
2965+
return cl.List(ctx, list, opts...)
2966+
},
2967+
})
2968+
j := &backupsv1alpha1.BackupJob{}
2969+
if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil {
2970+
t.Fatalf("get BackupJob: %v", err)
2971+
}
2972+
if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil {
2973+
t.Fatalf("reconcileCNPG swallowed the restore lookup error")
2974+
}
2975+
if got := f.liveDestination(); got != "s3://other/" {
2976+
t.Errorf("destination written although restores could not be listed: %q", got)
2977+
}
2978+
}
2979+
2980+
// Waiting for a restore resets the retry budget of the destination write,
2981+
// as waiting for another backup does, so the restore does not spend it.
2982+
func TestReconcileCNPG_RestoreResetsDestinationRetryBudget(t *testing.T) {
2983+
expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute))
2984+
f := newAttachedObjectStoreFixture(t, expired, "s3://other/")
2985+
f.setJobCondition(metav1.Condition{
2986+
Type: backupCondDestinationApplied, Status: metav1.ConditionFalse, Reason: "ClusterPluginApplyRetrying",
2987+
Message: "earlier failure", LastTransitionTime: metav1.NewTime(expired),
2988+
})
2989+
restore := f.seedRestoreOfApp()
2990+
2991+
j, _ := f.pass()
2992+
if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationApplied); cond != nil {
2993+
t.Fatalf("%s = %+v kept while waiting for a restore", backupCondDestinationApplied, cond)
2994+
}
2995+
2996+
restore.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded
2997+
if err := f.c.Status().Update(context.Background(), restore); err != nil {
2998+
t.Fatalf("finish the restore: %v", err)
2999+
}
3000+
f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
3001+
Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
3002+
if _, ok := obj.(*cnpgtypes.ObjectStore); ok {
3003+
return apierrors.NewInternalError(errors.New("webhook down"))
3004+
}
3005+
return cl.Patch(ctx, obj, patch, opts...)
3006+
},
3007+
})
3008+
if j, _ = f.pass(); j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
3009+
t.Fatalf("the first failure after the restore failed the job: %s", j.Status.Message)
3010+
}
3011+
}
3012+
28553013
func TestOtherCNPGBackupInFlight_IgnoresTheJobsOwnBackup(t *testing.T) {
28563014
f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/")
28573015
f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running")
@@ -2987,14 +3145,24 @@ func newRolloutInstancePod(created time.Time) *corev1.Pod {
29873145
// pass runs one reconcile and returns the persisted BackupJob and the
29883146
// cnpg.io/Backups that exist afterwards.
29893147
func (f *cnpgRolloutFixture) pass() (*backupsv1alpha1.BackupJob, []cnpgtypes.Backup) {
3148+
f.t.Helper()
3149+
j, backups, _ := f.passResult()
3150+
return j, backups
3151+
}
3152+
3153+
// passResult is pass that also returns the reconcile result: the controller
3154+
// watches only BackupJobs, so a pass that waits on anything else is woken
3155+
// only by its own requeue.
3156+
func (f *cnpgRolloutFixture) passResult() (*backupsv1alpha1.BackupJob, []cnpgtypes.Backup, ctrl.Result) {
29903157
f.t.Helper()
29913158
ctx := context.Background()
29923159
key := client.ObjectKey{Namespace: rolloutNS, Name: "bj"}
29933160
j := &backupsv1alpha1.BackupJob{}
29943161
if err := f.c.Get(ctx, key, j); err != nil {
29953162
f.t.Fatalf("get BackupJob: %v", err)
29963163
}
2997-
if _, err := f.r.reconcileCNPG(ctx, j, f.resolved); err != nil {
3164+
res, err := f.r.reconcileCNPG(ctx, j, f.resolved)
3165+
if err != nil {
29983166
f.t.Fatalf("reconcileCNPG: %v", err)
29993167
}
30003168
got := &backupsv1alpha1.BackupJob{}
@@ -3005,7 +3173,7 @@ func (f *cnpgRolloutFixture) pass() (*backupsv1alpha1.BackupJob, []cnpgtypes.Bac
30053173
if err := f.c.List(ctx, backups, client.InNamespace(rolloutNS)); err != nil {
30063174
f.t.Fatalf("list cnpg.io/Backups: %v", err)
30073175
}
3008-
return got, backups.Items
3176+
return got, backups.Items, res
30093177
}
30103178

30113179
// observeCNPG makes the next live read see cluster, as CNPG has written it.

0 commit comments

Comments
 (0)