@@ -22,6 +22,7 @@ import (
2222 "k8s.io/client-go/kubernetes/scheme"
2323 clienttesting "k8s.io/client-go/testing"
2424 "k8s.io/client-go/tools/record"
25+ ctrl "sigs.k8s.io/controller-runtime"
2526 "sigs.k8s.io/controller-runtime/pkg/client"
2627 clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
2728 "sigs.k8s.io/controller-runtime/pkg/client/interceptor"
@@ -2852,6 +2853,163 @@ func TestReconcileCNPG_QueueResetsDestinationRetryBudget(t *testing.T) {
28522853 }
28532854}
28542855
2856+ // seedRestoreOfApp starts a RestoreJob that writes into the fixture's app.
2857+ func (f * cnpgRolloutFixture ) seedRestoreOfApp () * backupsv1alpha1.RestoreJob {
2858+ f .t .Helper ()
2859+ appGroup := backupsv1alpha1 .DefaultApplicationAPIGroup
2860+ strategyGroup := strategyv1alpha1 .GroupVersion .Group
2861+ backup := & backupsv1alpha1.Backup {
2862+ ObjectMeta : metav1.ObjectMeta {Namespace : rolloutNS , Name : "app-backup" },
2863+ Spec : backupsv1alpha1.BackupSpec {
2864+ ApplicationRef : corev1.TypedLocalObjectReference {APIGroup : & appGroup , Kind : postgresAppKind , Name : rolloutApp },
2865+ StrategyRef : corev1.TypedLocalObjectReference {APIGroup : & strategyGroup , Kind : strategyv1alpha1 .CNPGStrategyKind , Name : "cnpg-strategy" },
2866+ },
2867+ }
2868+ restore := & backupsv1alpha1.RestoreJob {
2869+ ObjectMeta : metav1.ObjectMeta {Namespace : rolloutNS , Name : "app-restore" },
2870+ Spec : backupsv1alpha1.RestoreJobSpec {BackupRef : corev1.LocalObjectReference {Name : backup .Name }},
2871+ }
2872+ for _ , obj := range []client.Object {backup , restore } {
2873+ if err := f .c .Create (context .Background (), obj ); err != nil {
2874+ f .t .Fatalf ("seed %T: %v" , obj , err )
2875+ }
2876+ }
2877+ return restore
2878+ }
2879+
2880+ // A job past its start writes the WAL archive destination only while no
2881+ // restore rewrites its application. The restore gate in Reconcile looks only
2882+ // at a job that has not started, and a job released from the queue, or one
2883+ // still before its first write, has started long ago: a write landing during
2884+ // the restore can put the restored Cluster back on the source's WAL prefix.
2885+ func TestReconcileCNPG_HoldsTheDestinationWriteWhileTheAppIsRestored (t * testing.T ) {
2886+ for _ , queued := range []bool {true , false } {
2887+ t .Run (fmt .Sprintf ("released from the queue=%t" , queued ), func (t * testing.T ) {
2888+ f := newAttachedObjectStoreFixture (t , time .Now ().Add (- time .Hour ), "s3://other/" )
2889+ if queued {
2890+ f .seedOtherBackupJob ("bj-other" , backupsv1alpha1 .BackupJobPhaseRunning )
2891+ f .seedCNPGBackup ("bj-other-cnpg" , "bj-other" , rolloutCluster , "running" )
2892+ if j , _ := f .pass (); apimeta .FindStatusCondition (j .Status .Conditions , "Ready" ).Reason != "WaitingForOtherBackup" {
2893+ t .Fatalf ("Ready = %+v, want the job queued" , apimeta .FindStatusCondition (j .Status .Conditions , "Ready" ))
2894+ }
2895+ }
2896+ restore := f .seedRestoreOfApp ()
2897+ if queued {
2898+ held := & cnpgtypes.Backup {}
2899+ if err := f .c .Get (context .Background (), client.ObjectKey {Namespace : rolloutNS , Name : "bj-other-cnpg" }, held ); err != nil {
2900+ t .Fatalf ("get holder Backup: %v" , err )
2901+ }
2902+ held .Status .Phase = cnpgBackupPhaseComplete
2903+ if err := f .c .Update (context .Background (), held ); err != nil {
2904+ t .Fatalf ("complete holder Backup: %v" , err )
2905+ }
2906+ holder := & backupsv1alpha1.BackupJob {}
2907+ if err := f .c .Get (context .Background (), client.ObjectKey {Namespace : rolloutNS , Name : "bj-other" }, holder ); err != nil {
2908+ t .Fatalf ("get holder BackupJob: %v" , err )
2909+ }
2910+ holder .Status .Phase = backupsv1alpha1 .BackupJobPhaseSucceeded
2911+ if err := f .c .Status ().Update (context .Background (), holder ); err != nil {
2912+ t .Fatalf ("finish holder BackupJob: %v" , err )
2913+ }
2914+ }
2915+
2916+ j , backups , res := f .passResult ()
2917+ if got := f .liveDestination (); got != "s3://other/" {
2918+ t .Errorf ("destination written during a restore of the app: %q" , got )
2919+ }
2920+ for _ , b := range backups {
2921+ if b .Labels [backupsv1alpha1 .OwningJobNameLabel ] == "bj" {
2922+ t .Errorf ("a cnpg.io/Backup was created during a restore of the app: %s" , b .Name )
2923+ }
2924+ }
2925+ if ready := apimeta .FindStatusCondition (j .Status .Conditions , "Ready" ); ready == nil || ready .Reason != ConditionReasonRestoreInProgress {
2926+ t .Errorf ("Ready = %+v, want %s" , ready , ConditionReasonRestoreInProgress )
2927+ }
2928+ if res .RequeueAfter == 0 {
2929+ t .Errorf ("a held job is not requeued: %+v" , res )
2930+ }
2931+
2932+ restore .Status .Phase = backupsv1alpha1 .RestoreJobPhaseSucceeded
2933+ if err := f .c .Status ().Update (context .Background (), restore ); err != nil {
2934+ t .Fatalf ("finish the restore: %v" , err )
2935+ }
2936+ f .pass ()
2937+ if got := f .liveDestination (); got != "s3://bucket/" {
2938+ t .Errorf ("once the restore finished, destination = %q, want the job's own s3://bucket/" , got )
2939+ }
2940+ })
2941+ }
2942+ }
2943+
2944+ // A job whose Backup already runs writes nothing, so a restore does not hold
2945+ // it: holding would strand a backup that is already moving data.
2946+ func TestReconcileCNPG_RestoreDoesNotHoldARunningBackup (t * testing.T ) {
2947+ f := newAttachedObjectStoreFixture (t , time .Now ().Add (- time .Hour ), "s3://other/" )
2948+ f .seedCNPGBackup ("bj-cnpg" , "bj" , rolloutCluster , "running" )
2949+ f .seedRestoreOfApp ()
2950+
2951+ j , _ := f .pass ()
2952+ if ready := apimeta .FindStatusCondition (j .Status .Conditions , "Ready" ); ready == nil || ready .Reason != "WaitingForCNPGBackup" {
2953+ t .Errorf ("Ready = %+v, want the running backup still followed" , ready )
2954+ }
2955+ }
2956+
2957+ // An error looking for a restore is not taken as "no restore".
2958+ func TestReconcileCNPG_RestoreLookupErrorIsReturned (t * testing.T ) {
2959+ f := newAttachedObjectStoreFixture (t , time .Now ().Add (- time .Hour ), "s3://other/" )
2960+ f .r .Client = interceptor .NewClient (f .c .(client.WithWatch ), interceptor.Funcs {
2961+ List : func (ctx context.Context , cl client.WithWatch , list client.ObjectList , opts ... client.ListOption ) error {
2962+ if _ , ok := list .(* backupsv1alpha1.RestoreJobList ); ok {
2963+ return apierrors .NewServiceUnavailable ("cache not synced" )
2964+ }
2965+ return cl .List (ctx , list , opts ... )
2966+ },
2967+ })
2968+ j := & backupsv1alpha1.BackupJob {}
2969+ if err := f .c .Get (context .Background (), client.ObjectKey {Namespace : rolloutNS , Name : "bj" }, j ); err != nil {
2970+ t .Fatalf ("get BackupJob: %v" , err )
2971+ }
2972+ if _ , err := f .r .reconcileCNPG (context .Background (), j , f .resolved ); err == nil {
2973+ t .Fatalf ("reconcileCNPG swallowed the restore lookup error" )
2974+ }
2975+ if got := f .liveDestination (); got != "s3://other/" {
2976+ t .Errorf ("destination written although restores could not be listed: %q" , got )
2977+ }
2978+ }
2979+
2980+ // Waiting for a restore resets the retry budget of the destination write,
2981+ // as waiting for another backup does, so the restore does not spend it.
2982+ func TestReconcileCNPG_RestoreResetsDestinationRetryBudget (t * testing.T ) {
2983+ expired := time .Now ().Add (- (cnpgDefaultBackupDeadline + time .Minute ))
2984+ f := newAttachedObjectStoreFixture (t , expired , "s3://other/" )
2985+ f .setJobCondition (metav1.Condition {
2986+ Type : backupCondDestinationApplied , Status : metav1 .ConditionFalse , Reason : "ClusterPluginApplyRetrying" ,
2987+ Message : "earlier failure" , LastTransitionTime : metav1 .NewTime (expired ),
2988+ })
2989+ restore := f .seedRestoreOfApp ()
2990+
2991+ j , _ := f .pass ()
2992+ if cond := apimeta .FindStatusCondition (j .Status .Conditions , backupCondDestinationApplied ); cond != nil {
2993+ t .Fatalf ("%s = %+v kept while waiting for a restore" , backupCondDestinationApplied , cond )
2994+ }
2995+
2996+ restore .Status .Phase = backupsv1alpha1 .RestoreJobPhaseSucceeded
2997+ if err := f .c .Status ().Update (context .Background (), restore ); err != nil {
2998+ t .Fatalf ("finish the restore: %v" , err )
2999+ }
3000+ f .r .Client = interceptor .NewClient (f .c .(client.WithWatch ), interceptor.Funcs {
3001+ Patch : func (ctx context.Context , cl client.WithWatch , obj client.Object , patch client.Patch , opts ... client.PatchOption ) error {
3002+ if _ , ok := obj .(* cnpgtypes.ObjectStore ); ok {
3003+ return apierrors .NewInternalError (errors .New ("webhook down" ))
3004+ }
3005+ return cl .Patch (ctx , obj , patch , opts ... )
3006+ },
3007+ })
3008+ if j , _ = f .pass (); j .Status .Phase == backupsv1alpha1 .BackupJobPhaseFailed {
3009+ t .Fatalf ("the first failure after the restore failed the job: %s" , j .Status .Message )
3010+ }
3011+ }
3012+
28553013func TestOtherCNPGBackupInFlight_IgnoresTheJobsOwnBackup (t * testing.T ) {
28563014 f := newAttachedObjectStoreFixture (t , time .Now (), "s3://other/" )
28573015 f .seedCNPGBackup ("bj-cnpg" , "bj" , rolloutCluster , "running" )
@@ -2987,14 +3145,24 @@ func newRolloutInstancePod(created time.Time) *corev1.Pod {
29873145// pass runs one reconcile and returns the persisted BackupJob and the
29883146// cnpg.io/Backups that exist afterwards.
29893147func (f * cnpgRolloutFixture ) pass () (* backupsv1alpha1.BackupJob , []cnpgtypes.Backup ) {
3148+ f .t .Helper ()
3149+ j , backups , _ := f .passResult ()
3150+ return j , backups
3151+ }
3152+
3153+ // passResult is pass that also returns the reconcile result: the controller
3154+ // watches only BackupJobs, so a pass that waits on anything else is woken
3155+ // only by its own requeue.
3156+ func (f * cnpgRolloutFixture ) passResult () (* backupsv1alpha1.BackupJob , []cnpgtypes.Backup , ctrl.Result ) {
29903157 f .t .Helper ()
29913158 ctx := context .Background ()
29923159 key := client.ObjectKey {Namespace : rolloutNS , Name : "bj" }
29933160 j := & backupsv1alpha1.BackupJob {}
29943161 if err := f .c .Get (ctx , key , j ); err != nil {
29953162 f .t .Fatalf ("get BackupJob: %v" , err )
29963163 }
2997- if _ , err := f .r .reconcileCNPG (ctx , j , f .resolved ); err != nil {
3164+ res , err := f .r .reconcileCNPG (ctx , j , f .resolved )
3165+ if err != nil {
29983166 f .t .Fatalf ("reconcileCNPG: %v" , err )
29993167 }
30003168 got := & backupsv1alpha1.BackupJob {}
@@ -3005,7 +3173,7 @@ func (f *cnpgRolloutFixture) pass() (*backupsv1alpha1.BackupJob, []cnpgtypes.Bac
30053173 if err := f .c .List (ctx , backups , client .InNamespace (rolloutNS )); err != nil {
30063174 f .t .Fatalf ("list cnpg.io/Backups: %v" , err )
30073175 }
3008- return got , backups .Items
3176+ return got , backups .Items , res
30093177}
30103178
30113179// observeCNPG makes the next live read see cluster, as CNPG has written it.
0 commit comments