@@ -2538,9 +2538,16 @@ func TestReconcileFoundationDB_PrunesStoppedPriorBackups(t *testing.T) {
25382538 t .Errorf ("restore agents must not be pruned by the backup path" )
25392539 }
25402540 if len (names ) != 2 {
2541- t .Errorf ("no new backup may start while a stop is pending , got %v" , names )
2541+ t .Errorf ("no new backup may start while a restore runs into the cluster , got %v" , names )
25422542 }
25432543
2544+ // The restore finishes and its agents go.
2545+ if err := c .Delete (ctx , & foundationdbtypes.FoundationDBBackup {ObjectMeta : metav1.ObjectMeta {Namespace : "tenant" , Name : "restore-agents" }}); err != nil {
2546+ t .Fatalf ("delete restore agents: %v" , err )
2547+ }
2548+ if names := reconcileNames (); len (names ) != 1 || ! names ["stopped-pending" ] {
2549+ t .Errorf ("no new backup may start while a stop is pending, got %v" , names )
2550+ }
25442551 pending := & foundationdbtypes.FoundationDBBackup {}
25452552 if err := c .Get (ctx , types.NamespacedName {Namespace : "tenant" , Name : "stopped-pending" }, pending ); err != nil {
25462553 t .Fatalf ("get stopped-pending: %v" , err )
@@ -2552,11 +2559,67 @@ func TestReconcileFoundationDB_PrunesStoppedPriorBackups(t *testing.T) {
25522559 if err := c .Update (ctx , pending ); err != nil {
25532560 t .Fatalf ("mark stop reconciled: %v" , err )
25542561 }
2555- if names := reconcileNames (); len (names ) != 2 || ! names [ "restore-agents" ] || names ["stopped-pending" ] {
2562+ if names := reconcileNames (); len (names ) != 1 || names ["stopped-pending" ] {
25562563 t .Errorf ("once the stop is reconciled the prior CR goes and the new backup starts, got %v" , names )
25572564 }
25582565}
25592566
2567+ // A restore runs on a stopped agents CR; a backup started next to it would be
2568+ // discontinued the next time the operator reconciles that CR.
2569+ func TestReconcileFoundationDB_WaitsForRestoreIntoTheCluster (t * testing.T ) {
2570+ c , job , resolved := reconcileFoundationDBSeed (
2571+ t ,
2572+ foundationdbtypes.FoundationDBClusterSpec {Version : "7.3.63" },
2573+ foundationdbtypes.FoundationDBClusterStatus {Health : foundationdbtypes.FoundationDBClusterHealth {Available : true }},
2574+ )
2575+ ctx := context .Background ()
2576+ cluster := foundationdbClusterNameForApp ("fdb-src" )
2577+ if err := c .Create (ctx , & foundationdbtypes.FoundationDBBackup {
2578+ ObjectMeta : metav1.ObjectMeta {
2579+ Namespace : "tenant" , Name : "rj-agents" , Generation : 1 ,
2580+ Labels : map [string ]string {foundationdbClusterLabel : cluster , foundationdbRestoreAgentsLabel : "rj" },
2581+ },
2582+ Spec : foundationdbtypes.FoundationDBBackupSpec {ClusterName : cluster , Version : "7.3.79" , BackupState : foundationdbtypes .BackupStateStopped },
2583+ Status : foundationdbtypes.FoundationDBBackupStatus {Generations : & foundationdbtypes.BackupGenerationStatus {Reconciled : 1 }},
2584+ }); err != nil {
2585+ t .Fatalf ("seed agents: %v" , err )
2586+ }
2587+ r := & BackupJobReconciler {Client : c , Scheme : c .Scheme (), Recorder : record .NewFakeRecorder (10 )}
2588+ if _ , err := r .reconcileFoundationDB (ctx , job , resolved ); err != nil {
2589+ t .Fatalf ("reconcileFoundationDB: %v" , err )
2590+ }
2591+ if n := len (listFoundationDBBackups (t , c )); n != 1 {
2592+ t .Errorf ("no backup may start while a restore runs into the cluster, got %d CRs" , n )
2593+ }
2594+ }
2595+
2596+ // A BackupJob that fails to reach the API server is retried within its
2597+ // deadline: an earlier pass may already have stopped the cluster's backup.
2598+ func TestReconcileFoundationDB_TransientEnsureErrorIsRetried (t * testing.T ) {
2599+ c , job , resolved := reconcileFoundationDBSeed (
2600+ t ,
2601+ foundationdbtypes.FoundationDBClusterSpec {Version : "7.3.63" },
2602+ foundationdbtypes.FoundationDBClusterStatus {Health : foundationdbtypes.FoundationDBClusterHealth {Available : true }},
2603+ )
2604+ unavailable := apierrors .NewServiceUnavailable ("apiserver restarting" )
2605+ faulty := interceptor .NewClient (c .(client.WithWatch ), interceptor.Funcs {
2606+ Get : func (ctx context.Context , cl client.WithWatch , key client.ObjectKey , obj client.Object , opts ... client.GetOption ) error {
2607+ if _ , ok := obj .(* backupsv1alpha1.Backup ); ok {
2608+ return unavailable
2609+ }
2610+ return cl .Get (ctx , key , obj , opts ... )
2611+ },
2612+ })
2613+ r := & BackupJobReconciler {Client : faulty , Scheme : c .Scheme (), Recorder : record .NewFakeRecorder (10 )}
2614+
2615+ if _ , err := r .reconcileFoundationDB (context .Background (), job , resolved ); err == nil {
2616+ t .Fatalf ("a transient error must be returned for a requeue" )
2617+ }
2618+ if job .Status .Phase == backupsv1alpha1 .BackupJobPhaseFailed {
2619+ t .Errorf ("a transient error must not fail the BackupJob" )
2620+ }
2621+ }
2622+
25602623// The operator reports backup state for the whole cluster, so a stopped CR it
25612624// has reconciled can still show the previous backup running. A new CR created
25622625// then would make the operator move that backup into the new directory.
@@ -2714,46 +2777,57 @@ func TestReconcileFoundationDB_AbortsStuckBackupOfFinishedJob(t *testing.T) {
27142777// gets a backup: discontinue lets it reach a restorable snapshot and then ends
27152778// it, so the stopped CR reports its own directory restorable and not running.
27162779func TestReconcileFoundationDB_StoppedBackupThatCompletedSucceeds (t * testing.T ) {
2717- c , job , resolved := reconcileFoundationDBSeed (
2718- t ,
2719- foundationdbtypes.FoundationDBClusterSpec {Version : "7.3.63" },
2720- foundationdbtypes.FoundationDBClusterStatus {Health : foundationdbtypes.FoundationDBClusterHealth {Available : true }},
2721- )
2722- ctx := context .Background ()
2723- cluster := foundationdbClusterNameForApp ("fdb-src" )
2724- cfg := foundationdbtypes.BlobStoreConfiguration {AccountName : "@s3.example.com" , Bucket : "b" , BackupName : "tenant/bj" }
2725- url , err := foundationdbBlobURL (cfg )
2726- if err != nil {
2727- t .Fatalf ("foundationdbBlobURL: %v" , err )
2728- }
2729- completed := & foundationdbtypes.FoundationDBBackup {
2730- ObjectMeta : metav1.ObjectMeta {
2731- Namespace : "tenant" , Name : "bj-x" , Generation : 2 ,
2732- Labels : map [string ]string {
2733- foundationdbClusterLabel : cluster ,
2734- backupsv1alpha1 .OwningJobNameLabel : job .Name ,
2735- backupsv1alpha1 .OwningJobNamespaceLabel : job .Namespace ,
2736- },
2737- },
2738- Spec : foundationdbtypes.FoundationDBBackupSpec {ClusterName : cluster , Version : "7.3.79" , BackupState : foundationdbtypes .BackupStateStopped , BlobStoreConfiguration : cfg },
2739- Status : foundationdbtypes.FoundationDBBackupStatus {
2740- Generations : & foundationdbtypes.BackupGenerationStatus {Reconciled : 2 },
2741- BackupDetails : & foundationdbtypes.BackupDetails {Running : false , Restorable : true , URL : url },
2742- },
2743- }
2744- if err := c .Create (ctx , completed ); err != nil {
2745- t .Fatalf ("seed completed backup: %v" , err )
2746- }
2747- r := & BackupJobReconciler {Client : c , Scheme : c .Scheme (), Recorder : record .NewFakeRecorder (10 )}
2780+ for _ , tc := range []struct {
2781+ name string
2782+ reconciled int64
2783+ running bool
2784+ }{
2785+ {"completed after the stop" , 2 , false },
2786+ {"restorable while the operator has not acted on the stop yet" , 1 , true },
2787+ } {
2788+ t .Run (tc .name , func (t * testing.T ) {
2789+ c , job , resolved := reconcileFoundationDBSeed (
2790+ t ,
2791+ foundationdbtypes.FoundationDBClusterSpec {Version : "7.3.63" },
2792+ foundationdbtypes.FoundationDBClusterStatus {Health : foundationdbtypes.FoundationDBClusterHealth {Available : true }},
2793+ )
2794+ ctx := context .Background ()
2795+ cluster := foundationdbClusterNameForApp ("fdb-src" )
2796+ cfg := foundationdbtypes.BlobStoreConfiguration {AccountName : "@s3.example.com" , Bucket : "b" , BackupName : "tenant/bj" }
2797+ url , err := foundationdbBlobURL (cfg )
2798+ if err != nil {
2799+ t .Fatalf ("foundationdbBlobURL: %v" , err )
2800+ }
2801+ completed := & foundationdbtypes.FoundationDBBackup {
2802+ ObjectMeta : metav1.ObjectMeta {
2803+ Namespace : "tenant" , Name : "bj-x" , Generation : 2 ,
2804+ Labels : map [string ]string {
2805+ foundationdbClusterLabel : cluster ,
2806+ backupsv1alpha1 .OwningJobNameLabel : job .Name ,
2807+ backupsv1alpha1 .OwningJobNamespaceLabel : job .Namespace ,
2808+ },
2809+ },
2810+ Spec : foundationdbtypes.FoundationDBBackupSpec {ClusterName : cluster , Version : "7.3.79" , BackupState : foundationdbtypes .BackupStateStopped , BlobStoreConfiguration : cfg },
2811+ Status : foundationdbtypes.FoundationDBBackupStatus {
2812+ Generations : & foundationdbtypes.BackupGenerationStatus {Reconciled : tc .reconciled },
2813+ BackupDetails : & foundationdbtypes.BackupDetails {Running : tc .running , Restorable : true , URL : url },
2814+ },
2815+ }
2816+ if err := c .Create (ctx , completed ); err != nil {
2817+ t .Fatalf ("seed completed backup: %v" , err )
2818+ }
2819+ r := & BackupJobReconciler {Client : c , Scheme : c .Scheme (), Recorder : record .NewFakeRecorder (10 )}
27482820
2749- if _ , err := r .reconcileFoundationDB (ctx , job , resolved ); err != nil {
2750- t .Fatalf ("reconcileFoundationDB: %v" , err )
2751- }
2752- if job .Status .Phase != backupsv1alpha1 .BackupJobPhaseSucceeded || job .Status .BackupRef == nil {
2753- t .Fatalf ("a completed stopped backup in the job's own directory must succeed, got phase %q" , job .Status .Phase )
2754- }
2755- if err := c .Get (ctx , types.NamespacedName {Namespace : job .Namespace , Name : job .Status .BackupRef .Name }, & backupsv1alpha1.Backup {}); err != nil {
2756- t .Errorf ("the Backup that owns the directory must exist: %v" , err )
2821+ if _ , err := r .reconcileFoundationDB (ctx , job , resolved ); err != nil {
2822+ t .Fatalf ("reconcileFoundationDB: %v" , err )
2823+ }
2824+ if job .Status .Phase != backupsv1alpha1 .BackupJobPhaseSucceeded || job .Status .BackupRef == nil {
2825+ t .Fatalf ("a completed stopped backup in the job's own directory must succeed, got phase %q" , job .Status .Phase )
2826+ }
2827+ if err := c .Get (ctx , types.NamespacedName {Namespace : job .Namespace , Name : job .Status .BackupRef .Name }, & backupsv1alpha1.Backup {}); err != nil {
2828+ t .Errorf ("the Backup that owns the directory must exist: %v" , err )
2829+ }
2830+ })
27572831 }
27582832}
27592833
0 commit comments