Skip to content

Commit 91250fc

Browse files
committed
fix(backups): retry FoundationDB handover errors and wait for restores
- A BackupJob no longer starts a backup on a cluster a restore runs into. The restore runs on a stopped agents CR, and the operator would discontinue a backup started next to it the next time it reconciled that CR. - An API error while handing over the cluster's backup is retried within the deadline instead of failing the job: an earlier pass may already have stopped the running backup. A same-named Backup, an invalid or forbidden request, and the deadline still fail it. - A stopped CR whose own directory is restorable counts as ready before the operator has acted on the stop, so a job stopped just before its deadline is not aborted with a restorable directory. Assisted-by: LLM Signed-off-by: Andrey Kolkov <[email protected]>
1 parent edf2768 commit 91250fc

2 files changed

Lines changed: 142 additions & 44 deletions

File tree

‎internal/backupcontroller/foundationdbstrategy_controller.go‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -265,7 +265,15 @@ func (r *BackupJobReconciler) reconcileFoundationDB(ctx context.Context, j *back
265265

266266
fdbBackup, waiting, err := r.ensureFoundationDBBackup(ctx, j, clusterName, cluster.Spec.Version, rendered)
267267
if err != nil {
268-
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to ensure apps.foundationdb.org/FoundationDBBackup: %v", err))
268+
// A pass may already have stopped the cluster's running backup, so a
269+
// failure retrying can fix is retried within the deadline rather than
270+
// leaving the cluster without one until the next run.
271+
var refused *foundationdbBackupRefusedError
272+
if errors.As(err, &refused) || apierrors.IsInvalid(err) || apierrors.IsForbidden(err) ||
273+
foundationdbBackupDeadlineExceeded(j.Status.StartedAt, deadline) {
274+
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to ensure apps.foundationdb.org/FoundationDBBackup: %v", err))
275+
}
276+
return ctrl.Result{}, err
269277
}
270278
r.pruneStoppedFoundationDBBackups(ctx, j, clusterName)
271279
if waiting != "" {
@@ -358,7 +366,11 @@ func foundationdbBackupReady(b *foundationdbtypes.FoundationDBBackup) bool {
358366
if !b.Status.BackupDetails.Restorable {
359367
return false
360368
}
361-
if b.Status.Generations != nil && b.Status.Generations.Reconciled < b.Generation {
369+
// The stop is the only spec change a stopped CR gets, and a directory
370+
// that was restorable stays so, so its status counts before the
371+
// operator has acted on the stop.
372+
if b.Spec.BackupState != foundationdbtypes.BackupStateStopped &&
373+
b.Status.Generations != nil && b.Status.Generations.Reconciled < b.Generation {
362374
return false
363375
}
364376
// The operator reports backup state for the whole cluster, so a new CR
@@ -539,7 +551,7 @@ func (r *BackupJobReconciler) ensureFoundationDBBackup(ctx context.Context, j *b
539551
if prior.DeletionTimestamp != nil {
540552
return nil, fmt.Sprintf("waiting for Backup %s/%s, whose deletion removes the directory this BackupJob writes to, to be deleted", j.Namespace, j.Name), nil
541553
}
542-
return nil, "", fmt.Errorf("Backup %s/%s of an earlier BackupJob of this name already exists; delete it or use another BackupJob name", j.Namespace, j.Name)
554+
return nil, "", &foundationdbBackupRefusedError{fmt.Errorf("Backup %s/%s of an earlier BackupJob of this name already exists; delete it or use another BackupJob name", j.Namespace, j.Name)}
543555
} else if !apierrors.IsNotFound(err) {
544556
return nil, "", err
545557
}
@@ -599,6 +611,12 @@ func (r *BackupJobReconciler) ensureFoundationDBBackup(ctx context.Context, j *b
599611
return obj, "", nil
600612
}
601613

614+
// foundationdbBackupRefusedError is a BackupJob the driver will not run however
615+
// often it retries.
616+
type foundationdbBackupRefusedError struct{ error }
617+
618+
func (e *foundationdbBackupRefusedError) Unwrap() error { return e.error }
619+
602620
// foundationdbPreviousAttempt reports whether a CR carrying the job's name
603621
// was created by an earlier BackupJob of that name and has been stopped (a
604622
// stop on deadline expiry, or by a later job). A recreated job that reused it
@@ -634,6 +652,12 @@ func (r *BackupJobReconciler) waitForStoppedFoundationDBBackups(ctx context.Cont
634652
// wait; one the cache still shows running is a stop not seen yet.
635653
return fmt.Sprintf("waiting for the stop of FoundationDBBackup %s to show", b.Name), nil
636654
}
655+
// A restore into the cluster runs on a stopped agents CR, which the
656+
// operator would turn on a backup started next to it (a stopped CR
657+
// discontinues whatever the cluster runs) if it reconciled it again.
658+
if rj := b.Labels[foundationdbRestoreAgentsLabel]; rj != "" {
659+
return fmt.Sprintf("waiting for RestoreJob %s into this cluster to finish", rj), nil
660+
}
637661
if !foundationdbBackupQuiet(b) {
638662
if b.DeletionTimestamp != nil {
639663
return fmt.Sprintf("waiting for the operator to abort the backup of FoundationDBBackup %s", b.Name), nil

‎internal/backupcontroller/foundationdbstrategy_controller_test.go‎

Lines changed: 115 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -2538,9 +2538,16 @@ func TestReconcileFoundationDB_PrunesStoppedPriorBackups(t *testing.T) {
25382538
t.Errorf("restore agents must not be pruned by the backup path")
25392539
}
25402540
if len(names) != 2 {
2541-
t.Errorf("no new backup may start while a stop is pending, got %v", names)
2541+
t.Errorf("no new backup may start while a restore runs into the cluster, got %v", names)
25422542
}
25432543

2544+
// The restore finishes and its agents go.
2545+
if err := c.Delete(ctx, &foundationdbtypes.FoundationDBBackup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "restore-agents"}}); err != nil {
2546+
t.Fatalf("delete restore agents: %v", err)
2547+
}
2548+
if names := reconcileNames(); len(names) != 1 || !names["stopped-pending"] {
2549+
t.Errorf("no new backup may start while a stop is pending, got %v", names)
2550+
}
25442551
pending := &foundationdbtypes.FoundationDBBackup{}
25452552
if err := c.Get(ctx, types.NamespacedName{Namespace: "tenant", Name: "stopped-pending"}, pending); err != nil {
25462553
t.Fatalf("get stopped-pending: %v", err)
@@ -2552,11 +2559,67 @@ func TestReconcileFoundationDB_PrunesStoppedPriorBackups(t *testing.T) {
25522559
if err := c.Update(ctx, pending); err != nil {
25532560
t.Fatalf("mark stop reconciled: %v", err)
25542561
}
2555-
if names := reconcileNames(); len(names) != 2 || !names["restore-agents"] || names["stopped-pending"] {
2562+
if names := reconcileNames(); len(names) != 1 || names["stopped-pending"] {
25562563
t.Errorf("once the stop is reconciled the prior CR goes and the new backup starts, got %v", names)
25572564
}
25582565
}
25592566

2567+
// A restore runs on a stopped agents CR; a backup started next to it would be
2568+
// discontinued the next time the operator reconciles that CR.
2569+
func TestReconcileFoundationDB_WaitsForRestoreIntoTheCluster(t *testing.T) {
2570+
c, job, resolved := reconcileFoundationDBSeed(
2571+
t,
2572+
foundationdbtypes.FoundationDBClusterSpec{Version: "7.3.63"},
2573+
foundationdbtypes.FoundationDBClusterStatus{Health: foundationdbtypes.FoundationDBClusterHealth{Available: true}},
2574+
)
2575+
ctx := context.Background()
2576+
cluster := foundationdbClusterNameForApp("fdb-src")
2577+
if err := c.Create(ctx, &foundationdbtypes.FoundationDBBackup{
2578+
ObjectMeta: metav1.ObjectMeta{
2579+
Namespace: "tenant", Name: "rj-agents", Generation: 1,
2580+
Labels: map[string]string{foundationdbClusterLabel: cluster, foundationdbRestoreAgentsLabel: "rj"},
2581+
},
2582+
Spec: foundationdbtypes.FoundationDBBackupSpec{ClusterName: cluster, Version: "7.3.79", BackupState: foundationdbtypes.BackupStateStopped},
2583+
Status: foundationdbtypes.FoundationDBBackupStatus{Generations: &foundationdbtypes.BackupGenerationStatus{Reconciled: 1}},
2584+
}); err != nil {
2585+
t.Fatalf("seed agents: %v", err)
2586+
}
2587+
r := &BackupJobReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(10)}
2588+
if _, err := r.reconcileFoundationDB(ctx, job, resolved); err != nil {
2589+
t.Fatalf("reconcileFoundationDB: %v", err)
2590+
}
2591+
if n := len(listFoundationDBBackups(t, c)); n != 1 {
2592+
t.Errorf("no backup may start while a restore runs into the cluster, got %d CRs", n)
2593+
}
2594+
}
2595+
2596+
// A BackupJob that fails to reach the API server is retried within its
2597+
// deadline: an earlier pass may already have stopped the cluster's backup.
2598+
func TestReconcileFoundationDB_TransientEnsureErrorIsRetried(t *testing.T) {
2599+
c, job, resolved := reconcileFoundationDBSeed(
2600+
t,
2601+
foundationdbtypes.FoundationDBClusterSpec{Version: "7.3.63"},
2602+
foundationdbtypes.FoundationDBClusterStatus{Health: foundationdbtypes.FoundationDBClusterHealth{Available: true}},
2603+
)
2604+
unavailable := apierrors.NewServiceUnavailable("apiserver restarting")
2605+
faulty := interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
2606+
Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
2607+
if _, ok := obj.(*backupsv1alpha1.Backup); ok {
2608+
return unavailable
2609+
}
2610+
return cl.Get(ctx, key, obj, opts...)
2611+
},
2612+
})
2613+
r := &BackupJobReconciler{Client: faulty, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(10)}
2614+
2615+
if _, err := r.reconcileFoundationDB(context.Background(), job, resolved); err == nil {
2616+
t.Fatalf("a transient error must be returned for a requeue")
2617+
}
2618+
if job.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
2619+
t.Errorf("a transient error must not fail the BackupJob")
2620+
}
2621+
}
2622+
25602623
// The operator reports backup state for the whole cluster, so a stopped CR it
25612624
// has reconciled can still show the previous backup running. A new CR created
25622625
// then would make the operator move that backup into the new directory.
@@ -2714,46 +2777,57 @@ func TestReconcileFoundationDB_AbortsStuckBackupOfFinishedJob(t *testing.T) {
27142777
// gets a backup: discontinue lets it reach a restorable snapshot and then ends
27152778
// it, so the stopped CR reports its own directory restorable and not running.
27162779
func TestReconcileFoundationDB_StoppedBackupThatCompletedSucceeds(t *testing.T) {
2717-
c, job, resolved := reconcileFoundationDBSeed(
2718-
t,
2719-
foundationdbtypes.FoundationDBClusterSpec{Version: "7.3.63"},
2720-
foundationdbtypes.FoundationDBClusterStatus{Health: foundationdbtypes.FoundationDBClusterHealth{Available: true}},
2721-
)
2722-
ctx := context.Background()
2723-
cluster := foundationdbClusterNameForApp("fdb-src")
2724-
cfg := foundationdbtypes.BlobStoreConfiguration{AccountName: "@s3.example.com", Bucket: "b", BackupName: "tenant/bj"}
2725-
url, err := foundationdbBlobURL(cfg)
2726-
if err != nil {
2727-
t.Fatalf("foundationdbBlobURL: %v", err)
2728-
}
2729-
completed := &foundationdbtypes.FoundationDBBackup{
2730-
ObjectMeta: metav1.ObjectMeta{
2731-
Namespace: "tenant", Name: "bj-x", Generation: 2,
2732-
Labels: map[string]string{
2733-
foundationdbClusterLabel: cluster,
2734-
backupsv1alpha1.OwningJobNameLabel: job.Name,
2735-
backupsv1alpha1.OwningJobNamespaceLabel: job.Namespace,
2736-
},
2737-
},
2738-
Spec: foundationdbtypes.FoundationDBBackupSpec{ClusterName: cluster, Version: "7.3.79", BackupState: foundationdbtypes.BackupStateStopped, BlobStoreConfiguration: cfg},
2739-
Status: foundationdbtypes.FoundationDBBackupStatus{
2740-
Generations: &foundationdbtypes.BackupGenerationStatus{Reconciled: 2},
2741-
BackupDetails: &foundationdbtypes.BackupDetails{Running: false, Restorable: true, URL: url},
2742-
},
2743-
}
2744-
if err := c.Create(ctx, completed); err != nil {
2745-
t.Fatalf("seed completed backup: %v", err)
2746-
}
2747-
r := &BackupJobReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(10)}
2780+
for _, tc := range []struct {
2781+
name string
2782+
reconciled int64
2783+
running bool
2784+
}{
2785+
{"completed after the stop", 2, false},
2786+
{"restorable while the operator has not acted on the stop yet", 1, true},
2787+
} {
2788+
t.Run(tc.name, func(t *testing.T) {
2789+
c, job, resolved := reconcileFoundationDBSeed(
2790+
t,
2791+
foundationdbtypes.FoundationDBClusterSpec{Version: "7.3.63"},
2792+
foundationdbtypes.FoundationDBClusterStatus{Health: foundationdbtypes.FoundationDBClusterHealth{Available: true}},
2793+
)
2794+
ctx := context.Background()
2795+
cluster := foundationdbClusterNameForApp("fdb-src")
2796+
cfg := foundationdbtypes.BlobStoreConfiguration{AccountName: "@s3.example.com", Bucket: "b", BackupName: "tenant/bj"}
2797+
url, err := foundationdbBlobURL(cfg)
2798+
if err != nil {
2799+
t.Fatalf("foundationdbBlobURL: %v", err)
2800+
}
2801+
completed := &foundationdbtypes.FoundationDBBackup{
2802+
ObjectMeta: metav1.ObjectMeta{
2803+
Namespace: "tenant", Name: "bj-x", Generation: 2,
2804+
Labels: map[string]string{
2805+
foundationdbClusterLabel: cluster,
2806+
backupsv1alpha1.OwningJobNameLabel: job.Name,
2807+
backupsv1alpha1.OwningJobNamespaceLabel: job.Namespace,
2808+
},
2809+
},
2810+
Spec: foundationdbtypes.FoundationDBBackupSpec{ClusterName: cluster, Version: "7.3.79", BackupState: foundationdbtypes.BackupStateStopped, BlobStoreConfiguration: cfg},
2811+
Status: foundationdbtypes.FoundationDBBackupStatus{
2812+
Generations: &foundationdbtypes.BackupGenerationStatus{Reconciled: tc.reconciled},
2813+
BackupDetails: &foundationdbtypes.BackupDetails{Running: tc.running, Restorable: true, URL: url},
2814+
},
2815+
}
2816+
if err := c.Create(ctx, completed); err != nil {
2817+
t.Fatalf("seed completed backup: %v", err)
2818+
}
2819+
r := &BackupJobReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(10)}
27482820

2749-
if _, err := r.reconcileFoundationDB(ctx, job, resolved); err != nil {
2750-
t.Fatalf("reconcileFoundationDB: %v", err)
2751-
}
2752-
if job.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded || job.Status.BackupRef == nil {
2753-
t.Fatalf("a completed stopped backup in the job's own directory must succeed, got phase %q", job.Status.Phase)
2754-
}
2755-
if err := c.Get(ctx, types.NamespacedName{Namespace: job.Namespace, Name: job.Status.BackupRef.Name}, &backupsv1alpha1.Backup{}); err != nil {
2756-
t.Errorf("the Backup that owns the directory must exist: %v", err)
2821+
if _, err := r.reconcileFoundationDB(ctx, job, resolved); err != nil {
2822+
t.Fatalf("reconcileFoundationDB: %v", err)
2823+
}
2824+
if job.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded || job.Status.BackupRef == nil {
2825+
t.Fatalf("a completed stopped backup in the job's own directory must succeed, got phase %q", job.Status.Phase)
2826+
}
2827+
if err := c.Get(ctx, types.NamespacedName{Namespace: job.Namespace, Name: job.Status.BackupRef.Name}, &backupsv1alpha1.Backup{}); err != nil {
2828+
t.Errorf("the Backup that owns the directory must exist: %v", err)
2829+
}
2830+
})
27572831
}
27582832
}
27592833

0 commit comments

Comments
 (0)