Skip to content

Commit 05b96f6

Browse files
committed
fix(backups): close three FoundationDB handover gaps
- A BackupJob whose name already has a Backup fails before it starts a backup: the directory is named after the job, and that Backup's deletion would delete a new backup written there. - A Backup released with skip-artifact-cleanup only stops its operator CR. Deleting it at once could hide a backup still running from the next BackupJob of the cluster, which waits for every stopped backup. The next BackupJob prunes it once the stop has gone through. - Restore agents ignore a strategy's explicit agentCount of 0, which would make the operator start none; the restore then counted its own agents as an agentless destination and failed. Assisted-by: LLM Signed-off-by: Andrey Kolkov <[email protected]>
1 parent c7df9ff commit 05b96f6

4 files changed

Lines changed: 84 additions & 16 deletions

File tree

‎internal/backupcontroller/foundationdb_cleanup.go‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ func (r *BackupReconciler) cleanupFoundationDBBackup(ctx context.Context, backup
7070
// The data stays, but the agents must not keep writing into a
7171
// directory nothing owns any more. Best-effort and without waiting
7272
// on the operator: the annotation is the way out of a stuck delete.
73-
r.dropFoundationDBOperatorBackup(ctx, backup)
73+
r.stopFoundationDBOperatorBackup(ctx, backup)
7474
getLogger(ctx).Debug("skipping FoundationDB artifact cleanup per annotation; directory left in bucket", "backup", backup.Name, "url", url)
7575
return ctrl.Result{}, nil
7676
}
@@ -160,9 +160,13 @@ func (r *BackupReconciler) removeFoundationDBOperatorBackup(ctx context.Context,
160160
return false, client.IgnoreNotFound(r.Delete(ctx, fdbBackup))
161161
}
162162

163-
// dropFoundationDBOperatorBackup stops and deletes the Backup's operator-side
164-
// FoundationDBBackup without waiting for the operator. Errors are logged.
165-
func (r *BackupReconciler) dropFoundationDBOperatorBackup(ctx context.Context, backup *backupsv1alpha1.Backup) {
163+
// stopFoundationDBOperatorBackup stops the Backup's operator-side
164+
// FoundationDBBackup without waiting for the operator. The CR is left for the
165+
// cluster's next BackupJob to prune once the stop has gone through: deleting
166+
// it now would hide a backup that is still running from that job, which waits
167+
// for every stopped backup of the cluster before starting its own. Errors are
168+
// logged.
169+
func (r *BackupReconciler) stopFoundationDBOperatorBackup(ctx context.Context, backup *backupsv1alpha1.Backup) {
166170
md := backup.Spec.DriverMetadata
167171
name := md[foundationdbBackupNameKey]
168172
if name == "" {
@@ -176,15 +180,13 @@ func (r *BackupReconciler) dropFoundationDBOperatorBackup(ctx context.Context, b
176180
if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, fdbBackup); err != nil {
177181
return
178182
}
179-
if fdbBackup.Spec.BackupState != foundationdbtypes.BackupStateStopped {
180-
base := fdbBackup.DeepCopy()
181-
fdbBackup.Spec.BackupState = foundationdbtypes.BackupStateStopped
182-
if err := r.Patch(ctx, fdbBackup, client.MergeFrom(base)); err != nil {
183-
getLogger(ctx).Debug("failed to stop FoundationDBBackup on skipped cleanup", "fdbBackup", name, "error", err.Error())
184-
}
183+
if fdbBackup.Spec.BackupState == foundationdbtypes.BackupStateStopped {
184+
return
185185
}
186-
if err := r.Delete(ctx, fdbBackup); err != nil && !apierrors.IsNotFound(err) {
187-
getLogger(ctx).Debug("failed to delete FoundationDBBackup on skipped cleanup", "fdbBackup", name, "error", err.Error())
186+
base := fdbBackup.DeepCopy()
187+
fdbBackup.Spec.BackupState = foundationdbtypes.BackupStateStopped
188+
if err := r.Patch(ctx, fdbBackup, client.MergeFrom(base)); err != nil {
189+
getLogger(ctx).Debug("failed to stop FoundationDBBackup on skipped cleanup", "fdbBackup", name, "error", err.Error())
188190
}
189191
}
190192

‎internal/backupcontroller/foundationdb_cleanup_test.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -266,8 +266,12 @@ func TestCleanupFoundationDBBackup_SkipAnnotationReleases(t *testing.T) {
266266
if jobs := listCleanupJobs(t, c); len(jobs) != 0 {
267267
t.Errorf("skip annotation must not start a delete Job, got %d", len(jobs))
268268
}
269-
if err := c.Get(context.Background(), client.ObjectKey{Namespace: "tenant", Name: "bj-1-fdb"}, &foundationdbtypes.FoundationDBBackup{}); !apierrors.IsNotFound(err) {
270-
t.Errorf("the operator CR must not keep streaming into a released directory, got err=%v", err)
269+
got := &foundationdbtypes.FoundationDBBackup{}
270+
if err := c.Get(context.Background(), client.ObjectKey{Namespace: "tenant", Name: "bj-1-fdb"}, got); err != nil {
271+
t.Fatalf("the stopped operator CR stays for the next BackupJob to wait on and prune: %v", err)
272+
}
273+
if got.Spec.BackupState != foundationdbtypes.BackupStateStopped {
274+
t.Errorf("the operator CR must not keep streaming into a released directory, got %q", got.Spec.BackupState)
271275
}
272276
}
273277

‎internal/backupcontroller/foundationdbstrategy_controller.go‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -524,13 +524,16 @@ func (r *BackupJobReconciler) ensureFoundationDBBackup(ctx context.Context, j *b
524524
if waiting, err := r.waitForStoppedFoundationDBBackups(ctx, j, clusterName); err != nil || waiting != "" {
525525
return nil, waiting, err
526526
}
527-
// A Backup of the same name that is being deleted is about to delete the
528-
// directory this job would write to.
527+
// The Backup this job will record is named after it, and so is the
528+
// directory. One that already exists records an earlier job's backup in
529+
// that directory: wait while its deletion removes the directory, and
530+
// refuse to write into it otherwise.
529531
prior := &backupsv1alpha1.Backup{}
530532
if err := r.Get(ctx, types.NamespacedName{Namespace: j.Namespace, Name: j.Name}, prior); err == nil {
531533
if prior.DeletionTimestamp != nil {
532534
return nil, fmt.Sprintf("waiting for Backup %s/%s, whose deletion removes the directory this BackupJob writes to, to be deleted", j.Namespace, j.Name), nil
533535
}
536+
return nil, "", fmt.Errorf("Backup %s/%s of an earlier BackupJob of this name already exists; delete it or use another BackupJob name", j.Namespace, j.Name)
534537
} else if !apierrors.IsNotFound(err) {
535538
return nil, "", err
536539
}
@@ -1529,6 +1532,11 @@ func (r *RestoreJobReconciler) ensureFoundationDBRestoreAgents(
15291532
CustomParameters: append([]string(nil), rendered.CustomParameters...),
15301533
},
15311534
}
1535+
// These agents exist only to run the restore; an explicit zero would make
1536+
// the operator start none, so it falls back to the operator default.
1537+
if obj.Spec.AgentCount != nil && *obj.Spec.AgentCount == 0 {
1538+
obj.Spec.AgentCount = nil
1539+
}
15321540
if rendered.BackupDeploymentPodTemplateSpec != nil {
15331541
obj.Spec.PodTemplateSpec = rendered.BackupDeploymentPodTemplateSpec.DeepCopy()
15341542
}

‎internal/backupcontroller/foundationdbstrategy_controller_test.go‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2164,6 +2164,35 @@ func TestReconcileFoundationDBRestore_ToCopyStartsAgents(t *testing.T) {
21642164
}
21652165
}
21662166

2167+
// Restore agents exist only to run the restore, so a strategy's explicit
2168+
// agentCount: 0 must not leave the destination without any.
2169+
func TestReconcileFoundationDBRestore_AgentsIgnoreZeroAgentCount(t *testing.T) {
2170+
c, r, rj, backup := seedFoundationDBRestore(t, foundationdbRestoreSeed{toCopy: true})
2171+
ctx := context.Background()
2172+
strategy := &strategyv1alpha1.FoundationDB{}
2173+
if err := c.Get(ctx, client.ObjectKey{Name: "fdb-strategy"}, strategy); err != nil {
2174+
t.Fatalf("get strategy: %v", err)
2175+
}
2176+
zero := int32(0)
2177+
strategy.Spec.Template.AgentCount = &zero
2178+
if err := c.Update(ctx, strategy); err != nil {
2179+
t.Fatalf("update strategy: %v", err)
2180+
}
2181+
2182+
for range 2 {
2183+
if _, err := r.reconcileFoundationDBRestore(ctx, rj, backup); err != nil {
2184+
t.Fatalf("reconcileFoundationDBRestore: %v", err)
2185+
}
2186+
}
2187+
if rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed {
2188+
t.Fatalf("the restore's own agents must not count as agentless")
2189+
}
2190+
agents := listFoundationDBBackups(t, c)
2191+
if len(agents) != 1 || agents[0].Spec.AgentCount != nil {
2192+
t.Errorf("restore agents must fall back to the operator's agent count, got %+v", agents)
2193+
}
2194+
}
2195+
21672196
func TestReconcileFoundationDBRestore_CompletedRemovesAgents(t *testing.T) {
21682197
c, r, rj, backup := seedFoundationDBRestore(t, foundationdbRestoreSeed{toCopy: true})
21692198
if _, err := r.reconcileFoundationDBRestore(context.Background(), rj, backup); err != nil {
@@ -2724,6 +2753,31 @@ func TestReconcileFoundationDB_WaitsForSameNamedBackupDeletion(t *testing.T) {
27242753
}
27252754
}
27262755

2756+
// A Backup named after the job already records a backup in the directory the
2757+
// job would write; a new backup there would be deleted along with it.
2758+
func TestReconcileFoundationDB_RefusesSameNamedLiveBackup(t *testing.T) {
2759+
c, job, resolved := reconcileFoundationDBSeed(
2760+
t,
2761+
foundationdbtypes.FoundationDBClusterSpec{Version: "7.3.63"},
2762+
foundationdbtypes.FoundationDBClusterStatus{Health: foundationdbtypes.FoundationDBClusterHealth{Available: true}},
2763+
)
2764+
ctx := context.Background()
2765+
if err := c.Create(ctx, &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: job.Namespace, Name: job.Name}}); err != nil {
2766+
t.Fatalf("seed Backup: %v", err)
2767+
}
2768+
r := &BackupJobReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(10)}
2769+
2770+
if _, err := r.reconcileFoundationDB(ctx, job, resolved); err != nil {
2771+
t.Fatalf("reconcileFoundationDB: %v", err)
2772+
}
2773+
if job.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
2774+
t.Errorf("phase = %q, want Failed for a name whose Backup already exists", job.Status.Phase)
2775+
}
2776+
if n := len(listFoundationDBBackups(t, c)); n != 0 {
2777+
t.Errorf("no backup may start in that directory, got %d CRs", n)
2778+
}
2779+
}
2780+
27272781
func TestCreateFoundationDBBackupArtifact_RefusesBackupOfAnotherCR(t *testing.T) {
27282782
apps := foundationdbapp.GroupName
27292783
strategyAPIGroup := strategyv1alpha1.GroupVersion.Group

0 commit comments

Comments
 (0)