Skip to content

Commit 29b0149

Browse files
committed
fix(mongodb): keep a completed archive reachable and a legacy Backup unblocked
Once the operator CR is ready the archive exists, and the Backup object is the only thing that can ever reach it again. A transient failure creating that object (etcd timeout, quota, webhook) failed the job terminally with no artifact, and the terminal-phase guard stopped any retry, so the dump sat in the shared bucket with nothing to reclaim it. The failure is now retried with backoff for a window after the dump completed, and when it does give up the job names the operator CR that still holds the archive. Cleanup read the operator CR before deciding whether the Backup owned anything, so a legacy Backup — an unconditional no-op before this driver — could be held Terminating by a throttle or a revoked verb on a CR the driver never owned. The snapshot records the flow, so a legacy Backup now releases before any apiserver read; a missing or undecodable snapshot still falls through to the read. The opt-out refusal recognised the lingering injected entry only by the strategy's current credentialsSecret and only while the strategy still carried s3, so a platform change to the strategy after a tenant opted out let a legacy CR be minted onto the platform bucket. The platform default Secret name is now recognised as well, independent of the strategy's current block. Assisted-by: LLM Signed-off-by: Andrey Kolkov <[email protected]>
1 parent 1927670 commit 29b0149

2 files changed

Lines changed: 210 additions & 3 deletions

File tree

‎internal/backupcontroller/psmdbstrategy_controller.go‎

Lines changed: 52 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -366,8 +366,8 @@ func (r *BackupJobReconciler) reconcileMongoDB(ctx context.Context, j *backupsv1
366366
if err != nil {
367367
return ctrl.Result{}, err
368368
}
369-
if existing == nil && !useSystemBucket && rendered.S3 != nil {
370-
if _, cred := psmdbStorageS3(cluster.Spec.Backup.Storages[storageName]); cred != "" && cred == psmdbInjectedCredentialsSecret(rendered.S3) {
369+
if existing == nil && !useSystemBucket {
370+
if _, cred := psmdbStorageS3(cluster.Spec.Backup.Storages[storageName]); cred != "" && psmdbCredentialsSecretIsInjected(cred, rendered) {
371371
if psmdbBackupDeadlineExceeded(j.Status.StartedAt) {
372372
return r.markBackupJobFailed(ctx, j, fmt.Sprintf(
373373
"psmdb.percona.com/PerconaServerMongoDB %s/%s storage %q is still the entry injected for the system bucket (credentialsSecret %q) while backup.useSystemBucket=false, %s after the job started; delete that storage entry from the PerconaServerMongoDB by hand or set backup.useSystemBucket=true again (see docs/operations/backup-classes.md)",
@@ -405,7 +405,21 @@ func (r *BackupJobReconciler) reconcileMongoDB(ctx context.Context, j *backupsv1
405405
}
406406
artifact, err := r.createMongoDBBackupArtifact(ctx, j, resolved, mdbBackup, rendered, storageName, flowSystemBucket)
407407
if err != nil {
408-
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err))
408+
// The archive is already written; the Backup object is the only thing
409+
// that can ever reach it again. A transient Create failure (etcd
410+
// timeout, quota, webhook hiccup) must not turn that into a terminal
411+
// Failed with no artifact, so retry with backoff for a window after
412+
// the dump completed — the SSA-apply error above gets the same
413+
// treatment — and only then give up, naming the CR that still holds
414+
// the archive so it can be reclaimed by hand.
415+
if !psmdbArtifactWindowExceeded(mdbBackup, j.Status.StartedAt) {
416+
return ctrl.Result{}, err
417+
}
418+
if r.Recorder != nil && flowSystemBucket {
419+
r.Recorder.Eventf(j, corev1.EventTypeWarning, "BackupArtifactNotCreated",
420+
"the dump completed but no Backup object could be created (%v); the archive is reachable only through PerconaServerMongoDBBackup %s, which is left in place", err, mdbBackup.Name)
421+
}
422+
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact for %s after retrying for %s: %v", mdbBackup.Name, psmdbLiveReadGrace, err))
409423
}
410424
now := metav1.Now()
411425
j.Status.BackupRef = &corev1.LocalObjectReference{Name: artifact.Name}
@@ -702,6 +716,19 @@ func psmdbInjectedCredentialsSecret(s3 *strategyv1alpha1.MongoDBStorageS3) strin
702716
return psmdbDefaultCredentialsSecret
703717
}
704718

719+
// psmdbCredentialsSecretIsInjected reports whether a live storage entry's
720+
// credentialsSecret is one the driver would have written: the strategy's
721+
// current name, or the platform default. Checking the default as well keeps a
722+
// lingering injected entry recognisable after the strategy drops or renames
723+
// its s3 block — otherwise the opt-out refusal would stop seeing the entry it
724+
// exists to refuse, and a legacy CR would be minted onto the platform bucket.
725+
func psmdbCredentialsSecretIsInjected(cred string, rendered *strategyv1alpha1.MongoDBTemplate) bool {
726+
if cred == psmdbDefaultCredentialsSecret {
727+
return true
728+
}
729+
return rendered.S3 != nil && cred == psmdbInjectedCredentialsSecret(rendered.S3)
730+
}
731+
705732
// shouldInjectMongoDBSystemStorage reports whether the driver must SSA-inject
706733
// the system-bucket storage onto the live cluster: only when the app opted in
707734
// via backup.useSystemBucket AND the strategy actually carries S3 coordinates
@@ -803,6 +830,17 @@ func psmdbBackupTimedOut(state string, startedAt *metav1.Time, useSystemBucket b
803830
return psmdbBackupDeadlineExceeded(startedAt)
804831
}
805832

833+
// psmdbArtifactWindowExceeded bounds how long the driver keeps retrying to
834+
// materialise the Backup object for a completed dump: psmdbLiveReadGrace after
835+
// the operator marked the CR complete, or after the backup deadline when the
836+
// operator did not record a completion time.
837+
func psmdbArtifactWindowExceeded(b *psmdbtypes.PerconaServerMongoDBBackup, startedAt *metav1.Time) bool {
838+
if b.Status.Completed != nil {
839+
return time.Since(b.Status.Completed.Time) > psmdbLiveReadGrace
840+
}
841+
return startedAt != nil && time.Since(startedAt.Time) > psmdbDefaultBackupDeadline+psmdbLiveReadGrace
842+
}
843+
806844
// psmdbErrorIsUnresolvedStorage reports whether a PerconaServerMongoDBBackup
807845
// error is the operator failing to resolve the named storage — the message the
808846
// v1.22.0 backup reconciler sets ("unable to get storage '<name>'") when it
@@ -1062,6 +1100,17 @@ func (r *BackupReconciler) cleanupMongoDBBackup(ctx context.Context, backup *bac
10621100
return r.releaseMongoDBCleanup(ctx, backup, sourceBackupName, nil, "skip-artifact-cleanup annotation set")
10631101
}
10641102

1103+
// A legacy Backup owns nothing on the shared bucket, and before this driver
1104+
// its cleanup was an unconditional no-op that no apiserver error could hold
1105+
// up. The snapshot records the flow, so decide from it first: when it is our
1106+
// kind and says legacy, release without reading the operator CR at all, so a
1107+
// throttle or a revoked verb on a CR the driver does not own cannot pin the
1108+
// Backup (or a namespace being torn down) in Terminating. A missing or
1109+
// undecodable snapshot falls through to the read below, never the other way.
1110+
if snap, serr := unmarshalMongoDBBackupSnapshot(backup.Status.UnderlyingResources); serr == nil && snap != nil && snap.Kind == psmdbBackupSnapshotKind && !snap.UseSystemBucket {
1111+
return ctrl.Result{}, nil
1112+
}
1113+
10651114
live := &psmdbtypes.PerconaServerMongoDBBackup{}
10661115
err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: sourceBackupName}, live)
10671116
if apierrors.IsNotFound(err) || apimeta.IsNoMatchError(err) {

‎internal/backupcontroller/psmdbstrategy_controller_test.go‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2955,3 +2955,161 @@ func TestReconcileMongoDB_StorageRaceRetryBudgetExhausted(t *testing.T) {
29552955
t.Errorf("no further CR may be minted once the budget is spent, got %d", len(list.Items))
29562956
}
29572957
}
2958+
2959+
// The archive exists once the operator CR is ready; the Backup object is the
2960+
// only thing that can ever reach it. A transient Create failure must retry (the
2961+
// job stays Running) rather than fail terminally with no artifact, and when the
2962+
// window is spent the failure names the CR that still holds the archive.
2963+
func TestReconcileMongoDB_ArtifactCreateFailureRetriesThenNamesTheCR(t *testing.T) {
2964+
build := func(t *testing.T, completed time.Time) (*BackupJobReconciler, client.Client, *record.FakeRecorder, *ResolvedBackupConfig, *backupsv1alpha1.BackupJob) {
2965+
job, strategy, app, cluster, resolved := mongodbInjectFixture(true)
2966+
job.Status.StartedAt = &metav1.Time{Time: time.Now().Add(-10 * time.Minute)}
2967+
cluster.Spec.Backup.Storages = map[string]runtime.RawExtension{
2968+
"s3-storage": {Raw: []byte(`{"type":"s3","s3":{"bucket":"cozy-backups","credentialsSecret":"cozy-backups-creds"}}`)},
2969+
}
2970+
ready := &psmdbtypes.PerconaServerMongoDBBackup{
2971+
ObjectMeta: metav1.ObjectMeta{
2972+
Namespace: "tenant", Name: "op-ready", Finalizers: []string{psmdbDeleteBackupFinalizer},
2973+
Labels: map[string]string{
2974+
backupsv1alpha1.OwningJobNameLabel: job.Name,
2975+
backupsv1alpha1.OwningJobNamespaceLabel: job.Namespace,
2976+
},
2977+
},
2978+
Status: psmdbtypes.PerconaServerMongoDBBackupStatus{
2979+
State: psmdbtypes.StateReady, Completed: &metav1.Time{Time: completed},
2980+
Destination: "s3://cozy-backups/tenant/app1/2026-09-20",
2981+
S3: &psmdbtypes.BackupStorageS3{Bucket: "cozy-backups", CredentialsSecret: psmdbDefaultCredentialsSecret},
2982+
},
2983+
}
2984+
sch := runtime.NewScheme()
2985+
_ = scheme.AddToScheme(sch)
2986+
_ = backupsv1alpha1.AddToScheme(sch)
2987+
_ = strategyv1alpha1.AddToScheme(sch)
2988+
_ = psmdbtypes.AddToScheme(sch)
2989+
_ = mongodbapp.AddToScheme(sch)
2990+
c := clientfake.NewClientBuilder().WithScheme(sch).WithObjects(job, strategy, app, cluster, ready).
2991+
WithStatusSubresource(&backupsv1alpha1.BackupJob{}, &backupsv1alpha1.Backup{}).
2992+
WithInterceptorFuncs(interceptor.Funcs{
2993+
Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error {
2994+
if _, ok := obj.(*backupsv1alpha1.Backup); ok {
2995+
return apierrors.NewServiceUnavailable("etcd timeout")
2996+
}
2997+
return cl.Create(ctx, obj, opts...)
2998+
},
2999+
}).Build()
3000+
rec := record.NewFakeRecorder(10)
3001+
return &BackupJobReconciler{Client: c, Scheme: sch, Recorder: rec}, c, rec, resolved, job
3002+
}
3003+
getJob := func(t *testing.T, c client.Client, name string) *backupsv1alpha1.BackupJob {
3004+
p := &backupsv1alpha1.BackupJob{}
3005+
if err := c.Get(context.Background(), types.NamespacedName{Namespace: "tenant", Name: name}, p); err != nil {
3006+
t.Fatalf("get job: %v", err)
3007+
}
3008+
return p
3009+
}
3010+
3011+
t.Run("inside the window: retried, not failed", func(t *testing.T) {
3012+
r, c, _, resolved, job := build(t, time.Now())
3013+
if _, err := r.reconcileMongoDB(context.Background(), job.DeepCopy(), resolved); err == nil {
3014+
t.Fatalf("a transient artifact Create failure must be returned for backoff, got nil")
3015+
}
3016+
if p := getJob(t, c, job.Name); p.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
3017+
t.Errorf("a transient artifact Create failure must not fail the job while the window is open")
3018+
}
3019+
})
3020+
3021+
t.Run("window spent: fails and names the CR holding the archive", func(t *testing.T) {
3022+
r, c, rec, resolved, job := build(t, time.Now().Add(-2*psmdbLiveReadGrace))
3023+
if _, err := r.reconcileMongoDB(context.Background(), job.DeepCopy(), resolved); err != nil {
3024+
t.Fatalf("reconcileMongoDB: %v", err)
3025+
}
3026+
p := getJob(t, c, job.Name)
3027+
if p.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
3028+
t.Fatalf("expected Failed once the artifact window is spent, got phase=%q", p.Status.Phase)
3029+
}
3030+
if cond := apimeta.FindStatusCondition(p.Status.Conditions, "Ready"); cond == nil || !strings.Contains(cond.Message, "op-ready") {
3031+
t.Errorf("the terminal message must name the CR that still holds the archive, got %+v", cond)
3032+
}
3033+
select {
3034+
case ev := <-rec.Events:
3035+
if !strings.Contains(ev, "BackupArtifactNotCreated") || !strings.Contains(ev, "op-ready") {
3036+
t.Errorf("expected a BackupArtifactNotCreated Warning naming the CR, got %q", ev)
3037+
}
3038+
default:
3039+
t.Errorf("expected a Warning naming the orphaned archive's CR, got none")
3040+
}
3041+
})
3042+
}
3043+
3044+
// A legacy Backup owns nothing on the shared bucket; before this driver its
3045+
// cleanup was an unconditional no-op. With the flow recorded on the snapshot,
3046+
// cleanup must release a legacy Backup without any apiserver read, so a throttle
3047+
// or revoked verb on a CR the driver does not own cannot hold it Terminating.
3048+
func TestCleanupMongoDBBackup_LegacySnapshotSkipsTheRead(t *testing.T) {
3049+
legacySnap := &psmdbtypes.PerconaServerMongoDBBackup{Status: psmdbtypes.PerconaServerMongoDBBackupStatus{
3050+
Destination: "s3://tenant-own/x", S3: &psmdbtypes.BackupStorageS3{Bucket: "tenant-own", CredentialsSecret: "mongodb-app1-s3-creds"},
3051+
}}
3052+
raw, err := marshalMongoDBBackupSnapshot(legacySnap, &strategyv1alpha1.MongoDBTemplate{Type: "logical"}, "s3-storage", nil, false)
3053+
if err != nil {
3054+
t.Fatalf("marshal: %v", err)
3055+
}
3056+
s := runtime.NewScheme()
3057+
_ = scheme.AddToScheme(s)
3058+
_ = backupsv1alpha1.AddToScheme(s)
3059+
_ = psmdbtypes.AddToScheme(s)
3060+
reads := 0
3061+
c := clientfake.NewClientBuilder().WithScheme(s).WithInterceptorFuncs(interceptor.Funcs{
3062+
Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
3063+
if _, ok := obj.(*psmdbtypes.PerconaServerMongoDBBackup); ok {
3064+
reads++
3065+
return apierrors.NewTooManyRequests("apiserver throttled", 1)
3066+
}
3067+
return cl.Get(ctx, key, obj, opts...)
3068+
},
3069+
}).Build()
3070+
r := &BackupReconciler{Client: c, Recorder: record.NewFakeRecorder(10)}
3071+
backup := &backupsv1alpha1.Backup{
3072+
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "legacy-bk"},
3073+
Spec: backupsv1alpha1.BackupSpec{DriverMetadata: map[string]string{psmdbBackupNameKey: "op-legacy"}},
3074+
Status: backupsv1alpha1.BackupStatus{UnderlyingResources: raw},
3075+
}
3076+
res, err := r.cleanupMongoDBBackup(context.Background(), backup)
3077+
if err != nil || res.RequeueAfter != 0 {
3078+
t.Fatalf("a legacy Backup must release without depending on the operator CR read, got res=%+v err=%v", res, err)
3079+
}
3080+
if reads != 0 {
3081+
t.Errorf("a legacy Backup must not read the operator CR at all, got %d reads", reads)
3082+
}
3083+
}
3084+
3085+
// After a tenant opts out, the platform may drop or rename the strategy's s3
3086+
// block; the lingering injected entry still carries the platform default Secret
3087+
// and must still be refused, or a legacy CR is minted onto the platform bucket.
3088+
func TestReconcileMongoDB_OptOutRefusalSurvivesStrategyWithoutS3(t *testing.T) {
3089+
job, strategy, app, cluster, resolved := mongodbInjectFixture(true)
3090+
app.Spec.Backup.UseSystemBucket = false
3091+
strategy.Spec.Template.S3 = nil // platform dropped the coordinates after the opt-out
3092+
job.Status.StartedAt = &metav1.Time{Time: time.Now()}
3093+
cluster.Spec.Backup.Storages = map[string]runtime.RawExtension{
3094+
"s3-storage": {Raw: []byte(`{"type":"s3","s3":{"bucket":"cozy-backups","credentialsSecret":"cozy-backups-creds"}}`)},
3095+
}
3096+
c := newMongoDBStrategyTestClient(t, job, strategy, app, cluster)
3097+
r := &BackupJobReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(10)}
3098+
if _, err := r.reconcileMongoDB(context.Background(), job.DeepCopy(), resolved); err != nil {
3099+
t.Fatalf("reconcileMongoDB: %v", err)
3100+
}
3101+
list := &psmdbtypes.PerconaServerMongoDBBackupList{}
3102+
if err := c.List(context.Background(), list, client.InNamespace("tenant")); err != nil {
3103+
t.Fatalf("list: %v", err)
3104+
}
3105+
if len(list.Items) != 0 {
3106+
t.Errorf("the lingering injected entry must still be refused when the strategy carries no s3, got %d CRs minted", len(list.Items))
3107+
}
3108+
persisted := &backupsv1alpha1.BackupJob{}
3109+
if err := c.Get(context.Background(), types.NamespacedName{Namespace: "tenant", Name: job.Name}, persisted); err != nil {
3110+
t.Fatalf("get job: %v", err)
3111+
}
3112+
if cond := apimeta.FindStatusCondition(persisted.Status.Conditions, "Ready"); cond == nil || cond.Reason != "PerconaServerMongoDBStorageStale" {
3113+
t.Errorf("expected Ready=False PerconaServerMongoDBStorageStale, got %+v", cond)
3114+
}
3115+
}

0 commit comments

Comments
 (0)