Skip to content

Commit cc7f981

Browse files
fix(backups): read the keepOriginalPVC rename back past the cache
renamePVC created the -orig PVC, then read it back through the cached client to learn its UID. The informer had often not seen the PVC yet, so the read failed with NotFound. On a two-disk VM restore one disk was kept and the other was not; with that error turned into an event, its DataVolume was deleted, the original PVC stayed in place and Velero skipped it, and the restore reported success without restoring the disk. The -orig PVC was left Pending on the same PV, by then Retain. Create already returns the UID, so the read-back goes. The other reads the rename decides on, the -orig and original PVCs and the PV, go through the manager's uncached reader, and the PV rebind retries on conflict, since the PV controller writes the PV once the original PVC is deleted. A rename that fails for any reason other than an -orig name taken by another PVC now requeues the restore instead of failing it. The DataVolume is still there, and the retry finds its own -orig PVC by the RestoreJob UID and resumes where it stopped. Assisted-by: LLM Signed-off-by: Mattia Eleuteri <[email protected]>
1 parent d8d9608 commit cc7f981

3 files changed

Lines changed: 152 additions & 40 deletions

File tree

‎internal/backupcontroller/restorejob_controller.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,10 @@ type RestoreJobReconciler struct {
4949
// (missing pods/log RBAC) branch - is unit-testable without a live cluster.
5050
readPodLog func(ctx context.Context, namespace, podName, container string) (string, error)
5151
CredentialsConfig BackupCredentialsConfig
52+
// APIReader is the manager's uncached reader. The keepOriginalPVC rename
53+
// reads back objects it has just written, which the cache may not have
54+
// seen yet. Wired in SetupWithManager.
55+
APIReader client.Reader
5256
}
5357

5458
func (r *RestoreJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
@@ -174,6 +178,7 @@ func (r *RestoreJobReconciler) Reconcile(ctx context.Context, req ctrl.Request)
174178

175179
// SetupWithManager registers our controller with the Manager and sets up watches.
176180
func (r *RestoreJobReconciler) SetupWithManager(mgr ctrl.Manager) error {
181+
r.APIReader = mgr.GetAPIReader()
177182
cfg := mgr.GetConfig()
178183
var err error
179184
if r.Interface, err = dynamic.NewForConfig(cfg); err != nil {
@@ -194,6 +199,16 @@ func (r *RestoreJobReconciler) SetupWithManager(mgr ctrl.Manager) error {
194199
Complete(r)
195200
}
196201

202+
// apiReader returns the uncached reader. Reconcilers built by hand in unit
203+
// tests have none and fall back to their fake Client, which has no cache to
204+
// bypass.
205+
func (r *RestoreJobReconciler) apiReader() client.Reader {
206+
if r.APIReader != nil {
207+
return r.APIReader
208+
}
209+
return r.Client
210+
}
211+
197212
// handleProjectionError classifies a credentials-projection error as
198213
// transient (requeue with Ready=False/CredentialsProjectionPending) or
199214
// terminal (markRestoreJobFailed). Mirrors the BackupJob variant — both

‎internal/backupcontroller/velerostrategy_controller.go‎

Lines changed: 60 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
1919
"k8s.io/apimachinery/pkg/runtime"
2020
"k8s.io/apimachinery/pkg/runtime/schema"
21+
"k8s.io/client-go/util/retry"
2122
ctrl "sigs.k8s.io/controller-runtime"
2223
"sigs.k8s.io/controller-runtime/pkg/client"
2324
"sigs.k8s.io/controller-runtime/pkg/log"
@@ -1039,7 +1040,9 @@ func shortHash(input string) string {
10391040
// (e.g. restore requested when app was already deleted). Each action emits
10401041
// a Kubernetes Event on the RestoreJob for observability. The rename is the
10411042
// exception: the DataVolume deletion after it would hand a disk that was not
1042-
// kept to the restore, as keepOriginalPVC=false does, so it fails the restore.
1043+
// kept to the restore, as keepOriginalPVC=false does. A failed rename holds
1044+
// the restore back and is retried, and an -orig name another PVC holds fails
1045+
// the restore.
10431046
//
10441047
// postRestoreRename renames VMInstance HelmRelease after Velero Restore completes.
10451048
// Velero resource modifiers cannot change metadata.name, so this step creates
@@ -1136,7 +1139,13 @@ func (r *RestoreJobReconciler) prepareForRestore(ctx context.Context, restoreJob
11361139
// than fail halfway through a multi-disk rename.
11371140
if opts.GetKeepOriginalPVC() && vmRes != nil {
11381141
for _, dv := range vmRes.DataVolumes {
1139-
if _, err := r.checkOrigPVCName(ctx, restoreJob, ns, dv.DataVolumeName+origSuffix); err != nil {
1142+
origPVC, err := r.getOrigPVC(ctx, ns, dv.DataVolumeName+origSuffix)
1143+
if err != nil {
1144+
r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "PrepareForRestore",
1145+
fmt.Sprintf("Failed to check PVC %s: %v", dv.DataVolumeName+origSuffix, err))
1146+
return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
1147+
}
1148+
if err := foreignOrigPVC(origPVC, restoreJob); err != nil {
11401149
return false, ctrl.Result{}, err
11411150
}
11421151
}
@@ -1189,7 +1198,7 @@ func (r *RestoreJobReconciler) prepareForRestore(ctx context.Context, restoreJob
11891198
if err := r.renamePVC(ctx, restoreJob, ns, dv.DataVolumeName, dv.DataVolumeName+origSuffix); err != nil {
11901199
r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "PrepareForRestore",
11911200
fmt.Sprintf("Failed to keep old PVC %s: %v", dv.DataVolumeName, err))
1192-
return false, ctrl.Result{}, fmt.Errorf("failed to keep PVC %s: %w", dv.DataVolumeName, err)
1201+
return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
11931202
}
11941203
}
11951204
}
@@ -1279,21 +1288,26 @@ func (r *RestoreJobReconciler) deleteDataVolume(ctx context.Context, ns, name st
12791288
return nil
12801289
}
12811290

1282-
// checkOrigPVCName fails when newName is already taken by a PVC this
1283-
// RestoreJob did not create. Proceeding would hand the live disk to the
1284-
// restore under its own name, as keepOriginalPVC=false does, while the -orig
1285-
// name holds some other disk. It returns the PVC this RestoreJob did create,
1286-
// or nil.
1287-
func (r *RestoreJobReconciler) checkOrigPVCName(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, ns, newName string) (*corev1.PersistentVolumeClaim, error) {
1288-
existing := &corev1.PersistentVolumeClaim{}
1289-
if err := r.Get(ctx, client.ObjectKey{Namespace: ns, Name: newName}, existing); err != nil {
1291+
// getOrigPVC returns the PVC named name, or nil when there is none. It reads
1292+
// past the cache: a retried rename must see the -orig PVC it created, not
1293+
// take the name for free and fail to create it again.
1294+
func (r *RestoreJobReconciler) getOrigPVC(ctx context.Context, ns, name string) (*corev1.PersistentVolumeClaim, error) {
1295+
pvc := &corev1.PersistentVolumeClaim{}
1296+
if err := r.apiReader().Get(ctx, client.ObjectKey{Namespace: ns, Name: name}, pvc); err != nil {
12901297
return nil, client.IgnoreNotFound(err)
12911298
}
1292-
if existing.Annotations[origPVCRestoreJobUIDAnnotation] != string(restoreJob.UID) {
1293-
return nil, fmt.Errorf("PVC %s/%s already exists and was not created by this RestoreJob, so the current disk cannot be kept under that name: "+
1294-
"remove that PVC or create the RestoreJob under another name", ns, newName)
1299+
return pvc, nil
1300+
}
1301+
1302+
// foreignOrigPVC fails when pvc exists and restoreJob did not create it.
1303+
// Proceeding would hand the live disk to the restore under its own name, as
1304+
// keepOriginalPVC=false does, while the -orig name holds some other disk.
1305+
func foreignOrigPVC(pvc *corev1.PersistentVolumeClaim, restoreJob *backupsv1alpha1.RestoreJob) error {
1306+
if pvc == nil || pvc.Annotations[origPVCRestoreJobUIDAnnotation] == string(restoreJob.UID) {
1307+
return nil
12951308
}
1296-
return existing, nil
1309+
return fmt.Errorf("PVC %s/%s already exists and was not created by this RestoreJob, so the current disk cannot be kept under that name: "+
1310+
"remove that PVC or create the RestoreJob under another name", pvc.Namespace, pvc.Name)
12971311
}
12981312

12991313
// renamePVC preserves an existing PVC by rebinding it under a new name.
@@ -1304,14 +1318,17 @@ func (r *RestoreJobReconciler) checkOrigPVCName(ctx context.Context, restoreJob
13041318
func (r *RestoreJobReconciler) renamePVC(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, ns, oldName, newName string) error {
13051319
logger := getLogger(ctx)
13061320

1307-
newPVC, err := r.checkOrigPVCName(ctx, restoreJob, ns, newName)
1321+
newPVC, err := r.getOrigPVC(ctx, ns, newName)
13081322
if err != nil {
13091323
return err
13101324
}
1325+
if err := foreignOrigPVC(newPVC, restoreJob); err != nil {
1326+
return err
1327+
}
13111328
resuming := newPVC != nil
13121329

13131330
oldPVC := &corev1.PersistentVolumeClaim{}
1314-
if err := r.Get(ctx, client.ObjectKey{Namespace: ns, Name: oldName}, oldPVC); err != nil {
1331+
if err := r.apiReader().Get(ctx, client.ObjectKey{Namespace: ns, Name: oldName}, oldPVC); err != nil {
13151332
if !errors.IsNotFound(err) {
13161333
return err
13171334
}
@@ -1333,7 +1350,7 @@ func (r *RestoreJobReconciler) renamePVC(ctx context.Context, restoreJob *backup
13331350

13341351
// Patch PV reclaim policy to Retain so it survives PVC deletion.
13351352
// A policy already recorded is the one from before any rename.
1336-
if err := r.Get(ctx, client.ObjectKey{Name: pvName}, pv); err != nil {
1353+
if err := r.apiReader().Get(ctx, client.ObjectKey{Name: pvName}, pv); err != nil {
13371354
return fmt.Errorf("failed to get PV %s: %w", pvName, err)
13381355
}
13391356
originalPolicy, recorded := pv.Annotations[originalReclaimPolicyAnnotation]
@@ -1366,13 +1383,11 @@ func (r *RestoreJobReconciler) renamePVC(ctx context.Context, restoreJob *backup
13661383
VolumeName: pvName,
13671384
},
13681385
}
1386+
// Create fills in the UID the claimRef needs. Reading the PVC back
1387+
// through the cache can miss it, as it is this reconcile's own write.
13691388
if err := r.Create(ctx, newPVC); err != nil {
13701389
return fmt.Errorf("failed to create -orig PVC %s: %w", newName, err)
13711390
}
1372-
// Re-read to get UID for the claimRef
1373-
if err := r.Get(ctx, client.ObjectKey{Namespace: ns, Name: newName}, newPVC); err != nil {
1374-
return fmt.Errorf("failed to get -orig PVC %s: %w", newName, err)
1375-
}
13761391
}
13771392

13781393
// Delete the original PVC. Only while it still claims the kept PV: after
@@ -1386,22 +1401,32 @@ func (r *RestoreJobReconciler) renamePVC(ctx context.Context, restoreJob *backup
13861401

13871402
// Point the PV's claimRef directly to the new -orig PVC.
13881403
// This is atomic — no window where the PV is Available for other PVCs to grab.
1389-
if err := r.Get(ctx, client.ObjectKey{Name: pvName}, pv); err != nil {
1390-
return fmt.Errorf("failed to re-fetch PV %s: %w", pvName, err)
1404+
// The PV controller writes the PV too once the original PVC is gone, so
1405+
// the read is uncached and a conflict is retried.
1406+
rebound := false
1407+
err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
1408+
if err := r.apiReader().Get(ctx, client.ObjectKey{Name: pvName}, pv); err != nil {
1409+
return err
1410+
}
1411+
if ref := pv.Spec.ClaimRef; ref != nil && ref.Namespace == ns && ref.Name == newName && ref.UID == newPVC.UID {
1412+
return nil
1413+
}
1414+
pv.Spec.ClaimRef = &corev1.ObjectReference{
1415+
APIVersion: "v1",
1416+
Kind: "PersistentVolumeClaim",
1417+
Namespace: ns,
1418+
Name: newName,
1419+
UID: newPVC.UID,
1420+
}
1421+
rebound = true
1422+
return r.Update(ctx, pv)
1423+
})
1424+
if err != nil {
1425+
return fmt.Errorf("failed to rebind PV %s to %s: %w", pvName, newName, err)
13911426
}
1392-
if ref := pv.Spec.ClaimRef; ref != nil && ref.Namespace == ns && ref.Name == newName && ref.UID == newPVC.UID {
1427+
if !rebound {
13931428
return nil
13941429
}
1395-
pv.Spec.ClaimRef = &corev1.ObjectReference{
1396-
APIVersion: "v1",
1397-
Kind: "PersistentVolumeClaim",
1398-
Namespace: ns,
1399-
Name: newName,
1400-
UID: newPVC.UID,
1401-
}
1402-
if err := r.Update(ctx, pv); err != nil {
1403-
return fmt.Errorf("failed to rebind PV %s to %s: %w", pvName, newName, err)
1404-
}
14051430

14061431
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
14071432
fmt.Sprintf("Keep old PVC %s as %s (PV: %s)", oldName, newName, pvName))

‎internal/backupcontroller/velerostrategy_controller_test.go‎

Lines changed: 77 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"k8s.io/client-go/tools/record"
1919
"sigs.k8s.io/controller-runtime/pkg/client"
2020
clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
21+
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
2122

2223
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
2324
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
@@ -1726,17 +1727,88 @@ func TestRenamePVC_RecordsOriginalReclaimPolicy(t *testing.T) {
17261727
}
17271728
}
17281729

1729-
// Any failure to keep the disk must stop the restore before its DataVolume
1730-
// is deleted, not only a taken -orig name.
1730+
// Any failure to keep the disk must hold the restore back before its
1731+
// DataVolume is deleted, not only a taken -orig name, and the retry must
1732+
// finish the rename once the cause is gone.
17311733
func TestPrepareForRestore_KeepOriginalPVC_FailedRenameKeepsDataVolume(t *testing.T) {
17321734
f := newKeepPVCFixture(t)
17331735
r := f.reconciler(t, f.livePVC) // PV pv-current is missing
17341736
ctx := context.Background()
17351737

1736-
if _, _, err := r.prepareForRestore(ctx, f.restoreJob, f.backup, f.ur, f.target, f.opts); err == nil {
1737-
t.Error("prepareForRestore() succeeded although the PVC could not be kept")
1738+
ready, result, err := r.prepareForRestore(ctx, f.restoreJob, f.backup, f.ur, f.target, f.opts)
1739+
if err != nil || ready || result.RequeueAfter == 0 {
1740+
t.Errorf("prepareForRestore() = ready %v, %+v, error %v; want a requeue without failing the restore", ready, result, err)
17381741
}
17391742
if !f.dataVolumeExists(t, r) {
1740-
t.Error("DataVolume was deleted although the disk could not be kept")
1743+
t.Fatal("DataVolume was deleted although the disk could not be kept")
1744+
}
1745+
1746+
if err := r.Create(ctx, f.pv); err != nil {
1747+
t.Fatal(err)
1748+
}
1749+
ready, _, err = r.prepareForRestore(ctx, f.restoreJob, f.backup, f.ur, f.target, f.opts)
1750+
if err != nil || !ready {
1751+
t.Fatalf("retry: prepareForRestore() = ready %v, error %v; want ready", ready, err)
1752+
}
1753+
if err := r.Get(ctx, client.ObjectKey{Namespace: f.ns, Name: f.origName}, &corev1.PersistentVolumeClaim{}); err != nil {
1754+
t.Errorf("retry: -orig PVC: %v", err)
1755+
}
1756+
if f.dataVolumeExists(t, r) {
1757+
t.Error("retry: DataVolume was not deleted after the rename completed")
1758+
}
1759+
}
1760+
1761+
// laggingCacheReconciler serves every cached read from the objects as they
1762+
// were seeded, as an informer that has not caught up with this reconcile's
1763+
// own writes would; APIReader sees the writes.
1764+
func (f *keepPVCFixture) laggingCacheReconciler(t *testing.T, objects ...client.Object) *RestoreJobReconciler {
1765+
t.Helper()
1766+
r := f.reconciler(t, objects...)
1767+
seeded := append([]client.Object{f.restoreJob, f.backup}, objects...)
1768+
for i, obj := range seeded {
1769+
seeded[i] = obj.DeepCopyObject().(client.Object)
1770+
}
1771+
cache := clientfake.NewClientBuilder().WithScheme(r.Scheme).WithObjects(seeded...).Build()
1772+
apiServer := r.Client.(client.WithWatch)
1773+
r.Client = interceptor.NewClient(apiServer, interceptor.Funcs{
1774+
Get: func(ctx context.Context, _ client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
1775+
return cache.Get(ctx, key, obj, opts...)
1776+
},
1777+
})
1778+
r.APIReader = apiServer
1779+
return r
1780+
}
1781+
1782+
// On the lab, a two-disk restore failed one disk with "failed to get -orig
1783+
// PVC ...: not found": the PVC had just been created, and the cache had not
1784+
// seen it yet. The rename must not depend on the cache seeing its own writes.
1785+
func TestPrepareForRestore_KeepOriginalPVC_CacheLagsBehindWrites(t *testing.T) {
1786+
f := newKeepPVCFixture(t)
1787+
r := f.laggingCacheReconciler(t, f.livePVC, f.pv)
1788+
ctx := context.Background()
1789+
1790+
ready, _, err := r.prepareForRestore(ctx, f.restoreJob, f.backup, f.ur, f.target, f.opts)
1791+
if err != nil || !ready {
1792+
t.Fatalf("prepareForRestore() = ready %v, error %v; want ready", ready, err)
1793+
}
1794+
kept := &corev1.PersistentVolumeClaim{}
1795+
if err := r.APIReader.Get(ctx, client.ObjectKey{Namespace: f.ns, Name: f.origName}, kept); err != nil {
1796+
t.Fatalf("-orig PVC: %v", err)
1797+
}
1798+
if err := r.APIReader.Get(ctx, client.ObjectKeyFromObject(f.livePVC), &corev1.PersistentVolumeClaim{}); !errors.IsNotFound(err) {
1799+
t.Errorf("original PVC still exists (err %v)", err)
1800+
}
1801+
pv := &corev1.PersistentVolume{}
1802+
if err := r.APIReader.Get(ctx, client.ObjectKey{Name: "pv-current"}, pv); err != nil {
1803+
t.Fatal(err)
1804+
}
1805+
if ref := pv.Spec.ClaimRef; ref == nil || ref.Name != f.origName || ref.UID != kept.UID {
1806+
t.Errorf("PV claimRef = %+v, want %s", ref, f.origName)
1807+
}
1808+
if pv.Spec.PersistentVolumeReclaimPolicy != corev1.PersistentVolumeReclaimRetain {
1809+
t.Errorf("PV policy = %s, want Retain", pv.Spec.PersistentVolumeReclaimPolicy)
1810+
}
1811+
if f.dataVolumeExists(t, r) {
1812+
t.Error("DataVolume was not deleted after the rename completed")
17411813
}
17421814
}

0 commit comments

Comments
 (0)