@@ -18,6 +18,7 @@ import (
1818 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
1919 "k8s.io/apimachinery/pkg/runtime"
2020 "k8s.io/apimachinery/pkg/runtime/schema"
21+ "k8s.io/client-go/util/retry"
2122 ctrl "sigs.k8s.io/controller-runtime"
2223 "sigs.k8s.io/controller-runtime/pkg/client"
2324 "sigs.k8s.io/controller-runtime/pkg/log"
@@ -1039,7 +1040,9 @@ func shortHash(input string) string {
10391040// (e.g. restore requested when app was already deleted). Each action emits
10401041// a Kubernetes Event on the RestoreJob for observability. The rename is the
10411042// exception: the DataVolume deletion after it would hand a disk that was not
1042- // kept to the restore, as keepOriginalPVC=false does, so it fails the restore.
1043+ // kept to the restore, as keepOriginalPVC=false does. A failed rename holds
1044+ // the restore back and is retried, and an -orig name another PVC holds fails
1045+ // the restore.
10431046//
10441047// postRestoreRename renames VMInstance HelmRelease after Velero Restore completes.
10451048// Velero resource modifiers cannot change metadata.name, so this step creates
@@ -1136,7 +1139,13 @@ func (r *RestoreJobReconciler) prepareForRestore(ctx context.Context, restoreJob
11361139 // than fail halfway through a multi-disk rename.
11371140 if opts .GetKeepOriginalPVC () && vmRes != nil {
11381141 for _ , dv := range vmRes .DataVolumes {
1139- if _ , err := r .checkOrigPVCName (ctx , restoreJob , ns , dv .DataVolumeName + origSuffix ); err != nil {
1142+ origPVC , err := r .getOrigPVC (ctx , ns , dv .DataVolumeName + origSuffix )
1143+ if err != nil {
1144+ r .Recorder .Event (restoreJob , corev1 .EventTypeWarning , "PrepareForRestore" ,
1145+ fmt .Sprintf ("Failed to check PVC %s: %v" , dv .DataVolumeName + origSuffix , err ))
1146+ return false , ctrl.Result {RequeueAfter : defaultRestoreRequeueAfter }, nil
1147+ }
1148+ if err := foreignOrigPVC (origPVC , restoreJob ); err != nil {
11401149 return false , ctrl.Result {}, err
11411150 }
11421151 }
@@ -1189,7 +1198,7 @@ func (r *RestoreJobReconciler) prepareForRestore(ctx context.Context, restoreJob
11891198 if err := r .renamePVC (ctx , restoreJob , ns , dv .DataVolumeName , dv .DataVolumeName + origSuffix ); err != nil {
11901199 r .Recorder .Event (restoreJob , corev1 .EventTypeWarning , "PrepareForRestore" ,
11911200 fmt .Sprintf ("Failed to keep old PVC %s: %v" , dv .DataVolumeName , err ))
1192- return false , ctrl.Result {}, fmt . Errorf ( "failed to keep PVC %s: %w" , dv . DataVolumeName , err )
1201+ return false , ctrl.Result {RequeueAfter : defaultRestoreRequeueAfter }, nil
11931202 }
11941203 }
11951204 }
@@ -1279,21 +1288,26 @@ func (r *RestoreJobReconciler) deleteDataVolume(ctx context.Context, ns, name st
12791288 return nil
12801289}
12811290
1282- // checkOrigPVCName fails when newName is already taken by a PVC this
1283- // RestoreJob did not create. Proceeding would hand the live disk to the
1284- // restore under its own name, as keepOriginalPVC=false does, while the -orig
1285- // name holds some other disk. It returns the PVC this RestoreJob did create,
1286- // or nil.
1287- func (r * RestoreJobReconciler ) checkOrigPVCName (ctx context.Context , restoreJob * backupsv1alpha1.RestoreJob , ns , newName string ) (* corev1.PersistentVolumeClaim , error ) {
1288- existing := & corev1.PersistentVolumeClaim {}
1289- if err := r .Get (ctx , client.ObjectKey {Namespace : ns , Name : newName }, existing ); err != nil {
1291+ // getOrigPVC returns the PVC named name, or nil when there is none. It reads
1292+ // past the cache: a retried rename must see the -orig PVC it created, not
1293+ // take the name for free and fail to create it again.
1294+ func (r * RestoreJobReconciler ) getOrigPVC (ctx context.Context , ns , name string ) (* corev1.PersistentVolumeClaim , error ) {
1295+ pvc := & corev1.PersistentVolumeClaim {}
1296+ if err := r .apiReader ().Get (ctx , client.ObjectKey {Namespace : ns , Name : name }, pvc ); err != nil {
12901297 return nil , client .IgnoreNotFound (err )
12911298 }
1292- if existing .Annotations [origPVCRestoreJobUIDAnnotation ] != string (restoreJob .UID ) {
1293- return nil , fmt .Errorf ("PVC %s/%s already exists and was not created by this RestoreJob, so the current disk cannot be kept under that name: " +
1294- "remove that PVC or create the RestoreJob under another name" , ns , newName )
1299+ return pvc , nil
1300+ }
1301+
1302+ // foreignOrigPVC fails when pvc exists and restoreJob did not create it.
1303+ // Proceeding would hand the live disk to the restore under its own name, as
1304+ // keepOriginalPVC=false does, while the -orig name holds some other disk.
1305+ func foreignOrigPVC (pvc * corev1.PersistentVolumeClaim , restoreJob * backupsv1alpha1.RestoreJob ) error {
1306+ if pvc == nil || pvc .Annotations [origPVCRestoreJobUIDAnnotation ] == string (restoreJob .UID ) {
1307+ return nil
12951308 }
1296- return existing , nil
1309+ return fmt .Errorf ("PVC %s/%s already exists and was not created by this RestoreJob, so the current disk cannot be kept under that name: " +
1310+ "remove that PVC or create the RestoreJob under another name" , pvc .Namespace , pvc .Name )
12971311}
12981312
12991313// renamePVC preserves an existing PVC by rebinding it under a new name.
@@ -1304,14 +1318,17 @@ func (r *RestoreJobReconciler) checkOrigPVCName(ctx context.Context, restoreJob
13041318func (r * RestoreJobReconciler ) renamePVC (ctx context.Context , restoreJob * backupsv1alpha1.RestoreJob , ns , oldName , newName string ) error {
13051319 logger := getLogger (ctx )
13061320
1307- newPVC , err := r .checkOrigPVCName (ctx , restoreJob , ns , newName )
1321+ newPVC , err := r .getOrigPVC (ctx , ns , newName )
13081322 if err != nil {
13091323 return err
13101324 }
1325+ if err := foreignOrigPVC (newPVC , restoreJob ); err != nil {
1326+ return err
1327+ }
13111328 resuming := newPVC != nil
13121329
13131330 oldPVC := & corev1.PersistentVolumeClaim {}
1314- if err := r .Get (ctx , client.ObjectKey {Namespace : ns , Name : oldName }, oldPVC ); err != nil {
1331+ if err := r .apiReader (). Get (ctx , client.ObjectKey {Namespace : ns , Name : oldName }, oldPVC ); err != nil {
13151332 if ! errors .IsNotFound (err ) {
13161333 return err
13171334 }
@@ -1333,7 +1350,7 @@ func (r *RestoreJobReconciler) renamePVC(ctx context.Context, restoreJob *backup
13331350
13341351 // Patch PV reclaim policy to Retain so it survives PVC deletion.
13351352 // A policy already recorded is the one from before any rename.
1336- if err := r .Get (ctx , client.ObjectKey {Name : pvName }, pv ); err != nil {
1353+ if err := r .apiReader (). Get (ctx , client.ObjectKey {Name : pvName }, pv ); err != nil {
13371354 return fmt .Errorf ("failed to get PV %s: %w" , pvName , err )
13381355 }
13391356 originalPolicy , recorded := pv .Annotations [originalReclaimPolicyAnnotation ]
@@ -1366,13 +1383,11 @@ func (r *RestoreJobReconciler) renamePVC(ctx context.Context, restoreJob *backup
13661383 VolumeName : pvName ,
13671384 },
13681385 }
1386+ // Create fills in the UID the claimRef needs. Reading the PVC back
1387+ // through the cache can miss it, as it is this reconcile's own write.
13691388 if err := r .Create (ctx , newPVC ); err != nil {
13701389 return fmt .Errorf ("failed to create -orig PVC %s: %w" , newName , err )
13711390 }
1372- // Re-read to get UID for the claimRef
1373- if err := r .Get (ctx , client.ObjectKey {Namespace : ns , Name : newName }, newPVC ); err != nil {
1374- return fmt .Errorf ("failed to get -orig PVC %s: %w" , newName , err )
1375- }
13761391 }
13771392
13781393 // Delete the original PVC. Only while it still claims the kept PV: after
@@ -1386,22 +1401,32 @@ func (r *RestoreJobReconciler) renamePVC(ctx context.Context, restoreJob *backup
13861401
13871402 // Point the PV's claimRef directly to the new -orig PVC.
13881403 // This is atomic — no window where the PV is Available for other PVCs to grab.
1389- if err := r .Get (ctx , client.ObjectKey {Name : pvName }, pv ); err != nil {
1390- return fmt .Errorf ("failed to re-fetch PV %s: %w" , pvName , err )
1404+ // The PV controller writes the PV too once the original PVC is gone, so
1405+ // the read is uncached and a conflict is retried.
1406+ rebound := false
1407+ err = retry .RetryOnConflict (retry .DefaultRetry , func () error {
1408+ if err := r .apiReader ().Get (ctx , client.ObjectKey {Name : pvName }, pv ); err != nil {
1409+ return err
1410+ }
1411+ if ref := pv .Spec .ClaimRef ; ref != nil && ref .Namespace == ns && ref .Name == newName && ref .UID == newPVC .UID {
1412+ return nil
1413+ }
1414+ pv .Spec .ClaimRef = & corev1.ObjectReference {
1415+ APIVersion : "v1" ,
1416+ Kind : "PersistentVolumeClaim" ,
1417+ Namespace : ns ,
1418+ Name : newName ,
1419+ UID : newPVC .UID ,
1420+ }
1421+ rebound = true
1422+ return r .Update (ctx , pv )
1423+ })
1424+ if err != nil {
1425+ return fmt .Errorf ("failed to rebind PV %s to %s: %w" , pvName , newName , err )
13911426 }
1392- if ref := pv . Spec . ClaimRef ; ref != nil && ref . Namespace == ns && ref . Name == newName && ref . UID == newPVC . UID {
1427+ if ! rebound {
13931428 return nil
13941429 }
1395- pv .Spec .ClaimRef = & corev1.ObjectReference {
1396- APIVersion : "v1" ,
1397- Kind : "PersistentVolumeClaim" ,
1398- Namespace : ns ,
1399- Name : newName ,
1400- UID : newPVC .UID ,
1401- }
1402- if err := r .Update (ctx , pv ); err != nil {
1403- return fmt .Errorf ("failed to rebind PV %s to %s: %w" , pvName , newName , err )
1404- }
14051430
14061431 r .Recorder .Event (restoreJob , corev1 .EventTypeNormal , "PrepareForRestore" ,
14071432 fmt .Sprintf ("Keep old PVC %s as %s (PV: %s)" , oldName , newName , pvName ))
0 commit comments