Skip to content

Commit f17edc8

Browse files
fix(backups): delete the original PVC of an in-place VM restore without keepOriginalPVC
With keepOriginalPVC=false the restore deleted each DataVolume with Orphan propagation and left its PVC in place for Velero to overwrite. Velero never overwrites a PVC: its CSI restore action skips the data mover for a claim that already exists, and existingResourcePolicy=update fails to patch the immutable spec of a bound claim, so it only stamps its labels on it. The RestoreJob then reports Succeeded with the disk still holding its pre-restore data. The claim left behind also breaks the disk for good. It has lost its owner reference to the orphaned DataVolume, and it lacks the cdi.kubevirt.io/storage.populatedFor annotation that the backed-up copy carries, so the CDI webhook refuses the DataVolume the resumed HelmRelease creates ("Destination PVC already exists"), on every reconcile. Delete the PVC once its DataVolume is gone, and hold the Velero Restore until the claim has actually disappeared: one still mounted stays Terminating behind pvc-protection, and Velero would take it for an existing claim again. Assisted-by: LLM Signed-off-by: Mattia Eleuteri <[email protected]>
1 parent bc400df commit f17edc8

2 files changed

Lines changed: 126 additions & 29 deletions

File tree

‎internal/backupcontroller/velerostrategy_controller.go‎

Lines changed: 56 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,8 @@ type CommonRestoreOptions struct {
9797
// KeepOriginalIpAndMac) are only effective when the application kind is VMInstance.
9898
type RestoreOptions struct {
9999
CommonRestoreOptions `json:",inline"`
100-
// KeepOriginalPVC renames the original PVC to <name>-orig-<hash> before restore.
100+
// KeepOriginalPVC renames the original PVC to <name>-orig-<hash> before restore;
101+
// when false, the original PVC is deleted so that Velero recreates it.
101102
// Only effective for in-place VMInstance restore (no targetNamespace). Defaults to true when omitted.
102103
KeepOriginalPVC *bool `json:"keepOriginalPVC,omitempty"`
103104
// KeepOriginalIpAndMac preserves the original IP and MAC address via OVN
@@ -1182,6 +1183,31 @@ func (r *RestoreJobReconciler) prepareForRestore(ctx context.Context, restoreJob
11821183
}
11831184
}
11841185

1186+
// --- Step 5: Delete the PVCs a restore without keepOriginalPVC replaces ---
1187+
// Velero restores a PVC only by creating it: its CSI restore action skips the
1188+
// data mover for a PVC that already exists, and existingResourcePolicy=update
1189+
// cannot patch the immutable spec of a bound claim. A PVC left in place keeps
1190+
// its old data, and, orphaned from its DataVolume in step 4 and without the
1191+
// cdi.kubevirt.io/storage.populatedFor the backed-up copy carries, it makes
1192+
// CDI refuse the DataVolume the resumed HelmRelease creates.
1193+
if !opts.GetKeepOriginalPVC() && vmRes != nil {
1194+
var terminating []string
1195+
for _, dv := range vmRes.DataVolumes {
1196+
gone, err := r.deletePVC(ctx, ns, dv.DataVolumeName)
1197+
if err != nil {
1198+
return false, ctrl.Result{}, fmt.Errorf("failed to delete PVC %s: %w", dv.DataVolumeName, err)
1199+
}
1200+
if !gone {
1201+
terminating = append(terminating, dv.DataVolumeName)
1202+
}
1203+
}
1204+
if len(terminating) > 0 {
1205+
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
1206+
fmt.Sprintf("Waiting for PVCs %s to be deleted", strings.Join(terminating, ", ")))
1207+
return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
1208+
}
1209+
}
1210+
11851211
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore", "Pre-restore preparation complete")
11861212
return true, ctrl.Result{}, nil
11871213
}
@@ -1240,9 +1266,9 @@ func (r *RestoreJobReconciler) haltVirtualMachine(ctx context.Context, ns, vmNam
12401266
}
12411267

12421268
// deleteDataVolume deletes a DataVolume so CDI doesn't recreate the PVC after rename.
1243-
// Uses Orphan propagation to avoid cascade-deleting the PVC that the DV owns via
1244-
// ownerReference. Without this, keepOriginalPVC=false would silently destroy the
1245-
// original PVC through garbage collection instead of leaving it for Velero to overwrite.
1269+
// Uses Orphan propagation so that the PVC the DV owns via ownerReference is
1270+
// removed by prepareForRestore, which waits for it, rather than by background
1271+
// garbage collection, which nothing waits for before the Velero Restore starts.
12461272
func (r *RestoreJobReconciler) deleteDataVolume(ctx context.Context, ns, name string) error {
12471273
orphan := metav1.DeletePropagationOrphan
12481274
err := r.Resource(dataVolumeGVR).Namespace(ns).Delete(ctx, name, metav1.DeleteOptions{
@@ -1254,6 +1280,32 @@ func (r *RestoreJobReconciler) deleteDataVolume(ctx context.Context, ns, name st
12541280
return nil
12551281
}
12561282

1283+
// deletePVC deletes a PVC and reports whether it is gone. A claim still mounted
1284+
// stays Terminating behind kubernetes.io/pvc-protection, and Velero would find
1285+
// it still there.
1286+
func (r *RestoreJobReconciler) deletePVC(ctx context.Context, ns, name string) (bool, error) {
1287+
key := client.ObjectKey{Namespace: ns, Name: name}
1288+
pvc := &corev1.PersistentVolumeClaim{}
1289+
if err := r.Get(ctx, key, pvc); err != nil {
1290+
if errors.IsNotFound(err) {
1291+
return true, nil
1292+
}
1293+
return false, err
1294+
}
1295+
if pvc.DeletionTimestamp == nil {
1296+
if err := r.Delete(ctx, pvc); err != nil && !errors.IsNotFound(err) {
1297+
return false, err
1298+
}
1299+
}
1300+
if err := r.Get(ctx, key, pvc); err != nil {
1301+
if errors.IsNotFound(err) {
1302+
return true, nil
1303+
}
1304+
return false, err
1305+
}
1306+
return false, nil
1307+
}
1308+
12571309
// renamePVC preserves an existing PVC by rebinding it under a new name.
12581310
// The original PVC is deleted and a new one pointing to the same PV is created.
12591311
// Missing resources are silently skipped (non-fatal).

‎internal/backupcontroller/velerostrategy_controller_test.go‎

Lines changed: 70 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -402,18 +402,9 @@ func newTestRestoreJobReconciler(t *testing.T, objects ...client.Object) *Restor
402402
}
403403
}
404404

405-
func TestPrepareForRestore_KeepOriginalPVCFalse_SkipsRename(t *testing.T) {
406-
ns := "tenant-root"
407-
408-
// Create a PVC that would be renamed if keepOriginalPVC were true
409-
pvc := &corev1.PersistentVolumeClaim{
410-
ObjectMeta: metav1.ObjectMeta{
411-
Name: "vm-disk-ubuntu-source",
412-
Namespace: ns,
413-
},
414-
Spec: corev1.PersistentVolumeClaimSpec{},
415-
}
416-
405+
// inPlaceVMRestoreWithoutKeep is an in-place VMInstance restore of one disk,
406+
// vm-disk-ubuntu-source, with keepOriginalPVC=false.
407+
func inPlaceVMRestoreWithoutKeep(ns string) (*backupsv1alpha1.RestoreJob, *backupsv1alpha1.Backup, *runtime.RawExtension, restoreTarget, RestoreOptions) {
417408
restoreJob := &backupsv1alpha1.RestoreJob{
418409
ObjectMeta: metav1.ObjectMeta{
419410
Name: "restore-test",
@@ -438,14 +429,12 @@ func TestPrepareForRestore_KeepOriginalPVCFalse_SkipsRename(t *testing.T) {
438429
},
439430
}
440431

441-
// underlyingResources with a DataVolume referencing the PVC
442432
urData := vmInstanceResources{
443433
DataVolumes: []backupsv1alpha1.DataVolumeResource{
444434
{DataVolumeName: "vm-disk-ubuntu-source", ApplicationName: "ubuntu-source"},
445435
},
446436
}
447437
urRaw, _ := json.Marshal(urData)
448-
ur := &runtime.RawExtension{Raw: urRaw}
449438

450439
target := restoreTarget{
451440
Namespace: ns,
@@ -454,9 +443,21 @@ func TestPrepareForRestore_KeepOriginalPVCFalse_SkipsRename(t *testing.T) {
454443
IsCopy: false,
455444
}
456445

457-
// keepOriginalPVC = false → PVCs should NOT be renamed
458-
opts := RestoreOptions{
459-
KeepOriginalPVC: new(false),
446+
return restoreJob, backup, &runtime.RawExtension{Raw: urRaw}, target, RestoreOptions{KeepOriginalPVC: new(false)}
447+
}
448+
449+
// Velero only restores a PVC it creates: one still in place keeps its old data
450+
// and blocks the DataVolume the HelmRelease recreates.
451+
func TestPrepareForRestore_KeepOriginalPVCFalse_DeletesPVC(t *testing.T) {
452+
ns := "tenant-root"
453+
restoreJob, backup, ur, target, opts := inPlaceVMRestoreWithoutKeep(ns)
454+
455+
pvc := &corev1.PersistentVolumeClaim{
456+
ObjectMeta: metav1.ObjectMeta{
457+
Name: "vm-disk-ubuntu-source",
458+
Namespace: ns,
459+
},
460+
Spec: corev1.PersistentVolumeClaimSpec{},
460461
}
461462

462463
reconciler := newTestRestoreJobReconciler(t, pvc, restoreJob, backup)
@@ -470,22 +471,66 @@ func TestPrepareForRestore_KeepOriginalPVCFalse_SkipsRename(t *testing.T) {
470471
t.Fatal("expected ready=true, got false")
471472
}
472473

473-
// Verify the original PVC still exists with its original name (not renamed)
474-
origPVC := &corev1.PersistentVolumeClaim{}
475-
err = reconciler.Get(ctx, client.ObjectKey{Namespace: ns, Name: "vm-disk-ubuntu-source"}, origPVC)
476-
if err != nil {
477-
t.Errorf("original PVC should still exist with original name when keepOriginalPVC=false, got error: %v", err)
474+
err = reconciler.Get(ctx, client.ObjectKey{Namespace: ns, Name: "vm-disk-ubuntu-source"}, &corev1.PersistentVolumeClaim{})
475+
if !errors.IsNotFound(err) {
476+
t.Errorf("original PVC should be gone before Velero restores it when keepOriginalPVC=false, got error: %v", err)
478477
}
479478

480-
// Verify no -orig PVC was created
481479
origSuffix := "-orig-" + shortHash(restoreJob.Name)
482-
renamedPVC := &corev1.PersistentVolumeClaim{}
483-
err = reconciler.Get(ctx, client.ObjectKey{Namespace: ns, Name: "vm-disk-ubuntu-source" + origSuffix}, renamedPVC)
480+
err = reconciler.Get(ctx, client.ObjectKey{Namespace: ns, Name: "vm-disk-ubuntu-source" + origSuffix}, &corev1.PersistentVolumeClaim{})
484481
if err == nil {
485482
t.Error("PVC should NOT have been renamed when keepOriginalPVC=false, but found renamed PVC")
486483
}
487484
}
488485

486+
func TestPrepareForRestore_KeepOriginalPVCFalse_WaitsForTerminatingPVC(t *testing.T) {
487+
ns := "tenant-root"
488+
restoreJob, backup, ur, target, opts := inPlaceVMRestoreWithoutKeep(ns)
489+
490+
pvc := &corev1.PersistentVolumeClaim{
491+
ObjectMeta: metav1.ObjectMeta{
492+
Name: "vm-disk-ubuntu-source",
493+
Namespace: ns,
494+
Finalizers: []string{"kubernetes.io/pvc-protection"},
495+
},
496+
Spec: corev1.PersistentVolumeClaimSpec{},
497+
}
498+
499+
reconciler := newTestRestoreJobReconciler(t, pvc, restoreJob, backup)
500+
501+
ctx := context.Background()
502+
ready, result, err := reconciler.prepareForRestore(ctx, restoreJob, backup, ur, target, opts)
503+
if err != nil {
504+
t.Fatalf("prepareForRestore() error = %v", err)
505+
}
506+
if ready {
507+
t.Fatal("expected ready=false while the PVC is still terminating, got true")
508+
}
509+
if result.RequeueAfter == 0 {
510+
t.Error("expected a requeue while the PVC is still terminating")
511+
}
512+
513+
stuck := &corev1.PersistentVolumeClaim{}
514+
if err := reconciler.Get(ctx, client.ObjectKey{Namespace: ns, Name: "vm-disk-ubuntu-source"}, stuck); err != nil {
515+
t.Fatalf("terminating PVC should still be readable: %v", err)
516+
}
517+
if stuck.DeletionTimestamp == nil {
518+
t.Fatal("PVC should have been marked for deletion")
519+
}
520+
stuck.Finalizers = nil
521+
if err := reconciler.Update(ctx, stuck); err != nil {
522+
t.Fatalf("failed to release the PVC finalizer: %v", err)
523+
}
524+
525+
ready, _, err = reconciler.prepareForRestore(ctx, restoreJob, backup, ur, target, opts)
526+
if err != nil {
527+
t.Fatalf("prepareForRestore() error = %v", err)
528+
}
529+
if !ready {
530+
t.Fatal("expected ready=true once the PVC is gone, got false")
531+
}
532+
}
533+
489534
func TestResolveRestoreTarget_VMDisk_CommonRestoreOptions(t *testing.T) {
490535
tests := []struct {
491536
name string

0 commit comments

Comments
 (0)