Skip to content

Commit 6d064a8

Browse files
fix(backups): delete a restored disk's PVC only once the restore can run
Deleting the original PVC of a keepOriginalPVC=false restore is only safe when Velero can put the disk back. The Backup artifact outlives the Velero Backup it points to, which Velero garbage-collects at TTL, and nothing checked it before prepareForRestore halted the VM and deleted the disks: a restore of an expired backup lost the volume where it used to fail with the disk intact. The Velero Backup must now exist and be Completed before any preparation starts. A DataVolume that survives its deletion makes CDI recreate the claim under the original name, which Velero then skips as an existing PVC, so the restore would end Succeeded on a freshly imported disk. A failed DataVolume deletion now fails the restore, and the PVCs are deleted only once every DataVolume is gone. Reading the DataVolume back needs get on datavolumes, which the ClusterRole did not grant. Assisted-by: LLM Signed-off-by: Mattia Eleuteri <[email protected]>
1 parent 6ea713d commit 6d064a8

4 files changed

Lines changed: 300 additions & 33 deletions

File tree

‎internal/backupcontroller/velerostrategy_controller.go‎

Lines changed: 75 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -698,8 +698,26 @@ func (r *RestoreJobReconciler) reconcileVeleroRestore(ctx context.Context, resto
698698
}
699699
}
700700

701+
// The Backup artifact outlives the Velero Backup it points to, which
702+
// Velero garbage-collects once its TTL expires. Checked before
703+
// prepareForRestore halts the application and deletes its disks, so a
704+
// restore that cannot run leaves them in place.
705+
veleroBackup := &velerov1.Backup{}
706+
if err := r.Get(ctx, client.ObjectKey{Namespace: veleroNamespace, Name: veleroBackupName}, veleroBackup); err != nil {
707+
if errors.IsNotFound(err) {
708+
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf(
709+
"Velero Backup %s/%s not found; it may have expired", veleroNamespace, veleroBackupName))
710+
}
711+
return ctrl.Result{}, fmt.Errorf("failed to get Velero Backup %s: %w", veleroBackupName, err)
712+
}
713+
if veleroBackup.Status.Phase != velerov1.BackupPhaseCompleted {
714+
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf(
715+
"Velero Backup %s/%s is %q, not %q", veleroNamespace, veleroBackupName,
716+
veleroBackup.Status.Phase, velerov1.BackupPhaseCompleted))
717+
}
718+
701719
// Resolve underlying resources once; prefer Backup status, fall back to Velero annotation.
702-
ur := r.resolveUnderlyingResourcesForRestore(ctx, backup, veleroBackupName)
720+
ur := resolveUnderlyingResourcesForRestore(backup, veleroBackup)
703721

704722
// Pre-restore: graceful shutdown, suspend HRs, rename PVCs (skipped for copy)
705723
ready, result, err := r.prepareForRestore(ctx, restoreJob, backup, ur, target, restoreOpts)
@@ -961,14 +979,10 @@ func (r *RestoreJobReconciler) createResourceModifiersConfigMap(ctx context.Cont
961979
// resolveUnderlyingResourcesForRestore returns underlying resources for symmetric
962980
// restore label selectors. Velero Backup annotation is used when Backup.status was empty
963981
// (e.g. CRD without underlyingResources in schema).
964-
func (r *RestoreJobReconciler) resolveUnderlyingResourcesForRestore(ctx context.Context, backup *backupsv1alpha1.Backup, veleroBackupName string) *runtime.RawExtension {
982+
func resolveUnderlyingResourcesForRestore(backup *backupsv1alpha1.Backup, vb *velerov1.Backup) *runtime.RawExtension {
965983
if backup.Status.UnderlyingResources != nil && len(backup.Status.UnderlyingResources.Raw) > 0 {
966984
return backup.Status.UnderlyingResources
967985
}
968-
vb := &velerov1.Backup{}
969-
if err := r.Get(ctx, client.ObjectKey{Namespace: veleroNamespace, Name: veleroBackupName}, vb); err != nil {
970-
return backup.Status.UnderlyingResources
971-
}
972986
if urJSON, ok := vb.Annotations[underlyingResourcesAnnotation]; ok && urJSON != "" {
973987
return &runtime.RawExtension{Raw: []byte(urJSON)}
974988
}
@@ -1037,13 +1051,14 @@ func shortHash(input string) string {
10371051
// 5. Renames existing PVCs to <name>-orig-<hash> so Velero can create fresh
10381052
// ones via Data Movement.
10391053
//
1040-
// Failures in individual steps are non-fatal: missing resources are expected
1041-
// (e.g. restore requested when app was already deleted). Each action emits
1042-
// a Kubernetes Event on the RestoreJob for observability. The rename is the
1043-
// exception: the DataVolume deletion after it would hand a disk that was not
1044-
// kept to the restore, as keepOriginalPVC=false does. A failed rename holds
1045-
// the restore back and is retried, and an -orig name another PVC holds fails
1046-
// the restore.
1054+
// Missing resources are expected (e.g. restore requested when app was already
1055+
// deleted), and a failure to suspend or halt is non-fatal. A failed rename
1056+
// holds the restore back and is retried, since the DataVolume deletion after
1057+
// it would hand a disk that was not kept to the restore, as
1058+
// keepOriginalPVC=false does, and an -orig name another PVC holds fails the
1059+
// restore. A DataVolume or a replaced PVC that cannot be deleted fails the
1060+
// restore, since Velero would skip the claim left in place. Each action emits
1061+
// a Kubernetes Event on the RestoreJob for observability.
10471062
//
10481063
// postRestoreRename renames VMInstance HelmRelease after Velero Restore completes.
10491064
// Velero resource modifiers cannot change metadata.name, so this step creates
@@ -1205,15 +1220,28 @@ func (r *RestoreJobReconciler) prepareForRestore(ctx context.Context, restoreJob
12051220
}
12061221

12071222
// --- Step 4: Delete DataVolumes so CDI doesn't recreate PVCs ---
1223+
// A DataVolume still present makes CDI recreate its claim under the original
1224+
// name, which Velero then skips as an existing PVC: the restore would end
1225+
// Succeeded on a freshly imported disk, so nothing goes further until every
1226+
// DataVolume is gone.
12081227
if vmRes != nil {
1228+
var terminating []string
12091229
for _, dv := range vmRes.DataVolumes {
1210-
if err := r.deleteDataVolume(ctx, ns, dv.DataVolumeName); err != nil {
1211-
r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "PrepareForRestore",
1212-
fmt.Sprintf("Failed to delete DataVolume %s: %v", dv.DataVolumeName, err))
1213-
} else {
1214-
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
1215-
fmt.Sprintf("Deleted DataVolume %s", dv.DataVolumeName))
1230+
gone, err := r.deleteDataVolume(ctx, ns, dv.DataVolumeName)
1231+
if err != nil {
1232+
return false, ctrl.Result{}, fmt.Errorf("failed to delete DataVolume %s: %w", dv.DataVolumeName, err)
1233+
}
1234+
if !gone {
1235+
terminating = append(terminating, dv.DataVolumeName)
1236+
continue
12161237
}
1238+
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
1239+
fmt.Sprintf("Deleted DataVolume %s", dv.DataVolumeName))
1240+
}
1241+
if len(terminating) > 0 {
1242+
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
1243+
fmt.Sprintf("Waiting for DataVolumes %s to be deleted", strings.Join(terminating, ", ")))
1244+
return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
12171245
}
12181246
}
12191247

@@ -1299,19 +1327,35 @@ func (r *RestoreJobReconciler) haltVirtualMachine(ctx context.Context, ns, vmNam
12991327
return false, nil
13001328
}
13011329

1302-
// deleteDataVolume deletes a DataVolume so CDI doesn't recreate the PVC after rename.
1303-
// Uses Orphan propagation so that the PVC the DV owns via ownerReference is
1304-
// removed by prepareForRestore, which waits for it, rather than by background
1305-
// garbage collection, which nothing waits for before the Velero Restore starts.
1306-
func (r *RestoreJobReconciler) deleteDataVolume(ctx context.Context, ns, name string) error {
1307-
orphan := metav1.DeletePropagationOrphan
1308-
err := r.Resource(dataVolumeGVR).Namespace(ns).Delete(ctx, name, metav1.DeleteOptions{
1309-
PropagationPolicy: &orphan,
1310-
})
1311-
if err != nil && !errors.IsNotFound(err) {
1312-
return err
1330+
// deleteDataVolume deletes a DataVolume so CDI doesn't recreate the PVC after
1331+
// rename, and reports whether it is gone. Orphan propagation holds the DV behind
1332+
// the orphan finalizer until garbage collection has released its PVC, and leaves
1333+
// that PVC to prepareForRestore, which waits for its deletion, rather than to
1334+
// background garbage collection, which nothing waits for before the Velero
1335+
// Restore starts.
1336+
func (r *RestoreJobReconciler) deleteDataVolume(ctx context.Context, ns, name string) (bool, error) {
1337+
dvClient := r.Resource(dataVolumeGVR).Namespace(ns)
1338+
dv, err := dvClient.Get(ctx, name, metav1.GetOptions{})
1339+
if err != nil {
1340+
if errors.IsNotFound(err) {
1341+
return true, nil
1342+
}
1343+
return false, err
13131344
}
1314-
return nil
1345+
if dv.GetDeletionTimestamp() == nil {
1346+
orphan := metav1.DeletePropagationOrphan
1347+
err := dvClient.Delete(ctx, name, metav1.DeleteOptions{PropagationPolicy: &orphan})
1348+
if err != nil && !errors.IsNotFound(err) {
1349+
return false, err
1350+
}
1351+
}
1352+
if _, err := dvClient.Get(ctx, name, metav1.GetOptions{}); err != nil {
1353+
if errors.IsNotFound(err) {
1354+
return true, nil
1355+
}
1356+
return false, err
1357+
}
1358+
return false, nil
13151359
}
13161360

13171361
// getOrigPVC returns the PVC named name, or nil when there is none. It reads

‎internal/backupcontroller/velerostrategy_controller_test.go‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"k8s.io/apimachinery/pkg/runtime/schema"
1616
dynamicfake "k8s.io/client-go/dynamic/fake"
1717
"k8s.io/client-go/kubernetes/scheme"
18+
k8stesting "k8s.io/client-go/testing"
1819
"k8s.io/client-go/tools/record"
1920
"sigs.k8s.io/controller-runtime/pkg/client"
2021
clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
@@ -532,6 +533,203 @@ func TestPrepareForRestore_KeepOriginalPVCFalse_WaitsForTerminatingPVC(t *testin
532533
}
533534
}
534535

536+
func makeUnstructuredDataVolume(name, namespace string) *unstructured.Unstructured {
537+
dv := &unstructured.Unstructured{}
538+
dv.SetAPIVersion("cdi.kubevirt.io/v1beta1")
539+
dv.SetKind("DataVolume")
540+
dv.SetName(name)
541+
dv.SetNamespace(namespace)
542+
return dv
543+
}
544+
545+
// A DataVolume still in place makes CDI recreate the claim under its original
546+
// name, so the PVC must survive until the DataVolume is confirmed gone.
547+
func TestPrepareForRestore_KeepOriginalPVCFalse_KeepsPVCWhileDataVolumeRemains(t *testing.T) {
548+
tests := []struct {
549+
name string
550+
deleteErr error
551+
wantErr bool
552+
}{
553+
{
554+
name: "DataVolume deletion fails",
555+
deleteErr: errors.NewForbidden(dataVolumeGVR.GroupResource(), "vm-disk-ubuntu-source", nil),
556+
wantErr: true,
557+
},
558+
{
559+
name: "DataVolume deletion is still pending",
560+
},
561+
}
562+
for _, tt := range tests {
563+
t.Run(tt.name, func(t *testing.T) {
564+
ns := "tenant-root"
565+
restoreJob, backup, ur, target, opts := inPlaceVMRestoreWithoutKeep(ns)
566+
pvc := &corev1.PersistentVolumeClaim{
567+
ObjectMeta: metav1.ObjectMeta{Name: "vm-disk-ubuntu-source", Namespace: ns},
568+
}
569+
reconciler := newTestRestoreJobReconcilerWithDynamic(t,
570+
[]runtime.Object{makeUnstructuredDataVolume("vm-disk-ubuntu-source", ns)},
571+
pvc, restoreJob, backup)
572+
reconciler.Interface.(*dynamicfake.FakeDynamicClient).PrependReactor("delete", "datavolumes",
573+
func(k8stesting.Action) (bool, runtime.Object, error) {
574+
return true, nil, tt.deleteErr
575+
})
576+
577+
ctx := context.Background()
578+
ready, result, err := reconciler.prepareForRestore(ctx, restoreJob, backup, ur, target, opts)
579+
if tt.wantErr {
580+
if err == nil {
581+
t.Fatal("expected an error when the DataVolume cannot be deleted")
582+
}
583+
} else {
584+
if err != nil {
585+
t.Fatalf("prepareForRestore() error = %v", err)
586+
}
587+
if result.RequeueAfter == 0 {
588+
t.Error("expected a requeue while the DataVolume is still present")
589+
}
590+
}
591+
if ready {
592+
t.Fatal("expected ready=false while the DataVolume is still present")
593+
}
594+
595+
got := &corev1.PersistentVolumeClaim{}
596+
if err := reconciler.Get(ctx, client.ObjectKeyFromObject(pvc), got); err != nil {
597+
t.Fatalf("original PVC should survive while its DataVolume remains: %v", err)
598+
}
599+
if got.DeletionTimestamp != nil {
600+
t.Error("original PVC should not be marked for deletion while its DataVolume remains")
601+
}
602+
})
603+
}
604+
}
605+
606+
// keepOriginalPVC=true must never delete the original claim, even when the
607+
// rename that preserves it fails and leaves it under its original name.
608+
func TestPrepareForRestore_KeepOriginalPVCTrue_FailedRenameKeepsPVC(t *testing.T) {
609+
ns := "tenant-root"
610+
restoreJob, backup, ur, target, _ := inPlaceVMRestoreWithoutKeep(ns)
611+
opts := RestoreOptions{KeepOriginalPVC: new(true)}
612+
613+
pvc := &corev1.PersistentVolumeClaim{
614+
ObjectMeta: metav1.ObjectMeta{Name: "vm-disk-ubuntu-source", Namespace: ns},
615+
// The PV does not exist, so renamePVC fails before touching the claim.
616+
Spec: corev1.PersistentVolumeClaimSpec{VolumeName: "pv-missing"},
617+
}
618+
reconciler := newTestRestoreJobReconciler(t, pvc, restoreJob, backup)
619+
620+
ctx := context.Background()
621+
if _, _, err := reconciler.prepareForRestore(ctx, restoreJob, backup, ur, target, opts); err != nil {
622+
t.Fatalf("prepareForRestore() error = %v", err)
623+
}
624+
625+
if err := reconciler.Get(ctx, client.ObjectKeyFromObject(pvc), &corev1.PersistentVolumeClaim{}); err != nil {
626+
t.Errorf("original PVC should survive a failed rename when keepOriginalPVC=true, got error: %v", err)
627+
}
628+
}
629+
630+
// A Backup artifact can outlive its Velero Backup, which Velero garbage-collects
631+
// at TTL. The restore must fail before it halts the VM or deletes its disks.
632+
func TestReconcileVeleroRestore_UnusableVeleroBackupTouchesNothing(t *testing.T) {
633+
tests := []struct {
634+
name string
635+
veleroBackup *velerov1.Backup
636+
wantMessage string
637+
}{
638+
{
639+
name: "Velero Backup expired",
640+
wantMessage: "Velero Backup cozy-velero/vb not found",
641+
},
642+
{
643+
name: "Velero Backup being deleted",
644+
veleroBackup: &velerov1.Backup{
645+
ObjectMeta: metav1.ObjectMeta{Name: "vb", Namespace: veleroNamespace},
646+
Status: velerov1.BackupStatus{Phase: velerov1.BackupPhaseDeleting},
647+
},
648+
wantMessage: `Velero Backup cozy-velero/vb is "Deleting"`,
649+
},
650+
}
651+
for _, tt := range tests {
652+
t.Run(tt.name, func(t *testing.T) {
653+
ns := "tenant-root"
654+
restoreJob, backup, ur, _, opts := inPlaceVMRestoreWithoutKeep(ns)
655+
rawOpts, err := json.Marshal(opts)
656+
if err != nil {
657+
t.Fatalf("marshal options: %v", err)
658+
}
659+
started := metav1.Now()
660+
restoreJob.Spec.Options = &runtime.RawExtension{Raw: rawOpts}
661+
restoreJob.Status = backupsv1alpha1.RestoreJobStatus{
662+
Phase: backupsv1alpha1.RestoreJobPhaseRunning,
663+
StartedAt: &started,
664+
}
665+
backup.Spec.StrategyRef = corev1.TypedLocalObjectReference{Kind: "Velero", Name: "cozy-default-velero"}
666+
backup.Spec.DriverMetadata = map[string]string{veleroBackupNameMetadataKey: "vb"}
667+
backup.Status.UnderlyingResources = ur
668+
669+
pvc := &corev1.PersistentVolumeClaim{
670+
ObjectMeta: metav1.ObjectMeta{Name: "vm-disk-ubuntu-source", Namespace: ns},
671+
}
672+
strategy := &strategyv1alpha1.Velero{ObjectMeta: metav1.ObjectMeta{Name: "cozy-default-velero"}}
673+
objs := []client.Object{restoreJob, backup, pvc, strategy}
674+
if tt.veleroBackup != nil {
675+
objs = append(objs, tt.veleroBackup)
676+
}
677+
678+
vm := &unstructured.Unstructured{}
679+
vm.SetAPIVersion("kubevirt.io/v1")
680+
vm.SetKind("VirtualMachine")
681+
vm.SetName(vmNamePrefix + "test-vm")
682+
vm.SetNamespace(ns)
683+
if err := unstructured.SetNestedField(vm.Object, "Always", "spec", "runStrategy"); err != nil {
684+
t.Fatalf("set runStrategy: %v", err)
685+
}
686+
reconciler := newTestRestoreJobReconcilerWithDynamic(t,
687+
[]runtime.Object{vm, makeUnstructuredDataVolume("vm-disk-ubuntu-source", ns)})
688+
testScheme := runtime.NewScheme()
689+
_ = scheme.AddToScheme(testScheme)
690+
_ = backupsv1alpha1.AddToScheme(testScheme)
691+
_ = velerov1.AddToScheme(testScheme)
692+
_ = strategyv1alpha1.AddToScheme(testScheme)
693+
reconciler.Client = clientfake.NewClientBuilder().
694+
WithScheme(testScheme).
695+
WithObjects(objs...).
696+
WithStatusSubresource(&backupsv1alpha1.RestoreJob{}).
697+
Build()
698+
ctx := context.Background()
699+
700+
if _, err := reconciler.reconcileVeleroRestore(ctx, restoreJob, backup); err != nil {
701+
t.Fatalf("reconcileVeleroRestore() error = %v", err)
702+
}
703+
704+
got := &backupsv1alpha1.RestoreJob{}
705+
if err := reconciler.Get(ctx, client.ObjectKeyFromObject(restoreJob), got); err != nil {
706+
t.Fatalf("get RestoreJob: %v", err)
707+
}
708+
if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
709+
t.Fatalf("phase = %q, want %q", got.Status.Phase, backupsv1alpha1.RestoreJobPhaseFailed)
710+
}
711+
ready := meta.FindStatusCondition(got.Status.Conditions, "Ready")
712+
if ready == nil || !strings.Contains(ready.Message, tt.wantMessage) {
713+
t.Errorf("Ready condition = %+v, want a message containing %q", ready, tt.wantMessage)
714+
}
715+
716+
if err := reconciler.Get(ctx, client.ObjectKeyFromObject(pvc), &corev1.PersistentVolumeClaim{}); err != nil {
717+
t.Errorf("original PVC should survive a restore that cannot run: %v", err)
718+
}
719+
if _, err := reconciler.Resource(dataVolumeGVR).Namespace(ns).Get(ctx, "vm-disk-ubuntu-source", metav1.GetOptions{}); err != nil {
720+
t.Errorf("DataVolume should survive a restore that cannot run: %v", err)
721+
}
722+
gotVM, err := reconciler.Resource(virtualMachineGVR).Namespace(ns).Get(ctx, vm.GetName(), metav1.GetOptions{})
723+
if err != nil {
724+
t.Fatalf("get VirtualMachine: %v", err)
725+
}
726+
if rs, _, _ := unstructured.NestedString(gotVM.Object, "spec", "runStrategy"); rs != "Always" {
727+
t.Errorf("VirtualMachine runStrategy = %q, want it left at %q", rs, "Always")
728+
}
729+
})
730+
}
731+
}
732+
535733
func TestResolveRestoreTarget_VMDisk_CommonRestoreOptions(t *testing.T) {
536734
tests := []struct {
537735
name string

‎packages/system/backupstrategy-controller/templates/rbac.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,10 +57,12 @@ rules:
5757
- apiGroups: ["kubevirt.io"]
5858
resources: ["virtualmachineinstances"]
5959
verbs: ["get"]
60-
# CDI DataVolumes: pre-restore deletes DVs so CDI doesn't recreate PVCs after rename
60+
# CDI DataVolumes: pre-restore deletes DVs so CDI doesn't recreate PVCs after rename,
61+
# and reads each one back to hold the restore until it is gone (a point Get
62+
# through the dynamic client, so no list/watch)
6163
- apiGroups: ["cdi.kubevirt.io"]
6264
resources: ["datavolumes"]
63-
verbs: ["delete"]
65+
verbs: ["get", "delete"]
6466
# HelmReleases: pre-restore suspends HRs; post-restore rename creates new HR
6567
# and deletes old. patch additionally lets the DefaultObjectsGate stamp
6668
# reconcile.fluxcd.io/forceAt + requestedAt on THIS chart's own release to
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
suite: RBAC of the Velero restore preparation
2+
3+
# prepareForRestore reads every DataVolume back after deleting it and holds
4+
# the Velero Restore until it is gone. The read goes through the dynamic
5+
# client, which the Go unit tests run with no authorization, so a missing verb
6+
# only shows up on a cluster, where it fails every in-place VM restore.
7+
8+
templates:
9+
- templates/rbac.yaml
10+
11+
release:
12+
name: backupstrategy-controller
13+
namespace: cozy-backup-controller
14+
15+
tests:
16+
- it: "ClusterRole can read and delete DataVolumes"
17+
asserts:
18+
- contains:
19+
path: rules
20+
content:
21+
apiGroups: ["cdi.kubevirt.io"]
22+
resources: ["datavolumes"]
23+
verbs: ["get", "delete"]

0 commit comments

Comments
 (0)