Skip to content

Commit 4ddf320

Browse files
fix(backups): delete a restored disk's PVC only once the restore can run
Deleting the original PVC of a keepOriginalPVC=false restore is only safe when Velero can put the disk back. The Backup artifact outlives the Velero Backup it points to, which Velero garbage-collects at TTL, and nothing checked it before prepareForRestore halted the VM and deleted the disks: a restore of an expired backup lost the volume where it used to fail with the disk intact. The Velero Backup must now exist and be Completed before any preparation starts. A DataVolume that survives its deletion makes CDI recreate the claim under the original name, which Velero then skips as an existing PVC, so the restore would end Succeeded on a freshly imported disk. A failed DataVolume deletion now fails the restore, and the PVCs are deleted only once every DataVolume is gone. Reading the DataVolume back needs get on datavolumes, which the ClusterRole did not grant. Assisted-by: LLM Signed-off-by: Mattia Eleuteri <[email protected]>
1 parent f17edc8 commit 4ddf320

4 files changed

Lines changed: 297 additions & 29 deletions

File tree

‎internal/backupcontroller/velerostrategy_controller.go‎

Lines changed: 72 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -686,8 +686,26 @@ func (r *RestoreJobReconciler) reconcileVeleroRestore(ctx context.Context, resto
686686
}
687687
}
688688

689+
// The Backup artifact outlives the Velero Backup it points to, which
690+
// Velero garbage-collects once its TTL expires. Checked before
691+
// prepareForRestore halts the application and deletes its disks, so a
692+
// restore that cannot run leaves them in place.
693+
veleroBackup := &velerov1.Backup{}
694+
if err := r.Get(ctx, client.ObjectKey{Namespace: veleroNamespace, Name: veleroBackupName}, veleroBackup); err != nil {
695+
if errors.IsNotFound(err) {
696+
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf(
697+
"Velero Backup %s/%s not found; it may have expired", veleroNamespace, veleroBackupName))
698+
}
699+
return ctrl.Result{}, fmt.Errorf("failed to get Velero Backup %s: %w", veleroBackupName, err)
700+
}
701+
if veleroBackup.Status.Phase != velerov1.BackupPhaseCompleted {
702+
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf(
703+
"Velero Backup %s/%s is %q, not %q", veleroNamespace, veleroBackupName,
704+
veleroBackup.Status.Phase, velerov1.BackupPhaseCompleted))
705+
}
706+
689707
// Resolve underlying resources once; prefer Backup status, fall back to Velero annotation.
690-
ur := r.resolveUnderlyingResourcesForRestore(ctx, backup, veleroBackupName)
708+
ur := resolveUnderlyingResourcesForRestore(backup, veleroBackup)
691709

692710
// Pre-restore: graceful shutdown, suspend HRs, rename PVCs (skipped for copy)
693711
ready, result, err := r.prepareForRestore(ctx, restoreJob, backup, ur, target, restoreOpts)
@@ -949,14 +967,10 @@ func (r *RestoreJobReconciler) createResourceModifiersConfigMap(ctx context.Cont
949967
// resolveUnderlyingResourcesForRestore returns underlying resources for symmetric
950968
// restore label selectors. Velero Backup annotation is used when Backup.status was empty
951969
// (e.g. CRD without underlyingResources in schema).
952-
func (r *RestoreJobReconciler) resolveUnderlyingResourcesForRestore(ctx context.Context, backup *backupsv1alpha1.Backup, veleroBackupName string) *runtime.RawExtension {
970+
func resolveUnderlyingResourcesForRestore(backup *backupsv1alpha1.Backup, vb *velerov1.Backup) *runtime.RawExtension {
953971
if backup.Status.UnderlyingResources != nil && len(backup.Status.UnderlyingResources.Raw) > 0 {
954972
return backup.Status.UnderlyingResources
955973
}
956-
vb := &velerov1.Backup{}
957-
if err := r.Get(ctx, client.ObjectKey{Namespace: veleroNamespace, Name: veleroBackupName}, vb); err != nil {
958-
return backup.Status.UnderlyingResources
959-
}
960974
if urJSON, ok := vb.Annotations[underlyingResourcesAnnotation]; ok && urJSON != "" {
961975
return &runtime.RawExtension{Raw: []byte(urJSON)}
962976
}
@@ -1025,9 +1039,11 @@ func shortHash(input string) string {
10251039
// 5. Renames existing PVCs to <name>-orig-<hash> so Velero can create fresh
10261040
// ones via Data Movement.
10271041
//
1028-
// Failures in individual steps are non-fatal: missing resources are expected
1029-
// (e.g. restore requested when app was already deleted). Each action emits
1030-
// a Kubernetes Event on the RestoreJob for observability.
1042+
// Missing resources are expected (e.g. restore requested when app was already
1043+
// deleted), and a failure to suspend, halt or rename is non-fatal. A DataVolume
1044+
// or a replaced PVC that cannot be deleted fails the restore, since Velero would
1045+
// skip the claim left in place. Each action emits a Kubernetes Event on the
1046+
// RestoreJob for observability.
10311047
//
10321048
// postRestoreRename renames VMInstance HelmRelease after Velero Restore completes.
10331049
// Velero resource modifiers cannot change metadata.name, so this step creates
@@ -1171,15 +1187,28 @@ func (r *RestoreJobReconciler) prepareForRestore(ctx context.Context, restoreJob
11711187
}
11721188

11731189
// --- Step 4: Delete DataVolumes so CDI doesn't recreate PVCs ---
1190+
// A DataVolume still present makes CDI recreate its claim under the original
1191+
// name, which Velero then skips as an existing PVC: the restore would end
1192+
// Succeeded on a freshly imported disk, so nothing goes further until every
1193+
// DataVolume is gone.
11741194
if vmRes != nil {
1195+
var terminating []string
11751196
for _, dv := range vmRes.DataVolumes {
1176-
if err := r.deleteDataVolume(ctx, ns, dv.DataVolumeName); err != nil {
1177-
r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "PrepareForRestore",
1178-
fmt.Sprintf("Failed to delete DataVolume %s: %v", dv.DataVolumeName, err))
1179-
} else {
1180-
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
1181-
fmt.Sprintf("Deleted DataVolume %s", dv.DataVolumeName))
1197+
gone, err := r.deleteDataVolume(ctx, ns, dv.DataVolumeName)
1198+
if err != nil {
1199+
return false, ctrl.Result{}, fmt.Errorf("failed to delete DataVolume %s: %w", dv.DataVolumeName, err)
1200+
}
1201+
if !gone {
1202+
terminating = append(terminating, dv.DataVolumeName)
1203+
continue
11821204
}
1205+
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
1206+
fmt.Sprintf("Deleted DataVolume %s", dv.DataVolumeName))
1207+
}
1208+
if len(terminating) > 0 {
1209+
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
1210+
fmt.Sprintf("Waiting for DataVolumes %s to be deleted", strings.Join(terminating, ", ")))
1211+
return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil
11831212
}
11841213
}
11851214

@@ -1265,19 +1294,35 @@ func (r *RestoreJobReconciler) haltVirtualMachine(ctx context.Context, ns, vmNam
12651294
return false, nil
12661295
}
12671296

1268-
// deleteDataVolume deletes a DataVolume so CDI doesn't recreate the PVC after rename.
1269-
// Uses Orphan propagation so that the PVC the DV owns via ownerReference is
1270-
// removed by prepareForRestore, which waits for it, rather than by background
1271-
// garbage collection, which nothing waits for before the Velero Restore starts.
1272-
func (r *RestoreJobReconciler) deleteDataVolume(ctx context.Context, ns, name string) error {
1273-
orphan := metav1.DeletePropagationOrphan
1274-
err := r.Resource(dataVolumeGVR).Namespace(ns).Delete(ctx, name, metav1.DeleteOptions{
1275-
PropagationPolicy: &orphan,
1276-
})
1277-
if err != nil && !errors.IsNotFound(err) {
1278-
return err
1297+
// deleteDataVolume deletes a DataVolume so CDI doesn't recreate the PVC after
1298+
// rename, and reports whether it is gone. Orphan propagation holds the DV behind
1299+
// the orphan finalizer until garbage collection has released its PVC, and leaves
1300+
// that PVC to prepareForRestore, which waits for its deletion, rather than to
1301+
// background garbage collection, which nothing waits for before the Velero
1302+
// Restore starts.
1303+
func (r *RestoreJobReconciler) deleteDataVolume(ctx context.Context, ns, name string) (bool, error) {
1304+
dvClient := r.Resource(dataVolumeGVR).Namespace(ns)
1305+
dv, err := dvClient.Get(ctx, name, metav1.GetOptions{})
1306+
if err != nil {
1307+
if errors.IsNotFound(err) {
1308+
return true, nil
1309+
}
1310+
return false, err
12791311
}
1280-
return nil
1312+
if dv.GetDeletionTimestamp() == nil {
1313+
orphan := metav1.DeletePropagationOrphan
1314+
err := dvClient.Delete(ctx, name, metav1.DeleteOptions{PropagationPolicy: &orphan})
1315+
if err != nil && !errors.IsNotFound(err) {
1316+
return false, err
1317+
}
1318+
}
1319+
if _, err := dvClient.Get(ctx, name, metav1.GetOptions{}); err != nil {
1320+
if errors.IsNotFound(err) {
1321+
return true, nil
1322+
}
1323+
return false, err
1324+
}
1325+
return false, nil
12811326
}
12821327

12831328
// deletePVC deletes a PVC and reports whether it is gone. A claim still mounted

‎internal/backupcontroller/velerostrategy_controller_test.go‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"k8s.io/apimachinery/pkg/runtime/schema"
1616
dynamicfake "k8s.io/client-go/dynamic/fake"
1717
"k8s.io/client-go/kubernetes/scheme"
18+
k8stesting "k8s.io/client-go/testing"
1819
"k8s.io/client-go/tools/record"
1920
"sigs.k8s.io/controller-runtime/pkg/client"
2021
clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
@@ -531,6 +532,203 @@ func TestPrepareForRestore_KeepOriginalPVCFalse_WaitsForTerminatingPVC(t *testin
531532
}
532533
}
533534

535+
func makeUnstructuredDataVolume(name, namespace string) *unstructured.Unstructured {
536+
dv := &unstructured.Unstructured{}
537+
dv.SetAPIVersion("cdi.kubevirt.io/v1beta1")
538+
dv.SetKind("DataVolume")
539+
dv.SetName(name)
540+
dv.SetNamespace(namespace)
541+
return dv
542+
}
543+
544+
// A DataVolume still in place makes CDI recreate the claim under its original
545+
// name, so the PVC must survive until the DataVolume is confirmed gone.
546+
func TestPrepareForRestore_KeepOriginalPVCFalse_KeepsPVCWhileDataVolumeRemains(t *testing.T) {
547+
tests := []struct {
548+
name string
549+
deleteErr error
550+
wantErr bool
551+
}{
552+
{
553+
name: "DataVolume deletion fails",
554+
deleteErr: errors.NewForbidden(dataVolumeGVR.GroupResource(), "vm-disk-ubuntu-source", nil),
555+
wantErr: true,
556+
},
557+
{
558+
name: "DataVolume deletion is still pending",
559+
},
560+
}
561+
for _, tt := range tests {
562+
t.Run(tt.name, func(t *testing.T) {
563+
ns := "tenant-root"
564+
restoreJob, backup, ur, target, opts := inPlaceVMRestoreWithoutKeep(ns)
565+
pvc := &corev1.PersistentVolumeClaim{
566+
ObjectMeta: metav1.ObjectMeta{Name: "vm-disk-ubuntu-source", Namespace: ns},
567+
}
568+
reconciler := newTestRestoreJobReconcilerWithDynamic(t,
569+
[]runtime.Object{makeUnstructuredDataVolume("vm-disk-ubuntu-source", ns)},
570+
pvc, restoreJob, backup)
571+
reconciler.Interface.(*dynamicfake.FakeDynamicClient).PrependReactor("delete", "datavolumes",
572+
func(k8stesting.Action) (bool, runtime.Object, error) {
573+
return true, nil, tt.deleteErr
574+
})
575+
576+
ctx := context.Background()
577+
ready, result, err := reconciler.prepareForRestore(ctx, restoreJob, backup, ur, target, opts)
578+
if tt.wantErr {
579+
if err == nil {
580+
t.Fatal("expected an error when the DataVolume cannot be deleted")
581+
}
582+
} else {
583+
if err != nil {
584+
t.Fatalf("prepareForRestore() error = %v", err)
585+
}
586+
if result.RequeueAfter == 0 {
587+
t.Error("expected a requeue while the DataVolume is still present")
588+
}
589+
}
590+
if ready {
591+
t.Fatal("expected ready=false while the DataVolume is still present")
592+
}
593+
594+
got := &corev1.PersistentVolumeClaim{}
595+
if err := reconciler.Get(ctx, client.ObjectKeyFromObject(pvc), got); err != nil {
596+
t.Fatalf("original PVC should survive while its DataVolume remains: %v", err)
597+
}
598+
if got.DeletionTimestamp != nil {
599+
t.Error("original PVC should not be marked for deletion while its DataVolume remains")
600+
}
601+
})
602+
}
603+
}
604+
605+
// keepOriginalPVC=true must never delete the original claim, even when the
606+
// rename that preserves it fails and leaves it under its original name.
607+
func TestPrepareForRestore_KeepOriginalPVCTrue_FailedRenameKeepsPVC(t *testing.T) {
608+
ns := "tenant-root"
609+
restoreJob, backup, ur, target, _ := inPlaceVMRestoreWithoutKeep(ns)
610+
opts := RestoreOptions{KeepOriginalPVC: new(true)}
611+
612+
pvc := &corev1.PersistentVolumeClaim{
613+
ObjectMeta: metav1.ObjectMeta{Name: "vm-disk-ubuntu-source", Namespace: ns},
614+
// The PV does not exist, so renamePVC fails before touching the claim.
615+
Spec: corev1.PersistentVolumeClaimSpec{VolumeName: "pv-missing"},
616+
}
617+
reconciler := newTestRestoreJobReconciler(t, pvc, restoreJob, backup)
618+
619+
ctx := context.Background()
620+
if _, _, err := reconciler.prepareForRestore(ctx, restoreJob, backup, ur, target, opts); err != nil {
621+
t.Fatalf("prepareForRestore() error = %v", err)
622+
}
623+
624+
if err := reconciler.Get(ctx, client.ObjectKeyFromObject(pvc), &corev1.PersistentVolumeClaim{}); err != nil {
625+
t.Errorf("original PVC should survive a failed rename when keepOriginalPVC=true, got error: %v", err)
626+
}
627+
}
628+
629+
// A Backup artifact can outlive its Velero Backup, which Velero garbage-collects
630+
// at TTL. The restore must fail before it halts the VM or deletes its disks.
631+
func TestReconcileVeleroRestore_UnusableVeleroBackupTouchesNothing(t *testing.T) {
632+
tests := []struct {
633+
name string
634+
veleroBackup *velerov1.Backup
635+
wantMessage string
636+
}{
637+
{
638+
name: "Velero Backup expired",
639+
wantMessage: "Velero Backup cozy-velero/vb not found",
640+
},
641+
{
642+
name: "Velero Backup being deleted",
643+
veleroBackup: &velerov1.Backup{
644+
ObjectMeta: metav1.ObjectMeta{Name: "vb", Namespace: veleroNamespace},
645+
Status: velerov1.BackupStatus{Phase: velerov1.BackupPhaseDeleting},
646+
},
647+
wantMessage: `Velero Backup cozy-velero/vb is "Deleting"`,
648+
},
649+
}
650+
for _, tt := range tests {
651+
t.Run(tt.name, func(t *testing.T) {
652+
ns := "tenant-root"
653+
restoreJob, backup, ur, _, opts := inPlaceVMRestoreWithoutKeep(ns)
654+
rawOpts, err := json.Marshal(opts)
655+
if err != nil {
656+
t.Fatalf("marshal options: %v", err)
657+
}
658+
started := metav1.Now()
659+
restoreJob.Spec.Options = &runtime.RawExtension{Raw: rawOpts}
660+
restoreJob.Status = backupsv1alpha1.RestoreJobStatus{
661+
Phase: backupsv1alpha1.RestoreJobPhaseRunning,
662+
StartedAt: &started,
663+
}
664+
backup.Spec.StrategyRef = corev1.TypedLocalObjectReference{Kind: "Velero", Name: "cozy-default-velero"}
665+
backup.Spec.DriverMetadata = map[string]string{veleroBackupNameMetadataKey: "vb"}
666+
backup.Status.UnderlyingResources = ur
667+
668+
pvc := &corev1.PersistentVolumeClaim{
669+
ObjectMeta: metav1.ObjectMeta{Name: "vm-disk-ubuntu-source", Namespace: ns},
670+
}
671+
strategy := &strategyv1alpha1.Velero{ObjectMeta: metav1.ObjectMeta{Name: "cozy-default-velero"}}
672+
objs := []client.Object{restoreJob, backup, pvc, strategy}
673+
if tt.veleroBackup != nil {
674+
objs = append(objs, tt.veleroBackup)
675+
}
676+
677+
vm := &unstructured.Unstructured{}
678+
vm.SetAPIVersion("kubevirt.io/v1")
679+
vm.SetKind("VirtualMachine")
680+
vm.SetName(vmNamePrefix + "test-vm")
681+
vm.SetNamespace(ns)
682+
if err := unstructured.SetNestedField(vm.Object, "Always", "spec", "runStrategy"); err != nil {
683+
t.Fatalf("set runStrategy: %v", err)
684+
}
685+
reconciler := newTestRestoreJobReconcilerWithDynamic(t,
686+
[]runtime.Object{vm, makeUnstructuredDataVolume("vm-disk-ubuntu-source", ns)})
687+
testScheme := runtime.NewScheme()
688+
_ = scheme.AddToScheme(testScheme)
689+
_ = backupsv1alpha1.AddToScheme(testScheme)
690+
_ = velerov1.AddToScheme(testScheme)
691+
_ = strategyv1alpha1.AddToScheme(testScheme)
692+
reconciler.Client = clientfake.NewClientBuilder().
693+
WithScheme(testScheme).
694+
WithObjects(objs...).
695+
WithStatusSubresource(&backupsv1alpha1.RestoreJob{}).
696+
Build()
697+
ctx := context.Background()
698+
699+
if _, err := reconciler.reconcileVeleroRestore(ctx, restoreJob, backup); err != nil {
700+
t.Fatalf("reconcileVeleroRestore() error = %v", err)
701+
}
702+
703+
got := &backupsv1alpha1.RestoreJob{}
704+
if err := reconciler.Get(ctx, client.ObjectKeyFromObject(restoreJob), got); err != nil {
705+
t.Fatalf("get RestoreJob: %v", err)
706+
}
707+
if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
708+
t.Fatalf("phase = %q, want %q", got.Status.Phase, backupsv1alpha1.RestoreJobPhaseFailed)
709+
}
710+
ready := meta.FindStatusCondition(got.Status.Conditions, "Ready")
711+
if ready == nil || !strings.Contains(ready.Message, tt.wantMessage) {
712+
t.Errorf("Ready condition = %+v, want a message containing %q", ready, tt.wantMessage)
713+
}
714+
715+
if err := reconciler.Get(ctx, client.ObjectKeyFromObject(pvc), &corev1.PersistentVolumeClaim{}); err != nil {
716+
t.Errorf("original PVC should survive a restore that cannot run: %v", err)
717+
}
718+
if _, err := reconciler.Resource(dataVolumeGVR).Namespace(ns).Get(ctx, "vm-disk-ubuntu-source", metav1.GetOptions{}); err != nil {
719+
t.Errorf("DataVolume should survive a restore that cannot run: %v", err)
720+
}
721+
gotVM, err := reconciler.Resource(virtualMachineGVR).Namespace(ns).Get(ctx, vm.GetName(), metav1.GetOptions{})
722+
if err != nil {
723+
t.Fatalf("get VirtualMachine: %v", err)
724+
}
725+
if rs, _, _ := unstructured.NestedString(gotVM.Object, "spec", "runStrategy"); rs != "Always" {
726+
t.Errorf("VirtualMachine runStrategy = %q, want it left at %q", rs, "Always")
727+
}
728+
})
729+
}
730+
}
731+
534732
func TestResolveRestoreTarget_VMDisk_CommonRestoreOptions(t *testing.T) {
535733
tests := []struct {
536734
name string

‎packages/system/backupstrategy-controller/templates/rbac.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,10 +57,12 @@ rules:
5757
- apiGroups: ["kubevirt.io"]
5858
resources: ["virtualmachineinstances"]
5959
verbs: ["get"]
60-
# CDI DataVolumes: pre-restore deletes DVs so CDI doesn't recreate PVCs after rename
60+
# CDI DataVolumes: pre-restore deletes DVs so CDI doesn't recreate PVCs after rename,
61+
# and reads each one back to hold the restore until it is gone (a point Get
62+
# through the dynamic client, so no list/watch)
6163
- apiGroups: ["cdi.kubevirt.io"]
6264
resources: ["datavolumes"]
63-
verbs: ["delete"]
65+
verbs: ["get", "delete"]
6466
# HelmReleases: pre-restore suspends HRs; post-restore rename creates new HR
6567
# and deletes old. patch additionally lets the DefaultObjectsGate stamp
6668
# reconcile.fluxcd.io/forceAt + requestedAt on THIS chart's own release to
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
suite: RBAC of the Velero restore preparation
2+
3+
# prepareForRestore reads every DataVolume back after deleting it and holds
4+
# the Velero Restore until it is gone. The read goes through the dynamic
5+
# client, which the Go unit tests run with no authorization, so a missing verb
6+
# only shows up on a cluster, where it fails every in-place VM restore.
7+
8+
templates:
9+
- templates/rbac.yaml
10+
11+
release:
12+
name: backupstrategy-controller
13+
namespace: cozy-backup-controller
14+
15+
tests:
16+
- it: "ClusterRole can read and delete DataVolumes"
17+
asserts:
18+
- contains:
19+
path: rules
20+
content:
21+
apiGroups: ["cdi.kubevirt.io"]
22+
resources: ["datavolumes"]
23+
verbs: ["get", "delete"]

0 commit comments

Comments
 (0)