Skip to content

Commit 984936f

Browse files
committed
fix(vm-adoption-controller): keep the imported disk when adopting in place
Forklift owns every migrated PVC by the VirtualMachine it creates -- EnsureVM patches each one with a blockOwnerDeletion ownerReference, upstream comment "so that they'll be cleaned up when the VirtualMachine is removed". Adoption then deleted that VM to free the name the VMInstance takes, and the garbage collector took the disk with it: the clone lost its source mid-flight, the DataVolume sat in CloneWithoutSource and the VM in ErrorPvcNotFound, with a completed transfer destroyed by the adoption meant to consume it. The code assumed the opposite -- "KubeVirt does not delete PVCs with a VM" -- which holds for an ordinary VM but not for a Forklift import. Two changes, both scoped to same-namespace adoption: Detach before deleting. Only the volumes this adoption consumes, and only ownerReferences pointing at this VM, so an import that never reaches adoption keeps Forklift's cleanup of abandoned migrations. Stop cloning. On the raw-copy path Forklift already wrote the disk into the target namespace with the right size, StorageClass and content, so the clone copied a whole disk to stand still and left two full copies behind for good. The disks keep referencing the imported PVC; vm-instance already falls back to a bare PersistentVolumeClaim when no DataVolume of that name exists. The cross-namespace virt-v2v path is unchanged -- there the copy is the point. Assisted-By: Claude Signed-off-by: Andrei Kvapil <[email protected]>
1 parent 7dd95e9 commit 984936f

3 files changed

Lines changed: 261 additions & 8 deletions

File tree

‎packages/system/vm-adoption-controller/images/controller/controller_test.go‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@ var (
2222
migrationsGVR = schema.GroupVersionResource{Group: "forklift.konveyor.io", Version: "v1beta1", Resource: "migrations"}
2323
vmsGVR = schema.GroupVersionResource{Group: "kubevirt.io", Version: "v1", Resource: "virtualmachines"}
2424
vmInstancesGVR = schema.GroupVersionResource{Group: vmInstanceGroup, Version: vmInstanceVersion, Resource: "vminstances"}
25+
vmDisksGVR = schema.GroupVersionResource{Group: vmDiskGroup, Version: vmDiskVersion, Resource: "vmdisks"}
26+
dataVolumesGVR = schema.GroupVersionResource{Group: "cdi.kubevirt.io", Version: "v1beta1", Resource: "datavolumes"}
27+
pvcsGVR = schema.GroupVersionResource{Version: "v1", Resource: "persistentvolumeclaims"}
2528
)
2629

2730
func newForkliftObj(kind, namespace, name, uid string, ann map[string]string) *unstructured.Unstructured {
@@ -59,11 +62,37 @@ func fakeClient(objs ...runtime.Object) *dynamicfake.FakeDynamicClient {
5962
migrationsGVR: "MigrationList",
6063
vmsGVR: "VirtualMachineList",
6164
vmInstancesGVR: "VMInstanceList",
65+
vmDisksGVR: "VMDiskList",
66+
dataVolumesGVR: "DataVolumeList",
67+
pvcsGVR: "PersistentVolumeClaimList",
6268
},
6369
objs...,
6470
)
6571
}
6672

73+
// newPVC builds an imported disk the way Forklift leaves it: a bare PVC owned by
74+
// the VirtualMachine Forklift created, with blockOwnerDeletion set.
75+
func newPVC(namespace, name, ownerVM string) *unstructured.Unstructured {
76+
u := &unstructured.Unstructured{Object: map[string]interface{}{}}
77+
u.SetAPIVersion("v1")
78+
u.SetKind("PersistentVolumeClaim")
79+
u.SetNamespace(namespace)
80+
u.SetName(name)
81+
if ownerVM != "" {
82+
block := true
83+
u.SetOwnerReferences([]metav1.OwnerReference{{
84+
APIVersion: "kubevirt.io/v1",
85+
Kind: "VirtualMachine",
86+
Name: ownerVM,
87+
UID: types.UID("vm-uid"),
88+
BlockOwnerDeletion: &block,
89+
}})
90+
}
91+
_ = unstructured.SetNestedField(u.Object, "16Gi", "spec", "resources", "requests", "storage")
92+
_ = unstructured.SetNestedField(u.Object, "replicated", "spec", "storageClassName")
93+
return u
94+
}
95+
6796
func newVM(namespace, name string, labels map[string]string) *unstructured.Unstructured {
6897
u := &unstructured.Unstructured{Object: map[string]interface{}{}}
6998
u.SetAPIVersion("kubevirt.io/v1")
@@ -309,3 +338,126 @@ func TestIsAdoptionEnabled(t *testing.T) {
309338
})
310339
}
311340
}
341+
342+
// On the raw-copy path Forklift already wrote the disk into the target
343+
// namespace. Cloning it into a VMDisk there would copy the whole disk a second
344+
// time and leave two full copies behind for good, so the adoption must keep
345+
// referencing the imported PVC instead.
346+
func TestWrapDisksKeepsImportedPVCWhenAdoptingInPlace(t *testing.T) {
347+
client := fakeClient(newPVC("tenant-a", "web-disk-0", "web"))
348+
c := &AdoptionController{dynamicClient: client, recorder: record.NewFakeRecorder(10)}
349+
350+
disks := []interface{}{map[string]interface{}{"name": "imported-0", "dvName": "web-disk-0"}}
351+
if err := c.wrapDisksAsVMDisks(context.Background(), "tenant-a", "tenant-a", "web", disks); err != nil {
352+
t.Fatalf("wrapDisksAsVMDisks: %v", err)
353+
}
354+
355+
if got := disks[0].(map[string]interface{})["dvName"]; got != "web-disk-0" {
356+
t.Errorf("dvName = %v, want the imported PVC web-disk-0 (a clone was substituted)", got)
357+
}
358+
list, err := client.Resource(vmDisksGVR).Namespace("tenant-a").List(context.Background(), metav1.ListOptions{})
359+
if err != nil {
360+
t.Fatalf("listing VMDisks: %v", err)
361+
}
362+
if len(list.Items) != 0 {
363+
t.Errorf("created %d VMDisk(s) for an in-place adoption, want none", len(list.Items))
364+
}
365+
}
366+
367+
// The virt-v2v path still has to move the disk: conversion runs in a privileged
368+
// namespace the tenant cannot read from, so there the clone is the point.
369+
func TestWrapDisksClonesWhenCrossingNamespaces(t *testing.T) {
370+
client := fakeClient(newPVC("cozy-forklift", "web-disk-0", "web"))
371+
c := &AdoptionController{dynamicClient: client, recorder: record.NewFakeRecorder(10)}
372+
373+
disks := []interface{}{map[string]interface{}{"name": "imported-0", "dvName": "web-disk-0"}}
374+
if err := c.wrapDisksAsVMDisks(context.Background(), "cozy-forklift", "tenant-a", "web", disks); err != nil {
375+
t.Fatalf("wrapDisksAsVMDisks: %v", err)
376+
}
377+
378+
if got := disks[0].(map[string]interface{})["dvName"]; got != "vm-disk-web-imported-0" {
379+
t.Errorf("dvName = %v, want the VMDisk DataVolume vm-disk-web-imported-0", got)
380+
}
381+
if _, err := client.Resource(vmDisksGVR).Namespace("tenant-a").Get(context.Background(), "web-imported-0", metav1.GetOptions{}); err != nil {
382+
t.Errorf("VMDisk was not created for the cross-namespace adoption: %v", err)
383+
}
384+
}
385+
386+
// The imported PVC must be detached from the Forklift VM BEFORE that VM is
387+
// deleted. Forklift owns the PVC by the VM with blockOwnerDeletion precisely so
388+
// the two are collected together, so if the order slips the garbage collector
389+
// destroys a finished transfer -- the observed failure was a completed 16 GiB
390+
// disk removed 357ms into adoption. The fake client runs no garbage collector,
391+
// so the ordering itself is what this asserts.
392+
func TestReleaseSourceVMDetachesImportedPVCBeforeDeletingVM(t *testing.T) {
393+
client := fakeClient(
394+
newVM("tenant-a", "web", map[string]string{"plan": "uid-1"}),
395+
newPVC("tenant-a", "web-disk-0", "web"),
396+
)
397+
398+
c := &AdoptionController{dynamicClient: client, recorder: record.NewFakeRecorder(10)}
399+
vm := kubevirtv1.VirtualMachine{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-a", Name: "web"}}
400+
disks := []interface{}{map[string]interface{}{"name": "imported-0", "dvName": "web-disk-0"}}
401+
402+
if err := c.releaseSourceVM(context.Background(), vm, "tenant-a", "web", disks); err != nil {
403+
t.Fatalf("releaseSourceVM: %v", err)
404+
}
405+
406+
detachedAt, deletedAt := -1, -1
407+
for i, a := range client.Actions() {
408+
if a.Matches("update", "persistentvolumeclaims") && detachedAt < 0 {
409+
detachedAt = i
410+
}
411+
if a.Matches("delete", "virtualmachines") && deletedAt < 0 {
412+
deletedAt = i
413+
}
414+
}
415+
if detachedAt < 0 {
416+
t.Fatal("imported PVC was never detached from the source VM")
417+
}
418+
if deletedAt < 0 {
419+
t.Fatal("source VM was never deleted")
420+
}
421+
if detachedAt > deletedAt {
422+
t.Errorf("PVC detached at action %d but VM deleted at %d: the garbage collector would take the migrated disk with the VM", detachedAt, deletedAt)
423+
}
424+
425+
cur, err := client.Resource(pvcsGVR).Namespace("tenant-a").Get(context.Background(), "web-disk-0", metav1.GetOptions{})
426+
if err != nil {
427+
t.Fatalf("imported PVC did not survive adoption: %v", err)
428+
}
429+
for _, ref := range cur.GetOwnerReferences() {
430+
if ref.Kind == "VirtualMachine" {
431+
t.Errorf("imported PVC still owned by VirtualMachine %s after adoption", ref.Name)
432+
}
433+
}
434+
}
435+
436+
// Detaching is narrow on purpose: only ownerReferences pointing at the VM being
437+
// released are dropped, so any other owner keeps its claim on the volume.
438+
func TestReleaseImportedVolumesLeavesForeignOwnersAlone(t *testing.T) {
439+
pvc := newPVC("tenant-a", "web-disk-0", "web")
440+
pvc.SetOwnerReferences(append(pvc.GetOwnerReferences(), metav1.OwnerReference{
441+
APIVersion: "cdi.kubevirt.io/v1beta1",
442+
Kind: "DataVolume",
443+
Name: "someone-else",
444+
UID: types.UID("dv-uid"),
445+
}))
446+
client := fakeClient(pvc)
447+
c := &AdoptionController{dynamicClient: client, recorder: record.NewFakeRecorder(10)}
448+
449+
vm := kubevirtv1.VirtualMachine{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-a", Name: "web"}}
450+
disks := []interface{}{map[string]interface{}{"name": "imported-0", "dvName": "web-disk-0"}}
451+
if err := c.releaseImportedVolumes(context.Background(), vm, disks); err != nil {
452+
t.Fatalf("releaseImportedVolumes: %v", err)
453+
}
454+
455+
cur, err := client.Resource(pvcsGVR).Namespace("tenant-a").Get(context.Background(), "web-disk-0", metav1.GetOptions{})
456+
if err != nil {
457+
t.Fatalf("reading PVC: %v", err)
458+
}
459+
refs := cur.GetOwnerReferences()
460+
if len(refs) != 1 || refs[0].Kind != "DataVolume" {
461+
t.Errorf("ownerReferences = %+v, want only the unrelated DataVolume owner", refs)
462+
}
463+
}

‎packages/system/vm-adoption-controller/images/controller/main.go‎

Lines changed: 104 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -689,7 +689,21 @@ func (c *AdoptionController) ensureVMDisk(ctx context.Context, vmDiskNs, vmDiskN
689689
// wrapDisksAsVMDisks turns the raw imported PVC reference of each disk into a
690690
// managed Cozystack VMDisk (clone) in the target namespace, rewriting each
691691
// disk's dvName to the VMDisk's DataVolume (`vm-disk-<vmInstance>-<disk>`).
692+
//
693+
// This is only needed when the disk has to cross a namespace: on the virt-v2v
694+
// path Forklift converts in a privileged namespace and the data must be copied
695+
// into the tenant. On the raw-copy path Forklift wrote the disk straight into
696+
// the target namespace, with the right size, StorageClass and content, so there
697+
// is nothing to move. Cloning it there would copy the whole disk a second time
698+
// for no gain and leave two full copies behind permanently -- and, worse, race
699+
// the clone against releaseSourceVM deleting the VM that owns the source PVC.
700+
// The disks keep pointing at the imported PVC instead; vm-instance references a
701+
// bare PersistentVolumeClaim when no DataVolume of that name exists.
692702
func (c *AdoptionController) wrapDisksAsVMDisks(ctx context.Context, srcNs, targetNs, vmInstanceName string, disks []interface{}) error {
703+
if srcNs == targetNs {
704+
klog.V(2).Infof("Adoption target %s is the import namespace; referencing the imported PVCs directly instead of cloning", targetNs)
705+
return nil
706+
}
693707
for _, d := range disks {
694708
dm, ok := d.(map[string]interface{})
695709
if !ok {
@@ -981,7 +995,7 @@ func (c *AdoptionController) adoptVMViaVMDisks(ctx context.Context, vm kubevirtv
981995
// earlier pass and only the source VM is left to release.
982996
if _, err := c.dynamicClient.Resource(vmInstanceGVR).Namespace(targetNamespace).Get(ctx, vmInstanceName, metav1.GetOptions{}); err == nil {
983997
klog.Infof("VMInstance %s/%s already exists, ensuring source VM is handled", targetNamespace, vmInstanceName)
984-
return c.releaseSourceVM(ctx, vm, targetNamespace, vmInstanceName)
998+
return c.releaseSourceVM(ctx, vm, targetNamespace, vmInstanceName, disks)
985999
}
9861000

9871001
spec := map[string]interface{}{
@@ -1030,20 +1044,97 @@ func (c *AdoptionController) adoptVMViaVMDisks(ctx context.Context, vm kubevirtv
10301044
c.recorder.Eventf(vmRef(&vm), corev1.EventTypeNormal, "Adopted",
10311045
"Imported VM adopted as Cozystack VMInstance %s/%s", targetNamespace, vmInstanceName)
10321046

1033-
return c.releaseSourceVM(ctx, vm, targetNamespace, vmInstanceName)
1047+
return c.releaseSourceVM(ctx, vm, targetNamespace, vmInstanceName, disks)
1048+
}
1049+
1050+
// releaseImportedVolumes detaches the volumes this adoption consumes from the
1051+
// Forklift-created VirtualMachine, so deleting that VM does not take the
1052+
// migrated data with it.
1053+
//
1054+
// Forklift deliberately owns every migrated PVC by the VM it creates: EnsureVM
1055+
// (pkg/controller/plan/kubevirt.go) patches each PVC with an ownerReference
1056+
// carrying blockOwnerDeletion, described upstream as being there "so that
1057+
// they'll be cleaned up when the VirtualMachine is removed". That is the right
1058+
// default for Forklift and exactly wrong once Cozystack adopts the import: the
1059+
// VMInstance references those PVCs, so a cascading delete destroys a completed
1060+
// transfer.
1061+
//
1062+
// The detach is deliberately narrow -- only the volumes named by this VM's
1063+
// disks, only ownerReferences pointing at this VM. An import that never reaches
1064+
// adoption keeps Forklift's ownership, and with it Forklift's cleanup of
1065+
// abandoned migrations.
1066+
func (c *AdoptionController) releaseImportedVolumes(ctx context.Context, vm kubevirtv1.VirtualMachine, disks []interface{}) error {
1067+
gvrs := []schema.GroupVersionResource{
1068+
{Group: "cdi.kubevirt.io", Version: "v1beta1", Resource: "datavolumes"},
1069+
{Version: "v1", Resource: "persistentvolumeclaims"},
1070+
}
1071+
for _, d := range disks {
1072+
dm, ok := d.(map[string]interface{})
1073+
if !ok {
1074+
continue
1075+
}
1076+
name, _ := dm["dvName"].(string)
1077+
if name == "" {
1078+
continue
1079+
}
1080+
for _, gvr := range gvrs {
1081+
if err := c.dropVMOwnerReference(ctx, gvr, vm, name); err != nil {
1082+
return err
1083+
}
1084+
}
1085+
}
1086+
return nil
1087+
}
1088+
1089+
// dropVMOwnerReference removes the ownerReferences pointing at vm from a single
1090+
// object, leaving any other owner intact. A missing object is not an error:
1091+
// Forklift deletes the DataVolume once the transfer completes, so on a finished
1092+
// import only the PVC is normally still there.
1093+
func (c *AdoptionController) dropVMOwnerReference(ctx context.Context, gvr schema.GroupVersionResource,
1094+
vm kubevirtv1.VirtualMachine, name string) error {
1095+
1096+
obj, err := c.dynamicClient.Resource(gvr).Namespace(vm.Namespace).Get(ctx, name, metav1.GetOptions{})
1097+
if err != nil {
1098+
if apierrors.IsNotFound(err) {
1099+
return nil
1100+
}
1101+
return fmt.Errorf("failed to read %s %s/%s: %w", gvr.Resource, vm.Namespace, name, err)
1102+
}
1103+
1104+
refs := obj.GetOwnerReferences()
1105+
kept := make([]metav1.OwnerReference, 0, len(refs))
1106+
for _, ref := range refs {
1107+
// Same namespace and same name make this the VM being released; the
1108+
// typed VM carries no UID here, so kind+name is the available identity.
1109+
if ref.Kind == "VirtualMachine" && ref.Name == vm.Name {
1110+
continue
1111+
}
1112+
kept = append(kept, ref)
1113+
}
1114+
if len(kept) == len(refs) {
1115+
return nil
1116+
}
1117+
1118+
obj.SetOwnerReferences(kept)
1119+
if _, err := c.dynamicClient.Resource(gvr).Namespace(vm.Namespace).Update(ctx, obj, metav1.UpdateOptions{}); err != nil {
1120+
return fmt.Errorf("failed to detach %s %s/%s from VM %s: %w", gvr.Resource, vm.Namespace, name, vm.Name, err)
1121+
}
1122+
klog.Infof("Detached %s %s/%s from Forklift VM %s/%s so it survives adoption", gvr.Resource, vm.Namespace, name, vm.Namespace, vm.Name)
1123+
return nil
10341124
}
10351125

10361126
// releaseSourceVM hands the Forklift-created VM over to the managed VMInstance,
10371127
// once that VMInstance exists. In the same namespace the VMInstance renders a
10381128
// VirtualMachine under the same name, so the Forklift VM has to go; the imported
1039-
// PVC stays behind as the VMDisk clone source, since KubeVirt does not delete
1040-
// PVCs with a VM. In a different namespace nothing collides, so the VM is only
1041-
// labeled adopted to keep it out of the next reconcile.
1129+
// PVCs it referenced are detached from it first (see releaseImportedVolumes) so
1130+
// they survive the delete and back the adopted VMInstance. In a different
1131+
// namespace nothing collides, so the VM is only labeled adopted to keep it out
1132+
// of the next reconcile.
10421133
//
10431134
// Failing here is retryable: the VMInstance is already created, so the next loop
10441135
// takes the idempotency path and calls this again.
10451136
func (c *AdoptionController) releaseSourceVM(ctx context.Context, vm kubevirtv1.VirtualMachine,
1046-
targetNamespace, vmInstanceName string) error {
1137+
targetNamespace, vmInstanceName string, disks []interface{}) error {
10471138

10481139
helmReleaseName := "vm-instance-" + vmInstanceName
10491140
if targetNamespace != vm.Namespace {
@@ -1053,6 +1144,13 @@ func (c *AdoptionController) releaseSourceVM(ctx context.Context, vm kubevirtv1.
10531144
return nil
10541145
}
10551146

1147+
// Must happen before the delete, not after: the garbage collector acts on
1148+
// the ownerReference as soon as the VM goes, and a PVC it has already
1149+
// removed cannot be recovered.
1150+
if err := c.releaseImportedVolumes(ctx, vm, disks); err != nil {
1151+
return fmt.Errorf("failed to detach imported volumes from source VM %s/%s: %w", vm.Namespace, vm.Name, err)
1152+
}
1153+
10561154
vmGVR := schema.GroupVersionResource{Group: "kubevirt.io", Version: "v1", Resource: "virtualmachines"}
10571155
if err := c.dynamicClient.Resource(vmGVR).Namespace(vm.Namespace).Delete(ctx, vm.Name, metav1.DeleteOptions{}); err != nil {
10581156
if apierrors.IsNotFound(err) {

‎packages/system/vm-adoption-controller/templates/clusterrole.yaml‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,13 @@ rules:
2828
resources: ["datavolumes/source"]
2929
verbs: ["create"]
3030

31-
# Read source PVCs to size the cross-namespace clone
31+
# Read source PVCs to size the cross-namespace clone, and detach the imported
32+
# PVCs from the Forklift VM before it is deleted (Forklift owns them by that VM
33+
# with blockOwnerDeletion, so without this the migrated data is garbage-collected
34+
# along with the VM the adoption replaces)
3235
- apiGroups: [""]
3336
resources: ["persistentvolumeclaims"]
34-
verbs: ["get", "list"]
37+
verbs: ["get", "list", "update", "patch"]
3538

3639
# Emit Events on imported VMs so adoption outcomes are visible via
3740
# `kubectl describe` / the dashboard, not only in the controller log

0 commit comments

Comments
 (0)