Skip to content

Commit 9317fed

Browse files
committed
fix(kubeovn): retry VM migration setup after scheduling
Signed-off-by: Yan Bondarenko <[email protected]>
1 parent ae7a711 commit 9317fed

5 files changed

Lines changed: 125 additions & 0 deletions

File tree

‎Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ build: build-deps
2323
make -C packages/system/cilium image
2424
make -C packages/system/linstor image
2525
make -C packages/system/linstor-gui image
26+
make -C packages/system/kubeovn image
2627
make -C packages/system/kubeovn-webhook image
2728
make -C packages/system/kubeovn-plunger image
2829
make -C packages/system/dashboard image

‎hack/build-matrix_test.bats‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,14 @@
5050
[ "$out" = '["packages/apps/mariadb"]' ]
5151
}
5252

53+
@test "kubeovn source patch selects the kubeovn image build" {
54+
tmp=$(mktemp)
55+
echo "packages/system/kubeovn/images/kubeovn/patches/fix-vmim-scheduling-retry.diff" > "$tmp"
56+
out=$(hack/build-matrix.sh "$tmp")
57+
[ "$out" = '["packages/system/kubeovn"]' ]
58+
rm -f "$tmp"
59+
}
60+
5361
@test "two-package diff selects both units" {
5462
tmp=$(mktemp); trap 'rm -f "$tmp"' EXIT
5563
printf 'packages/apps/mariadb/values.yaml\npackages/system/dashboard/values.yaml\n' > "$tmp"

‎packages/system/kubeovn/images/kubeovn/Dockerfile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ WORKDIR /source
1111

1212
COPY patches /patches
1313
RUN git apply /patches/*.diff
14+
COPY tests/kubevirt_migration_test.go.txt /source/pkg/controller/kubevirt_migration_test.go
15+
RUN go test ./pkg/controller -run '^TestVMIMigrationSchedulingRetry$' -count=1
1416
RUN make build-go
1517

1618
WORKDIR /source/dist/images
Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
Backport of kubeovn/kube-ovn#6834 (a229dec3d5150f55ca8344d0e296a8a02d52b726).
2+
Remove when the image is updated to Kube-OVN v1.15.14 or later.
3+
4+
diff --git a/pkg/controller/kubevirt.go b/pkg/controller/kubevirt.go
5+
--- a/pkg/controller/kubevirt.go
6+
+++ b/pkg/controller/kubevirt.go
7+
@@ -186,6 +186,15 @@ func (c *Controller) handleAddOrUpdateVMIMigration(key string) error {
8+
if sourceNode == "" || targetPod.Spec.NodeName == "" || sourceNode == targetPod.Spec.NodeName {
9+
klog.Warningf("VM pod %s/%s migration setup skipped, source node: %s, target node: %s (migration job UID: %s)",
10+
targetPod.Namespace, targetPod.Name, sourceNode, targetPod.Spec.NodeName, vmiMigration.UID)
11+
+ // The source or target node may not be known yet while the migration is still
12+
+ // in the Scheduling phase; this produces no further phase-change event, so
13+
+ // return an error to requeue with rate limiting and retry once scheduling
14+
+ // completes, instead of dropping the event.
15+
+ // https://github.com/kubeovn/kube-ovn/issues/6823
16+
+ if sourceNode == "" || targetPod.Spec.NodeName == "" {
17+
+ return fmt.Errorf("VM pod %s/%s migration setup deferred, source node %q, target node %q not ready yet (migration job UID %s)",
18+
+ targetPod.Namespace, targetPod.Name, sourceNode, targetPod.Spec.NodeName, vmiMigration.UID)
19+
+ }
20+
return nil
21+
}
22+
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
package controller
2+
3+
import (
4+
"testing"
5+
"time"
6+
7+
"github.com/stretchr/testify/require"
8+
"go.uber.org/mock/gomock"
9+
corev1 "k8s.io/api/core/v1"
10+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
11+
corelisters "k8s.io/client-go/listers/core/v1"
12+
"k8s.io/client-go/tools/cache"
13+
"k8s.io/client-go/util/workqueue"
14+
kubevirtv1 "kubevirt.io/api/core/v1"
15+
"kubevirt.io/client-go/kubecli"
16+
17+
mockovs "github.com/kubeovn/kube-ovn/mocks/pkg/ovs"
18+
"github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb"
19+
)
20+
21+
func TestVMIMigrationSchedulingRetry(t *testing.T) {
22+
for _, tc := range []struct {
23+
name string
24+
sourceNode string
25+
targetNode string
26+
wantRetry bool
27+
wantSetup bool
28+
}{
29+
{"target not scheduled", "source", "", true, true},
30+
{"source not known", "", "target", true, true},
31+
{"both nodes unknown", "", "", true, true},
32+
{"already scheduled", "source", "target", false, true},
33+
{"same node", "source", "source", false, false},
34+
} {
35+
t.Run(tc.name, func(t *testing.T) {
36+
mockCtrl := gomock.NewController(t)
37+
virtClient := kubecli.NewMockKubevirtClient(mockCtrl)
38+
migrationClient := kubecli.NewMockVirtualMachineInstanceMigrationInterface(mockCtrl)
39+
vmiClient := kubecli.NewMockVirtualMachineInstanceInterface(mockCtrl)
40+
nbClient := mockovs.NewMockNbClient(mockCtrl)
41+
migration := &kubevirtv1.VirtualMachineInstanceMigration{
42+
ObjectMeta: metav1.ObjectMeta{Name: "migration", Namespace: "default", UID: "migration-uid"},
43+
Spec: kubevirtv1.VirtualMachineInstanceMigrationSpec{VMIName: "vm"},
44+
Status: kubevirtv1.VirtualMachineInstanceMigrationStatus{
45+
Phase: kubevirtv1.MigrationScheduling,
46+
MigrationState: &kubevirtv1.VirtualMachineInstanceMigrationState{},
47+
},
48+
}
49+
vmi := &kubevirtv1.VirtualMachineInstance{
50+
ObjectMeta: metav1.ObjectMeta{Name: "vm", Namespace: "default"},
51+
Status: kubevirtv1.VirtualMachineInstanceStatus{NodeName: tc.sourceNode},
52+
}
53+
pod := &corev1.Pod{
54+
ObjectMeta: metav1.ObjectMeta{Name: "target", Namespace: "default", Labels: map[string]string{
55+
kubevirtv1.MigrationJobLabel: string(migration.UID),
56+
}},
57+
Spec: corev1.PodSpec{NodeName: tc.targetNode},
58+
}
59+
indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
60+
require.NoError(t, indexer.Add(pod))
61+
virtClient.EXPECT().VirtualMachineInstanceMigration("default").Return(migrationClient).AnyTimes()
62+
virtClient.EXPECT().VirtualMachineInstance("default").Return(vmiClient).AnyTimes()
63+
migrationClient.EXPECT().Get(gomock.Any(), "migration", metav1.GetOptions{}).Return(migration, nil).AnyTimes()
64+
vmiClient.EXPECT().Get(gomock.Any(), "vm", metav1.GetOptions{}).Return(vmi, nil).AnyTimes()
65+
nbClient.EXPECT().ListNormalLogicalSwitchPorts(false, map[string]string{"pod": "default/vm"}).Return([]ovnnb.LogicalSwitchPort{{Name: "vm-port"}}, nil).AnyTimes()
66+
if tc.wantSetup {
67+
nbClient.EXPECT().SetLogicalSwitchPortMigrateOptions("vm-port", "source", "target").Return(nil).Times(1)
68+
}
69+
queue := workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[string]())
70+
t.Cleanup(queue.ShutDown)
71+
controller := &Controller{
72+
config: &Configuration{KubevirtClient: virtClient}, podsLister: corelisters.NewPodLister(indexer),
73+
OVNNbClient: nbClient, addOrUpdateVMIMigrationQueue: queue,
74+
}
75+
const key = "default/migration"
76+
queue.Add(key)
77+
require.True(t, processNextWorkItem("migration", queue, controller.handleAddOrUpdateVMIMigration, getWorkItemKey))
78+
if tc.wantRetry {
79+
require.Equal(t, 1, queue.NumRequeues(key), "an unscheduled target must not drop the migration event")
80+
vmi.Status.NodeName = "source"
81+
scheduledPod := pod.DeepCopy()
82+
scheduledPod.Spec.NodeName = "target"
83+
require.NoError(t, indexer.Update(scheduledPod))
84+
// The VMIM stays in Scheduling: only the rate-limited retry can finish setup.
85+
require.Eventually(t, func() bool { return queue.Len() == 1 }, time.Second, time.Millisecond)
86+
require.True(t, processNextWorkItem("migration", queue, controller.handleAddOrUpdateVMIMigration, getWorkItemKey))
87+
}
88+
require.Zero(t, queue.NumRequeues(key))
89+
require.Zero(t, queue.Len())
90+
})
91+
}
92+
}

0 commit comments

Comments
 (0)