Skip to content

Commit 93faa7a

Browse files
mattia-eleuterimattia-eleuteri
committed
fix(lineage, kubernetes): resolve CAPK node VMs to their cluster
The Cluster API KubeVirt provider creates a node's VirtualMachine without an ownerReference and links it to its KubevirtMachine through the capk.cluster.x-k8s.io/kubevirt-machine-{name,namespace} labels only. The ownership walk therefore stopped at the VM, found no ancestor, and the lineage webhook left every object below it unmanaged: the virt-launcher pods of worker nodes never got internal.cozystack.io/managed-by-cozystack, while still carrying the application lineage labels copied from the machine template. Anything that relies on the webhook having resolved those pods misses them. The securitygroup-controller caches only managed pods, so a SecurityGroup attached to a Kubernetes application labelled the control-plane pods and never the nodes. Follow the CAPK labels from the VM to its KubevirtMachine, whose ownerReferences reach the CAPI Cluster and from there the cluster's HelmRelease. Only a KubevirtMachine in the VM's own namespace is followed: CAPK can target an external infrastructure cluster, and the walk must never attribute an object across namespaces. Signed-off-by: Mattia Eleuteri <[email protected]> Assisted-by: LLM
1 parent 7dc5c0b commit 93faa7a

2 files changed

Lines changed: 139 additions & 0 deletions

File tree

‎pkg/lineage/capk_test.go‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
package lineage
2+
3+
import (
4+
"context"
5+
"testing"
6+
7+
helmv2 "github.com/fluxcd/helm-controller/api/v2"
8+
"k8s.io/apimachinery/pkg/api/meta"
9+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
10+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
11+
"k8s.io/apimachinery/pkg/runtime"
12+
"k8s.io/apimachinery/pkg/runtime/schema"
13+
dynamicfake "k8s.io/client-go/dynamic/fake"
14+
)
15+
16+
const capkNamespace = "tenant-demo"
17+
18+
// kubernetesAppMapper maps the cluster's HelmRelease to its Kubernetes
19+
// application, as the webhook's ApplicationDefinition mapper does.
20+
type kubernetesAppMapper struct{}
21+
22+
func (kubernetesAppMapper) Map(*helmv2.HelmRelease) (string, string, string, error) {
23+
return "apps.cozystack.io/v1alpha1", "Kubernetes", "kubernetes-", nil
24+
}
25+
26+
func capkObject(apiVersion, kind, name string, labels map[string]string, owner *unstructured.Unstructured) *unstructured.Unstructured {
27+
o := &unstructured.Unstructured{}
28+
o.SetAPIVersion(apiVersion)
29+
o.SetKind(kind)
30+
o.SetNamespace(capkNamespace)
31+
o.SetName(name)
32+
o.SetLabels(labels)
33+
if owner != nil {
34+
o.SetOwnerReferences([]metav1.OwnerReference{{APIVersion: owner.GetAPIVersion(), Kind: owner.GetKind(), Name: owner.GetName()}})
35+
}
36+
return o
37+
}
38+
39+
// capkNodeChain builds the objects behind the virt-launcher pod of a
40+
// Kubernetes worker node, as observed on a live cluster: the pod is owned by
41+
// its VMI and the VMI by its VM, but CAPK leaves the VM without an
42+
// ownerReference and points at its KubevirtMachine through labels only. From
43+
// the KubevirtMachine up, ownerReferences run to the CAPI Cluster, which the
44+
// cluster's HelmRelease labels.
45+
func capkNodeChain(vmLabels map[string]string) (*unstructured.Unstructured, []runtime.Object, meta.RESTMapper) {
46+
app := capkObject("apps.cozystack.io/v1alpha1", "Kubernetes", "cluster-a", nil, nil)
47+
hr := capkObject(HRAPIVersion, HRKind, "kubernetes-cluster-a", nil, nil)
48+
cluster := capkObject("cluster.x-k8s.io/v1beta1", "Cluster", "kubernetes-cluster-a", map[string]string{HRLabel: "kubernetes-cluster-a"}, nil)
49+
md := capkObject("cluster.x-k8s.io/v1beta1", "MachineDeployment", "kubernetes-cluster-a-md0", nil, cluster)
50+
ms := capkObject("cluster.x-k8s.io/v1beta1", "MachineSet", "kubernetes-cluster-a-md0-x", nil, md)
51+
machine := capkObject("cluster.x-k8s.io/v1beta1", "Machine", "kubernetes-cluster-a-md0-x-y", nil, ms)
52+
kvm := capkObject(KubevirtMachineAPIVersion, KubevirtMachineKind, "kubernetes-cluster-a-md0-x-y", nil, machine)
53+
vm := capkObject(VMAPIVersion, VMKind, "kubernetes-cluster-a-md0-x-y", vmLabels, nil)
54+
vmi := capkObject("kubevirt.io/v1", "VirtualMachineInstance", "kubernetes-cluster-a-md0-x-y", nil, vm)
55+
pod := capkObject("v1", "Pod", "virt-launcher-kubernetes-cluster-a-md0-x-y-abcde", nil, vmi)
56+
57+
objs := []runtime.Object{app, hr, cluster, md, ms, machine, kvm, vm, vmi}
58+
mapper := meta.NewDefaultRESTMapper(nil)
59+
for _, o := range objs {
60+
mapper.Add(o.GetObjectKind().GroupVersionKind(), meta.RESTScopeNamespace)
61+
}
62+
mapper.Add(schema.GroupVersionKind{Version: "v1", Kind: "Pod"}, meta.RESTScopeNamespace)
63+
return pod, objs, mapper
64+
}
65+
66+
func walkNodePod(t *testing.T, vmLabels map[string]string) []ObjectID {
67+
t.Helper()
68+
pod, objs, mapper := capkNodeChain(vmLabels)
69+
dyn := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), objs...)
70+
return WalkOwnershipGraph(context.Background(), dyn, mapper, kubernetesAppMapper{}, pod)
71+
}
72+
73+
func TestWalkResolvesCAPKNodeVMToKubernetesApp(t *testing.T) {
74+
got := walkNodePod(t, map[string]string{
75+
KubevirtMachineNameLabel: "kubernetes-cluster-a-md0-x-y",
76+
KubevirtMachineNamespaceLabel: capkNamespace,
77+
})
78+
want := ObjectID{APIVersion: "apps.cozystack.io/v1alpha1", Kind: "Kubernetes", Namespace: capkNamespace, Name: "cluster-a"}
79+
if len(got) != 1 || got[0] != want {
80+
t.Fatalf("WalkOwnershipGraph(node pod) = %+v, want [%+v]", got, want)
81+
}
82+
}
83+
84+
func TestWalkIgnoresKubevirtMachineOfAnotherNamespace(t *testing.T) {
85+
// A VM pointing at a KubevirtMachine outside its own namespace (an external
86+
// infrastructure cluster, or a forged label) must not be resolved through it.
87+
got := walkNodePod(t, map[string]string{
88+
KubevirtMachineNameLabel: "kubernetes-cluster-a-md0-x-y",
89+
KubevirtMachineNamespaceLabel: "tenant-other",
90+
})
91+
if len(got) != 0 {
92+
t.Fatalf("WalkOwnershipGraph(node pod) = %+v, want no ancestors", got)
93+
}
94+
}
95+
96+
func TestWalkStopsAtUnlabelledVM(t *testing.T) {
97+
// A standalone VM with no CAPK labels and no ownerReference has no ancestors.
98+
if got := walkNodePod(t, nil); len(got) != 0 {
99+
t.Fatalf("WalkOwnershipGraph(pod of standalone VM) = %+v, want no ancestors", got)
100+
}
101+
}

‎pkg/lineage/lineage.go‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,16 @@ const (
2020
HRAPIVersion = "helm.toolkit.fluxcd.io/v2"
2121
HRKind = "HelmRelease"
2222
HRLabel = "helm.toolkit.fluxcd.io/name"
23+
24+
// KubeVirt VirtualMachines created by the Cluster API KubeVirt provider
25+
// (CAPK) for cluster nodes carry no ownerReference: CAPK links them to
26+
// their KubevirtMachine through these labels only.
27+
VMAPIVersion = "kubevirt.io/v1"
28+
VMKind = "VirtualMachine"
29+
KubevirtMachineAPIVersion = "infrastructure.cluster.x-k8s.io/v1alpha1"
30+
KubevirtMachineKind = "KubevirtMachine"
31+
KubevirtMachineNameLabel = "capk.cluster.x-k8s.io/kubevirt-machine-name"
32+
KubevirtMachineNamespaceLabel = "capk.cluster.x-k8s.io/kubevirt-machine-namespace"
2333
)
2434

2535
// AppMapper maps HelmRelease to application metadata.
@@ -91,6 +101,18 @@ func WalkOwnershipGraph(
91101
return
92102
}
93103

104+
// A node VM of a Kubernetes cluster has no ownerReference, so without this
105+
// step the walk would stop at it and never reach the cluster's application.
106+
if name, ok := kubevirtMachineName(obj); ok {
107+
ownerObj, err := getUnstructuredObject(ctx, client, mapper, state.cache, KubevirtMachineAPIVersion, KubevirtMachineKind, obj.GetNamespace(), name)
108+
if err != nil {
109+
l.Error(err, "couldn't get KubevirtMachine of VirtualMachine", "namespace", obj.GetNamespace(), "virtualMachine", obj.GetName(), "kubevirtMachine", name)
110+
return
111+
}
112+
out = append(out, WalkOwnershipGraph(ctx, client, mapper, appMapper, ownerObj, state)...)
113+
return
114+
}
115+
94116
// I want "if err1 != nil go to next block, if err2 != nil, go to next block, etc semantics",
95117
// like an early return from a function, but if all checks succeed, I don't want to do the rest
96118
// of the function, so it's a `for { if err { break } if othererr { break } if allgood { return }
@@ -138,6 +160,22 @@ func WalkOwnershipGraph(
138160
return
139161
}
140162

163+
// kubevirtMachineName returns the KubevirtMachine a CAPK-created
164+
// VirtualMachine belongs to. It only follows a KubevirtMachine in the VM's own
165+
// namespace: CAPK can also run against an external infrastructure cluster,
166+
// where the KubevirtMachine lives elsewhere and is not ours to resolve.
167+
func kubevirtMachineName(obj *unstructured.Unstructured) (string, bool) {
168+
if obj.GetAPIVersion() != VMAPIVersion || obj.GetKind() != VMKind {
169+
return "", false
170+
}
171+
labels := obj.GetLabels()
172+
name := labels[KubevirtMachineNameLabel]
173+
if name == "" || labels[KubevirtMachineNamespaceLabel] != obj.GetNamespace() {
174+
return "", false
175+
}
176+
return name, true
177+
}
178+
141179
func parseWalkMemory(memory []interface{}) (*walkState, error) {
142180
switch len(memory) {
143181
case 0:

0 commit comments

Comments
 (0)