Skip to content

Commit e8fa1ac

Browse files
committed
[cozystack-controller] Ancestor tracking webhook
Many resources created as part of managed apps in cozystack (pods, secrets, etc) do not carry predictable labels that unambiguously indicate which app originally triggered their creation. Some resources are managed by controllers and other custom resources and this indirection can lead to loss of information. Other controllers sometimes simply do not allow setting labels on controlled resources and the latter do not inherit labels from the owner. This patch implements a webhook that sidesteps this problem with a universal solution. On creation of a pod/secret/PVC etc it walks through the owner references until a HelmRelease is found that can be matched with a managed app dynamically registered in the Cozystack API server. The pod is mutated with labels identifying the managed app. ```release-note [cozystack-controller] Add a mutating webhook to identify the Cozystack managed app that ultimately owns low-level resources created in the cluster and label these resources with a reference to said app. ``` Signed-off-by: Timofei Larkin <[email protected]>
1 parent 9f89ef3 commit e8fa1ac

12 files changed

Lines changed: 677 additions & 1 deletion

File tree

‎cmd/cozystack-controller/main.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ import (
3838

3939
cozystackiov1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
4040
"github.com/cozystack/cozystack/internal/controller"
41+
lcw "github.com/cozystack/cozystack/internal/lineagecontrollerwebhook"
4142
"github.com/cozystack/cozystack/internal/telemetry"
4243

4344
helmv2 "github.com/fluxcd/helm-controller/api/v2"
@@ -214,6 +215,20 @@ func main() {
214215
os.Exit(1)
215216
}
216217

218+
// special one that's both a webhook and a reconciler
219+
lineageControllerWebhook := &lcw.LineageControllerWebhook{
220+
Client: mgr.GetClient(),
221+
Scheme: mgr.GetScheme(),
222+
}
223+
if err := lineageControllerWebhook.SetupWithManagerAsController(mgr); err != nil {
224+
setupLog.Error(err, "unable to setup controller", "controller", "LineageController")
225+
os.Exit(1)
226+
}
227+
if err := lineageControllerWebhook.SetupWithManagerAsWebhook(mgr); err != nil {
228+
setupLog.Error(err, "unable to setup webhook", "webhook", "LineageWebhook")
229+
os.Exit(1)
230+
}
231+
217232
// +kubebuilder:scaffold:builder
218233

219234
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
package lineagecontrollerwebhook
2+
3+
import (
4+
"fmt"
5+
6+
helmv2 "github.com/fluxcd/helm-controller/api/v2"
7+
)
8+
9+
type chartRef struct {
10+
repo string
11+
chart string
12+
}
13+
14+
type appRef struct {
15+
groupVersion string
16+
kind string
17+
prefix string
18+
}
19+
20+
type runtimeConfig struct {
21+
chartAppMap map[chartRef]appRef
22+
}
23+
24+
func (l *LineageControllerWebhook) initConfig() {
25+
l.initOnce.Do(func() {
26+
if l.config.Load() == nil {
27+
l.config.Store(&runtimeConfig{chartAppMap: make(map[chartRef]appRef)})
28+
}
29+
})
30+
}
31+
32+
func (l *LineageControllerWebhook) Map(hr *helmv2.HelmRelease) (string, string, string, error) {
33+
cfg := l.config.Load().(*runtimeConfig).chartAppMap
34+
s := &hr.Spec.Chart.Spec
35+
val, ok := cfg[chartRef{s.SourceRef.Name, s.Chart}]
36+
if !ok {
37+
return "", "", "", fmt.Errorf("cannot map helm release %s/%s to dynamic app", hr.Namespace, hr.Name)
38+
}
39+
return val.groupVersion, val.kind, val.prefix, nil
40+
}
Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package lineagecontrollerwebhook
2+
3+
import (
4+
"context"
5+
6+
cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
7+
ctrl "sigs.k8s.io/controller-runtime"
8+
"sigs.k8s.io/controller-runtime/pkg/client"
9+
"sigs.k8s.io/controller-runtime/pkg/log"
10+
)
11+
12+
// +kubebuilder:rbac:groups=cozystack.io,resources=cozystackresourcedefinitions,verbs=list;watch
13+
14+
func (c *LineageControllerWebhook) SetupWithManagerAsController(mgr ctrl.Manager) error {
15+
return ctrl.NewControllerManagedBy(mgr).
16+
For(&cozyv1alpha1.CozystackResourceDefinition{}).
17+
Complete(c)
18+
}
19+
20+
func (c *LineageControllerWebhook) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
21+
l := log.FromContext(ctx)
22+
crds := &cozyv1alpha1.CozystackResourceDefinitionList{}
23+
if err := c.List(ctx, crds, &client.ListOptions{Namespace: "cozy-system"}); err != nil {
24+
l.Error(err, "failed reading CozystackResourceDefinitions")
25+
return ctrl.Result{}, err
26+
}
27+
newConfig := make(map[chartRef]appRef)
28+
for _, crd := range crds.Items {
29+
k := chartRef{
30+
crd.Spec.Release.Chart.SourceRef.Name,
31+
crd.Spec.Release.Chart.Name,
32+
}
33+
newRef := appRef{"apps.cozystack.io/v1alpha1", crd.Spec.Application.Kind, crd.Spec.Release.Prefix}
34+
if oldRef, exists := newConfig[k]; exists {
35+
l.Info("duplicate chart mapping detected; ignoring subsequent entry", "key", k, "retained value", oldRef, "ignored value", newRef)
36+
continue
37+
}
38+
newConfig[k] = newRef
39+
}
40+
c.config.Store(&runtimeConfig{newConfig})
41+
return ctrl.Result{}, nil
42+
}
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
package lineagecontrollerwebhook
2+
3+
import (
4+
"sync"
5+
"sync/atomic"
6+
7+
"k8s.io/apimachinery/pkg/api/meta"
8+
"k8s.io/apimachinery/pkg/runtime"
9+
"k8s.io/client-go/dynamic"
10+
"sigs.k8s.io/controller-runtime/pkg/client"
11+
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
12+
)
13+
14+
// +kubebuilder:webhook:path=/mutate-lineage,mutating=true,failurePolicy=Fail,sideEffects=None,groups="",resources=pods,secrets,services,persistentvolumeclaims,verbs=create;update,versions=v1,name=mlineage.cozystack.io,admissionReviewVersions={v1}
15+
type LineageControllerWebhook struct {
16+
client.Client
17+
Scheme *runtime.Scheme
18+
decoder admission.Decoder
19+
dynClient dynamic.Interface
20+
mapper meta.RESTMapper
21+
config atomic.Value
22+
initOnce sync.Once
23+
}
Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
package lineagecontrollerwebhook
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"errors"
7+
"fmt"
8+
9+
"github.com/cozystack/cozystack/pkg/lineage"
10+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
11+
"k8s.io/apimachinery/pkg/runtime/schema"
12+
"k8s.io/client-go/discovery"
13+
"k8s.io/client-go/discovery/cached/memory"
14+
"k8s.io/client-go/dynamic"
15+
"k8s.io/client-go/rest"
16+
"k8s.io/client-go/restmapper"
17+
ctrl "sigs.k8s.io/controller-runtime"
18+
"sigs.k8s.io/controller-runtime/pkg/client"
19+
"sigs.k8s.io/controller-runtime/pkg/log"
20+
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
21+
)
22+
23+
var (
24+
NoAncestors = fmt.Errorf("no managed apps found in lineage")
25+
AncestryAmbiguous = fmt.Errorf("object ancestry is ambiguous")
26+
)
27+
28+
// SetupWithManager registers the handler with the webhook server.
29+
func (h *LineageControllerWebhook) SetupWithManagerAsWebhook(mgr ctrl.Manager) error {
30+
cfg := rest.CopyConfig(mgr.GetConfig())
31+
32+
var err error
33+
h.dynClient, err = dynamic.NewForConfig(cfg)
34+
if err != nil {
35+
return err
36+
}
37+
38+
discoClient, err := discovery.NewDiscoveryClientForConfig(cfg)
39+
if err != nil {
40+
return err
41+
}
42+
43+
cachedDisco := memory.NewMemCacheClient(discoClient)
44+
h.mapper = restmapper.NewDeferredDiscoveryRESTMapper(cachedDisco)
45+
46+
h.initConfig()
47+
// Register HTTP path -> handler.
48+
mgr.GetWebhookServer().Register("/mutate-lineage", &admission.Webhook{Handler: h})
49+
50+
return nil
51+
}
52+
53+
// InjectDecoder lets controller-runtime give us a decoder for AdmissionReview requests.
54+
func (h *LineageControllerWebhook) InjectDecoder(d admission.Decoder) error {
55+
h.decoder = d
56+
return nil
57+
}
58+
59+
// Handle is called for each AdmissionReview that matches the webhook config.
60+
func (h *LineageControllerWebhook) Handle(ctx context.Context, req admission.Request) admission.Response {
61+
logger := log.FromContext(ctx).WithValues(
62+
"gvk", req.Kind.String(),
63+
"namespace", req.Namespace,
64+
"name", req.Name,
65+
"operation", req.Operation,
66+
)
67+
warn := make(admission.Warnings, 0)
68+
69+
obj := &unstructured.Unstructured{}
70+
if err := h.decodeUnstructured(req, obj); err != nil {
71+
return admission.Errored(400, fmt.Errorf("decode object: %w", err))
72+
}
73+
74+
labels, err := h.computeLabels(ctx, obj)
75+
for {
76+
if err != nil && errors.Is(err, NoAncestors) {
77+
return admission.Allowed("object not managed by app")
78+
}
79+
if err != nil && errors.Is(err, AncestryAmbiguous) {
80+
warn = append(warn, "object ancestry ambiguous, using first ancestor found")
81+
break
82+
}
83+
if err != nil {
84+
return admission.Errored(500, fmt.Errorf("error computing lineage labels: %w", err))
85+
}
86+
if err == nil {
87+
break
88+
}
89+
}
90+
91+
h.applyLabels(obj, labels)
92+
93+
mutated, err := json.Marshal(obj)
94+
if err != nil {
95+
return admission.Errored(500, fmt.Errorf("marshal mutated pod: %w", err))
96+
}
97+
logger.V(1).Info("mutated pod", "namespace", obj.GetNamespace(), "name", obj.GetName())
98+
return admission.PatchResponseFromRaw(req.Object.Raw, mutated).WithWarnings(warn...)
99+
}
100+
101+
func (h *LineageControllerWebhook) computeLabels(ctx context.Context, o *unstructured.Unstructured) (map[string]string, error) {
102+
owners := lineage.WalkOwnershipGraph(ctx, h.dynClient, h.mapper, h, o)
103+
if len(owners) == 0 {
104+
return nil, NoAncestors
105+
}
106+
obj, err := owners[0].GetUnstructured(ctx, h.dynClient, h.mapper)
107+
if err != nil {
108+
return nil, err
109+
}
110+
gv, err := schema.ParseGroupVersion(obj.GetAPIVersion())
111+
if err != nil {
112+
// should never happen, we got an APIVersion right from the API
113+
return nil, fmt.Errorf("could not parse APIVersion %s to a group and version: %w", obj.GetAPIVersion(), err)
114+
}
115+
if len(owners) > 1 {
116+
err = AncestryAmbiguous
117+
}
118+
return map[string]string{
119+
// truncate apigroup to first 63 chars
120+
"apps.cozystack.io/application.group": func(s string) string {
121+
if len(s) < 63 {
122+
return s
123+
}
124+
s = s[:63]
125+
for b := s[62]; !((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9')); s = s[:len(s)-1] {
126+
b = s[len(s)-1]
127+
}
128+
return s
129+
}(gv.Group),
130+
"apps.cozystack.io/application.kind": obj.GetKind(),
131+
"apps.cozystack.io/application.name": obj.GetName(),
132+
}, err
133+
}
134+
135+
func (h *LineageControllerWebhook) applyLabels(o client.Object, labels map[string]string) {
136+
existing := o.GetLabels()
137+
if existing == nil {
138+
existing = make(map[string]string)
139+
}
140+
for k, v := range labels {
141+
existing[k] = v
142+
}
143+
o.SetLabels(existing)
144+
}
145+
146+
func (h *LineageControllerWebhook) decodeUnstructured(req admission.Request, out *unstructured.Unstructured) error {
147+
if h.decoder != nil {
148+
if err := h.decoder.Decode(req, out); err == nil {
149+
return nil
150+
}
151+
if req.Kind.Group != "" || req.Kind.Kind != "" || req.Kind.Version != "" {
152+
out.SetGroupVersionKind(schema.GroupVersionKind{
153+
Group: req.Kind.Group,
154+
Version: req.Kind.Version,
155+
Kind: req.Kind.Kind,
156+
})
157+
if err := h.decoder.Decode(req, out); err == nil {
158+
return nil
159+
}
160+
}
161+
}
162+
if len(req.Object.Raw) == 0 {
163+
return errors.New("empty admission object")
164+
}
165+
return json.Unmarshal(req.Object.Raw, &out.Object)
166+
}
Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
apiVersion: cert-manager.io/v1
2+
kind: Issuer
3+
metadata:
4+
name: cozystack-controller-webhook-selfsigned
5+
namespace: {{ .Release.Namespace }}
6+
spec:
7+
selfSigned: {}
8+
---
9+
apiVersion: cert-manager.io/v1
10+
kind: Certificate
11+
metadata:
12+
name: cozystack-controller-webhook-ca
13+
namespace: {{ .Release.Namespace }}
14+
spec:
15+
secretName: cozystack-controller-webhook-ca
16+
duration: 43800h # 5 years
17+
commonName: cozystack-controller-webhook-ca
18+
issuerRef:
19+
name: cozystack-controller-webhook-selfsigned
20+
isCA: true
21+
---
22+
apiVersion: cert-manager.io/v1
23+
kind: Issuer
24+
metadata:
25+
name: cozystack-controller-webhook-ca
26+
namespace: {{ .Release.Namespace }}
27+
spec:
28+
ca:
29+
secretName: cozystack-controller-webhook-ca
30+
---
31+
apiVersion: cert-manager.io/v1
32+
kind: Certificate
33+
metadata:
34+
name: cozystack-controller-webhook
35+
namespace: {{ .Release.Namespace }}
36+
spec:
37+
secretName: cozystack-controller-webhook-cert
38+
duration: 8760h
39+
renewBefore: 720h
40+
issuerRef:
41+
name: cozystack-controller-webhook-ca
42+
commonName: cozystack-controller
43+
dnsNames:
44+
- cozystack-controller
45+
- cozystack-controller.{{ .Release.Namespace }}.svc

‎packages/system/cozystack-controller/templates/deployment.yaml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ apiVersion: apps/v1
22
kind: Deployment
33
metadata:
44
name: cozystack-controller
5-
namespace: cozy-system
65
labels:
76
app: cozystack-controller
87
spec:
@@ -29,3 +28,6 @@ spec:
2928
{{- if .Values.cozystackController.disableTelemetry }}
3029
- --disable-telemetry
3130
{{- end }}
31+
ports:
32+
- name: webhook
33+
containerPort: 9443
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
apiVersion: admissionregistration.k8s.io/v1
2+
kind: MutatingWebhookConfiguration
3+
metadata:
4+
name: lineage
5+
annotations:
6+
cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/cozystack-controller-webhook
7+
labels:
8+
app: cozystack-controller
9+
webhooks:
10+
- name: lineage.cozystack.io
11+
admissionReviewVersions: ["v1"]
12+
sideEffects: None
13+
clientConfig:
14+
service:
15+
name: cozystack-controller
16+
namespace: {{ .Release.Namespace }}
17+
path: /mutate-lineage
18+
rules:
19+
- operations: ["CREATE"]
20+
apiGroups: [""]
21+
apiVersions: ["v1"]
22+
resources: ["pods","secrets", "services", "persistentvolumeclaims"]
23+
failurePolicy: Fail
24+
namespaceSelector:
25+
matchExpressions:
26+
- key: cozystack.io/system
27+
operator: NotIn
28+
values:
29+
- "true"
30+
- key: kubernetes.io/metadata.name
31+
operator: NotIn
32+
values:
33+
- kube-system

0 commit comments

Comments
 (0)