Skip to content

Commit 1a21fc2

Browse files
authored
Merge pull request feast-dev#88 from ugiordan/fix/oom-informer-cache-flooding
fix: Harden informer cache with label selectors and memory optimizations
2 parents 9fba4fb + ce37cc3 commit 1a21fc2

9 files changed

Lines changed: 78 additions & 13 deletions

File tree

‎infra/feast-operator/cmd/main.go‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,16 +29,23 @@ import (
2929
// to ensure that exec-entrypoint and run can make use of them.
3030
_ "k8s.io/client-go/plugin/pkg/client/auth"
3131

32+
appsv1 "k8s.io/api/apps/v1"
33+
autoscalingv2 "k8s.io/api/autoscaling/v2"
34+
batchv1 "k8s.io/api/batch/v1"
3235
corev1 "k8s.io/api/core/v1"
36+
policyv1 "k8s.io/api/policy/v1"
37+
rbacv1 "k8s.io/api/rbac/v1"
3338
apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
3439
apierrors "k8s.io/apimachinery/pkg/api/errors"
3540
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
41+
"k8s.io/apimachinery/pkg/labels"
3642
"k8s.io/apimachinery/pkg/runtime"
3743
"k8s.io/apimachinery/pkg/runtime/schema"
3844
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
3945
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
4046
"k8s.io/client-go/rest"
4147
ctrl "sigs.k8s.io/controller-runtime"
48+
"sigs.k8s.io/controller-runtime/pkg/cache"
4249
"sigs.k8s.io/controller-runtime/pkg/client"
4350
"sigs.k8s.io/controller-runtime/pkg/healthz"
4451
"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -69,6 +76,29 @@ func init() {
6976
// +kubebuilder:scaffold:scheme
7077
}
7178

79+
func newCacheOptions() cache.Options {
80+
managedBySelector := labels.SelectorFromSet(labels.Set{
81+
services.ManagedByLabelKey: services.ManagedByLabelValue,
82+
})
83+
managedByFilter := cache.ByObject{Label: managedBySelector}
84+
85+
return cache.Options{
86+
DefaultTransform: cache.TransformStripManagedFields(),
87+
ByObject: map[client.Object]cache.ByObject{
88+
&corev1.ConfigMap{}: managedByFilter,
89+
&appsv1.Deployment{}: managedByFilter,
90+
&corev1.Service{}: managedByFilter,
91+
&corev1.ServiceAccount{}: managedByFilter,
92+
&corev1.PersistentVolumeClaim{}: managedByFilter,
93+
&rbacv1.RoleBinding{}: managedByFilter,
94+
&rbacv1.Role{}: managedByFilter,
95+
&batchv1.CronJob{}: managedByFilter,
96+
&autoscalingv2.HorizontalPodAutoscaler{}: managedByFilter,
97+
&policyv1.PodDisruptionBudget{}: managedByFilter,
98+
},
99+
}
100+
}
101+
72102
func main() {
73103
var metricsAddr string
74104
var enableLeaderElection bool
@@ -155,6 +185,7 @@ func main() {
155185
// if you are doing or is intended to do any operation such as perform cleanups
156186
// after the manager stops then its usage might be unsafe.
157187
// LeaderElectionReleaseOnCancel: true,
188+
Cache: newCacheOptions(),
158189
Client: client.Options{
159190
Cache: &client.CacheOptions{
160191
DisableFor: []client.Object{
Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,16 @@
1+
- op: test
2+
path: "/spec/template/spec/containers/0/env/1/name"
3+
value: RELATED_IMAGE_FEATURE_SERVER
14
- op: replace
2-
path: "/spec/template/spec/containers/0/env/0"
5+
path: "/spec/template/spec/containers/0/env/1"
36
value:
47
name: RELATED_IMAGE_FEATURE_SERVER
58
value: ${FS_IMG}
9+
- op: test
10+
path: "/spec/template/spec/containers/0/env/2/name"
11+
value: RELATED_IMAGE_CRON_JOB
612
- op: replace
7-
path: "/spec/template/spec/containers/0/env/1"
13+
path: "/spec/template/spec/containers/0/env/2"
814
value:
915
name: RELATED_IMAGE_CRON_JOB
1016
value: ${CJ_IMG}
Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,16 @@
1+
- op: test
2+
path: "/spec/template/spec/containers/0/env/1/name"
3+
value: RELATED_IMAGE_FEATURE_SERVER
14
- op: replace
2-
path: "/spec/template/spec/containers/0/env/0"
5+
path: "/spec/template/spec/containers/0/env/1"
36
value:
47
name: RELATED_IMAGE_FEATURE_SERVER
58
value: quay.io/feastdev/feature-server:0.61.0
9+
- op: test
10+
path: "/spec/template/spec/containers/0/env/2/name"
11+
value: RELATED_IMAGE_CRON_JOB
612
- op: replace
7-
path: "/spec/template/spec/containers/0/env/1"
13+
path: "/spec/template/spec/containers/0/env/2"
814
value:
915
name: RELATED_IMAGE_CRON_JOB
1016
value: quay.io/openshift/origin-cli:4.17

‎infra/feast-operator/config/manager/manager.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,8 @@ spec:
7373
drop:
7474
- "ALL"
7575
env:
76+
- name: GOMEMLIMIT
77+
value: "230MiB"
7678
- name: RELATED_IMAGE_FEATURE_SERVER
7779
value: feast:latest
7880
- name: RELATED_IMAGE_CRON_JOB

‎infra/feast-operator/dist/install.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20615,6 +20615,8 @@ spec:
2061520615
command:
2061620616
- /manager
2061720617
env:
20618+
- name: GOMEMLIMIT
20619+
value: 230MiB
2061820620
- name: RELATED_IMAGE_FEATURE_SERVER
2061920621
value: quay.io/feastdev/feature-server:0.61.0
2062020622
- name: RELATED_IMAGE_CRON_JOB

‎infra/feast-operator/internal/controller/notebook_configmap_controller.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,7 @@ func (r *NotebookConfigMapReconciler) setNotebookConfigMapData(
214214
if cm.Labels == nil {
215215
cm.Labels = make(map[string]string)
216216
}
217-
cm.Labels["managed-by"] = "feast-operator"
217+
cm.Labels[services.ManagedByLabelKey] = services.ManagedByLabelValue
218218
cm.Labels["source-resource"] = notebook.GetName()
219219
cm.Labels["source-kind"] = notebook.GetObjectKind().GroupVersionKind().Kind
220220

‎infra/feast-operator/internal/controller/notebook_configmap_controller_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -381,7 +381,7 @@ var _ = Describe("NotebookConfigMap Controller", func() {
381381
Expect(err).NotTo(HaveOccurred())
382382
Expect(cm.Data).To(HaveKey(projectName))
383383
Expect(cm.Data[projectName]).To(Equal(testYAMLContent))
384-
Expect(cm.Labels["managed-by"]).To(Equal("feast-operator"))
384+
Expect(cm.Labels[services.ManagedByLabelKey]).To(Equal(services.ManagedByLabelValue))
385385
Expect(cm.Labels["source-resource"]).To(Equal(notebookName))
386386
Expect(cm.Labels["source-kind"]).To(Equal("Notebook"))
387387
Expect(cm.OwnerReferences).To(HaveLen(1))

‎infra/feast-operator/internal/controller/services/services.go‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -445,9 +445,10 @@ func (feast *FeastServices) setDeployment(deploy *appsv1.Deployment) error {
445445
}
446446

447447
deploy.Labels = feast.getLabels()
448+
selectorLabels := feast.getSelectorLabels()
448449
deploy.Spec = appsv1.DeploymentSpec{
449450
Replicas: replicas,
450-
Selector: metav1.SetAsLabelSelector(deploy.GetLabels()),
451+
Selector: metav1.SetAsLabelSelector(selectorLabels),
451452
Strategy: feast.getDeploymentStrategy(),
452453
Template: corev1.PodTemplateSpec{
453454
ObjectMeta: metav1.ObjectMeta{
@@ -864,7 +865,7 @@ func (feast *FeastServices) setService(svc *corev1.Service, feastType FeastServi
864865
}
865866

866867
svc.Spec = corev1.ServiceSpec{
867-
Selector: feast.getLabels(),
868+
Selector: feast.getSelectorLabels(),
868869
Type: corev1.ServiceTypeClusterIP,
869870
Ports: []corev1.ServicePort{
870871
{
@@ -1022,7 +1023,7 @@ func (feast *FeastServices) applyTopologySpread(podSpec *corev1.PodSpec) {
10221023
MaxSkew: 1,
10231024
TopologyKey: "topology.kubernetes.io/zone",
10241025
WhenUnsatisfiable: corev1.ScheduleAnyway,
1025-
LabelSelector: metav1.SetAsLabelSelector(feast.getLabels()),
1026+
LabelSelector: metav1.SetAsLabelSelector(feast.getSelectorLabels()),
10261027
}}
10271028
}
10281029

@@ -1045,7 +1046,7 @@ func (feast *FeastServices) applyAffinity(podSpec *corev1.PodSpec) {
10451046
Weight: 100,
10461047
PodAffinityTerm: corev1.PodAffinityTerm{
10471048
TopologyKey: "kubernetes.io/hostname",
1048-
LabelSelector: metav1.SetAsLabelSelector(feast.getLabels()),
1049+
LabelSelector: metav1.SetAsLabelSelector(feast.getSelectorLabels()),
10491050
},
10501051
}},
10511052
},
@@ -1106,12 +1107,24 @@ func (feast *FeastServices) getFeastTypeLabels(feastType FeastServiceType) map[s
11061107
return labels
11071108
}
11081109

1109-
func (feast *FeastServices) getLabels() map[string]string {
1110+
// getSelectorLabels returns the minimal label set used for immutable selectors
1111+
// (Deployment spec.selector, Service spec.selector, TopologySpreadConstraints, PodAffinity).
1112+
// This must NOT change after initial resource creation.
1113+
func (feast *FeastServices) getSelectorLabels() map[string]string {
11101114
return map[string]string{
11111115
NameLabelKey: feast.Handler.FeatureStore.Name,
11121116
}
11131117
}
11141118

1119+
// getLabels returns the full label set for mutable metadata (ObjectMeta.Labels).
1120+
// Includes the managed-by label used by the informer cache filter.
1121+
func (feast *FeastServices) getLabels() map[string]string {
1122+
return map[string]string{
1123+
NameLabelKey: feast.Handler.FeatureStore.Name,
1124+
ManagedByLabelKey: ManagedByLabelValue,
1125+
}
1126+
}
1127+
11151128
func (feast *FeastServices) setServiceHostnames() error {
11161129
feast.Handler.FeatureStore.Status.ServiceHostnames = feastdevv1.ServiceHostnames{}
11171130
domain := svcDomain + ":"
@@ -1489,10 +1502,10 @@ func IsDeploymentAvailable(conditions []appsv1.DeploymentCondition) bool {
14891502
// container that is in a failing state. Returns empty string if no failure found.
14901503
func (feast *FeastServices) GetPodContainerFailureMessage(deploy appsv1.Deployment) string {
14911504
podList := corev1.PodList{}
1492-
labels := feast.getLabels()
1505+
selectorLabels := feast.getSelectorLabels()
14931506
if err := feast.Handler.Client.List(feast.Handler.Context, &podList,
14941507
client.InNamespace(deploy.Namespace),
1495-
client.MatchingLabels(labels),
1508+
client.MatchingLabels(selectorLabels),
14961509
); err != nil {
14971510
return ""
14981511
}

‎infra/feast-operator/internal/controller/services/services_types.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,11 @@ const (
9797
OidcMissingSecretError string = "missing OIDC secret: %s"
9898
)
9999

100+
const (
101+
ManagedByLabelKey = "app.kubernetes.io/managed-by"
102+
ManagedByLabelValue = "feast-operator"
103+
)
104+
100105
var (
101106
DefaultImage = "quay.io/feastdev/feature-server:" + feastversion.FeastVersion
102107
DefaultCronJobImage = "quay.io/openshift/origin-cli:4.17"

0 commit comments

Comments
 (0)