Skip to content

Commit a6d7db4

Browse files
fix(lineage-webhook): keep the tenant class when an application's is refused
The webhook fell back to the namespace class only when an application left spec.schedulingClass empty. A non-empty value naming a class that does not exist reached the not-found return and the pod got no scheduler at all, so a typo, or a deliberate bogus name, in any application lifted the placement constraint the tenant administrator had set. An application class now applies only when the namespace lists it in its allowed classes and the SchedulingClass exists; otherwise the pod keeps the tenant class. Admission already refuses a class outside the list, but checking it again here also covers a class removed from the list after the application was admitted. Naming the tenant's own class needs no entry, being the same as leaving the field empty. Assisted-by: LLM Signed-off-by: Mattia Eleuteri <[email protected]>
1 parent 4bf46f3 commit a6d7db4

3 files changed

Lines changed: 164 additions & 26 deletions

File tree

‎internal/lineagecontrollerwebhook/webhook.go‎

Lines changed: 65 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"errors"
77
"fmt"
88
"maps"
9+
"slices"
910
"strings"
1011

1112
"github.com/cozystack/cozystack/pkg/lineage"
@@ -231,49 +232,62 @@ func hasAllLineageLabels(o *unstructured.Unstructured) bool {
231232
return true
232233
}
233234

235+
// AllowedSchedulingClassesAnnotation lists, comma-separated, the
236+
// SchedulingClasses an application in the namespace may select through
237+
// spec.schedulingClass. The tenant chart writes it from the tenant's
238+
// allowedSchedulingClasses; an absent or empty annotation allows none.
239+
const AllowedSchedulingClassesAnnotation = "namespace.cozystack.io/allowed-scheduling-classes"
240+
234241
// applySchedulingClass injects schedulerName, scheduling-class annotation, and
235-
// scheduling-class label into Pods whose namespace carries the
236-
// scheduler.cozystack.io/scheduling-class label.
242+
// scheduling-class label into Pods whose application or namespace selects a
243+
// SchedulingClass.
237244
//
238245
// The label (in addition to the annotation) lets SchedulingClass authors write
239246
// podAffinity rules whose labelSelector matches pods across applications that
240247
// share the same class.
241248
//
242-
// If the referenced SchedulingClass CR does not exist (e.g. the scheduler
249+
// The tenant class comes from the namespace label and is set by whoever
250+
// administers the tenant, so an application may replace it only with a class
251+
// the namespace allows and that exists. Anything else falls back to the tenant
252+
// class rather than dropping the injection, which would let a typo in an
253+
// application lift the tenant's placement constraint. Admission of the
254+
// application already refuses a class outside the allowed list; checking it
255+
// again here covers a class removed from the list after the application was
256+
// admitted.
257+
//
258+
// If the resolved SchedulingClass CR does not exist (e.g. the scheduler
243259
// package is not installed), the injection is silently skipped so that pods
244260
// are not left Pending.
245261
func (h *LineageControllerWebhook) applySchedulingClass(ctx context.Context, obj, owner *unstructured.Unstructured, namespace string) error {
246262
if obj.GetKind() != "Pod" {
247263
return nil
248264
}
265+
logger := log.FromContext(ctx)
249266

250-
// Determine scheduling class: owner Application field takes priority,
251-
// then fall back to namespace label.
252-
var schedulingClass string
253-
if owner != nil {
254-
var app appsv1alpha1.Application
255-
if err := runtime.DefaultUnstructuredConverter.FromUnstructured(owner.Object, &app); err == nil {
256-
schedulingClass = app.SchedulingClass()
257-
}
267+
ns := &corev1.Namespace{}
268+
if err := h.Get(ctx, client.ObjectKey{Name: namespace}, ns); err != nil {
269+
return fmt.Errorf("getting namespace %s: %w", namespace, err)
258270
}
259-
if schedulingClass == "" {
260-
ns := &corev1.Namespace{}
261-
if err := h.Get(ctx, client.ObjectKey{Name: namespace}, ns); err != nil {
262-
return fmt.Errorf("getting namespace %s: %w", namespace, err)
271+
schedulingClass := ns.Labels[schedulerapi.SchedulingClassLabel]
272+
273+
if appClass := ownerSchedulingClass(owner); appClass != "" && appClass != schedulingClass {
274+
switch {
275+
case !slices.Contains(allowedSchedulingClasses(ns), appClass):
276+
logger.Info("application SchedulingClass is not allowed in this namespace, keeping the tenant class",
277+
"schedulingClass", appClass, "tenantSchedulingClass", schedulingClass, "namespace", namespace)
278+
case !h.schedulingClassExists(ctx, appClass):
279+
logger.Info("application SchedulingClass not found, keeping the tenant class",
280+
"schedulingClass", appClass, "tenantSchedulingClass", schedulingClass, "namespace", namespace)
281+
default:
282+
schedulingClass = appClass
263283
}
264-
schedulingClass = ns.Labels[schedulerapi.SchedulingClassLabel]
265284
}
266285

267286
if schedulingClass == "" {
268287
return nil
269288
}
270289

271-
// Verify that the referenced SchedulingClass CR exists.
272-
// If the CRD is not installed or the CR is missing, skip injection
273-
// so that pods are not stuck Pending on a non-existent scheduler.
274-
_, err := h.dynClient.Resource(schedulingClassGVR).Get(ctx, schedulingClass, metav1.GetOptions{})
275-
if err != nil {
276-
logger := log.FromContext(ctx)
290+
if !h.schedulingClassExists(ctx, schedulingClass) {
277291
logger.Info("SchedulingClass not found, skipping scheduler injection",
278292
"schedulingClass", schedulingClass, "namespace", namespace)
279293
return nil
@@ -295,7 +309,7 @@ func (h *LineageControllerWebhook) applySchedulingClass(ctx context.Context, obj
295309
// schema-capped at 63, but a pre-existing long-named class must degrade to
296310
// annotation-only injection rather than have the apiserver reject the Pod.
297311
if errs := validation.IsValidLabelValue(schedulingClass); len(errs) > 0 {
298-
log.FromContext(ctx).Info("SchedulingClass name is not a valid label value, skipping co-affinity label",
312+
logger.Info("SchedulingClass name is not a valid label value, skipping co-affinity label",
299313
"schedulingClass", schedulingClass, "namespace", namespace)
300314
return nil
301315
}
@@ -310,6 +324,34 @@ func (h *LineageControllerWebhook) applySchedulingClass(ctx context.Context, obj
310324
return nil
311325
}
312326

327+
func ownerSchedulingClass(owner *unstructured.Unstructured) string {
328+
if owner == nil {
329+
return ""
330+
}
331+
var app appsv1alpha1.Application
332+
if err := runtime.DefaultUnstructuredConverter.FromUnstructured(owner.Object, &app); err != nil {
333+
return ""
334+
}
335+
return app.SchedulingClass()
336+
}
337+
338+
func allowedSchedulingClasses(ns *corev1.Namespace) []string {
339+
var allowed []string
340+
for _, name := range strings.Split(ns.Annotations[AllowedSchedulingClassesAnnotation], ",") {
341+
if name = strings.TrimSpace(name); name != "" {
342+
allowed = append(allowed, name)
343+
}
344+
}
345+
return allowed
346+
}
347+
348+
// schedulingClassExists treats any lookup error as absence: the CRD may not be
349+
// installed, and either way the class cannot be scheduled against.
350+
func (h *LineageControllerWebhook) schedulingClassExists(ctx context.Context, name string) bool {
351+
_, err := h.dynClient.Resource(schedulingClassGVR).Get(ctx, name, metav1.GetOptions{})
352+
return err == nil
353+
}
354+
313355
var schedulingClassGVR = schema.GroupVersionResource{
314356
Group: schedulerapi.Group,
315357
Version: schedulerapi.Version,

‎internal/lineagecontrollerwebhook/webhook_test.go‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"k8s.io/apimachinery/pkg/runtime/schema"
2020
dynamicfake "k8s.io/client-go/dynamic/fake"
2121
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
22+
"sigs.k8s.io/controller-runtime/pkg/client"
2223
"sigs.k8s.io/controller-runtime/pkg/client/fake"
2324
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
2425
)
@@ -481,10 +482,23 @@ func addSchedulingClassCR(t *testing.T, h *LineageControllerWebhook, name string
481482
}
482483
}
483484

485+
func allowSchedulingClasses(t *testing.T, h *LineageControllerWebhook, namespace, allowed string) {
486+
t.Helper()
487+
ns := &corev1.Namespace{}
488+
if err := h.Get(context.Background(), client.ObjectKey{Name: namespace}, ns); err != nil {
489+
t.Fatalf("getting namespace %s: %v", namespace, err)
490+
}
491+
ns.Annotations = map[string]string{AllowedSchedulingClassesAnnotation: allowed}
492+
if err := h.Update(context.Background(), ns); err != nil {
493+
t.Fatalf("updating namespace %s: %v", namespace, err)
494+
}
495+
}
496+
484497
func TestApplySchedulingClass_OwnerApplicationClassOverridesNamespace(t *testing.T) {
485498
ctx := context.Background()
486499
h := newWebhookTestEnv(t, "tenant-acme", testSchedulingClass, true)
487500
addSchedulingClassCR(t, h, "app-class")
501+
allowSchedulingClasses(t, h, "tenant-acme", "other-class, app-class")
488502
owner := newOwnerApplication("tenant-acme", "db", map[string]interface{}{
489503
"replicas": int64(2),
490504
"schedulingClass": "app-class",
@@ -536,6 +550,7 @@ func TestApplySchedulingClass_ClassNameTooLongForLabelSkipsOnlyLabel(t *testing.
536550
longClass := strings.Repeat("a", 64)
537551
h := newWebhookTestEnv(t, "tenant-acme", "", false)
538552
addSchedulingClassCR(t, h, longClass)
553+
allowSchedulingClasses(t, h, "tenant-acme", longClass)
539554
owner := newOwnerApplication("tenant-acme", "db", map[string]interface{}{"schedulingClass": longClass})
540555
pod := newPod("tenant-acme", "db-0")
541556

@@ -553,3 +568,84 @@ func TestApplySchedulingClass_ClassNameTooLongForLabelSkipsOnlyLabel(t *testing.
553568
t.Errorf("expected pod spec.schedulerName=%s, got %q", schedulerapi.SchedulerName, sched)
554569
}
555570
}
571+
572+
// The tenant class is the administrator's placement constraint. An application
573+
// class that the namespace does not allow, or that does not exist, must leave
574+
// it in force instead of dropping the injection altogether.
575+
func TestApplySchedulingClass_RejectedOwnerApplicationClassKeepsTenantClass(t *testing.T) {
576+
for _, tc := range []struct {
577+
name string
578+
appClass string
579+
allowed string
580+
existing []string
581+
}{
582+
{name: "class does not exist", appClass: "does-not-exist", allowed: "does-not-exist"},
583+
{name: "class not in the allowed list", appClass: "app-class", allowed: "other-class", existing: []string{"app-class"}},
584+
{name: "no allowed list", appClass: "app-class", existing: []string{"app-class"}},
585+
} {
586+
t.Run(tc.name, func(t *testing.T) {
587+
ctx := context.Background()
588+
h := newWebhookTestEnv(t, "tenant-acme", testSchedulingClass, true)
589+
for _, name := range tc.existing {
590+
addSchedulingClassCR(t, h, name)
591+
}
592+
if tc.allowed != "" {
593+
allowSchedulingClasses(t, h, "tenant-acme", tc.allowed)
594+
}
595+
owner := newOwnerApplication("tenant-acme", "db", map[string]interface{}{"schedulingClass": tc.appClass})
596+
pod := newPod("tenant-acme", "db-0")
597+
598+
if err := h.applySchedulingClass(ctx, pod, owner, "tenant-acme"); err != nil {
599+
t.Fatalf("applySchedulingClass returned error: %v", err)
600+
}
601+
602+
if got := pod.GetLabels()[schedulerapi.SchedulingClassLabel]; got != testSchedulingClass {
603+
t.Errorf("expected pod label %s=%s, got %q", schedulerapi.SchedulingClassLabel, testSchedulingClass, got)
604+
}
605+
if got := pod.GetAnnotations()[schedulerapi.SchedulingClassAnnotation]; got != testSchedulingClass {
606+
t.Errorf("expected pod annotation %s=%s, got %q", schedulerapi.SchedulingClassAnnotation, testSchedulingClass, got)
607+
}
608+
if sched, _, _ := unstructured.NestedString(pod.Object, "spec", "schedulerName"); sched != schedulerapi.SchedulerName {
609+
t.Errorf("expected pod spec.schedulerName=%s, got %q", schedulerapi.SchedulerName, sched)
610+
}
611+
})
612+
}
613+
}
614+
615+
// Naming the tenant's own class is the same as leaving the field empty, so it
616+
// needs no entry in the allowed list.
617+
func TestApplySchedulingClass_OwnerApplicationNamingTenantClassNeedsNoAllowedList(t *testing.T) {
618+
ctx := context.Background()
619+
h := newWebhookTestEnv(t, "tenant-acme", testSchedulingClass, true)
620+
owner := newOwnerApplication("tenant-acme", "db", map[string]interface{}{"schedulingClass": testSchedulingClass})
621+
pod := newPod("tenant-acme", "db-0")
622+
623+
if err := h.applySchedulingClass(ctx, pod, owner, "tenant-acme"); err != nil {
624+
t.Fatalf("applySchedulingClass returned error: %v", err)
625+
}
626+
627+
if got := pod.GetLabels()[schedulerapi.SchedulingClassLabel]; got != testSchedulingClass {
628+
t.Errorf("expected pod label %s=%s, got %q", schedulerapi.SchedulingClassLabel, testSchedulingClass, got)
629+
}
630+
}
631+
632+
// Without a tenant class there is nothing to fall back to: a rejected
633+
// application class leaves the pod on the default scheduler.
634+
func TestApplySchedulingClass_RejectedOwnerApplicationClassWithoutTenantClassIsNoOp(t *testing.T) {
635+
ctx := context.Background()
636+
h := newWebhookTestEnv(t, "tenant-acme", "", false)
637+
addSchedulingClassCR(t, h, "app-class")
638+
owner := newOwnerApplication("tenant-acme", "db", map[string]interface{}{"schedulingClass": "app-class"})
639+
pod := newPod("tenant-acme", "db-0")
640+
641+
if err := h.applySchedulingClass(ctx, pod, owner, "tenant-acme"); err != nil {
642+
t.Fatalf("applySchedulingClass returned error: %v", err)
643+
}
644+
645+
if _, ok := pod.GetAnnotations()[schedulerapi.SchedulingClassAnnotation]; ok {
646+
t.Errorf("pod annotation should not be set for a class the namespace does not allow")
647+
}
648+
if sched, _, _ := unstructured.NestedString(pod.Object, "spec", "schedulerName"); sched != "" {
649+
t.Errorf("pod spec.schedulerName should not be set, got %q", sched)
650+
}
651+
}

‎pkg/apis/apps/v1alpha1/types.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,9 +60,9 @@ type ApplicationStatus struct {
6060
}
6161

6262
// SchedulingClass returns the scheduling class requested by this Application,
63-
// parsed from its Spec JSON under the top-level `schedulingClass` key. When set,
64-
// the named SchedulingClass takes precedence over any tenant-level class for
65-
// the workloads of this Application.
63+
// parsed from its Spec JSON under the top-level `schedulingClass` key. It is a
64+
// request: the lineage webhook applies it in place of the tenant-level class
65+
// only when the namespace allows it and the SchedulingClass exists.
6666
//
6767
// A missing or empty field returns "" — the caller then falls back to the
6868
// namespace-level scheduling class.

0 commit comments

Comments
 (0)