Skip to content

Commit 81df1db

Browse files
fix(lineage-webhook): fail admission when a SchedulingClass lookup errors
schedulingClassExists treated every lookup error as absence. A timeout or a 5xx from the apiserver during Pod admission therefore skipped the injection, and the Pod ran on the default scheduler, outside the tenant's placement constraint, until it was recreated. Only NotFound and NoMatch (the SchedulingClass CRD is not installed) now count as absence. Any other error is returned, the webhook answers 500, and with failurePolicy Fail the Pod's controller retries the creation. Assisted-by: LLM Signed-off-by: Mattia Eleuteri <[email protected]>
1 parent 809bd0d commit 81df1db

2 files changed

Lines changed: 81 additions & 9 deletions

File tree

‎internal/lineagecontrollerwebhook/webhook.go‎

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ import (
1111

1212
"github.com/cozystack/cozystack/pkg/lineage"
1313
corev1 "k8s.io/api/core/v1"
14+
apierrors "k8s.io/apimachinery/pkg/api/errors"
15+
"k8s.io/apimachinery/pkg/api/meta"
1416
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1517
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
1618
"k8s.io/apimachinery/pkg/runtime"
@@ -271,14 +273,15 @@ func (h *LineageControllerWebhook) applySchedulingClass(ctx context.Context, obj
271273
schedulingClass := ns.Labels[schedulerapi.SchedulingClassLabel]
272274

273275
if appClass := ownerSchedulingClass(owner); appClass != "" && appClass != schedulingClass {
274-
switch {
275-
case !slices.Contains(allowedSchedulingClasses(ns), appClass):
276+
if !slices.Contains(allowedSchedulingClasses(ns), appClass) {
276277
logger.Info("application SchedulingClass is not allowed in this namespace, keeping the tenant class",
277278
"schedulingClass", appClass, "tenantSchedulingClass", schedulingClass, "namespace", namespace)
278-
case !h.schedulingClassExists(ctx, appClass):
279+
} else if exists, err := h.schedulingClassExists(ctx, appClass); err != nil {
280+
return err
281+
} else if !exists {
279282
logger.Info("application SchedulingClass not found, keeping the tenant class",
280283
"schedulingClass", appClass, "tenantSchedulingClass", schedulingClass, "namespace", namespace)
281-
default:
284+
} else {
282285
schedulingClass = appClass
283286
}
284287
}
@@ -287,7 +290,11 @@ func (h *LineageControllerWebhook) applySchedulingClass(ctx context.Context, obj
287290
return nil
288291
}
289292

290-
if !h.schedulingClassExists(ctx, schedulingClass) {
293+
exists, err := h.schedulingClassExists(ctx, schedulingClass)
294+
if err != nil {
295+
return err
296+
}
297+
if !exists {
291298
logger.Info("SchedulingClass not found, skipping scheduler injection",
292299
"schedulingClass", schedulingClass, "namespace", namespace)
293300
return nil
@@ -345,11 +352,21 @@ func allowedSchedulingClasses(ns *corev1.Namespace) []string {
345352
return allowed
346353
}
347354

348-
// schedulingClassExists treats any lookup error as absence: the CRD may not be
349-
// installed, and either way the class cannot be scheduled against.
350-
func (h *LineageControllerWebhook) schedulingClassExists(ctx context.Context, name string) bool {
355+
// schedulingClassExists reports a missing class, or a missing SchedulingClass
356+
// CRD, as absence. Any other lookup error is returned: treating a timeout or a
357+
// 5xx as absence would admit the Pod without the tenant's placement constraint,
358+
// which it then keeps until it is recreated. Failing the admission lets the
359+
// Pod's controller retry instead.
360+
func (h *LineageControllerWebhook) schedulingClassExists(ctx context.Context, name string) (bool, error) {
351361
_, err := h.dynClient.Resource(schedulingClassGVR).Get(ctx, name, metav1.GetOptions{})
352-
return err == nil
362+
switch {
363+
case err == nil:
364+
return true, nil
365+
case apierrors.IsNotFound(err) || meta.IsNoMatchError(err):
366+
return false, nil
367+
default:
368+
return false, fmt.Errorf("getting SchedulingClass %s: %w", name, err)
369+
}
353370
}
354371

355372
var schedulingClassGVR = schema.GroupVersionResource{

‎internal/lineagecontrollerwebhook/webhook_test.go‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package lineagecontrollerwebhook
33
import (
44
"context"
55
"encoding/json"
6+
"errors"
67
"sort"
78
"strings"
89
"testing"
@@ -12,13 +13,15 @@ import (
1213
corev1alpha1 "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1"
1314
admissionv1 "k8s.io/api/admission/v1"
1415
corev1 "k8s.io/api/core/v1"
16+
apierrors "k8s.io/apimachinery/pkg/api/errors"
1517
"k8s.io/apimachinery/pkg/api/meta"
1618
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1719
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
1820
"k8s.io/apimachinery/pkg/runtime"
1921
"k8s.io/apimachinery/pkg/runtime/schema"
2022
dynamicfake "k8s.io/client-go/dynamic/fake"
2123
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
24+
k8stesting "k8s.io/client-go/testing"
2225
"sigs.k8s.io/controller-runtime/pkg/client"
2326
"sigs.k8s.io/controller-runtime/pkg/client/fake"
2427
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
@@ -649,3 +652,55 @@ func TestApplySchedulingClass_RejectedOwnerApplicationClassWithoutTenantClassIsN
649652
t.Errorf("pod spec.schedulerName should not be set, got %q", sched)
650653
}
651654
}
655+
656+
// failSchedulingClassGet makes every SchedulingClass lookup return err.
657+
func failSchedulingClassGet(h *LineageControllerWebhook, err error) {
658+
h.dynClient.(*dynamicfake.FakeDynamicClient).PrependReactor("get", schedulerapi.Resource,
659+
func(k8stesting.Action) (bool, runtime.Object, error) { return true, nil, err })
660+
}
661+
662+
// A lookup that fails for any reason other than absence must fail the
663+
// admission, so the Pod's controller retries it. Admitting the Pod without
664+
// the tenant class would leave it on the default scheduler until recreated.
665+
func TestApplySchedulingClass_TransientLookupErrorFailsAdmission(t *testing.T) {
666+
transient := apierrors.NewInternalError(errors.New("etcdserver: request timed out"))
667+
for _, tc := range []struct {
668+
name string
669+
owner *unstructured.Unstructured
670+
}{
671+
{name: "tenant class", owner: nil},
672+
{name: "application class", owner: newOwnerApplication("tenant-acme", "db", map[string]interface{}{"schedulingClass": "app-class"})},
673+
} {
674+
t.Run(tc.name, func(t *testing.T) {
675+
ctx := context.Background()
676+
h := newWebhookTestEnv(t, "tenant-acme", testSchedulingClass, true)
677+
allowSchedulingClasses(t, h, "tenant-acme", "app-class")
678+
failSchedulingClassGet(h, transient)
679+
pod := newPod("tenant-acme", "db-0")
680+
681+
err := h.applySchedulingClass(ctx, pod, tc.owner, "tenant-acme")
682+
if !apierrors.IsInternalError(err) {
683+
t.Fatalf("expected the lookup error to be returned, got %v", err)
684+
}
685+
if _, ok := pod.GetAnnotations()[schedulerapi.SchedulingClassAnnotation]; ok {
686+
t.Errorf("pod must not be mutated when the lookup fails")
687+
}
688+
})
689+
}
690+
}
691+
692+
// Without the SchedulingClass CRD there is no scheduler to send the Pod to, so
693+
// a NoMatch error is absence, like NotFound, and the injection is skipped.
694+
func TestApplySchedulingClass_MissingCRDSkipsInjection(t *testing.T) {
695+
ctx := context.Background()
696+
h := newWebhookTestEnv(t, "tenant-acme", testSchedulingClass, true)
697+
failSchedulingClassGet(h, &meta.NoKindMatchError{GroupKind: schema.GroupKind{Group: schedulerapi.Group, Kind: "SchedulingClass"}})
698+
pod := newPod("tenant-acme", "db-0")
699+
700+
if err := h.applySchedulingClass(ctx, pod, nil, "tenant-acme"); err != nil {
701+
t.Fatalf("applySchedulingClass returned error: %v", err)
702+
}
703+
if sched, _, _ := unstructured.NestedString(pod.Object, "spec", "schedulerName"); sched != "" {
704+
t.Errorf("pod spec.schedulerName should not be set without the CRD, got %q", sched)
705+
}
706+
}

0 commit comments

Comments
 (0)