Skip to content

Commit 870062d

Browse files
committed
adjusts DRA extended resource quota to include devices usages from regular resource claims
1 parent 11787f5 commit 870062d

16 files changed

Lines changed: 841 additions & 75 deletions

File tree

cmd/kube-controller-manager/app/core.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -597,7 +597,7 @@ func newResourceQuotaController(ctx context.Context, controllerContext Controlle
597597

598598
discoveryFunc := resourceQuotaControllerDiscoveryClient.ServerPreferredNamespacedResources
599599
listerFuncForResource := generic.ListerFuncForResourceFunc(controllerContext.InformerFactory.ForResource)
600-
quotaConfiguration := quotainstall.NewQuotaConfigurationForControllers(listerFuncForResource)
600+
quotaConfiguration := quotainstall.NewQuotaConfigurationForControllers(listerFuncForResource, controllerContext.InformerFactory)
601601

602602
resourceQuotaControllerOptions := &resourcequotacontroller.ControllerOptions{
603603
QuotaClient: resourceQuotaControllerClient.CoreV1(),

pkg/controller/resourcequota/resource_quota_controller_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ type quotaController struct {
111111

112112
func setupQuotaController(t *testing.T, kubeClient kubernetes.Interface, lister quota.ListerForResourceFunc, discoveryFunc NamespacedResourcesFunc) quotaController {
113113
informerFactory := informers.NewSharedInformerFactory(kubeClient, controller.NoResyncPeriodFunc())
114-
quotaConfiguration := install.NewQuotaConfigurationForControllers(lister)
114+
quotaConfiguration := install.NewQuotaConfigurationForControllers(lister, informerFactory)
115115
alwaysStarted := make(chan struct{})
116116
close(alwaysStarted)
117117
resourceQuotaControllerOptions := &ControllerOptions{

pkg/controlplane/apiserver/admission/config.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ func (c *Config) New(proxyTransport *http.Transport, egressSelector *egressselec
4343
webhookPluginInitializer := webhookinit.NewPluginInitializer(webhookAuthResolverWrapper, serviceResolver)
4444

4545
kubePluginInitializer := NewPluginInitializer(
46-
quotainstall.NewQuotaConfigurationForAdmission(),
46+
quotainstall.NewQuotaConfigurationForAdmission(c.ExternalInformers),
4747
exclusion.Excluded(),
4848
)
4949

pkg/quota/v1/evaluator/core/pods.go

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"time"
2323

2424
corev1 "k8s.io/api/core/v1"
25+
resourceapi "k8s.io/api/resource/v1"
2526
"k8s.io/apimachinery/pkg/api/resource"
2627
"k8s.io/apimachinery/pkg/labels"
2728
"k8s.io/apimachinery/pkg/runtime"
@@ -39,10 +40,12 @@ import (
3940
"k8s.io/kubernetes/pkg/apis/core/v1/helper"
4041
"k8s.io/kubernetes/pkg/apis/core/v1/helper/qos"
4142
"k8s.io/kubernetes/pkg/features"
43+
schedutil "k8s.io/kubernetes/pkg/scheduler/util"
4244
)
4345

4446
// the name used for object count quota
4547
var podObjectCountName = generic.ObjectCountQuotaResourceNameFor(corev1.SchemeGroupVersion.WithResource("pods").GroupResource())
48+
var resourceRequestsDeviceClassPrefix = corev1.DefaultResourceRequestsPrefix + resourceapi.ResourceDeviceClassPrefix
4649

4750
// podResources are the set of resources managed by quota associated with pods.
4851
var podResources = []corev1.ResourceName{
@@ -82,8 +85,15 @@ func maskResourceWithPrefix(resource corev1.ResourceName, prefix string) corev1.
8285
// has the quota related resource prefix.
8386
func isExtendedResourceNameForQuota(name corev1.ResourceName) bool {
8487
// As overcommit is not supported by extended resources for now,
85-
// only quota objects in format of "requests.resourceName" is allowed.
86-
return !helper.IsNativeResource(name) && strings.HasPrefix(string(name), corev1.DefaultResourceRequestsPrefix)
88+
// quota objects in format of "requests.resourceName" is allowed
89+
nonNative := !helper.IsNativeResource(name)
90+
// allow the implicit extended resource name in the format of
91+
// requests.deviceclass.resource.kubernetes.io/device-class-name
92+
implicitExtendedResource := strings.HasPrefix(string(name), resourceRequestsDeviceClassPrefix)
93+
// name starts with 'requests.'
94+
isQuotaRequest := strings.HasPrefix(string(name), corev1.DefaultResourceRequestsPrefix)
95+
96+
return (nonNative || implicitExtendedResource) && isQuotaRequest
8797
}
8898

8999
// NOTE: it was a mistake, but if a quota tracks cpu or memory related resources,
@@ -312,7 +322,7 @@ func podComputeUsageHelper(requests corev1.ResourceList, limits corev1.ResourceL
312322
result[maskResourceWithPrefix(resource, corev1.DefaultResourceRequestsPrefix)] = request
313323
}
314324
// for extended resources
315-
if helper.IsExtendedResourceName(resource) {
325+
if schedutil.IsDRAExtendedResourceName(resource) {
316326
// only quota objects in format of "requests.resourceName" is allowed for extended resource.
317327
result[maskResourceWithPrefix(resource, corev1.DefaultResourceRequestsPrefix)] = request
318328
}

pkg/quota/v1/evaluator/core/pods_test.go

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -276,6 +276,23 @@ func TestPodEvaluatorUsage(t *testing.T) {
276276
generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "pods"}): resource.MustParse("1"),
277277
},
278278
},
279+
"init container implicit extended resources": {
280+
pod: &api.Pod{
281+
Spec: api.PodSpec{
282+
InitContainers: []api.Container{{
283+
Resources: api.ResourceRequirements{
284+
Requests: api.ResourceList{api.ResourceName("deviceclass.resource.kubernetes.io/dongle"): resource.MustParse("3")},
285+
Limits: api.ResourceList{api.ResourceName("deviceclass.resource.kubernetes.io/dongle"): resource.MustParse("3")},
286+
},
287+
}},
288+
},
289+
},
290+
usage: corev1.ResourceList{
291+
corev1.ResourceName("requests.deviceclass.resource.kubernetes.io/dongle"): resource.MustParse("3"),
292+
corev1.ResourcePods: resource.MustParse("1"),
293+
generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "pods"}): resource.MustParse("1"),
294+
},
295+
},
279296
"container CPU": {
280297
pod: &api.Pod{
281298
Spec: api.PodSpec{
@@ -367,6 +384,23 @@ func TestPodEvaluatorUsage(t *testing.T) {
367384
generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "pods"}): resource.MustParse("1"),
368385
},
369386
},
387+
"container implicit extended resources": {
388+
pod: &api.Pod{
389+
Spec: api.PodSpec{
390+
Containers: []api.Container{{
391+
Resources: api.ResourceRequirements{
392+
Requests: api.ResourceList{api.ResourceName("deviceclass.resource.kubernetes.io/dongle"): resource.MustParse("3")},
393+
Limits: api.ResourceList{api.ResourceName("deviceclass.resource.kubernetes.io/dongle"): resource.MustParse("3")},
394+
},
395+
}},
396+
},
397+
},
398+
usage: corev1.ResourceList{
399+
corev1.ResourceName("requests.deviceclass.resource.kubernetes.io/dongle"): resource.MustParse("3"),
400+
corev1.ResourcePods: resource.MustParse("1"),
401+
generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "pods"}): resource.MustParse("1"),
402+
},
403+
},
370404
"terminated generic count still appears": {
371405
pod: &api.Pod{
372406
Spec: api.PodSpec{

pkg/quota/v1/evaluator/core/registry.go

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,17 @@ limitations under the License.
1717
package core
1818

1919
import (
20+
"context"
21+
2022
corev1 "k8s.io/api/core/v1"
2123
"k8s.io/apimachinery/pkg/runtime/schema"
2224
quota "k8s.io/apiserver/pkg/quota/v1"
2325
"k8s.io/apiserver/pkg/quota/v1/generic"
2426
utilfeature "k8s.io/apiserver/pkg/util/feature"
27+
"k8s.io/client-go/informers"
28+
corev1listers "k8s.io/client-go/listers/core/v1"
29+
"k8s.io/dynamic-resource-allocation/deviceclass/extendedresourcecache"
30+
"k8s.io/klog/v2"
2531
"k8s.io/kubernetes/pkg/features"
2632
"k8s.io/utils/clock"
2733
)
@@ -35,15 +41,25 @@ var legacyObjectCountAliases = map[schema.GroupVersionResource]corev1.ResourceNa
3541
}
3642

3743
// NewEvaluators returns the list of static evaluators that manage more than counts
38-
func NewEvaluators(f quota.ListerForResourceFunc) []quota.Evaluator {
44+
func NewEvaluators(f quota.ListerForResourceFunc, i informers.SharedInformerFactory) []quota.Evaluator {
3945
// these evaluators have special logic
4046
result := []quota.Evaluator{
4147
NewPodEvaluator(f, clock.RealClock{}),
4248
NewServiceEvaluator(f),
4349
NewPersistentVolumeClaimEvaluator(f),
4450
}
4551
if utilfeature.DefaultFeatureGate.Enabled(features.DynamicResourceAllocation) {
46-
result = append(result, NewResourceClaimEvaluator(f))
52+
var podLister corev1listers.PodLister
53+
var deviceClassMapping *extendedresourcecache.ExtendedResourceCache
54+
if utilfeature.DefaultFeatureGate.Enabled(features.DRAExtendedResource) {
55+
podLister = i.Core().V1().Pods().Lister()
56+
logger := klog.FromContext(context.Background())
57+
deviceClassMapping = extendedresourcecache.NewExtendedResourceCache(logger)
58+
if _, err := i.Resource().V1().DeviceClasses().Informer().AddEventHandler(deviceClassMapping); err != nil {
59+
logger.Error(err, "failed to add device class informer event handler")
60+
}
61+
}
62+
result = append(result, NewResourceClaimEvaluator(f, deviceClassMapping, podLister))
4763
}
4864

4965
// these evaluators require an alias for backwards compatibility

pkg/quota/v1/evaluator/core/resource_claims.go

Lines changed: 97 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,19 @@ import (
2323
corev1 "k8s.io/api/core/v1"
2424
resourceapi "k8s.io/api/resource/v1"
2525
"k8s.io/apimachinery/pkg/api/resource"
26+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2627
"k8s.io/apimachinery/pkg/runtime"
2728
"k8s.io/apimachinery/pkg/runtime/schema"
2829
"k8s.io/apiserver/pkg/admission"
2930
quota "k8s.io/apiserver/pkg/quota/v1"
3031
"k8s.io/apiserver/pkg/quota/v1/generic"
32+
utilfeature "k8s.io/apiserver/pkg/util/feature"
33+
corev1listers "k8s.io/client-go/listers/core/v1"
34+
"k8s.io/dynamic-resource-allocation/deviceclass/extendedresourcecache"
3135
resourceinternal "k8s.io/kubernetes/pkg/apis/resource"
3236
resourceversioned "k8s.io/kubernetes/pkg/apis/resource/v1"
37+
"k8s.io/kubernetes/pkg/features"
38+
"k8s.io/utils/clock"
3339
)
3440

3541
// The name used for object count quota. This evaluator takes over counting
@@ -38,21 +44,26 @@ import (
3844
var ClaimObjectCountName = generic.ObjectCountQuotaResourceNameFor(resourceapi.SchemeGroupVersion.WithResource("resourceclaims").GroupResource())
3945

4046
// V1ResourceByDeviceClass returns a quota resource name by device class.
47+
// gpuclass -> gpuclass.deviceclass.resource.k8s.io/devices
4148
func V1ResourceByDeviceClass(className string) corev1.ResourceName {
4249
return corev1.ResourceName(className + corev1.ResourceClaimsPerClass)
4350
}
4451

4552
// NewResourceClaimEvaluator returns an evaluator that can evaluate resource claims
46-
func NewResourceClaimEvaluator(f quota.ListerForResourceFunc) quota.Evaluator {
53+
func NewResourceClaimEvaluator(f quota.ListerForResourceFunc, m *extendedresourcecache.ExtendedResourceCache, podsGetter corev1listers.PodLister) quota.Evaluator {
4754
listFuncByNamespace := generic.ListResourceUsingListerFunc(f, resourceapi.SchemeGroupVersion.WithResource("resourceclaims"))
48-
claimEvaluator := &claimEvaluator{listFuncByNamespace: listFuncByNamespace}
55+
claimEvaluator := &claimEvaluator{listFuncByNamespace: listFuncByNamespace, deviceClassMapping: m, podsGetter: podsGetter}
4956
return claimEvaluator
5057
}
5158

5259
// claimEvaluator knows how to evaluate quota usage for resource claims
5360
type claimEvaluator struct {
5461
// listFuncByNamespace knows how to list resource claims
5562
listFuncByNamespace generic.ListFuncByNamespace
63+
// a global cache of device class and extended resource mapping
64+
deviceClassMapping *extendedresourcecache.ExtendedResourceCache
65+
// podsGetter is used to get pods
66+
podsGetter corev1listers.PodLister
5667
}
5768

5869
// Constraints verifies that all required resources are present on the item.
@@ -98,11 +109,91 @@ func (p *claimEvaluator) MatchingResources(items []corev1.ResourceName) []corev1
98109
if item == ClaimObjectCountName /* object count quota fields */ ||
99110
strings.HasSuffix(string(item), corev1.ResourceClaimsPerClass /* by device class */) {
100111
result = append(result, item)
112+
continue
113+
}
114+
if utilfeature.DefaultFeatureGate.Enabled(features.DRAExtendedResource) {
115+
if strings.HasPrefix(string(item), corev1.ResourceImplicitExtendedClaimsPerClass /* by implicit extended resource name */) {
116+
className := string(item[len(corev1.ResourceImplicitExtendedClaimsPerClass):])
117+
if p.deviceClassMapping.GetExtendedResource(className) != "" {
118+
result = append(result, item)
119+
continue
120+
}
121+
}
122+
if isExtendedResourceNameForQuota(item) /* by extended resource name */ {
123+
resourceName := string(item[len(corev1.DefaultResourceRequestsPrefix):])
124+
if p.deviceClassMapping.GetDeviceClass(corev1.ResourceName(resourceName)) != nil {
125+
result = append(result, item)
126+
}
127+
}
101128
}
102129
}
103130
return result
104131
}
105132

133+
func (p *claimEvaluator) addExtendedResourceQuota(resourceClaimUsage map[corev1.ResourceName]resource.Quantity, podUsage corev1.ResourceList) {
134+
extendedResourceUsage := make(map[corev1.ResourceName]resource.Quantity)
135+
for name, quantity := range resourceClaimUsage {
136+
// e.g. myclass
137+
deviceClassName, isDeviceClassUsage := strings.CutSuffix(string(name), corev1.ResourceClaimsPerClass)
138+
if !isDeviceClassUsage || len(deviceClassName) == 0 {
139+
continue
140+
}
141+
142+
// requests.deviceclass.resource.kubernetes.io/myclass
143+
extendedResourceUsage[corev1.ResourceName(corev1.ResourceImplicitExtendedClaimsPerClass+deviceClassName)] = quantity
144+
145+
// e.g. example.com/mygpu
146+
if extendedResourceName := p.deviceClassMapping.GetExtendedResource(deviceClassName); len(extendedResourceName) > 0 {
147+
// requests.example.com/gpu
148+
extendedResourceUsage[corev1.ResourceName(corev1.DefaultResourceRequestsPrefix+extendedResourceName)] = quantity
149+
}
150+
}
151+
152+
for name, quantity := range extendedResourceUsage {
153+
// Subtract any amount already accounted for in the pod
154+
if podQuantity, found := podUsage[name]; found {
155+
quantity.Sub(podQuantity)
156+
}
157+
// Add any remaining amount to the resource claim resources
158+
if quantity.CmpInt64(0) > 0 {
159+
resourceClaimUsage[name] = quantity
160+
}
161+
}
162+
}
163+
164+
// Verify extended resource claim owning pod exists, and the pod's ExtendedResourceClaimStatus points
165+
// back to the claim if it's not nil, and returns the pod's quota usage. If any error is encountered, nil is returned.
166+
func (p *claimEvaluator) getVerifiedPodUsage(claim *resourceapi.ResourceClaim) corev1.ResourceList {
167+
if claim.Annotations[resourceapi.ExtendedResourceClaimAnnotation] != "true" {
168+
return nil
169+
}
170+
controllerRef := metav1.GetControllerOf(claim)
171+
if controllerRef == nil {
172+
return nil
173+
}
174+
if controllerRef.Kind != "Pod" || controllerRef.APIVersion != "v1" {
175+
return nil
176+
}
177+
if p.podsGetter == nil {
178+
return nil
179+
}
180+
pod, err := p.podsGetter.Pods(claim.Namespace).Get(controllerRef.Name)
181+
if err != nil {
182+
return nil
183+
}
184+
if controllerRef.UID != pod.UID {
185+
return nil
186+
}
187+
if pod.Status.ExtendedResourceClaimStatus != nil && pod.Status.ExtendedResourceClaimStatus.ResourceClaimName != claim.Name {
188+
return nil
189+
}
190+
quotaReqs, err := PodUsageFunc(pod, clock.RealClock{})
191+
if err != nil {
192+
return nil
193+
}
194+
return quotaReqs
195+
}
196+
106197
// Usage knows how to measure usage associated with item.
107198
func (p *claimEvaluator) Usage(item runtime.Object) (corev1.ResourceList, error) {
108199
result := corev1.ResourceList{}
@@ -168,6 +259,10 @@ func (p *claimEvaluator) Usage(item runtime.Object) (corev1.ResourceList, error)
168259
}
169260
}
170261

262+
if utilfeature.DefaultFeatureGate.Enabled(features.DRAExtendedResource) {
263+
p.addExtendedResourceQuota(result, p.getVerifiedPodUsage(claim))
264+
}
265+
171266
return result, nil
172267
}
173268

0 commit comments

Comments
 (0)