Skip to content

Commit 02d1264

Browse files
committed
feat(api): check GPU budgets when a VM or a worker pool is written
A tenant's GPU budget is declared per class as <class>.gpuclass.cozystack.io/devices and over every class as gpuclass.cozystack.io/devices. These keys are not pod resources, so no ResourceQuota enforces them. The proposal hands the check to the pool-wide admission gate that counts declared sizes; until that gate is on, the Cozystack API server does it for GPU keys only, with the inputs and the pool arithmetic that gate reads, so the switch changes no verdict. On a VMInstance or KubernetesNodes create or update, every class entry reserves its count, times maxReplicas for a worker pool, read from the chart-defaulted values. The tenant pool arithmetic is applied to the declared resourceQuotas and these entries. Only growth is checked on an update, so shrinking and unchanged entries pass even in a pool that is already over budget. Like the gate it stands in for, this is a read-check-write without a transaction. The tenant declaration gate already treats GPU keys like any other; it now reads their pool usage from the class entries, since the ResourceQuota it reads for other keys never carries them. It also refuses a key in the GPU budget domain that is not a budget key: the tenant chart keeps such a key out of the ResourceQuota, so nothing would enforce it. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <[email protected]>
1 parent 6e05fb0 commit 02d1264

5 files changed

Lines changed: 750 additions & 1 deletion

File tree

‎packages/apps/tenant/README.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,12 @@ The `resourceQuotas` parameter allows you to limit resources available to the te
116116
- `secrets` - Maximum number of Secrets
117117
- `persistentvolumeclaims` - Maximum number of PVCs
118118

119+
**GPU budgets** (checked when a VM or a worker pool is written, never rendered into the namespace ResourceQuota):
120+
- `<class>.gpuclass.cozystack.io/devices` - GPUs of one GPUClass, each vGPU counting as one
121+
- `gpuclass.cozystack.io/devices` - GPUs of every class together
122+
123+
A `VMInstance` reserves the `count` of each class entry and a `KubernetesNodes` pool reserves `count` times `maxReplicas`, stopped VMs included. A tenant with only GPU budgets gets no ResourceQuota and no default LimitRange.
124+
119125
**Example:**
120126
```yaml
121127
resourceQuotas:
Lines changed: 290 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,290 @@
1+
/*
2+
Copyright 2026 The Cozystack Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package application
18+
19+
import (
20+
"context"
21+
"encoding/json"
22+
"fmt"
23+
"math"
24+
"slices"
25+
"sort"
26+
"strings"
27+
"sync"
28+
29+
helmv2 "github.com/fluxcd/helm-controller/api/v2"
30+
structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema"
31+
"k8s.io/apimachinery/pkg/api/resource"
32+
"k8s.io/apimachinery/pkg/labels"
33+
"k8s.io/apimachinery/pkg/util/validation/field"
34+
"k8s.io/klog/v2"
35+
"sigs.k8s.io/controller-runtime/pkg/client"
36+
37+
"github.com/cozystack/cozystack/internal/gpu"
38+
"github.com/cozystack/cozystack/internal/gpu/model"
39+
appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1"
40+
"github.com/cozystack/cozystack/pkg/apis/apps/validation"
41+
)
42+
43+
// The application kinds whose gpus entries reserve GPU budget.
44+
const (
45+
vmInstanceKind = "VMInstance"
46+
kubernetesNodesKind = "KubernetesNodes"
47+
)
48+
49+
// tenantReleasePrefix is the name prefix of a Tenant's HelmRelease and of
50+
// every tenant namespace, which computeTenantNamespace spells out too.
51+
const tenantReleasePrefix = "tenant-"
52+
53+
// specSchemas maps an application kind to its spec schema. NewREST fills
54+
// it, so a check served by one kind can default the stored values of
55+
// another, the way the chart reads them.
56+
var specSchemas sync.Map
57+
58+
type gpuEntry struct {
59+
ClassName string `json:"className"`
60+
Count *int64 `json:"count"`
61+
}
62+
63+
type gpuValues struct {
64+
GPUs []gpuEntry `json:"gpus"`
65+
MaxReplicas *int64 `json:"maxReplicas"`
66+
}
67+
68+
// gpuDemand returns the GPU budget units an application's values reserve,
69+
// by budget key: every class entry reserves its count under its class key
70+
// and under the all-class key, times maxReplicas for a worker pool. Raw
71+
// name entries reserve nothing.
72+
func gpuDemand(kind string, raw []byte) (map[string]int64, error) {
73+
if kind != vmInstanceKind && kind != kubernetesNodesKind {
74+
return nil, nil
75+
}
76+
values := map[string]any{}
77+
if len(raw) > 0 {
78+
if err := json.Unmarshal(raw, &values); err != nil {
79+
return nil, fmt.Errorf("decode %s values: %w", kind, err)
80+
}
81+
}
82+
if s, ok := specSchemas.Load(kind); ok {
83+
if err := defaultLikeKubernetes(&values, s.(*structuralschema.Structural)); err != nil {
84+
return nil, fmt.Errorf("default %s values: %w", kind, err)
85+
}
86+
}
87+
defaulted, err := json.Marshal(values)
88+
if err != nil {
89+
return nil, err
90+
}
91+
var v gpuValues
92+
if err := json.Unmarshal(defaulted, &v); err != nil {
93+
return nil, fmt.Errorf("decode %s gpus: %w", kind, err)
94+
}
95+
classEntries := slices.ContainsFunc(v.GPUs, func(e gpuEntry) bool { return e.ClassName != "" })
96+
replicas := int64(1)
97+
if kind == kubernetesNodesKind && classEntries {
98+
if v.MaxReplicas == nil {
99+
return nil, fmt.Errorf("decode %s values: no maxReplicas", kind)
100+
}
101+
replicas = *v.MaxReplicas
102+
if replicas < 0 {
103+
return nil, fmt.Errorf("decode %s values: maxReplicas %d is negative", kind, replicas)
104+
}
105+
}
106+
demand := map[string]int64{}
107+
for _, e := range v.GPUs {
108+
if e.ClassName == "" {
109+
continue
110+
}
111+
count := int64(1)
112+
if e.Count != nil {
113+
count = *e.Count
114+
}
115+
if count < 1 {
116+
return nil, fmt.Errorf("decode %s gpus: class %s has count %d, below 1", kind, e.ClassName, count)
117+
}
118+
if replicas > 0 && count > math.MaxInt64/replicas {
119+
return nil, fmt.Errorf("decode %s gpus: class %s reserves more than %d units", kind, e.ClassName, int64(math.MaxInt64))
120+
}
121+
units := count * replicas
122+
for _, k := range []string{string(model.ClassQuotaKey(e.ClassName)), gpu.QuotaKeyTotal} {
123+
if units > math.MaxInt64-demand[k] {
124+
return nil, fmt.Errorf("decode %s gpus: more than %d units under %s", kind, int64(math.MaxInt64), k)
125+
}
126+
demand[k] += units
127+
}
128+
}
129+
return demand, nil
130+
}
131+
132+
// tenantQuotas returns the declared resourceQuotas of every tenant, keyed
133+
// by the namespace the tenant owns.
134+
func (r *REST) tenantQuotas(ctx context.Context) (map[string]map[string]resource.Quantity, error) {
135+
releases := &helmv2.HelmReleaseList{}
136+
selector := labels.SelectorFromSet(labels.Set{
137+
ApplicationKindLabel: validation.TenantKind,
138+
ApplicationGroupLabel: appsv1alpha1.GroupName,
139+
})
140+
if err := r.c.List(ctx, releases, &client.ListOptions{LabelSelector: selector}); err != nil {
141+
return nil, err
142+
}
143+
out := make(map[string]map[string]resource.Quantity, len(releases.Items))
144+
for i := range releases.Items {
145+
hr := &releases.Items[i]
146+
ns := r.computeTenantNamespace(hr.Namespace, strings.TrimPrefix(hr.Name, tenantReleasePrefix))
147+
quotas, err := declaredQuotasFromHelmRelease(hr)
148+
if err != nil {
149+
klog.Warningf("skipping tenant %s/%s with unparseable quotas while checking GPU budgets: %v", hr.Namespace, hr.Name, err)
150+
quotas = nil
151+
}
152+
out[ns] = quotas
153+
}
154+
return out, nil
155+
}
156+
157+
func boundedTenants(quotas map[string]map[string]resource.Quantity) map[string]bool {
158+
bounded := make(map[string]bool, len(quotas))
159+
for ns, q := range quotas {
160+
if len(q) > 0 {
161+
bounded[ns] = true
162+
}
163+
}
164+
return bounded
165+
}
166+
167+
// gpuPoolUsage sums the GPU budget the class entries in the pool rooted at
168+
// root reserve: the root's namespace plus every namespace whose nearest
169+
// bounded ancestor is root, the same members parentPoolUsage reads, minus
170+
// excludeNS. A release of the kind and name in skip is left out, so an
171+
// update does not count against itself.
172+
func (r *REST) gpuPoolUsage(ctx context.Context, root, excludeNS string, bounded map[string]bool, skip client.ObjectKey) (map[string]int64, error) {
173+
used := map[string]int64{}
174+
for _, kind := range []string{vmInstanceKind, kubernetesNodesKind} {
175+
releases := &helmv2.HelmReleaseList{}
176+
selector := labels.SelectorFromSet(labels.Set{
177+
ApplicationKindLabel: kind,
178+
ApplicationGroupLabel: appsv1alpha1.GroupName,
179+
})
180+
if err := r.c.List(ctx, releases, &client.ListOptions{LabelSelector: selector}); err != nil {
181+
return nil, err
182+
}
183+
for i := range releases.Items {
184+
hr := &releases.Items[i]
185+
if hr.Namespace == excludeNS || client.ObjectKeyFromObject(hr) == skip || r.poolRootOf(hr.Namespace, bounded) != root {
186+
continue
187+
}
188+
var raw []byte
189+
if hr.Spec.Values != nil {
190+
raw = hr.Spec.Values.Raw
191+
}
192+
demand, err := gpuDemand(kind, raw)
193+
if err != nil {
194+
klog.Warningf("skipping %s %s/%s with unreadable gpus while checking GPU budgets: %v", kind, hr.Namespace, hr.Name, err)
195+
continue
196+
}
197+
for k, v := range demand {
198+
used[k] = min(used[k], math.MaxInt64-v) + v
199+
}
200+
}
201+
}
202+
return used, nil
203+
}
204+
205+
// validateGPUBudget refuses a VMInstance or KubernetesNodes write whose
206+
// class entries would take its pool over a declared GPU budget. It applies
207+
// the tenant pool arithmetic to the declared resourceQuotas and the
208+
// declared class entries, the inputs the pool-wide admission gate of the
209+
// design reads. On an update only growth is checked, so shrinking and
210+
// unchanged entries always pass.
211+
func (r *REST) validateGPUBudget(ctx context.Context, app, old *appsv1alpha1.Application) field.ErrorList {
212+
fldPath := field.NewPath("spec", "gpus")
213+
demand, err := gpuDemand(r.kindName, specRaw(app))
214+
if err != nil {
215+
return field.ErrorList{field.Invalid(fldPath, "", err.Error())}
216+
}
217+
var previous map[string]int64
218+
if old != nil {
219+
// The stored values passed this check before, so a read failure
220+
// counts them as reserving nothing and checks the full new demand.
221+
previous, _ = gpuDemand(r.kindName, specRaw(old))
222+
}
223+
var grown []string
224+
for k, v := range demand {
225+
if v > previous[k] {
226+
grown = append(grown, k)
227+
}
228+
}
229+
if len(grown) == 0 {
230+
return nil
231+
}
232+
sort.Strings(grown)
233+
234+
quotas, err := r.tenantQuotas(ctx)
235+
if err != nil {
236+
return field.ErrorList{field.InternalError(fldPath, err)}
237+
}
238+
bounded := boundedTenants(quotas)
239+
root := r.poolRootOf(app.Namespace, bounded)
240+
if root == "" {
241+
return nil
242+
}
243+
budget := quotas[root]
244+
245+
carved := map[string]resource.Quantity{}
246+
for ns, q := range quotas {
247+
if r.poolRootOf(r.parentNamespaceOf(ns), bounded) == root {
248+
addQuotas(carved, q)
249+
}
250+
}
251+
used, err := r.gpuPoolUsage(ctx, root, "", bounded, client.ObjectKey{Namespace: app.Namespace, Name: r.releaseConfig.Prefix + app.Name})
252+
if err != nil {
253+
return field.ErrorList{field.InternalError(fldPath, err)}
254+
}
255+
256+
var errs field.ErrorList
257+
for _, k := range grown {
258+
limit, ok := budget[k]
259+
if !ok {
260+
continue
261+
}
262+
reserved := carved[k].DeepCopy()
263+
reserved.Add(*resource.NewQuantity(used[k], resource.DecimalSI))
264+
total := reserved.DeepCopy()
265+
total.Add(*resource.NewQuantity(demand[k], resource.DecimalSI))
266+
if total.Cmp(limit) > 0 {
267+
errs = append(errs, field.Forbidden(fldPath, fmt.Sprintf(
268+
"GPU budget %q of tenant pool %s is %s and %s is already reserved; this application needs %d",
269+
k, root, limit.String(), reserved.String(), demand[k])))
270+
}
271+
}
272+
return errs
273+
}
274+
275+
// parentPoolGPUUsage is the GPU counterpart of parentPoolUsage for the
276+
// tenant declaration gate.
277+
func (r *REST) parentPoolGPUUsage(ctx context.Context, parentNS, excludeNS string) (map[string]int64, error) {
278+
quotas, err := r.tenantQuotas(ctx)
279+
if err != nil {
280+
return nil, err
281+
}
282+
return r.gpuPoolUsage(ctx, parentNS, excludeNS, boundedTenants(quotas), client.ObjectKey{})
283+
}
284+
285+
func specRaw(app *appsv1alpha1.Application) []byte {
286+
if app == nil || app.Spec == nil {
287+
return nil
288+
}
289+
return app.Spec.Raw
290+
}

0 commit comments

Comments
 (0)