Skip to content

Commit c0776b8

Browse files
committed
feat(gpu): add the shared GPU placement and budget model
The binder re-checks what admission checked, and cozystack-api shows the remaining budget a later order is checked against; computing these in three places would let them disagree. The model is pure, so each caller builds a snapshot from its own cache. GPU budgets cap the whole subtree of the tenant that declares them, walked over the namespace ancestry and independent of the tenant quota pools. A raw GPU key blocks a class until the same tenant also declares a GPU key, and then caps devices and raw-name pods together. Pod usage follows the scheduler's arithmetic, sidecars included, so the binder never books a unit a raw-name pod holds. A node takes binds only when it is schedulable, Ready and carries no NoSchedule or NoExecute taint the pod does not tolerate; the node.kubernetes.io taints are left to the first two checks, and a taint never counts as a cordon. Gt and Lt tolerations do not count, since the scheduler ignores them where their feature gate is off. Template device names are derived from the VMI and shortened with a hash when the VMI name leaves no room, so they stay valid subdomains. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <[email protected]>
1 parent a12ac32 commit c0776b8

15 files changed

Lines changed: 1973 additions & 0 deletions

‎internal/gpu/model/class.go‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
/*
2+
Copyright 2026 The Cozystack Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package model
18+
19+
import (
20+
"sort"
21+
"strings"
22+
23+
gpuv1alpha1 "github.com/cozystack/cozystack/api/gpu/v1alpha1"
24+
)
25+
26+
// EffectiveMaxCount returns how many GPUs of the class one VM may take,
27+
// and false when nothing beyond one node's GPUs limits it.
28+
func EffectiveMaxCount(spec *gpuv1alpha1.GPUClassSpec) (int32, bool) {
29+
if spec.MaxCount != nil {
30+
return *spec.MaxCount, true
31+
}
32+
if spec.Mode == gpuv1alpha1.GPUModeVGPU {
33+
return 1, true
34+
}
35+
return 0, false
36+
}
37+
38+
// ResourceNames is the effective GPU resource-name set, sorted: the
39+
// classes' resource names, the hoster's list and the names node agents
40+
// report.
41+
func ResourceNames(classes []gpuv1alpha1.GPUClass, configured []string, nodes []gpuv1alpha1.GPUNode) []string {
42+
set := map[string]struct{}{}
43+
add := func(n string) {
44+
if n != "" {
45+
set[n] = struct{}{}
46+
}
47+
}
48+
for i := range classes {
49+
if dp := classes[i].Spec.Source.DevicePlugin; dp != nil {
50+
add(dp.ResourceName)
51+
}
52+
}
53+
for _, n := range configured {
54+
add(n)
55+
}
56+
for i := range nodes {
57+
for _, n := range nodes[i].Status.ResourceNames {
58+
add(n)
59+
}
60+
}
61+
out := make([]string, 0, len(set))
62+
for n := range set {
63+
out = append(out, n)
64+
}
65+
sort.Strings(out)
66+
return out
67+
}
68+
69+
// FormatResourceNames renders names for the settings ConfigMap.
70+
func FormatResourceNames(names []string) string {
71+
return strings.Join(names, ",")
72+
}
73+
74+
// ParseResourceNames reads the settings ConfigMap value.
75+
func ParseResourceNames(s string) []string {
76+
var out []string
77+
for _, n := range strings.Split(s, ",") {
78+
if n = strings.TrimSpace(n); n != "" {
79+
out = append(out, n)
80+
}
81+
}
82+
return out
83+
}

‎internal/gpu/model/class_test.go‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
/*
2+
Copyright 2026 The Cozystack Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package model
18+
19+
import (
20+
"reflect"
21+
"testing"
22+
23+
"k8s.io/utils/ptr"
24+
25+
gpuv1alpha1 "github.com/cozystack/cozystack/api/gpu/v1alpha1"
26+
)
27+
28+
func TestEffectiveMaxCount(t *testing.T) {
29+
tests := []struct {
30+
name string
31+
spec gpuv1alpha1.GPUClassSpec
32+
want int32
33+
wantLimit bool
34+
}{
35+
{"vgpu unset defaults to 1", gpuv1alpha1.GPUClassSpec{Mode: gpuv1alpha1.GPUModeVGPU}, 1, true},
36+
{"vgpu set", gpuv1alpha1.GPUClassSpec{Mode: gpuv1alpha1.GPUModeVGPU, MaxCount: ptr.To[int32](2)}, 2, true},
37+
{"passthrough unset is no limit", gpuv1alpha1.GPUClassSpec{Mode: gpuv1alpha1.GPUModePassthrough}, 0, false},
38+
{"passthrough set", gpuv1alpha1.GPUClassSpec{Mode: gpuv1alpha1.GPUModePassthrough, MaxCount: ptr.To[int32](4)}, 4, true},
39+
}
40+
for _, tc := range tests {
41+
got, limited := EffectiveMaxCount(&tc.spec)
42+
if got != tc.want || limited != tc.wantLimit {
43+
t.Errorf("%s: EffectiveMaxCount = (%d, %v), want (%d, %v)", tc.name, got, limited, tc.want, tc.wantLimit)
44+
}
45+
}
46+
}
47+
48+
func TestResourceNames(t *testing.T) {
49+
classes := []gpuv1alpha1.GPUClass{
50+
{Spec: gpuv1alpha1.GPUClassSpec{Source: gpuv1alpha1.GPUClassSource{DevicePlugin: &gpuv1alpha1.DevicePluginSource{ResourceName: "nvidia.com/NVIDIA_L40S-24C"}}}},
51+
{Spec: gpuv1alpha1.GPUClassSpec{Source: gpuv1alpha1.GPUClassSource{DevicePlugin: &gpuv1alpha1.DevicePluginSource{ResourceName: "nvidia.com/GH100_H100"}}}},
52+
{Spec: gpuv1alpha1.GPUClassSpec{}},
53+
}
54+
nodes := []gpuv1alpha1.GPUNode{
55+
{Status: gpuv1alpha1.GPUNodeStatus{ResourceNames: []string{"nvidia.com/gpu", "nvidia.com/GH100_H100"}}},
56+
}
57+
got := ResourceNames(classes, []string{"nvidia.com/AD102GL_L40S", "", "nvidia.com/GH100_H100"}, nodes)
58+
want := []string{"nvidia.com/AD102GL_L40S", "nvidia.com/GH100_H100", "nvidia.com/NVIDIA_L40S-24C", "nvidia.com/gpu"}
59+
if !reflect.DeepEqual(got, want) {
60+
t.Errorf("ResourceNames = %v, want %v", got, want)
61+
}
62+
}
63+
64+
func TestResourceNamesFormatRoundTrip(t *testing.T) {
65+
names := []string{"nvidia.com/AD102GL_L40S", "nvidia.com/gpu"}
66+
s := FormatResourceNames(names)
67+
if s != "nvidia.com/AD102GL_L40S,nvidia.com/gpu" {
68+
t.Errorf("FormatResourceNames = %q", s)
69+
}
70+
if got := ParseResourceNames(s); !reflect.DeepEqual(got, names) {
71+
t.Errorf("ParseResourceNames = %v, want %v", got, names)
72+
}
73+
if got := ParseResourceNames(""); len(got) != 0 {
74+
t.Errorf("ParseResourceNames(\"\") = %v, want empty", got)
75+
}
76+
if got := ParseResourceNames(" a/b , ,c/d"); !reflect.DeepEqual(got, []string{"a/b", "c/d"}) {
77+
t.Errorf("ParseResourceNames tolerates spaces and empties: got %v", got)
78+
}
79+
}

‎internal/gpu/model/node.go‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/*
2+
Copyright 2026 The Cozystack Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package model
18+
19+
import (
20+
"strings"
21+
22+
"github.com/go-logr/logr"
23+
corev1 "k8s.io/api/core/v1"
24+
)
25+
26+
// systemTaintPrefix marks the taints Kubernetes sets for a cordon or a
27+
// node that is not Ready, which Cordoned and Ready already decide.
28+
const systemTaintPrefix = "node.kubernetes.io/"
29+
30+
// Cordoned reports whether the node is cordoned. Only spec.unschedulable
31+
// counts: a taint, such as the one Fence Agents Remediation sets, is not a
32+
// cordon and does not trigger the class's onCordon policy.
33+
func Cordoned(n *corev1.Node) bool {
34+
return n.Spec.Unschedulable
35+
}
36+
37+
// Ready reports whether the node's Ready condition is True.
38+
func Ready(n *corev1.Node) bool {
39+
for _, c := range n.Status.Conditions {
40+
if c.Type == corev1.NodeReady {
41+
return c.Status == corev1.ConditionTrue
42+
}
43+
}
44+
return false
45+
}
46+
47+
// UntoleratedTaint returns the first NoSchedule or NoExecute taint outside
48+
// node.kubernetes.io/ that tolerations do not tolerate, or nil. Gt and Lt
49+
// tolerations never match: where their feature gate is off the scheduler
50+
// does not honour them, and a node is better skipped than bound to in vain.
51+
func UntoleratedTaint(n *corev1.Node, tolerations []corev1.Toleration) *corev1.Taint {
52+
for i := range n.Spec.Taints {
53+
taint := &n.Spec.Taints[i]
54+
if taint.Effect != corev1.TaintEffectNoSchedule && taint.Effect != corev1.TaintEffectNoExecute {
55+
continue
56+
}
57+
if strings.HasPrefix(taint.Key, systemTaintPrefix) {
58+
continue
59+
}
60+
tolerated := false
61+
for j := range tolerations {
62+
if tolerations[j].ToleratesTaint(logr.Discard(), taint, false) {
63+
tolerated = true
64+
break
65+
}
66+
}
67+
if !tolerated {
68+
return taint
69+
}
70+
}
71+
return nil
72+
}
73+
74+
// Eligible reports whether new binds, moves, rebinds and gathers may use
75+
// the node, and whether its units count in a class's capacity.
76+
// tolerations are those of the virt-launcher pod, nil when no VMI is
77+
// known.
78+
func Eligible(n *corev1.Node, tolerations []corev1.Toleration) bool {
79+
return !Cordoned(n) && Ready(n) && UntoleratedTaint(n, tolerations) == nil
80+
}

‎internal/gpu/model/node_test.go‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
/*
2+
Copyright 2026 The Cozystack Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package model
18+
19+
import (
20+
"testing"
21+
22+
corev1 "k8s.io/api/core/v1"
23+
)
24+
25+
func readyNode(mutate func(*corev1.Node)) *corev1.Node {
26+
n := &corev1.Node{Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}}}
27+
n.Name = "gpu-node-3"
28+
if mutate != nil {
29+
mutate(n)
30+
}
31+
return n
32+
}
33+
34+
func withTaint(key string, effect corev1.TaintEffect) func(*corev1.Node) {
35+
return func(n *corev1.Node) {
36+
n.Spec.Taints = append(n.Spec.Taints, corev1.Taint{Key: key, Effect: effect})
37+
}
38+
}
39+
40+
const farTaint = "medik8s.io/fence-agents-remediation"
41+
42+
func TestCordoned(t *testing.T) {
43+
tests := []struct {
44+
name string
45+
node *corev1.Node
46+
want bool
47+
}{
48+
{"unschedulable", readyNode(func(n *corev1.Node) { n.Spec.Unschedulable = true }), true},
49+
{"schedulable", readyNode(nil), false},
50+
{"Fence Agents Remediation taint alone", readyNode(withTaint(farTaint, corev1.TaintEffectNoSchedule)), false},
51+
{"unschedulable taint without the field", readyNode(withTaint(corev1.TaintNodeUnschedulable, corev1.TaintEffectNoSchedule)), false},
52+
}
53+
for _, tc := range tests {
54+
if got := Cordoned(tc.node); got != tc.want {
55+
t.Errorf("%s: Cordoned = %v, want %v", tc.name, got, tc.want)
56+
}
57+
}
58+
}
59+
60+
func TestReady(t *testing.T) {
61+
tests := []struct {
62+
name string
63+
node *corev1.Node
64+
want bool
65+
}{
66+
{"ready", readyNode(nil), true},
67+
{"not ready", readyNode(func(n *corev1.Node) { n.Status.Conditions[0].Status = corev1.ConditionFalse }), false},
68+
{"unknown", readyNode(func(n *corev1.Node) { n.Status.Conditions[0].Status = corev1.ConditionUnknown }), false},
69+
{"no condition", readyNode(func(n *corev1.Node) { n.Status.Conditions = nil }), false},
70+
}
71+
for _, tc := range tests {
72+
if got := Ready(tc.node); got != tc.want {
73+
t.Errorf("%s: Ready = %v, want %v", tc.name, got, tc.want)
74+
}
75+
}
76+
}
77+
78+
func TestEligible(t *testing.T) {
79+
tolerateFAR := []corev1.Toleration{{Key: farTaint, Operator: corev1.TolerationOpExists}}
80+
tests := []struct {
81+
name string
82+
node *corev1.Node
83+
tolerations []corev1.Toleration
84+
want bool
85+
}{
86+
{name: "ready, schedulable, untainted", node: readyNode(nil), want: true},
87+
{name: "cordoned", node: readyNode(func(n *corev1.Node) { n.Spec.Unschedulable = true }), want: false},
88+
{name: "not ready", node: readyNode(func(n *corev1.Node) { n.Status.Conditions[0].Status = corev1.ConditionFalse }), want: false},
89+
{name: "untolerated NoSchedule", node: readyNode(withTaint(farTaint, corev1.TaintEffectNoSchedule)), want: false},
90+
{name: "untolerated NoExecute", node: readyNode(withTaint("example.com/maintenance", corev1.TaintEffectNoExecute)), want: false},
91+
{name: "tolerated NoSchedule", node: readyNode(withTaint(farTaint, corev1.TaintEffectNoSchedule)), tolerations: tolerateFAR, want: true},
92+
{name: "PreferNoSchedule does not count", node: readyNode(withTaint("example.com/soft", corev1.TaintEffectPreferNoSchedule)), want: true},
93+
// The node.kubernetes.io taints mirror cordon and readiness, which
94+
// the field and the condition already decide.
95+
{name: "node.kubernetes.io/unschedulable taint on a schedulable node", node: readyNode(withTaint(corev1.TaintNodeUnschedulable, corev1.TaintEffectNoSchedule)), want: true},
96+
{name: "node.kubernetes.io/not-ready taint on a ready node", node: readyNode(withTaint(corev1.TaintNodeNotReady, corev1.TaintEffectNoExecute)), want: true},
97+
{
98+
name: "Gt toleration does not count",
99+
node: readyNode(func(n *corev1.Node) {
100+
n.Spec.Taints = []corev1.Taint{{Key: "example.com/gpu-gen", Value: "5", Effect: corev1.TaintEffectNoSchedule}}
101+
}),
102+
tolerations: []corev1.Toleration{{Key: "example.com/gpu-gen", Operator: corev1.TolerationOpGt, Value: "3", Effect: corev1.TaintEffectNoSchedule}},
103+
want: false,
104+
},
105+
{name: "toleration of another effect", node: readyNode(withTaint(farTaint, corev1.TaintEffectNoExecute)), tolerations: []corev1.Toleration{{Key: farTaint, Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}}, want: false},
106+
}
107+
for _, tc := range tests {
108+
if got := Eligible(tc.node, tc.tolerations); got != tc.want {
109+
t.Errorf("%s: Eligible = %v, want %v", tc.name, got, tc.want)
110+
}
111+
}
112+
}

‎internal/gpu/model/phase.go‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
Copyright 2026 The Cozystack Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
// Package model is the GPU logic shared by the binder, the admission
18+
// webhooks and cozystack-api: the phase table, quota keys, the tenant
19+
// ancestor walk, budgets, entitlement, capacity and node eligibility. It
20+
// holds no client; callers build a Snapshot from their own caches.
21+
package model
22+
23+
import gpuv1alpha1 "github.com/cozystack/cozystack/api/gpu/v1alpha1"
24+
25+
// HoldsGPUs reports whether a device in phase p holds units on a node.
26+
func HoldsGPUs(p gpuv1alpha1.GPUDevicePhase) bool {
27+
return p == gpuv1alpha1.GPUDeviceBound || p == gpuv1alpha1.GPUDeviceAttached
28+
}
29+
30+
// CountsAgainstBudget reports whether a device in phase p counts against
31+
// GPU budgets. A Lost device counts so that its rebind never fails a
32+
// budget check.
33+
func CountsAgainstBudget(p gpuv1alpha1.GPUDevicePhase) bool {
34+
return HoldsGPUs(p) || p == gpuv1alpha1.GPUDeviceLost
35+
}
36+
37+
// InUsageRecords reports whether a device in phase p has a usage record.
38+
func InUsageRecords(p gpuv1alpha1.GPUDevicePhase) bool {
39+
return HoldsGPUs(p)
40+
}

0 commit comments

Comments
 (0)