Skip to content

Commit 2dfe380

Browse files
committed
fix(kubernetes): raise kubevirt-csi API client rate limits
The kubevirt-csi controller built its infra- and tenant-cluster Kubernetes clients on a rest.Config that left QPS/Burst unset, so client-go applied its 5 QPS / 10 burst default. ControllerPublishVolume polls the infra PVC once per second for up to two minutes; when a tenant scales out and several volumes attach at once, the default limiter starves and the poll fails with "client rate limiter Wait returned an error: context deadline exceeded", which the tenant kubelet surfaces as FailedAttachVolume. Set QPS/Burst to 100/200, overridable via --kube-api-qps and --kube-api-burst, on both the infra and tenant configs so a concurrent-attach burst is admitted promptly. Reject non-positive flag values, which client-go would otherwise turn into the starving default (0) or unlimited (<0). Assisted-By: Claude <[email protected]> Signed-off-by: Aleksei Sviridkin <[email protected]>
1 parent 8bffa34 commit 2dfe380

2 files changed

Lines changed: 152 additions & 0 deletions

File tree

‎packages/apps/kubernetes/images/kubevirt-csi-driver/main.go‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,17 @@ var (
3737

3838
runNodeService = flag.Bool("run-node-service", true, "Specifies rather or not to run the node service, the default is true")
3939
runControllerService = flag.Bool("run-controller-service", true, "Specifies rather or not to run the controller service, the default is true")
40+
41+
kubeAPIQPS = flag.Float64("kube-api-qps", defaultKubeAPIQPS, "QPS limit for the infra- and tenant-cluster Kubernetes API clients. client-go defaults to 5, which starves the once-per-second infra PVC-bound poll in the NFS ControllerPublishVolume path under a burst of concurrent attaches.")
42+
kubeAPIBurst = flag.Int("kube-api-burst", defaultKubeAPIBurst, "Burst limit for the infra- and tenant-cluster Kubernetes API clients (client-go default is 10).")
43+
)
44+
45+
// Chosen well above the client-go defaults (5 QPS / 10 burst) so a scale-out
46+
// burst of PVC attaches does not starve the bound poll, while staying well
47+
// within the infra API server's capacity.
48+
const (
49+
defaultKubeAPIQPS = 100
50+
defaultKubeAPIBurst = 200
4051
)
4152

4253
func init() {
@@ -65,6 +76,9 @@ func handle() {
6576
if volumePrefix == nil || *volumePrefix == "" {
6677
klog.Fatal("volume-prefix must be set")
6778
}
79+
if err := validateRateLimitFlags(*kubeAPIQPS, *kubeAPIBurst); err != nil {
80+
klog.Fatal(err)
81+
}
6882

6983
inClusterConfig, err := rest.InClusterConfig()
7084
if err != nil {
@@ -89,6 +103,9 @@ func handle() {
89103
infraRestConfig = inClusterConfig
90104
}
91105

106+
applyKubeAPIRateLimits(tenantRestConfig, *kubeAPIQPS, *kubeAPIBurst)
107+
applyKubeAPIRateLimits(infraRestConfig, *kubeAPIQPS, *kubeAPIBurst)
108+
92109
tenantClientSet, err := kubernetes.NewForConfig(tenantRestConfig)
93110
if err != nil {
94111
klog.Fatalf("Failed to build tenant client set: %v", err)
@@ -184,6 +201,32 @@ func handle() {
184201
s.Wait()
185202
}
186203

204+
// validateRateLimitFlags rejects non-positive limits. client-go treats QPS == 0
205+
// as "use the 5 QPS default" and QPS < 0 as "no rate limiting at all", so an
206+
// operator passing 0 expecting "unlimited" would silently get the starving
207+
// default this fix exists to avoid; refuse both rather than surprise them.
208+
func validateRateLimitFlags(qps float64, burst int) error {
209+
if qps <= 0 {
210+
return fmt.Errorf("kube-api-qps must be positive, got %v", qps)
211+
}
212+
if burst <= 0 {
213+
return fmt.Errorf("kube-api-burst must be positive, got %d", burst)
214+
}
215+
return nil
216+
}
217+
218+
// applyKubeAPIRateLimits raises a rest.Config's client-side rate limiter above
219+
// the client-go default of 5 QPS / 10 burst. The NFS-volume ControllerPublishVolume
220+
// path polls the infra PersistentVolumeClaim once per second for up to two minutes;
221+
// when a tenant scales out and several volumes attach at once, the default limiter
222+
// starves and the poll fails with "client rate limiter Wait returned an error:
223+
// context deadline exceeded", which the tenant kubelet surfaces as
224+
// FailedAttachVolume.
225+
func applyKubeAPIRateLimits(cfg *rest.Config, qps float64, burst int) {
226+
cfg.QPS = float32(qps)
227+
cfg.Burst = burst
228+
}
229+
187230
func configureStorageClassEnforcement(infraStorageClassEnforcement string) util.StorageClassEnforcement {
188231
var storageClassEnforcement util.StorageClassEnforcement
189232

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"testing"
6+
"time"
7+
8+
"k8s.io/client-go/rest"
9+
"k8s.io/client-go/util/flowcontrol"
10+
)
11+
12+
// client-go applies these limits to a rest.Config that leaves QPS/Burst unset.
13+
// They are what the driver ran with before this fix.
14+
const (
15+
clientGoDefaultQPS float32 = 5
16+
clientGoDefaultBurst int = 10
17+
)
18+
19+
// applyKubeAPIRateLimits must raise the client both above the client-go default,
20+
// otherwise the ControllerPublishVolume PVC-bound poll keeps starving under a
21+
// burst of concurrent attaches.
22+
func TestApplyKubeAPIRateLimitsRaisesAboveClientGoDefault(t *testing.T) {
23+
cfg := &rest.Config{}
24+
applyKubeAPIRateLimits(cfg, defaultKubeAPIQPS, defaultKubeAPIBurst)
25+
26+
if cfg.QPS <= clientGoDefaultQPS {
27+
t.Errorf("QPS = %v, want > %v (client-go default)", cfg.QPS, clientGoDefaultQPS)
28+
}
29+
if cfg.Burst <= clientGoDefaultBurst {
30+
t.Errorf("Burst = %d, want > %d (client-go default)", cfg.Burst, clientGoDefaultBurst)
31+
}
32+
}
33+
34+
// A non-default qps/burst must actually reach the config, so --kube-api-qps and
35+
// --kube-api-burst are real tuning knobs and not just decoration over the constants.
36+
func TestApplyKubeAPIRateLimitsPropagatesValues(t *testing.T) {
37+
cfg := &rest.Config{}
38+
applyKubeAPIRateLimits(cfg, 42, 84)
39+
40+
if cfg.QPS != 42 {
41+
t.Errorf("QPS = %v, want 42", cfg.QPS)
42+
}
43+
if cfg.Burst != 84 {
44+
t.Errorf("Burst = %d, want 84", cfg.Burst)
45+
}
46+
}
47+
48+
// The default flag values must pass validation, and a non-positive value — which
49+
// client-go would silently turn into the starving default (0) or unlimited
50+
// (<0) — must be rejected.
51+
func TestValidateRateLimitFlags(t *testing.T) {
52+
if err := validateRateLimitFlags(defaultKubeAPIQPS, defaultKubeAPIBurst); err != nil {
53+
t.Errorf("defaults must be valid, got %v", err)
54+
}
55+
56+
cases := []struct {
57+
name string
58+
qps float64
59+
burst int
60+
}{
61+
{"zero qps falls back to the starving default", 0, defaultKubeAPIBurst},
62+
{"negative qps disables rate limiting", -1, defaultKubeAPIBurst},
63+
{"zero burst", defaultKubeAPIQPS, 0},
64+
{"negative burst", defaultKubeAPIQPS, -5},
65+
}
66+
for _, tc := range cases {
67+
t.Run(tc.name, func(t *testing.T) {
68+
if err := validateRateLimitFlags(tc.qps, tc.burst); err == nil {
69+
t.Errorf("validateRateLimitFlags(%v, %d) = nil, want error", tc.qps, tc.burst)
70+
}
71+
})
72+
}
73+
}
74+
75+
// Behavioural regression for the FailedAttachVolume flake. The NFS-volume
76+
// ControllerPublishVolume path polls the infra PVC once per second for up to two
77+
// minutes; when several volumes attach at once the default 5 QPS / 10 burst limiter starves and Wait returns
78+
// "context deadline exceeded" — the exact error the tenant kubelet reports as
79+
// FailedAttachVolume. The limits applyKubeAPIRateLimits sets must admit that burst.
80+
func TestRateLimitedConfigAdmitsBurstThatDefaultStarves(t *testing.T) {
81+
// A burst larger than any refill the deadline can supply: 5 QPS over 500ms
82+
// tops up ~2.5 tokens on top of a 10-token bucket, so the default limiter
83+
// cannot admit 200 waiters before the deadline, while a bucket sized for the
84+
// burst admits them immediately. Both outcomes are deterministic.
85+
const burst = 200
86+
const deadline = 500 * time.Millisecond
87+
88+
drain := func(qps float32, b int) error {
89+
lim := flowcontrol.NewTokenBucketRateLimiter(qps, b)
90+
ctx, cancel := context.WithTimeout(context.Background(), deadline)
91+
defer cancel()
92+
for range burst {
93+
if err := lim.Wait(ctx); err != nil {
94+
return err
95+
}
96+
}
97+
return nil
98+
}
99+
100+
if err := drain(clientGoDefaultQPS, clientGoDefaultBurst); err == nil {
101+
t.Fatal("default 5 QPS / 10 burst limiter admitted the burst; the test can no longer prove the fix")
102+
}
103+
104+
cfg := &rest.Config{}
105+
applyKubeAPIRateLimits(cfg, defaultKubeAPIQPS, defaultKubeAPIBurst)
106+
if err := drain(cfg.QPS, cfg.Burst); err != nil {
107+
t.Errorf("configured limiter (%v QPS / %d burst) still starved on the attach burst: %v", cfg.QPS, cfg.Burst, err)
108+
}
109+
}

0 commit comments

Comments
 (0)