Skip to content

Commit 181f920

Browse files
nadaverellnadaverell
authored
Add HPA diagnosis insights (#916)
1 parent 6faadb3 commit 181f920

34 files changed

Lines changed: 2125 additions & 161 deletions

‎internal/k8s/detect_test.go‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,8 @@ func TestDetectProblems_PopulatesGroup(t *testing.T) {
6767
NumberUnavailable: 2,
6868
},
6969
},
70-
// HPA at its replica ceiling — DetectHPAProblems flags
71-
// "maxed" when current and desired both hit MaxReplicas.
70+
// HPA capped by maxReplicas — DetectHPAProblems flags
71+
// "maxed" when the controller reports TooManyReplicas.
7272
// The wrapper sets Group="autoscaling".
7373
&autoscalingv2.HorizontalPodAutoscaler{
7474
ObjectMeta: metav1.ObjectMeta{Name: "api", Namespace: "prod"},
@@ -79,6 +79,9 @@ func TestDetectProblems_PopulatesGroup(t *testing.T) {
7979
Status: autoscalingv2.HorizontalPodAutoscalerStatus{
8080
CurrentReplicas: 10,
8181
DesiredReplicas: 10,
82+
Conditions: []autoscalingv2.HorizontalPodAutoscalerCondition{
83+
{Type: autoscalingv2.ScalingLimited, Status: corev1.ConditionTrue, Reason: "TooManyReplicas", Message: "the desired replica count is more than the maximum replica count"},
84+
},
8285
},
8386
},
8487
// Job stuck Active>0 for >1h with no completions.

‎internal/k8s/detect_workload.go‎

Lines changed: 53 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,15 @@ import (
77

88
autoscalingv2 "k8s.io/api/autoscaling/v2"
99
batchv1 "k8s.io/api/batch/v1"
10-
corev1 "k8s.io/api/core/v1"
10+
11+
"github.com/skyhook-io/radar/pkg/hpadiag"
1112
)
1213

1314
// HPAProblem describes a detected issue with an HPA.
1415
type HPAProblem struct {
1516
Name string
1617
Namespace string
17-
Problem string // "maxed"
18+
Problem string // "maxed" or "cannot-scale"
1819
Reason string
1920
}
2021

@@ -29,42 +30,67 @@ type HPAProblem struct {
2930
func DetectHPAProblems(hpas []*autoscalingv2.HorizontalPodAutoscaler) []HPAProblem {
3031
var problems []HPAProblem
3132
for _, hpa := range hpas {
32-
// "maxed" — at replica ceiling and wanting more.
33-
if hpa.Spec.MaxReplicas > 0 && hpa.Status.CurrentReplicas >= hpa.Spec.MaxReplicas && hpa.Status.DesiredReplicas >= hpa.Spec.MaxReplicas {
33+
diagnosis := hpadiag.Analyze(hpa)
34+
if diagnosis == nil {
35+
continue
36+
}
37+
38+
if reason, ok := firstHPAReason(diagnosis, hpadiag.ReasonLimitedMax); ok {
3439
problems = append(problems, HPAProblem{
3540
Name: hpa.Name,
3641
Namespace: hpa.Namespace,
3742
Problem: "maxed",
38-
Reason: fmt.Sprintf("%d/%d replicas (wants %d)", hpa.Status.CurrentReplicas, hpa.Spec.MaxReplicas, hpa.Status.DesiredReplicas),
43+
Reason: maxedReasonText(diagnosis, reason),
3944
})
4045
}
41-
// "cannot scale" — the autoscaler controller reports it can't get
42-
// metrics or scale calls are failing. Emitted as a separate problem
43-
// so the maxed-check above isn't masked by an unrelated metrics
44-
// outage on the same HPA.
45-
for _, cond := range hpa.Status.Conditions {
46-
if cond.Type == autoscalingv2.ScalingActive && cond.Status == corev1.ConditionFalse {
47-
reason := cond.Reason
48-
if reason == "" {
49-
reason = "ScalingActive=False"
50-
}
51-
msg := cond.Message
52-
if msg == "" {
53-
msg = "HPA controller reports it cannot scale this workload"
54-
}
55-
problems = append(problems, HPAProblem{
56-
Name: hpa.Name,
57-
Namespace: hpa.Namespace,
58-
Problem: "cannot-scale",
59-
Reason: fmt.Sprintf("%s: %s", reason, msg),
60-
})
61-
break
62-
}
46+
47+
if reason, ok := firstHPAReason(diagnosis, hpadiag.ReasonUnableToScale, hpadiag.ReasonMetricsUnavailable); ok {
48+
problems = append(problems, HPAProblem{
49+
Name: hpa.Name,
50+
Namespace: hpa.Namespace,
51+
Problem: "cannot-scale",
52+
Reason: reasonText(reason),
53+
})
6354
}
6455
}
6556
return problems
6657
}
6758

59+
func firstHPAReason(diagnosis *hpadiag.Diagnosis, ids ...hpadiag.ReasonID) (hpadiag.Reason, bool) {
60+
for _, id := range ids {
61+
for _, reason := range diagnosis.Reasons {
62+
if reason.ID == id {
63+
return reason, true
64+
}
65+
}
66+
}
67+
return hpadiag.Reason{}, false
68+
}
69+
70+
func reasonText(reason hpadiag.Reason) string {
71+
if reason.ConditionReason != "" && reason.Message != "" {
72+
return reason.ConditionReason + ": " + reason.Message
73+
}
74+
if reason.Message != "" {
75+
return reason.Message
76+
}
77+
return string(reason.ID)
78+
}
79+
80+
func maxedReasonText(diagnosis *hpadiag.Diagnosis, reason hpadiag.Reason) string {
81+
if diagnosis == nil || diagnosis.Bounds.Max <= 0 {
82+
return reasonText(reason)
83+
}
84+
text := fmt.Sprintf("%d/%d replicas", diagnosis.Bounds.Current, diagnosis.Bounds.Max)
85+
if diagnosis.Bounds.Desired > 0 {
86+
text += fmt.Sprintf(" (wants %d)", diagnosis.Bounds.Desired)
87+
}
88+
if detail := reasonText(reason); detail != "" {
89+
return text + ": " + detail
90+
}
91+
return text
92+
}
93+
6894
// CronJobProblem describes a detected issue with a CronJob.
6995
type CronJobProblem struct {
7096
Name string

‎internal/k8s/detect_workload_test.go‎

Lines changed: 140 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
package k8s
22

33
import (
4+
"strings"
5+
"testing"
6+
"time"
7+
48
autoscalingv2 "k8s.io/api/autoscaling/v2"
59
batchv1 "k8s.io/api/batch/v1"
10+
corev1 "k8s.io/api/core/v1"
611
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
7-
"testing"
8-
"time"
912
)
1013

1114
func TestDetectHPAProblems(t *testing.T) {
@@ -14,18 +17,37 @@ func TestDetectHPAProblems(t *testing.T) {
1417
hpas []*autoscalingv2.HorizontalPodAutoscaler
1518
wantCount int
1619
wantProblem string
20+
wantReason string
1721
}{
1822
{
1923
name: "maxed HPA",
2024
hpas: []*autoscalingv2.HorizontalPodAutoscaler{
2125
{
2226
ObjectMeta: metav1.ObjectMeta{Name: "web", Namespace: "default"},
2327
Spec: autoscalingv2.HorizontalPodAutoscalerSpec{MaxReplicas: 10},
24-
Status: autoscalingv2.HorizontalPodAutoscalerStatus{CurrentReplicas: 10, DesiredReplicas: 10},
28+
Status: autoscalingv2.HorizontalPodAutoscalerStatus{
29+
CurrentReplicas: 10,
30+
DesiredReplicas: 10,
31+
Conditions: []autoscalingv2.HorizontalPodAutoscalerCondition{
32+
{Type: autoscalingv2.ScalingLimited, Status: corev1.ConditionTrue, Reason: "TooManyReplicas", Message: "the desired replica count is more than the maximum replica count"},
33+
},
34+
},
2535
},
2636
},
2737
wantCount: 1,
2838
wantProblem: "maxed",
39+
wantReason: "10/10 replicas (wants 10): TooManyReplicas: the desired replica count is more than the maximum replica count",
40+
},
41+
{
42+
name: "at max without controller limit condition is not maxed",
43+
hpas: []*autoscalingv2.HorizontalPodAutoscaler{
44+
{
45+
ObjectMeta: metav1.ObjectMeta{Name: "web", Namespace: "default"},
46+
Spec: autoscalingv2.HorizontalPodAutoscalerSpec{MaxReplicas: 10},
47+
Status: autoscalingv2.HorizontalPodAutoscalerStatus{CurrentReplicas: 10, DesiredReplicas: 10},
48+
},
49+
},
50+
wantCount: 0,
2951
},
3052
{
3153
name: "not maxed",
@@ -60,6 +82,113 @@ func TestDetectHPAProblems(t *testing.T) {
6082
},
6183
wantCount: 0,
6284
},
85+
{
86+
name: "metrics unavailable",
87+
hpas: []*autoscalingv2.HorizontalPodAutoscaler{
88+
{
89+
ObjectMeta: metav1.ObjectMeta{Name: "web", Namespace: "default"},
90+
Spec: autoscalingv2.HorizontalPodAutoscalerSpec{MaxReplicas: 10},
91+
Status: autoscalingv2.HorizontalPodAutoscalerStatus{
92+
CurrentReplicas: 5,
93+
DesiredReplicas: 5,
94+
Conditions: []autoscalingv2.HorizontalPodAutoscalerCondition{
95+
{Type: autoscalingv2.ScalingActive, Status: corev1.ConditionFalse, Reason: "FailedGetResourceMetric", Message: "missing cpu request"},
96+
},
97+
},
98+
},
99+
},
100+
wantCount: 1,
101+
wantProblem: "cannot-scale",
102+
},
103+
{
104+
name: "maxed and metrics unavailable emit two distinct issues",
105+
hpas: []*autoscalingv2.HorizontalPodAutoscaler{
106+
{
107+
ObjectMeta: metav1.ObjectMeta{Name: "web", Namespace: "default"},
108+
Spec: autoscalingv2.HorizontalPodAutoscalerSpec{MaxReplicas: 10},
109+
Status: autoscalingv2.HorizontalPodAutoscalerStatus{
110+
CurrentReplicas: 10,
111+
DesiredReplicas: 10,
112+
Conditions: []autoscalingv2.HorizontalPodAutoscalerCondition{
113+
{Type: autoscalingv2.ScalingActive, Status: corev1.ConditionFalse, Reason: "FailedGetResourceMetric", Message: "missing cpu request"},
114+
{Type: autoscalingv2.ScalingLimited, Status: corev1.ConditionTrue, Reason: "TooManyReplicas", Message: "the desired replica count is more than the maximum replica count"},
115+
},
116+
},
117+
},
118+
},
119+
wantCount: 2,
120+
},
121+
{
122+
name: "scaling disabled is not a metrics outage",
123+
hpas: []*autoscalingv2.HorizontalPodAutoscaler{
124+
{
125+
ObjectMeta: metav1.ObjectMeta{Name: "paused", Namespace: "default"},
126+
Spec: autoscalingv2.HorizontalPodAutoscalerSpec{MaxReplicas: 10},
127+
Status: autoscalingv2.HorizontalPodAutoscalerStatus{
128+
CurrentReplicas: 0,
129+
DesiredReplicas: 0,
130+
Conditions: []autoscalingv2.HorizontalPodAutoscalerCondition{
131+
{Type: autoscalingv2.ScalingActive, Status: corev1.ConditionFalse, Reason: "ScalingDisabled", Message: "scaling is disabled since the replica count of the target is zero"},
132+
},
133+
},
134+
},
135+
},
136+
wantCount: 0,
137+
},
138+
{
139+
name: "pinned min equals max is not maxed",
140+
hpas: []*autoscalingv2.HorizontalPodAutoscaler{
141+
{
142+
ObjectMeta: metav1.ObjectMeta{Name: "fixed", Namespace: "default"},
143+
Spec: autoscalingv2.HorizontalPodAutoscalerSpec{
144+
MinReplicas: ptrInt32(5),
145+
MaxReplicas: 5,
146+
},
147+
Status: autoscalingv2.HorizontalPodAutoscalerStatus{
148+
CurrentReplicas: 5,
149+
DesiredReplicas: 5,
150+
},
151+
},
152+
},
153+
wantCount: 0,
154+
},
155+
{
156+
name: "min limited is drawer context only",
157+
hpas: []*autoscalingv2.HorizontalPodAutoscaler{
158+
{
159+
ObjectMeta: metav1.ObjectMeta{Name: "idle", Namespace: "default"},
160+
Spec: autoscalingv2.HorizontalPodAutoscalerSpec{
161+
MinReplicas: ptrInt32(2),
162+
MaxReplicas: 10,
163+
},
164+
Status: autoscalingv2.HorizontalPodAutoscalerStatus{
165+
CurrentReplicas: 2,
166+
DesiredReplicas: 2,
167+
Conditions: []autoscalingv2.HorizontalPodAutoscalerCondition{
168+
{Type: autoscalingv2.ScalingLimited, Status: corev1.ConditionTrue, Reason: "TooFewReplicas", Message: "the desired replica count is less than the minimum replica count"},
169+
},
170+
},
171+
},
172+
},
173+
wantCount: 0,
174+
},
175+
{
176+
name: "scale down stabilization is drawer context only",
177+
hpas: []*autoscalingv2.HorizontalPodAutoscaler{
178+
{
179+
ObjectMeta: metav1.ObjectMeta{Name: "web", Namespace: "default"},
180+
Spec: autoscalingv2.HorizontalPodAutoscalerSpec{MaxReplicas: 10},
181+
Status: autoscalingv2.HorizontalPodAutoscalerStatus{
182+
CurrentReplicas: 5,
183+
DesiredReplicas: 5,
184+
Conditions: []autoscalingv2.HorizontalPodAutoscalerCondition{
185+
{Type: autoscalingv2.ScalingLimited, Status: corev1.ConditionTrue, Reason: "ScaleDownStabilized"},
186+
},
187+
},
188+
},
189+
},
190+
wantCount: 0,
191+
},
63192
}
64193

65194
for _, tt := range tests {
@@ -68,15 +197,22 @@ func TestDetectHPAProblems(t *testing.T) {
68197
if len(problems) != tt.wantCount {
69198
t.Errorf("DetectHPAProblems() returned %d problems, want %d", len(problems), tt.wantCount)
70199
}
71-
if tt.wantCount > 0 && len(problems) > 0 {
200+
if tt.wantProblem != "" && len(problems) > 0 {
72201
if problems[0].Problem != tt.wantProblem {
73202
t.Errorf("problem = %q, want %q", problems[0].Problem, tt.wantProblem)
74203
}
75204
}
205+
if tt.wantReason != "" && len(problems) > 0 && !strings.Contains(problems[0].Reason, tt.wantReason) {
206+
t.Errorf("reason = %q, want to contain %q", problems[0].Reason, tt.wantReason)
207+
}
76208
})
77209
}
78210
}
79211

212+
func ptrInt32(v int32) *int32 {
213+
return &v
214+
}
215+
80216
func TestDetectCronJobProblems(t *testing.T) {
81217
now := time.Now()
82218
suspended := true

‎internal/server/server.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"github.com/go-chi/chi/v5"
2222
"github.com/go-chi/chi/v5/middleware"
2323
"github.com/go-chi/cors"
24+
autoscalingv2 "k8s.io/api/autoscaling/v2"
2425
corev1 "k8s.io/api/core/v1"
2526
apierrors "k8s.io/apimachinery/pkg/api/errors"
2627
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -42,6 +43,7 @@ import (
4243
"github.com/skyhook-io/radar/internal/timeline"
4344
"github.com/skyhook-io/radar/internal/updater"
4445
"github.com/skyhook-io/radar/internal/version"
46+
"github.com/skyhook-io/radar/pkg/hpadiag"
4547
"github.com/skyhook-io/radar/pkg/perfstats"
4648
"github.com/skyhook-io/radar/pkg/rbac"
4749
topology "github.com/skyhook-io/radar/pkg/topology"
@@ -1609,6 +1611,14 @@ func setTypeMeta(resource any) {
16091611
k8s.SetTypeMeta(resource)
16101612
}
16111613

1614+
func hpaDiagnosisFor(resource any) *hpadiag.Diagnosis {
1615+
hpa, ok := resource.(*autoscalingv2.HorizontalPodAutoscaler)
1616+
if !ok {
1617+
return nil
1618+
}
1619+
return hpadiag.Analyze(hpa)
1620+
}
1621+
16121622
// preflightResourceGet runs the per-user RBAC gates that must pass before any
16131623
// single-resource GET fetch. Mirrors the kind/scope-aware logic used by both
16141624
// the REST handler (handleGetResource) and the AI handler (handleAIGetResource)
@@ -1748,6 +1758,7 @@ func (s *Server) handleGetResource(w http.ResponseWriter, r *http.Request) {
17481758
s.writeJSON(w, topology.ResourceWithRelationships{
17491759
Resource: resource,
17501760
Relationships: relationships,
1761+
HPADiagnosis: hpaDiagnosisFor(resource),
17511762
})
17521763
return
17531764
}
@@ -1963,6 +1974,7 @@ func (s *Server) handleGetResource(w http.ResponseWriter, r *http.Request) {
19631974
response := topology.ResourceWithRelationships{
19641975
Resource: resource,
19651976
Relationships: relationships,
1977+
HPADiagnosis: hpaDiagnosisFor(resource),
19661978
}
19671979

19681980
// Enrich TLS secrets with parsed certificate info

0 commit comments

Comments
 (0)