Skip to content

Commit ec4a2e4

Browse files
nadaverellnadaverell
authored
Stabilize pod root cause guidance (#918)
1 parent 2b676aa commit ec4a2e4

4 files changed

Lines changed: 48 additions & 12 deletions

File tree

‎internal/k8s/detect.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1207,7 +1207,7 @@ func imagePullDiagnosis(reason, message string) (cause, action string) {
12071207
case containsAny(lower, "unauthorized", "forbidden", "denied", "authentication required", "pull access denied") ||
12081208
containsStatusCode(lower, "401") || containsStatusCode(lower, "403"):
12091209
return imageCause("Not authorized to pull image", ref),
1210-
"Check imagePullSecrets, the pod service account, and registry permissions for this namespace."
1210+
"Check imagePullSecrets, the pod service account, registry permissions for this namespace, and whether the repository/tag exists."
12111211
case containsAny(lower, "toomanyrequests", "too many requests", "rate limit"):
12121212
return imageCause("Registry rate-limited", ref),
12131213
"Use an authenticated pull secret, reduce pull frequency, or mirror/cache the image."

‎internal/k8s/detect_test.go‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -913,42 +913,42 @@ func TestImagePullDiagnosis(t *testing.T) {
913913
reason string
914914
message string
915915
wantCause string
916-
wantAct string
916+
wantActs []string
917917
}{
918918
{
919919
name: "not found",
920920
reason: "ImagePullBackOff",
921921
message: `Back-off pulling image "reg.io/team/api:v2": failed to resolve reference "reg.io/team/api:v2": not found`,
922922
wantCause: "Image not found: reg.io/team/api:v2",
923-
wantAct: "repository and tag",
923+
wantActs: []string{"repository and tag"},
924924
},
925925
{
926926
name: "auth wins over not found",
927927
reason: "ErrImagePull",
928928
message: `failed to pull image "priv.io/app:v1": not found: authentication required`,
929929
wantCause: "Not authorized to pull image: priv.io/app:v1",
930-
wantAct: "imagePullSecrets",
930+
wantActs: []string{"imagePullSecrets", "repository/tag"},
931931
},
932932
{
933933
name: "registry unreachable",
934934
reason: "ImagePullBackOff",
935935
message: `failed to pull image "reg.io/app:v1": dial tcp: lookup reg.io: no such host`,
936936
wantCause: "Registry unreachable: reg.io/app:v1",
937-
wantAct: "DNS",
937+
wantActs: []string{"DNS"},
938938
},
939939
{
940940
name: "rate limited",
941941
reason: "ImagePullBackOff",
942942
message: `toomanyrequests: rate limit exceeded for image "reg.io/app:v1"`,
943943
wantCause: "Registry rate-limited: reg.io/app:v1",
944-
wantAct: "authenticated",
944+
wantActs: []string{"authenticated"},
945945
},
946946
{
947947
name: "invalid reference",
948948
reason: "InvalidImageName",
949949
message: `Failed to apply default image tag "bad image": invalid reference format`,
950950
wantCause: "Image reference is invalid",
951-
wantAct: "syntax",
951+
wantActs: []string{"syntax"},
952952
},
953953
{
954954
name: "unknown shape",
@@ -972,14 +972,16 @@ func TestImagePullDiagnosis(t *testing.T) {
972972
if cause != tc.wantCause {
973973
t.Fatalf("cause = %q, want %q", cause, tc.wantCause)
974974
}
975-
if tc.wantAct == "" {
975+
if len(tc.wantActs) == 0 {
976976
if action != "" {
977977
t.Fatalf("action = %q, want empty", action)
978978
}
979979
return
980980
}
981-
if !strings.Contains(action, tc.wantAct) {
982-
t.Fatalf("action = %q, want substring %q", action, tc.wantAct)
981+
for _, wantAct := range tc.wantActs {
982+
if !strings.Contains(action, wantAct) {
983+
t.Fatalf("action = %q, want substring %q", action, wantAct)
984+
}
983985
}
984986
})
985987
}

‎internal/k8s/health.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -438,7 +438,7 @@ func shortRunContext(term *corev1.ContainerStateTerminated) string {
438438
if runDuration <= 0 || runDuration >= shortCrashRunWindow {
439439
return ""
440440
}
441-
return fmt.Sprintf(" It ran for %s before exiting.", FormatAge(runDuration))
441+
return " It exited within seconds of starting."
442442
}
443443

444444
// PodProblemReason returns a short reason string for a problematic pod.

‎internal/k8s/health_test.go‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -581,7 +581,7 @@ func TestPodCrashLoopDiagnosisUsesActiveCrashCandidate(t *testing.T) {
581581
}
582582

583583
cause, action := podCrashLoopDiagnosis(pod, now)
584-
if !strings.Contains(cause, `container "app"`) || !strings.Contains(cause, "code 127") || !strings.Contains(cause, "before exiting") {
584+
if !strings.Contains(cause, `container "app"`) || !strings.Contains(cause, "code 127") || !strings.Contains(cause, "within seconds") {
585585
t.Fatalf("cause = %q, want app exit-code diagnosis with short-run context", cause)
586586
}
587587
if strings.Contains(cause, "sidecar") || strings.Contains(cause, "139") {
@@ -592,6 +592,40 @@ func TestPodCrashLoopDiagnosisUsesActiveCrashCandidate(t *testing.T) {
592592
}
593593
}
594594

595+
func TestPodCrashLoopDiagnosisShortRunContextStableAcrossReplicas(t *testing.T) {
596+
now := time.Now()
597+
mkPod := func(name string, runFor time.Duration) *corev1.Pod {
598+
finished := metav1.NewTime(now.Add(-1 * time.Second))
599+
started := metav1.NewTime(finished.Time.Add(-runFor))
600+
return &corev1.Pod{
601+
ObjectMeta: metav1.ObjectMeta{Name: name},
602+
Status: corev1.PodStatus{
603+
Phase: corev1.PodRunning,
604+
ContainerStatuses: []corev1.ContainerStatus{{
605+
Name: "app",
606+
RestartCount: 2,
607+
State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "CrashLoopBackOff"}},
608+
LastTerminationState: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{
609+
Reason: "Error", ExitCode: 139, StartedAt: started, FinishedAt: finished,
610+
}},
611+
}},
612+
},
613+
}
614+
}
615+
616+
fastCause, fastAction := podCrashLoopDiagnosis(mkPod("web-a", 2*time.Second), now)
617+
slowerCause, slowerAction := podCrashLoopDiagnosis(mkPod("web-b", 4*time.Second), now)
618+
if fastCause == "" || fastCause != slowerCause {
619+
t.Fatalf("short-run crash causes should be identical across replicas, got %q vs %q", fastCause, slowerCause)
620+
}
621+
if fastAction != slowerAction {
622+
t.Fatalf("short-run crash actions should be identical across replicas, got %q vs %q", fastAction, slowerAction)
623+
}
624+
if strings.Contains(fastCause, "2s") || strings.Contains(slowerCause, "4s") {
625+
t.Fatalf("short-run cause should not include per-replica duration: %q / %q", fastCause, slowerCause)
626+
}
627+
}
628+
595629
func TestPodCrashLoopDiagnosisOrdersMultipleCandidates(t *testing.T) {
596630
now := time.Now()
597631
older := metav1.NewTime(now.Add(-30 * time.Second))

0 commit comments

Comments
 (0)