Skip to content

Commit b1124fd

Browse files
nadaverellnadaverell
authored
Surface native Helm release incidents to agents (#1036)
1 parent 7e3cde8 commit b1124fd

24 files changed

Lines changed: 789 additions & 102 deletions

‎docs/helm.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ Use `get_helm_release` for detail:
6363
- `include=notes_diff` returns release notes diff.
6464
- `include=resource_diff` returns added/removed/unchanged rendered resource identities between revisions.
6565

66-
`get_changes` is intentionally Kubernetes-resource timeline data, not Helm release history. For Helm deployments, use the Helm MCP tools above.
66+
`get_changes` includes the Kubernetes-resource timeline plus recent native Helm release deployment and operation history (`source: helm`). Use it when an agent asks what changed before an incident. For full Helm revision history, operation details, hook diagnostics, values, and diffs, use the Helm MCP tools above.
6767

6868
## Known Limits
6969

‎docs/mcp.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -184,8 +184,8 @@ Add to `~/.gemini/settings.json`:
184184

185185
| Tool | Description | Parameters |
186186
|------|-------------|------------|
187-
| `issues` | "What's broken right now?" — a ranked, curated stream of live operational failures: failing workloads/pods, dangling references, pod-startup blockers (unschedulable / admission-rejected / stuck post-bind), and False CRD conditions. No source filter; each row carries a `source` label sliceable via `filter`. Recovered Helm rollbacks are deploy history, not live issues; use `list_helm_releases` / `get_helm_release` for Helm release history. For static posture use `get_cluster_audit`; for raw events use `get_events`. | `namespace` (optional), `severity` (optional: `critical,warning`), `kind` (optional), `filter` (optional CEL), `limit` (optional, default 200, max 1000) |
188-
| `diagnose` | Root-cause one workload or GitOps reconciler in a single call. Pod/Deployment/StatefulSet/DaemonSet get minified resource + `resourceContext` + current AND previous container logs across pods + filtered events + `startupBlockers`; Application/Kustomization/HelmRelease get reconciler status + related parsed issues. | `kind` (required: workload or GitOps reconciler), `namespace` (required), `name` (required) |
187+
| `issues` | "What's broken right now?" — a ranked, curated stream of live operational failures: failing workloads/pods, active native Helm release failures or stuck pending operations (`kind=HelmRelease`, `group=helm.sh`), dangling references, pod-startup blockers (unschedulable / admission-rejected / stuck post-bind), and False CRD conditions. No source filter; each row carries a `source` label sliceable via `filter`. Recovered Helm rollbacks are deploy history, not live issues; use `get_changes` for Helm deployment history and `get_helm_release` for native Helm full revision/history/hook diagnostics. Flux `HelmRelease` rows (`group=helm.toolkit.fluxcd.io`) are GitOps reconcilers and should use `diagnose`. For static posture use `get_cluster_audit`; for raw events use `get_events`. | `namespace` (optional), `severity` (optional: `critical,warning`), `kind` (optional), `filter` (optional CEL), `limit` (optional, default 200, max 1000) |
188+
| `diagnose` | Root-cause one workload or GitOps reconciler in a single call. Pod/Deployment/StatefulSet/DaemonSet get minified resource + `resourceContext` + current AND previous container logs across pods + filtered events + `startupBlockers`; Application/Kustomization/Flux HelmRelease get reconciler status + related parsed issues. | `kind` (required: workload or GitOps reconciler), `namespace` (required), `name` (required) |
189189
| `get_dashboard` | Cluster/namespace health overview — resource counts, failing pods, unhealthy workloads, warning events, Helm status. Inventory-style triage before drilling in. | `namespace` (optional) |
190190
| `top_resources` | Live metrics ranked like `kubectl top | sort`, joined with K8s context (status, restarts, owner, requests/limits). Use for CPU/memory/OOM/load symptoms. | `kind` (optional: `pods` default, `workloads`, `nodes`), `namespace` (optional), `sort` (optional: `cpu` default, `memory`), `limit` (optional, default 20, max 100) |
191191
| `list_resources` | List resources of a kind with minified summaries + per-row `summaryContext` (managedBy / health / issueCount). | `kind` (required), `group` (optional), `namespace` (optional), `context` (optional: default / `none`) |
@@ -194,7 +194,7 @@ Add to `~/.gemini/settings.json`:
194194
| `get_topology` | Whole-namespace/cluster topology graph (nodes + edges). Use `summary` format for LLM-friendly text chains. Once you have a suspect root, prefer `get_neighborhood`. | `namespace` (optional), `view` (optional: `traffic` or `resources`), `format` (optional: `graph` or `summary`) |
195195
| `get_neighborhood` | BFS-expanded topology neighborhood around one known root — cheaper and clearer than `get_topology` for cross-resource failures (routing, selector/endpoint, refs, owner chains). RBAC-filtered. | `kind` (required), `namespace` (optional), `name` (required), `profile` (optional: `auto` default / `all`), `hops` (optional, default 1, max 2) |
196196
| `get_events` | Recent Kubernetes Warning events, deduplicated and sorted by recency. Filter by resource kind/name to scope. | `namespace` (optional), `limit` (optional, default 20, max 100), `kind` (optional), `name` (optional) |
197-
| `get_changes` | Recent resource changes (creates, updates, deletes) from the Kubernetes cluster timeline. Use to investigate what changed before an incident. Helm release deployment history is separate; use `list_helm_releases` or `get_helm_release include=history,operations` for failed upgrades and rollbacks. | `namespace` (optional), `kind` (optional), `name` (optional), `since` (optional, e.g. `1h`, `30m`; default `1h`), `limit` (optional, default 20, max 50) |
197+
| `get_changes` | Recent meaningful changes from the Kubernetes cluster timeline plus native Helm release deployment/operation history (`source: helm`). Use to investigate what changed before an incident, including failed upgrades, rollbacks, and current Helm revisions. If the response includes `sourcesErrored`, treat it as partial data for those sources. Use `get_helm_release include=history,operations` for the full Helm revision trail. | `namespace` (optional), `kind` (optional), `name` (optional), `since` (optional, e.g. `1h`, `30m`; default `1h`), `limit` (optional, default 20, max 50) |
198198
| `get_pod_logs` | Filtered pod logs prioritizing errors/warnings, with secret redaction. Set `grep` for server-side filtering. | `namespace` (required), `name` (required), `container` (optional), `tail_lines` (optional, default 200), `grep` (optional) |
199199
| `get_workload_logs` | Aggregated, AI-filtered logs from all pods of a workload (Deployment, StatefulSet, DaemonSet) | `kind` (required), `namespace` (required), `name` (required), `container` (optional), `tail_lines` (optional, default 100 per pod), `grep` (optional) |
200200
| `get_cluster_audit` | Static config posture — best-practice findings (Security / Reliability / Efficiency) with remediation. INDEPENDENT of operational health; for "what's broken right now?" use `issues`. | `namespace` (optional), `category` (optional), `severity` (optional) |

‎internal/issues/category.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,9 @@ func classifyProblem(in classifyInput) issuesapi.Category {
274274
if g == "kustomize.toolkit.fluxcd.io" || g == "helm.toolkit.fluxcd.io" {
275275
return classifyGitOpsReason(in.Reason, issuesapi.CategoryGitOpsSyncFailed)
276276
}
277+
if in.Kind == "HelmRelease" && g == NativeHelmGroup {
278+
return issuesapi.CategoryHelmReleaseFailed
279+
}
277280
return issuesapi.CategoryUnknown
278281

279282
case "Cluster", "KubeadmControlPlane":

‎internal/issues/category_test.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@ func TestClassify(t *testing.T) {
5959
{"argo unknown reason falls back to sync_failed", classifyInput{Source: SourceProblem, Kind: "Application", APIGroup: "argoproj.io", Reason: "SomethingNovel"}, issuesapi.CategoryGitOpsSyncFailed},
6060
{"flux kustomization problem", classifyInput{Source: SourceProblem, Kind: "Kustomization", APIGroup: "kustomize.toolkit.fluxcd.io", Reason: "ReconciliationFailed"}, issuesapi.CategoryGitOpsOperationFailed},
6161
{"flux helmrelease problem", classifyInput{Source: SourceProblem, Kind: "HelmRelease", APIGroup: "helm.toolkit.fluxcd.io", Reason: "InstallFailed"}, issuesapi.CategoryGitOpsOperationFailed},
62+
{"native helm release failed", classifyInput{Source: SourceProblem, Kind: "HelmRelease", APIGroup: "helm.sh", Reason: "HelmReleaseFailed"}, issuesapi.CategoryHelmReleaseFailed},
63+
{"native helm upgrade failed", classifyInput{Source: SourceProblem, Kind: "HelmRelease", APIGroup: "helm.sh", Reason: "HelmUpgradeFailed"}, issuesapi.CategoryHelmReleaseFailed},
64+
{"native helm pending", classifyInput{Source: SourceProblem, Kind: "HelmRelease", APIGroup: "helm.sh", Reason: "HelmReleasePending"}, issuesapi.CategoryHelmReleaseFailed},
6265
{"flux artifact failed is render", classifyInput{Source: SourceProblem, Kind: "Kustomization", APIGroup: "kustomize.toolkit.fluxcd.io", Reason: "ArtifactFailed"}, issuesapi.CategoryGitOpsRenderFailed},
6366
{"flux chart not ready is render", classifyInput{Source: SourceProblem, Kind: "HelmRelease", APIGroup: "helm.toolkit.fluxcd.io", Reason: "ChartNotReady"}, issuesapi.CategoryGitOpsRenderFailed},
6467
{"flux-looking kustomization group is not gitops", classifyInput{Source: SourceProblem, Kind: "Kustomization", APIGroup: "custom-fluxcd.io", Reason: "ReconciliationFailed"}, issuesapi.CategoryUnknown},

‎internal/issues/compose.go‎

Lines changed: 85 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -108,13 +108,8 @@ func ComposeWithStats(p Provider, f Filters) ([]Issue, ComposeStats) {
108108
// doesn't silently drop counts for resources whose issues fall in
109109
// the tail beyond MaxLimit. Zero still maps to DefaultLimit so the
110110
// public /api/issues + MCP issues_list keep their tight caps.
111-
uncapped := f.Limit < 0
112-
if f.Limit == 0 {
113-
f.Limit = DefaultLimit
114-
}
115-
if !uncapped && f.Limit > MaxLimit {
116-
f.Limit = MaxLimit
117-
}
111+
limit, uncapped := normalizedLimit(f.Limit)
112+
f.Limit = limit
118113

119114
out := make([]Issue, 0, 64)
120115
now := time.Now()
@@ -172,44 +167,56 @@ func ComposeWithStats(p Provider, f Filters) ([]Issue, ComposeStats) {
172167
}
173168
out = enrichDiagnosticContext(out, flatForContext, groupedForContext, p)
174169

175-
// ---- 4. Public filters on the shaped rows ------------------------
176-
out = applyFilters(out, f) // severity + kind, against subject (grouped) or evidence (flat)
177-
var stats ComposeStats
178-
if f.Filter != nil {
179-
// CEL evaluated last so it sees the public shape. Eval errors count as
180-
// non-match ("missing field" semantics: zero hits + clean response, not
181-
// a 500); ComposeStats forwards the count + first sample so the caller
182-
// can distinguish "filter excluded everything" from "nothing matched."
183-
filtered := out[:0]
184-
var firstErr error
185-
errCount := 0
186-
for _, i := range out {
187-
ok, err := f.Filter.Match(issueToActivation(i))
188-
if err != nil {
189-
errCount++
190-
if firstErr == nil {
191-
firstErr = err
192-
}
193-
continue
194-
}
195-
if ok {
196-
filtered = append(filtered, i)
197-
}
198-
}
199-
if errCount > 0 {
200-
log.Printf("[issues] CEL filter eval errors: %d/%d rows; first=%v", errCount, len(out), firstErr)
201-
stats.FilterErrors = errCount
202-
if firstErr != nil {
203-
stats.FilterErrorSample = firstErr.Error()
204-
}
170+
return finalizeShapedIssues(out, f, !f.Grouped, uncapped)
171+
}
172+
173+
// MergeExternalIssues adds already-shaped issue rows from handler-owned data
174+
// sources, then applies the public sort/cap. `base` must already have gone
175+
// through ComposeWithStats with the same filters except Limit=NoLimit.
176+
func MergeExternalIssues(base []Issue, baseStats ComposeStats, f Filters, extras []Issue) ([]Issue, ComposeStats) {
177+
limit, uncapped := normalizedLimit(f.Limit)
178+
f.Limit = limit
179+
180+
if len(extras) == 0 {
181+
baseStats.TotalMatched = len(base)
182+
if !uncapped && len(base) > f.Limit {
183+
base = base[:f.Limit]
205184
}
206-
out = filtered
185+
return base, baseStats
186+
}
187+
188+
filteredExtras, extraStats := filterShapedIssues(append([]Issue(nil), extras...), f)
189+
out := make([]Issue, 0, len(base)+len(filteredExtras))
190+
out = append(out, base...)
191+
out = append(out, filteredExtras...)
192+
sort.SliceStable(out, func(i, j int) bool { return lessIssue(out[i], out[j]) })
193+
194+
stats := baseStats
195+
stats.FilterErrors += extraStats.FilterErrors
196+
if stats.FilterErrorSample == "" {
197+
stats.FilterErrorSample = extraStats.FilterErrorSample
198+
}
199+
stats.TotalMatched = len(out)
200+
if !uncapped && len(out) > f.Limit {
201+
out = out[:f.Limit]
207202
}
203+
return out, stats
204+
}
208205

209-
// ---- 5. Sort, count, cap -----------------------------------------
210-
// GroupIssues already sorted deterministically; the flat path sorts here
211-
// with the same comparator so both orders agree.
212-
if !f.Grouped {
206+
func normalizedLimit(limit int) (int, bool) {
207+
uncapped := limit < 0
208+
if limit == 0 {
209+
limit = DefaultLimit
210+
}
211+
if !uncapped && limit > MaxLimit {
212+
limit = MaxLimit
213+
}
214+
return limit, uncapped
215+
}
216+
217+
func finalizeShapedIssues(out []Issue, f Filters, sortOut bool, uncapped bool) ([]Issue, ComposeStats) {
218+
out, stats := filterShapedIssues(out, f)
219+
if sortOut {
213220
sort.SliceStable(out, func(i, j int) bool { return lessIssue(out[i], out[j]) })
214221
}
215222
stats.TotalMatched = len(out)
@@ -218,3 +225,39 @@ func ComposeWithStats(p Provider, f Filters) ([]Issue, ComposeStats) {
218225
}
219226
return out, stats
220227
}
228+
229+
func filterShapedIssues(out []Issue, f Filters) ([]Issue, ComposeStats) {
230+
out = applyFilters(out, f) // severity + kind, against subject (grouped) or evidence (flat)
231+
var stats ComposeStats
232+
if f.Filter == nil {
233+
return out, stats
234+
}
235+
// CEL evaluated last so it sees the public shape. Eval errors count as
236+
// non-match ("missing field" semantics: zero hits + clean response, not
237+
// a 500); ComposeStats forwards the count + first sample so the caller
238+
// can distinguish "filter excluded everything" from "nothing matched."
239+
filtered := out[:0]
240+
var firstErr error
241+
errCount := 0
242+
for _, i := range out {
243+
ok, err := f.Filter.Match(issueToActivation(i))
244+
if err != nil {
245+
errCount++
246+
if firstErr == nil {
247+
firstErr = err
248+
}
249+
continue
250+
}
251+
if ok {
252+
filtered = append(filtered, i)
253+
}
254+
}
255+
if errCount > 0 {
256+
log.Printf("[issues] CEL filter eval errors: %d/%d rows; first=%v", errCount, len(out), firstErr)
257+
stats.FilterErrors = errCount
258+
if firstErr != nil {
259+
stats.FilterErrorSample = firstErr.Error()
260+
}
261+
}
262+
return filtered, stats
263+
}

‎internal/issues/external_merge_test.go‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
package issues
2+
3+
import (
4+
"testing"
5+
"time"
6+
)
7+
8+
func TestMergeExternalIssuesFiltersSortsAndCaps(t *testing.T) {
9+
now := time.Date(2026, 6, 28, 12, 0, 0, 0, time.UTC)
10+
base := []Issue{
11+
testIssueForMerge("Deployment", "apps", "web", SeverityCritical, now.Add(-30*time.Minute)),
12+
}
13+
extras := []Issue{
14+
testIssueForMerge("HelmRelease", "apps", "cart", SeverityCritical, now.Add(-5*time.Minute)),
15+
testIssueForMerge("Pod", "apps", "ignored", SeverityCritical, now.Add(-1*time.Minute)),
16+
}
17+
18+
got, stats := MergeExternalIssues(base, ComposeStats{TotalMatched: len(base)}, Filters{
19+
Kinds: []string{"Deployment", "HelmRelease"},
20+
Limit: 1,
21+
}, extras)
22+
23+
if stats.TotalMatched != 2 {
24+
t.Fatalf("TotalMatched = %d, want 2", stats.TotalMatched)
25+
}
26+
if len(got) != 1 {
27+
t.Fatalf("len(got) = %d, want cap 1", len(got))
28+
}
29+
if got[0].Kind != "HelmRelease" || got[0].Name != "cart" {
30+
t.Fatalf("first issue = %s/%s, want HelmRelease/cart", got[0].Kind, got[0].Name)
31+
}
32+
}
33+
34+
func testIssueForMerge(kind, namespace, name string, severity Severity, firstSeen time.Time) Issue {
35+
iss := Issue{
36+
Severity: severity,
37+
Source: SourceProblem,
38+
Kind: kind,
39+
Group: resolveGroup("", kind),
40+
Namespace: namespace,
41+
Name: name,
42+
Reason: "TestReason",
43+
FirstSeen: firstSeen,
44+
LastSeen: firstSeen,
45+
}
46+
classifyIssue(&iss)
47+
enrichIdentity(&iss)
48+
return iss
49+
}

‎internal/issues/helm.go‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
package issues
2+
3+
import (
4+
"fmt"
5+
"time"
6+
7+
"github.com/skyhook-io/radar/internal/helm"
8+
"github.com/skyhook-io/radar/pkg/helmhistory"
9+
)
10+
11+
const NativeHelmGroup = "helm.sh"
12+
13+
func NativeHelmReleaseIssues(releases []helm.HelmRelease, now time.Time) []Issue {
14+
out := make([]Issue, 0, len(releases))
15+
for _, rel := range releases {
16+
if rel.ManagedByFluxHelmRelease != "" || rel.LastOperation == nil {
17+
continue
18+
}
19+
op := rel.LastOperation
20+
severity, reason, ok := nativeHelmIssueReason(*op)
21+
if !ok {
22+
continue
23+
}
24+
firstSeen := op.Updated
25+
if firstSeen.IsZero() {
26+
firstSeen = rel.Updated
27+
}
28+
if firstSeen.IsZero() {
29+
firstSeen = now
30+
}
31+
storageNamespace := rel.StorageNamespace
32+
if storageNamespace == "" {
33+
storageNamespace = rel.Namespace
34+
}
35+
iss := Issue{
36+
Severity: severity,
37+
Source: SourceProblem,
38+
Kind: "HelmRelease",
39+
Group: NativeHelmGroup,
40+
Namespace: storageNamespace,
41+
Name: rel.Name,
42+
Reason: reason,
43+
Message: nativeHelmIssueMessage(rel, *op),
44+
Cause: nativeHelmIssueCause(*op),
45+
Action: nativeHelmIssueAction(*op),
46+
Stuck: true,
47+
FirstSeen: firstSeen,
48+
LastSeen: now,
49+
}
50+
classifyIssue(&iss)
51+
enrichIdentity(&iss)
52+
out = append(out, iss)
53+
}
54+
return out
55+
}
56+
57+
func nativeHelmIssueReason(op helm.HelmOperation) (Severity, string, bool) {
58+
switch op.Kind {
59+
case helmhistory.KindReleaseFailed:
60+
return SeverityCritical, "HelmReleaseFailed", true
61+
case helmhistory.KindUpgradeFailed:
62+
return SeverityCritical, "HelmUpgradeFailed", true
63+
case helmhistory.KindPending:
64+
if op.Status == helmhistory.StatusStuck {
65+
return SeverityWarning, "HelmReleasePending", true
66+
}
67+
}
68+
return "", "", false
69+
}
70+
71+
func nativeHelmIssueMessage(rel helm.HelmRelease, op helm.HelmOperation) string {
72+
if op.Message != "" {
73+
return op.Message
74+
}
75+
switch op.Kind {
76+
case helmhistory.KindPending:
77+
return fmt.Sprintf("Helm release %q is stuck in %s at rev %d.", rel.Name, op.PendingStatus, op.Revision)
78+
default:
79+
return fmt.Sprintf("Helm release %q failed at rev %d.", rel.Name, op.Revision)
80+
}
81+
}
82+
83+
func nativeHelmIssueCause(op helm.HelmOperation) string {
84+
if op.Kind == helmhistory.KindPending {
85+
return "A Helm install, upgrade, or rollback has remained pending past the stuck-operation threshold."
86+
}
87+
return "The latest native Helm release revision is failed."
88+
}
89+
90+
func nativeHelmIssueAction(op helm.HelmOperation) string {
91+
if op.Kind == helmhistory.KindPending {
92+
return "Open the Helm release details, check whether the operation is still running, then inspect hooks, Jobs, Pods, events, logs, and owned resources."
93+
}
94+
return "Open the Helm release history and hook diagnostics, then inspect the failed revision, values, hooks, and owned resources."
95+
}

0 commit comments

Comments
 (0)