Skip to content

Commit 6e36385

Browse files
committed
feat(api): expose HelmRelease failure state on Application status
Tenants usually read Applications but not the HelmReleases behind them, and only Ready and Released were copied over. A client could therefore see that a release was failing but not how often it had failed, nor whether helm-controller had stopped retrying it. Copy the Stalled condition, which helm-controller sets once it stops reconciling a release (remediation retries exhausted, rollback target missing, invalid CEL, access denied), and surface the install and upgrade failure counters with the last attempted release action. Stalled is dropped when it was observed for an older generation than the HelmRelease's current one: the Application carries no generation, so after a spec update a client would otherwise read the previous spec's failure as terminal for the new values. With the RetryOnFailure policy the platform generates today, a failed release is retried rather than stalled, so the counters are the signal. That does not hold for every state: a release missing from storage while an install failure count is retained still stalls, and a release on a remediation policy stalls once its retries run out. Assisted-by: LLM Signed-off-by: Alexey Artamonov <[email protected]>
1 parent e1e925a commit 6e36385

5 files changed

Lines changed: 211 additions & 2 deletions

File tree

‎pkg/apis/apps/v1alpha1/types.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,23 @@ type ApplicationStatus struct {
5555
// ExternalIPsCount holds the number of LoadBalancer services with assigned external IPs for Tenant applications.
5656
// +optional
5757
ExternalIPsCount int32 `json:"externalIPsCount,omitempty"`
58+
// InstallFailures is the install failure count of the underlying HelmRelease.
59+
// It counts only attempts that recorded a Helm release, so a failure before
60+
// Helm records one, such as a chart that fails to render, shows only as
61+
// Ready=False; a failure to apply the rendered resources is counted. It
62+
// resets when the desired state (generation, chart version, values) changes,
63+
// and on success only under the RetryOnFailure strategy, so a Ready release
64+
// may carry it.
65+
// +optional
66+
InstallFailures int64 `json:"installFailures,omitempty"`
67+
// UpgradeFailures is the upgrade failure count of the underlying HelmRelease,
68+
// with the same counting and reset rules as InstallFailures.
69+
// +optional
70+
UpgradeFailures int64 `json:"upgradeFailures,omitempty"`
71+
// LastAttemptedReleaseAction is the last release action (install or upgrade)
72+
// the underlying HelmRelease attempted.
73+
// +optional
74+
LastAttemptedReleaseAction string `json:"lastAttemptedReleaseAction,omitempty"`
5875
}
5976

6077
// SchedulingClass returns the scheduling class requested by this Application.

‎pkg/generated/applyconfiguration/apps/v1alpha1/applicationstatus.go‎

Lines changed: 38 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/generated/openapi/zz_generated.openapi.go‎

Lines changed: 21 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/registry/apps/application/rest.go‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1573,13 +1573,23 @@ func (r *REST) convertHelmReleaseToApplication(ctx context.Context, hr *helmv2.H
15731573
},
15741574
Spec: filteredSpec,
15751575
Status: appsv1alpha1.ApplicationStatus{
1576-
Version: hr.Status.LastAttemptedRevision,
1576+
Version: hr.Status.LastAttemptedRevision,
1577+
InstallFailures: hr.Status.InstallFailures,
1578+
UpgradeFailures: hr.Status.UpgradeFailures,
1579+
LastAttemptedReleaseAction: string(hr.Status.LastAttemptedReleaseAction),
15771580
},
15781581
}
15791582

15801583
var conditions []metav1.Condition
15811584
for _, hrCondition := range hr.GetConditions() {
1582-
if hrCondition.Type == "Ready" || hrCondition.Type == "Released" {
1585+
// The Application carries no generation, so a client cannot tell a Stalled
1586+
// left over from the previous spec from one about the current spec. Compare
1587+
// the condition's own ObservedGeneration, not Status.ObservedGeneration: the
1588+
// question is which spec this condition describes.
1589+
if hrCondition.Type == "Stalled" && hrCondition.ObservedGeneration < hr.Generation {
1590+
continue
1591+
}
1592+
if hrCondition.Type == "Ready" || hrCondition.Type == "Released" || hrCondition.Type == "Stalled" {
15831593
conditions = append(conditions, metav1.Condition{
15841594
LastTransitionTime: hrCondition.LastTransitionTime,
15851595
Reason: hrCondition.Reason,
Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
package application
2+
3+
import (
4+
"context"
5+
"testing"
6+
7+
helmv2 "github.com/fluxcd/helm-controller/api/v2"
8+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
9+
)
10+
11+
func TestConvertReleaseFailure_StalledAndCountersExposed(t *testing.T) {
12+
r := newTestRESTWithSchemes()
13+
hr := makeHelmRelease("postgresql-mydb", "default")
14+
hr.Generation = 2
15+
hr.Status.Conditions = []metav1.Condition{
16+
{Type: "Stalled", Status: metav1.ConditionTrue, Reason: "RetriesExceeded", Message: "Failed to install after 1 attempt(s)", ObservedGeneration: 2},
17+
{Type: "Ready", Status: metav1.ConditionFalse, Reason: "InstallFailed", Message: "install failed", ObservedGeneration: 2},
18+
{Type: "Released", Status: metav1.ConditionFalse, Reason: "InstallFailed", Message: "install failed", ObservedGeneration: 2},
19+
}
20+
hr.Status.Failures = 7
21+
hr.Status.InstallFailures = 1
22+
hr.Status.UpgradeFailures = 3
23+
hr.Status.LastAttemptedReleaseAction = helmv2.ReleaseActionInstall
24+
25+
app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil)
26+
if err != nil {
27+
t.Fatalf("unexpected error: %v", err)
28+
}
29+
30+
sc := findCondition(app.GetConditions(), "Stalled")
31+
if sc == nil {
32+
t.Fatal("expected Stalled condition to be copied from the HelmRelease")
33+
}
34+
if sc.Status != metav1.ConditionTrue || sc.Reason != "RetriesExceeded" {
35+
t.Errorf("expected Stalled=True/RetriesExceeded, got %s/%s", sc.Status, sc.Reason)
36+
}
37+
if app.Status.InstallFailures != 1 {
38+
t.Errorf("expected installFailures=1, got %d", app.Status.InstallFailures)
39+
}
40+
if app.Status.UpgradeFailures != 3 {
41+
t.Errorf("expected upgradeFailures=3, got %d", app.Status.UpgradeFailures)
42+
}
43+
if app.Status.LastAttemptedReleaseAction != "install" {
44+
t.Errorf("expected lastAttemptedReleaseAction=install, got %q", app.Status.LastAttemptedReleaseAction)
45+
}
46+
}
47+
48+
// After a spec update the HelmRelease keeps the Stalled condition of the
49+
// previous generation until helm-controller reconciles it; reported as is, it
50+
// would read as a terminal failure of the new values.
51+
func TestConvertReleaseFailure_StalledFromOlderGenerationOmitted(t *testing.T) {
52+
r := newTestRESTWithSchemes()
53+
hr := makeHelmRelease("postgresql-mydb", "default")
54+
hr.Generation = 2
55+
hr.Status.Conditions = []metav1.Condition{
56+
{Type: "Stalled", Status: metav1.ConditionTrue, Reason: "RetriesExceeded", Message: "Failed to install after 1 attempt(s)", ObservedGeneration: 1},
57+
{Type: "Ready", Status: metav1.ConditionFalse, Reason: "InstallFailed", Message: "install failed", ObservedGeneration: 1},
58+
}
59+
hr.Status.InstallFailures = 1
60+
hr.Status.LastAttemptedReleaseAction = helmv2.ReleaseActionInstall
61+
62+
app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil)
63+
if err != nil {
64+
t.Fatalf("unexpected error: %v", err)
65+
}
66+
67+
if findCondition(app.GetConditions(), "Stalled") != nil {
68+
t.Error("expected Stalled observed at an older generation not to be copied")
69+
}
70+
if findCondition(app.GetConditions(), "Ready") == nil {
71+
t.Error("expected Ready to still be copied")
72+
}
73+
}
74+
75+
// helm-controller clears the counters on success only under the RetryOnFailure
76+
// strategy, so a Ready release can still carry them; they must not be hidden.
77+
func TestConvertReleaseFailure_CountersKeptOnReadyRelease(t *testing.T) {
78+
r := newTestRESTWithSchemes()
79+
hr := makeHelmRelease("postgresql-mydb", "default")
80+
hr.Status.Conditions = []metav1.Condition{
81+
{Type: "Ready", Status: metav1.ConditionTrue, Reason: "UpgradeSucceeded", Message: "ok"},
82+
{Type: "Released", Status: metav1.ConditionTrue, Reason: "UpgradeSucceeded", Message: "ok"},
83+
}
84+
hr.Status.InstallFailures = 2
85+
hr.Status.UpgradeFailures = 1
86+
hr.Status.LastAttemptedReleaseAction = helmv2.ReleaseActionUpgrade
87+
88+
app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil)
89+
if err != nil {
90+
t.Fatalf("unexpected error: %v", err)
91+
}
92+
93+
if findCondition(app.GetConditions(), "Stalled") != nil {
94+
t.Error("expected no Stalled condition on a release that does not carry one")
95+
}
96+
if app.Status.InstallFailures != 2 || app.Status.UpgradeFailures != 1 {
97+
t.Errorf("expected install=2 upgrade=1, got install=%d upgrade=%d", app.Status.InstallFailures, app.Status.UpgradeFailures)
98+
}
99+
if app.Status.LastAttemptedReleaseAction != "upgrade" {
100+
t.Errorf("expected lastAttemptedReleaseAction=upgrade, got %q", app.Status.LastAttemptedReleaseAction)
101+
}
102+
}
103+
104+
func TestConvertReleaseFailure_ReconcilingNotCopied(t *testing.T) {
105+
r := newTestRESTWithSchemes()
106+
hr := makeHelmRelease("postgresql-mydb", "default")
107+
hr.Status.Conditions = []metav1.Condition{
108+
{Type: "Reconciling", Status: metav1.ConditionTrue, Reason: "Progressing", Message: "Running 'install' action"},
109+
{Type: "Ready", Status: metav1.ConditionUnknown, Reason: "Progressing", Message: "Running 'install' action"},
110+
}
111+
112+
app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil)
113+
if err != nil {
114+
t.Fatalf("unexpected error: %v", err)
115+
}
116+
117+
if findCondition(app.GetConditions(), "Reconciling") != nil {
118+
t.Error("expected Reconciling not to be copied")
119+
}
120+
if findCondition(app.GetConditions(), "Ready") == nil {
121+
t.Error("expected Ready to be copied")
122+
}
123+
}

0 commit comments

Comments
 (0)