Skip to content

Commit 5df219b

Browse files
fix(api): keep a suspended HelmRelease suspended across an app update
Update rebuilds the HelmRelease from the Application and writes it back whole, and the Application carries no suspension, so every edit made through the API resumed a release that an operator or a controller had suspended. The CNPG restore driver depends on that suspension. It suspends the target release, patches the Postgres app through this API, purges the Cluster, and resumes only once the purge is done, so the chart's next render lands bootstrap.recovery on an empty namespace. Resumed by the patch instead, helm-controller upgrades straight away: the chart can render the recovery Cluster while the purge is still running, the driver then deletes it as the Cluster being replaced, and nothing renders it again because the values have not changed since. The RestoreJob sits in Running until its deadline. On main this has been masked by timing rather than prevented. The same write also dropped the Flux finalizer, so helm-controller's first pass after it only put the finalizer back and requeued, which held the upgrade back by at least its 750ms minimum retry delay. A branch that keeps the finalizer removes that delay, and the backup round-trip failed on both of its runs there. The suspension is re-read on a conflict retry as well, since the write that caused the conflict may be the suspend itself. Assisted-by: LLM Signed-off-by: Myasnikov Daniil <[email protected]>
1 parent eec069b commit 5df219b

3 files changed

Lines changed: 219 additions & 10 deletions

File tree

‎pkg/registry/apps/application/rest.go‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -561,8 +561,8 @@ func (r *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObje
561561
}
562562

563563
// Fetch the live HelmRelease: it backs the ResourceVersion when the
564-
// converted object carries none, and runtime-managed labels are carried
565-
// over from it below.
564+
// converted object carries none, and runtime-managed labels and the
565+
// suspension are carried over from it below.
566566
cur := &helmv2.HelmRelease{}
567567
if err := r.c.Get(ctx, client.ObjectKey{Namespace: helmRelease.Namespace, Name: helmRelease.Name}, cur, &client.GetOptions{Raw: &metav1.GetOptions{}}); err != nil {
568568
return nil, false, registry.WrapPreservingStatus("failed to fetch current HelmRelease", err, r.gvr.GroupResource(), name)
@@ -593,6 +593,17 @@ func (r *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObje
593593
helmRelease.Labels[fluxshard.ShardKeyLabel] = shard
594594
}
595595

596+
// Suspension is not part of the Application, so the rebuilt object always
597+
// says suspend=false, and sending it as is resumes a release that an
598+
// operator or a controller suspended. The CNPG restore driver relies on the
599+
// suspension holding across its own patch of the Postgres app through this
600+
// API: it purges the Cluster while the release is suspended and resumes it
601+
// only once the purge is done, so the chart's next render lands
602+
// bootstrap.recovery on an empty namespace. Resumed early, the release
603+
// renders while the purge is still running, and the purge then deletes
604+
// what it rendered.
605+
helmRelease.Spec.Suspend = cur.Spec.Suspend
606+
596607
klog.V(6).Infof("Updating HelmRelease %s in namespace %s", helmRelease.Name, helmRelease.Namespace)
597608

598609
// Update the HelmRelease in Kubernetes.
@@ -601,10 +612,11 @@ func (r *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObje
601612
// HelmRelease's status, which shares the object's resourceVersion. When a
602613
// caller updates an app CR while a prior reconcile is still in flight, the
603614
// resourceVersion read above goes stale and the Update is rejected with a
604-
// 409 Conflict. The HelmRelease spec is fully derived from the Application
605-
// the caller just applied, so a stale-resourceVersion conflict is never a
606-
// real spec conflict here: refresh the resourceVersion from the live object
607-
// and retry.
615+
// 409 Conflict. Apart from suspend, the HelmRelease spec is derived from the
616+
// Application the caller just applied, so a stale-resourceVersion conflict
617+
// is never a real spec conflict here: refresh the resourceVersion from the
618+
// live object and retry. Suspend is refreshed with it, because the write
619+
// that caused the conflict may be the one that suspended the release.
608620
err = retry.RetryOnConflict(retry.DefaultBackoff, func() error {
609621
updateErr := r.c.Update(ctx, helmRelease, registry.ClientUpdateOptions(options))
610622
if apierrors.IsConflict(updateErr) {
@@ -613,6 +625,7 @@ func (r *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObje
613625
return getErr
614626
}
615627
helmRelease.SetResourceVersion(cur.GetResourceVersion())
628+
helmRelease.Spec.Suspend = cur.Spec.Suspend
616629
}
617630
return updateErr
618631
})

‎pkg/registry/apps/application/rest_conflict_test.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,11 @@ import (
4141
// Flux's helm-controller continuously rewrites a HelmRelease's status, which
4242
// shares the object's resourceVersion. When a caller updates an Application CR
4343
// while a prior reconcile is still in flight, the resourceVersion read by
44-
// Update goes stale and the write is rejected with a 409 Conflict. The
45-
// HelmRelease spec is fully derived from the Application the caller just
46-
// applied, so a stale-resourceVersion conflict is never a real spec conflict:
47-
// Update must refresh the resourceVersion and retry rather than surface the 409.
44+
// Update goes stale and the write is rejected with a 409 Conflict. Apart from
45+
// suspend, which TestUpdate_RefreshesSuspendOnConflictRetry covers, the
46+
// HelmRelease spec is derived from the Application the caller just applied, so
47+
// a stale-resourceVersion conflict is never a real spec conflict: Update must
48+
// refresh the resourceVersion and retry rather than surface the 409.
4849
//
4950
// The fake client returns one Conflict on the first HelmRelease Update, then
5051
// lets the second through. A regression that drops the retry makes the first
Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
/*
2+
Copyright 2026 The Cozystack Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package application
18+
19+
import (
20+
"context"
21+
"errors"
22+
"testing"
23+
24+
helmv2 "github.com/fluxcd/helm-controller/api/v2"
25+
apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
26+
apierrors "k8s.io/apimachinery/pkg/api/errors"
27+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
28+
"k8s.io/apimachinery/pkg/runtime"
29+
"k8s.io/apimachinery/pkg/runtime/schema"
30+
"k8s.io/apiserver/pkg/endpoints/request"
31+
"sigs.k8s.io/controller-runtime/pkg/client"
32+
"sigs.k8s.io/controller-runtime/pkg/client/fake"
33+
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
34+
35+
appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1"
36+
"github.com/cozystack/cozystack/pkg/config"
37+
)
38+
39+
func newSuspendTestREST(t *testing.T, existing *helmv2.HelmRelease, funcs *interceptor.Funcs) (*REST, client.Client) {
40+
t.Helper()
41+
scheme := runtime.NewScheme()
42+
if err := helmv2.AddToScheme(scheme); err != nil {
43+
t.Fatalf("register helmv2 scheme: %v", err)
44+
}
45+
resourceCfg := &config.ResourceConfig{
46+
Resources: []config.Resource{
47+
{Application: config.ApplicationConfig{Kind: "Postgres"}},
48+
},
49+
}
50+
if err := appsv1alpha1.RegisterDynamicTypes(scheme, resourceCfg); err != nil {
51+
t.Fatalf("register dynamic types: %v", err)
52+
}
53+
builder := fake.NewClientBuilder().WithScheme(scheme).WithObjects(existing)
54+
if funcs != nil {
55+
builder = builder.WithInterceptorFuncs(*funcs)
56+
}
57+
c := builder.Build()
58+
return &REST{
59+
c: c,
60+
gvr: schema.GroupVersionResource{
61+
Group: appsv1alpha1.GroupName,
62+
Version: "v1alpha1",
63+
Resource: "postgreses",
64+
},
65+
gvk: schema.GroupVersionKind{
66+
Group: appsv1alpha1.GroupName,
67+
Version: "v1alpha1",
68+
Kind: "Postgres",
69+
},
70+
kindName: "Postgres",
71+
releaseConfig: config.ReleaseConfig{Prefix: "postgres-"},
72+
}, c
73+
}
74+
75+
func suspendTestHelmRelease(suspend bool) *helmv2.HelmRelease {
76+
return &helmv2.HelmRelease{
77+
ObjectMeta: metav1.ObjectMeta{
78+
Name: "postgres-pg",
79+
Namespace: "tenant-foo",
80+
Labels: map[string]string{
81+
ApplicationKindLabel: "Postgres",
82+
ApplicationGroupLabel: appsv1alpha1.GroupName,
83+
ApplicationNameLabel: "pg",
84+
},
85+
},
86+
Spec: helmv2.HelmReleaseSpec{
87+
Suspend: suspend,
88+
// Differs from the Application below, so a carry-over that took
89+
// more than suspension off the live object shows up as stale values.
90+
Values: &apiextv1.JSON{Raw: []byte(`{"replicas":5}`)},
91+
},
92+
}
93+
}
94+
95+
func suspendTestApplication() *appsv1alpha1.Application {
96+
return &appsv1alpha1.Application{
97+
TypeMeta: metav1.TypeMeta{
98+
APIVersion: "apps.cozystack.io/v1alpha1",
99+
Kind: "Postgres",
100+
},
101+
ObjectMeta: metav1.ObjectMeta{
102+
Name: "pg",
103+
Namespace: "tenant-foo",
104+
},
105+
Spec: &apiextv1.JSON{Raw: []byte(`{"replicas":2}`)},
106+
}
107+
}
108+
109+
// TestUpdate_PreservesSuspend pins that an Application edit leaves the
110+
// release's suspension exactly as it found it. The rebuilt HelmRelease carries
111+
// no suspension of its own, so a regression resumes a release someone
112+
// suspended: the CNPG restore driver suspends the target release, patches the
113+
// Postgres app through this API and purges the Cluster, and a release resumed
114+
// by that patch renders the recovery Cluster before the purge, which then
115+
// deletes it. The unsuspended case guards the other direction, a fix that
116+
// suspends every release it touches.
117+
func TestUpdate_PreservesSuspend(t *testing.T) {
118+
for _, suspend := range []bool{true, false} {
119+
r, c := newSuspendTestREST(t, suspendTestHelmRelease(suspend), nil)
120+
121+
ctx := request.WithNamespace(context.Background(), "tenant-foo")
122+
if _, _, err := r.Update(ctx, "pg", newDefaultUpdatedObjectInfo(suspendTestApplication()),
123+
nil, nil, false, &metav1.UpdateOptions{}); err != nil {
124+
t.Fatalf("suspend=%v: Update failed: %v", suspend, err)
125+
}
126+
127+
got := &helmv2.HelmRelease{}
128+
if err := c.Get(ctx, client.ObjectKey{Namespace: "tenant-foo", Name: "postgres-pg"}, got); err != nil {
129+
t.Fatalf("suspend=%v: fetch updated HelmRelease: %v", suspend, err)
130+
}
131+
if got.Spec.Suspend != suspend {
132+
t.Errorf("an Application update changed spec.suspend from %v to %v", suspend, got.Spec.Suspend)
133+
}
134+
if got.Spec.Values == nil || string(got.Spec.Values.Raw) != `{"replicas":2}` {
135+
var raw string
136+
if got.Spec.Values != nil {
137+
raw = string(got.Spec.Values.Raw)
138+
}
139+
t.Errorf("suspend=%v: values must come from the Application, not the live object, got %s", suspend, raw)
140+
}
141+
}
142+
}
143+
144+
// TestUpdate_RefreshesSuspendOnConflictRetry covers the window between Update's
145+
// read of the live HelmRelease and its write. A 409 there means something wrote
146+
// the release in between, and that write can be the one that suspended it: the
147+
// restore driver suspends immediately before it patches the app. Re-sending the
148+
// suspension from the stale read would resume the release on the retry.
149+
//
150+
// The interceptor suspends the stored release and then fails the first write
151+
// with a Conflict, which is the order a real racing suspend produces.
152+
func TestUpdate_RefreshesSuspendOnConflictRetry(t *testing.T) {
153+
var updateCalls int
154+
funcs := &interceptor.Funcs{
155+
Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error {
156+
if _, ok := obj.(*helmv2.HelmRelease); ok {
157+
updateCalls++
158+
if updateCalls == 1 {
159+
live := &helmv2.HelmRelease{}
160+
if err := c.Get(ctx, client.ObjectKeyFromObject(obj), live); err != nil {
161+
return err
162+
}
163+
live.Spec.Suspend = true
164+
if err := c.Update(ctx, live); err != nil {
165+
return err
166+
}
167+
return apierrors.NewConflict(
168+
schema.GroupResource{Group: "helm.toolkit.fluxcd.io", Resource: "helmreleases"},
169+
obj.GetName(),
170+
errors.New("simulated suspend between read and write"),
171+
)
172+
}
173+
}
174+
return c.Update(ctx, obj, opts...)
175+
},
176+
}
177+
r, c := newSuspendTestREST(t, suspendTestHelmRelease(false), funcs)
178+
179+
ctx := request.WithNamespace(context.Background(), "tenant-foo")
180+
if _, _, err := r.Update(ctx, "pg", newDefaultUpdatedObjectInfo(suspendTestApplication()),
181+
nil, nil, false, &metav1.UpdateOptions{}); err != nil {
182+
t.Fatalf("Update failed: %v", err)
183+
}
184+
if updateCalls != 2 {
185+
t.Fatalf("expected one conflicting write and one retry, got %d HelmRelease writes", updateCalls)
186+
}
187+
188+
got := &helmv2.HelmRelease{}
189+
if err := c.Get(ctx, client.ObjectKey{Namespace: "tenant-foo", Name: "postgres-pg"}, got); err != nil {
190+
t.Fatalf("fetch updated HelmRelease: %v", err)
191+
}
192+
if !got.Spec.Suspend {
193+
t.Errorf("the retry resumed a release that was suspended between Update's read and its write")
194+
}
195+
}

0 commit comments

Comments
 (0)