99
1010 helmv2 "github.com/fluxcd/helm-controller/api/v2"
1111 corev1 "k8s.io/api/core/v1"
12+ apierrors "k8s.io/apimachinery/pkg/api/errors"
1213 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1314 "k8s.io/apimachinery/pkg/types"
1415 ctrl "sigs.k8s.io/controller-runtime"
@@ -19,6 +20,11 @@ import (
1920
2021const (
2122 annotationLastProcessedVersion = "flux-plunger.cozystack.io/last-processed-version"
23+ // annotationSuspendedByPlunger marks a suspension that flux-plunger itself
24+ // set, so recovery can be told apart from a suspension an operator or another
25+ // controller owns. It is written atomically with spec.suspend=true and
26+ // removed atomically with spec.suspend=false.
27+ annotationSuspendedByPlunger = "flux-plunger.cozystack.io/suspended"
2228 errorMessageNoDeployedReleases = "has no deployed releases"
2329 fieldManager = "flux-client-side-apply"
2430)
@@ -41,50 +47,69 @@ func (r *FluxPlunger) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Res
4147 return ctrl.Result {}, client .IgnoreNotFound (err )
4248 }
4349
44- // Check if HelmRelease is suspended
45- if hr .Spec .Suspend {
46- logger .Info ("HelmRelease is suspended, checking if we need to unsuspend" )
47-
48- // Get the list of Helm release secrets
49- secrets , err := r .listHelmReleaseSecrets (ctx , hr .Namespace , hr .Name )
50- if err != nil {
51- logger .Error (err , "Failed to list Helm release secrets" )
52- return ctrl.Result {}, err
50+ // A HelmRelease that is being deleted must never stay suspended. helm-controller
51+ // v1.5.x skips the Helm uninstall and drops its finalizer for a suspended
52+ // HelmRelease (https://github.com/fluxcd/helm-controller/blob/v1.5.0/internal/controller/helmrelease_controller.go#L447-L475),
53+ // which orphans every resource the release owns. If flux-plunger owns the
54+ // suspension, release it so the uninstall proceeds; in every case do not begin
55+ // recovery on a terminating object.
56+ if ! hr .DeletionTimestamp .IsZero () {
57+ if hr .Spec .Suspend && ownsSuspension (hr ) {
58+ logger .Info ("HelmRelease is being deleted while suspended by flux-plunger, clearing suspension so uninstall can proceed" )
59+ if err := r .unsuspendHelmRelease (ctx , hr ); err != nil {
60+ // Requeue with backoff: leaving the suspension is exactly what
61+ // orphans the release, so retry rather than wait for another event.
62+ logger .Info ("Could not clear suspension on deleted HelmRelease, requeuing" , "error" , err .Error ())
63+ return ctrl.Result {}, err
64+ }
5365 }
66+ return ctrl.Result {}, nil
67+ }
5468
55- // If no secrets, treat latest version as 0
56- latestVersion := 0
57- if len (secrets ) > 0 {
58- latestSecret := getLatestSecret (secrets )
59- latestVersion = extractVersionNumber (latestSecret .Name )
60- } else {
61- logger .Info ("No Helm release secrets found while suspended, treating as version 0" )
69+ // Check if HelmRelease is suspended
70+ if hr .Spec .Suspend {
71+ // Ownership is decided solely by the marker flux-plunger writes atomically
72+ // with the suspension. A suspension we own but see in a fresh reconcile means
73+ // recovery crashed mid-flight rather than another instance running now: the
74+ // Deployment pins replicas: 1 (packages/system/flux-plunger, which is what
75+ // makes this annotation-as-state safe — see the comment there), and
76+ // controller-runtime serialises reconciles per key within that one process,
77+ // so there is no concurrent owner to race. Releasing it un-sticks the
78+ // release and lets error-recovery start over. A suspension without the marker
79+ // belongs to an operator or another controller and must be left untouched —
80+ // the processed-version annotation is deliberately NOT consulted here, because
81+ // it is never cleared and would misread a much later, unrelated operator
82+ // suspension as ours once the release's latest revision stops advancing.
83+ if ! ownsSuspension (hr ) {
84+ logger .Info ("HelmRelease is suspended by external process, skipping" )
85+ return ctrl.Result {}, nil
6286 }
6387
64- // Check if version is previous to just processed (latestVersion+1 == processedVersion)
65- // This is the ONLY condition when we unsuspend
66- shouldUnsuspend := false
67- if hr .Annotations != nil {
68- if processedVersionStr , exists := hr .Annotations [annotationLastProcessedVersion ]; exists {
69- processedVersion , err := strconv .Atoi (processedVersionStr )
70- if err == nil && latestVersion + 1 == processedVersion {
71- shouldUnsuspend = true
72- }
73- }
88+ logger .Info ("Removing suspension owned by flux-plunger" )
89+ if err := r .unsuspendHelmRelease (ctx , hr ); err != nil {
90+ // Requeue with backoff so a stuck owned suspension is not left until the
91+ // next informer resync, symmetric with the deletion and recovery paths.
92+ logger .Info ("Could not unsuspend HelmRelease, requeuing" , "error" , err .Error ())
93+ return ctrl.Result {}, err
7494 }
95+ return ctrl.Result {}, nil
96+ }
7597
76- if shouldUnsuspend {
77- // Unsuspend the HelmRelease
78- logger .Info ("Secret was already deleted in previous run, removing suspend" , "latest" , latestVersion , "processed" , latestVersion + 1 )
79- if err := r .unsuspendHelmRelease (ctx , hr ); err != nil {
80- logger .Info ("Could not unsuspend HelmRelease, will retry on next reconcile" , "error" , err .Error ())
81- return ctrl.Result {}, nil
82- }
83- return ctrl.Result {}, nil
98+ // A suspension flux-plunger set is marked with its ownership annotation and
99+ // cleared atomically with it. But an external actor — an operator, `cozyhr
100+ // resume`, the backup controller's own unsuspend — can lift the suspension by
101+ // writing spec.suspend=false WITHOUT removing the marker, since nothing else
102+ // removes it. A marker left on a non-suspended release would make a LATER,
103+ // unrelated operator suspension look plunger-owned and get cleared out from
104+ // under them: the one boundary this controller must never cross. Reap the stale
105+ // marker the moment we observe it on a release that is no longer suspended,
106+ // before it can be inherited. The watch predicate wakes us for this transition.
107+ if ownsSuspension (hr ) {
108+ logger .Info ("Reaping stale flux-plunger ownership marker on a non-suspended HelmRelease" )
109+ if err := r .clearOwnershipMarker (ctx , hr ); err != nil {
110+ logger .Info ("Could not reap stale ownership marker, requeuing" , "error" , err .Error ())
111+ return ctrl.Result {}, err
84112 }
85-
86- // If not previous to processed, skip all actions
87- logger .Info ("HelmRelease is suspended by external process, skipping" , "latest" , latestVersion )
88113 return ctrl.Result {}, nil
89114 }
90115
@@ -132,10 +157,76 @@ func (r *FluxPlunger) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Res
132157
133158 // Suspend the HelmRelease
134159 logger .Info ("Suspending HelmRelease" )
135- if err := r .suspendHelmRelease (ctx , hr ); err != nil {
136- // Optimistic lock conflicts are normal - FluxCD also updates HelmRelease
137- // Don't return error, just log and let controller-runtime requeue on next update
138- logger .Info ("Could not suspend HelmRelease, will retry on next reconcile" , "error" , err .Error ())
160+ acquired , err := r .suspendHelmRelease (ctx , hr )
161+ if err != nil {
162+ // An optimistic-lock conflict is expected here — helm-controller and Flux
163+ // also patch the HelmRelease — and the conflicting write is itself an update
164+ // event that requeues us, so a conflict needs no error return. Any OTHER
165+ // error (a transient API failure) produces no such follow-up event, so
166+ // return it to requeue with backoff, symmetric with every step below;
167+ // swallowing it would strand a stuck release until the next informer resync.
168+ if apierrors .IsConflict (err ) {
169+ logger .Info ("Suspend conflicted with a concurrent write, will retry on the resulting update" , "error" , err .Error ())
170+ return ctrl.Result {}, nil
171+ }
172+ logger .Info ("Could not suspend HelmRelease, requeuing" , "error" , err .Error ())
173+ return ctrl.Result {}, err
174+ }
175+ if ! acquired {
176+ // The release was suspended by someone else, or started terminating, between
177+ // our read and our write, so flux-plunger does not own it. Do not delete its
178+ // storage Secret; let the next reconcile handle it (the suspended branch, or
179+ // the deletion branch which lets helm-controller uninstall).
180+ logger .Info ("HelmRelease was not acquired for recovery, deferring to next reconcile" )
181+ return ctrl.Result {}, nil
182+ }
183+
184+ // From here flux-plunger owns the suspension (acquired above). Guarantee it is
185+ // released before returning from any path below: a step that fails must never
186+ // leave the HelmRelease suspended, because a delete arriving on a suspended
187+ // release makes helm-controller skip the uninstall and orphan the release's
188+ // resources. The unsuspend is idempotent, so the happy path running it
189+ // explicitly and this deferred fallback are safe together.
190+ completed := false
191+ defer func () {
192+ if completed {
193+ return
194+ }
195+ if err := r .unsuspendHelmRelease (ctx , hr ); err != nil {
196+ logger .Info ("Could not clear suspension after failed recovery step, will retry on next reconcile" , "error" , err .Error ())
197+ }
198+ }()
199+
200+ // Record the version we are about to reclaim BEFORE deleting its storage
201+ // Secret. The "already processed" guard above keys on this annotation, so
202+ // writing it first means a failed write can never authorise deleting the NEXT
203+ // revision on the following requeue: on failure we return here, the deferred
204+ // unsuspend clears the suspension, and the retry re-deletes the SAME revision
205+ // (idempotent) instead of walking down the release history one Secret per
206+ // requeue until none is left — which would itself orphan the release. Requeue
207+ // with backoff on failure, symmetric with every other step.
208+ logger .Info ("Updating annotation with processed version" , "version" , latestVersion )
209+ if err := r .updateProcessedVersionAnnotation (ctx , hr , latestVersion ); err != nil {
210+ logger .Info ("Could not update annotation, requeuing" , "error" , err .Error ())
211+ return ctrl.Result {}, err
212+ }
213+
214+ // A delete can land between acquiring the suspension and here. Deleting the
215+ // storage Secret of a terminating release removes the only revision
216+ // helm-controller could uninstall; it then treats the release as already gone,
217+ // drops its finalizer, and orphans the resources — the exact failure this
218+ // recovery exists to prevent, reached by a narrower race than the suspended one.
219+ // The top-of-Reconcile check and suspendHelmRelease's re-fetch only see a delete
220+ // visible earlier, so re-check immediately before the delete. On termination,
221+ // leave the Secret intact and return: the deferred unsuspend clears our
222+ // suspension so helm-controller runs the uninstall against the intact revision.
223+ terminating , err := r .releaseIsTerminating (ctx , hr )
224+ if err != nil {
225+ logger .Info ("Could not re-check deletion before deleting the storage Secret, requeuing" , "error" , err .Error ())
226+ return ctrl.Result {}, err
227+ }
228+ if terminating {
229+ logger .Info ("HelmRelease started terminating during recovery, leaving its storage Secret for helm-controller to uninstall" )
139230 return ctrl.Result {}, nil
140231 }
141232
@@ -146,24 +237,30 @@ func (r *FluxPlunger) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Res
146237 return ctrl.Result {}, err
147238 }
148239
149- // Update annotation with processed version
150- logger .Info ("Updating annotation with processed version" , "version" , latestVersion )
151- if err := r .updateProcessedVersionAnnotation (ctx , hr , latestVersion ); err != nil {
152- logger .Info ("Could not update annotation, will retry on next reconcile" , "error" , err .Error ())
153- return ctrl.Result {}, nil
154- }
155-
156- // Unsuspend the HelmRelease
240+ // Unsuspend the HelmRelease. Force a backoff requeue on failure, symmetric with
241+ // the deletion path: a persistently failing unsuspend (e.g. an admission webhook
242+ // rejecting the patch) would otherwise leave the release suspended-by-plunger
243+ // until the informer's periodic resync, far longer than necessary.
157244 logger .Info ("Unsuspending HelmRelease" )
158245 if err := r .unsuspendHelmRelease (ctx , hr ); err != nil {
159- logger .Info ("Could not unsuspend HelmRelease, will retry on next reconcile" , "error" , err .Error ())
160- return ctrl.Result {}, nil
246+ logger .Info ("Could not unsuspend HelmRelease, requeuing" , "error" , err .Error ())
247+ // The returned error requeues and retries; skip the redundant deferred attempt.
248+ completed = true
249+ return ctrl.Result {}, err
161250 }
162251
252+ completed = true
163253 logger .Info ("Successfully processed HelmRelease" , "version" , latestVersion )
164254 return ctrl.Result {}, nil
165255}
166256
257+ // ownsSuspension reports whether the current suspension was set by flux-plunger,
258+ // as opposed to an operator or another controller. Recovery and delete handling
259+ // must only clear a suspension flux-plunger owns.
260+ func ownsSuspension (hr * helmv2.HelmRelease ) bool {
261+ return hr .Annotations [annotationSuspendedByPlunger ] == "true"
262+ }
263+
167264// hasNoDeployedReleasesError checks if the HelmRelease has the specific error
168265func hasNoDeployedReleasesError (hr * helmv2.HelmRelease ) bool {
169266 for _ , condition := range hr .Status .Conditions {
@@ -240,27 +337,84 @@ func extractVersionNumber(secretName string) int {
240337 return num
241338}
242339
243- // suspendHelmRelease sets suspend to true on the HelmRelease
244- func (r * FluxPlunger ) suspendHelmRelease (ctx context.Context , hr * helmv2.HelmRelease ) error {
340+ // suspendHelmRelease sets suspend to true on the HelmRelease and marks the
341+ // suspension as owned by flux-plunger. It reports whether it acquired the
342+ // suspension: false means the release was already suspended by someone else, or
343+ // it started terminating after the top-of-Reconcile check, so the caller must
344+ // not proceed as if it owns it.
345+ func (r * FluxPlunger ) suspendHelmRelease (ctx context.Context , hr * helmv2.HelmRelease ) (bool , error ) {
245346 // Re-fetch the HelmRelease to get the latest state
246347 key := types.NamespacedName {Namespace : hr .Namespace , Name : hr .Name }
247348 latestHR := & helmv2.HelmRelease {}
248349 if err := r .Get (ctx , key , latestHR ); err != nil {
249- return fmt .Errorf ("failed to get latest HelmRelease: %w" , err )
350+ return false , fmt .Errorf ("failed to get latest HelmRelease: %w" , err )
351+ }
352+
353+ // Never suspend a terminating release. A delete can arrive between the
354+ // top-of-Reconcile deletion check and this re-fetch, and suspending a release
355+ // helm-controller is about to uninstall is exactly what makes it skip the
356+ // uninstall and orphan the release's resources.
357+ if ! latestHR .DeletionTimestamp .IsZero () {
358+ return false , nil
250359 }
251360
252- // If already suspended, nothing to do
361+ // If already suspended, do not take ownership of a suspension we did not set.
253362 if latestHR .Spec .Suspend {
254- return nil
363+ return false , nil
255364 }
256365
257366 patch := client .MergeFromWithOptions (latestHR .DeepCopy (), client.MergeFromWithOptimisticLock {})
258367 latestHR .Spec .Suspend = true
368+ if latestHR .Annotations == nil {
369+ latestHR .Annotations = make (map [string ]string )
370+ }
371+ latestHR .Annotations [annotationSuspendedByPlunger ] = "true"
372+
373+ if err := r .Patch (ctx , latestHR , patch , client .FieldOwner (fieldManager )); err != nil {
374+ return false , err
375+ }
376+ return true , nil
377+ }
378+
379+ // releaseIsTerminating re-fetches the HelmRelease and reports whether it has begun
380+ // terminating. Recovery calls this immediately before deleting the storage Secret
381+ // so a delete that landed after the earlier checks cannot make it remove the only
382+ // revision helm-controller could uninstall.
383+ func (r * FluxPlunger ) releaseIsTerminating (ctx context.Context , hr * helmv2.HelmRelease ) (bool , error ) {
384+ key := types.NamespacedName {Namespace : hr .Namespace , Name : hr .Name }
385+ latestHR := & helmv2.HelmRelease {}
386+ if err := r .Get (ctx , key , latestHR ); err != nil {
387+ return false , fmt .Errorf ("failed to get latest HelmRelease: %w" , err )
388+ }
389+ return ! latestHR .DeletionTimestamp .IsZero (), nil
390+ }
259391
392+ // clearOwnershipMarker removes a stale ownership marker from a HelmRelease that is
393+ // no longer suspended. It never touches spec.suspend: if the release was suspended
394+ // again between the caller's read and this re-fetch, the marker is left in place
395+ // for the suspend path to evaluate and this is a no-op. Optimistic locking makes
396+ // the marker delete lose to a concurrent write rather than clobber it.
397+ func (r * FluxPlunger ) clearOwnershipMarker (ctx context.Context , hr * helmv2.HelmRelease ) error {
398+ key := types.NamespacedName {Namespace : hr .Namespace , Name : hr .Name }
399+ latestHR := & helmv2.HelmRelease {}
400+ if err := r .Get (ctx , key , latestHR ); err != nil {
401+ return fmt .Errorf ("failed to get latest HelmRelease: %w" , err )
402+ }
403+ if _ , ok := latestHR .Annotations [annotationSuspendedByPlunger ]; ! ok {
404+ return nil
405+ }
406+ if latestHR .Spec .Suspend {
407+ return nil
408+ }
409+ patch := client .MergeFromWithOptions (latestHR .DeepCopy (), client.MergeFromWithOptimisticLock {})
410+ delete (latestHR .Annotations , annotationSuspendedByPlunger )
260411 return r .Patch (ctx , latestHR , patch , client .FieldOwner (fieldManager ))
261412}
262413
263- // unsuspendHelmRelease sets suspend to false on the HelmRelease
414+ // unsuspendHelmRelease clears a flux-plunger-owned suspension and its marker.
415+ // Callers already gate on ownership; the re-fetched ownership check here is
416+ // defense-in-depth against a future caller that forgets to, so an operator's
417+ // suspension is never cleared by accident.
264418func (r * FluxPlunger ) unsuspendHelmRelease (ctx context.Context , hr * helmv2.HelmRelease ) error {
265419 // Re-fetch the HelmRelease to get the latest state
266420 key := types.NamespacedName {Namespace : hr .Namespace , Name : hr .Name }
@@ -269,13 +423,28 @@ func (r *FluxPlunger) unsuspendHelmRelease(ctx context.Context, hr *helmv2.HelmR
269423 return fmt .Errorf ("failed to get latest HelmRelease: %w" , err )
270424 }
271425
272- // If already unsuspended, nothing to do
426+ // If already unsuspended, clear any stale ownership marker and stop.
273427 if ! latestHR .Spec .Suspend {
428+ if _ , ok := latestHR .Annotations [annotationSuspendedByPlunger ]; ! ok {
429+ return nil
430+ }
431+ patch := client .MergeFromWithOptions (latestHR .DeepCopy (), client.MergeFromWithOptimisticLock {})
432+ delete (latestHR .Annotations , annotationSuspendedByPlunger )
433+ return r .Patch (ctx , latestHR , patch , client .FieldOwner (fieldManager ))
434+ }
435+
436+ // Deliberate redundancy with the caller-side ownership gate in Reconcile: this
437+ // re-checks against the freshly re-fetched object, which the caller cannot,
438+ // and it is the guard pinned directly by
439+ // TestUnsuspendHelmRelease_LeavesSuspensionWithoutMarker so a future refactor
440+ // that drops the caller-side gate still cannot clear an operator's suspension.
441+ if ! ownsSuspension (latestHR ) {
274442 return nil
275443 }
276444
277445 patch := client .MergeFromWithOptions (latestHR .DeepCopy (), client.MergeFromWithOptimisticLock {})
278446 latestHR .Spec .Suspend = false
447+ delete (latestHR .Annotations , annotationSuspendedByPlunger )
279448
280449 return r .Patch (ctx , latestHR , patch , client .FieldOwner (fieldManager ))
281450}
@@ -303,7 +472,10 @@ func (r *FluxPlunger) updateProcessedVersionAnnotation(ctx context.Context, hr *
303472func (r * FluxPlunger ) SetupWithManager (mgr ctrl.Manager ) error {
304473 // Watch HelmReleases that either:
305474 // 1. Have the specific error, OR
306- // 2. Are suspended with our annotation (to handle crash recovery)
475+ // 2. Carry flux-plunger's ownership marker — whether still suspended (crash
476+ // recovery, and a delete arriving mid-recovery, both need to wake a
477+ // reconcile) or already resumed by an external actor (reap the stale marker
478+ // before a later operator suspension can inherit it).
307479 pred := predicate .NewPredicateFuncs (func (obj client.Object ) bool {
308480 hr , ok := obj .(* helmv2.HelmRelease )
309481 if ! ok {
@@ -315,11 +487,9 @@ func (r *FluxPlunger) SetupWithManager(mgr ctrl.Manager) error {
315487 return true
316488 }
317489
318- // Also process suspended HelmReleases with our annotation (crash recovery)
319- if hr .Spec .Suspend && hr .Annotations != nil {
320- if _ , exists := hr .Annotations [annotationLastProcessedVersion ]; exists {
321- return true
322- }
490+ // Also process any HelmRelease still carrying our ownership marker.
491+ if ownsSuspension (hr ) {
492+ return true
323493 }
324494
325495 return false
0 commit comments