@@ -19,6 +19,11 @@ import (
1919
2020const (
2121 annotationLastProcessedVersion = "flux-plunger.cozystack.io/last-processed-version"
22+ // annotationSuspendedByPlunger marks a suspension that flux-plunger itself
23+ // set, so recovery can be told apart from a suspension an operator or another
24+ // controller owns. It is written atomically with spec.suspend=true and
25+ // removed atomically with spec.suspend=false.
26+ annotationSuspendedByPlunger = "flux-plunger.cozystack.io/suspended"
2227 errorMessageNoDeployedReleases = "has no deployed releases"
2328 fieldManager = "flux-client-side-apply"
2429)
@@ -41,50 +46,48 @@ func (r *FluxPlunger) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Res
4146 return ctrl.Result {}, client .IgnoreNotFound (err )
4247 }
4348
44- // Check if HelmRelease is suspended
45- if hr .Spec .Suspend {
46- logger .Info ("HelmRelease is suspended, checking if we need to unsuspend" )
47-
48- // Get the list of Helm release secrets
49- secrets , err := r .listHelmReleaseSecrets (ctx , hr .Namespace , hr .Name )
50- if err != nil {
51- logger .Error (err , "Failed to list Helm release secrets" )
52- return ctrl.Result {}, err
53- }
54-
55- // If no secrets, treat latest version as 0
56- latestVersion := 0
57- if len (secrets ) > 0 {
58- latestSecret := getLatestSecret (secrets )
59- latestVersion = extractVersionNumber (latestSecret .Name )
60- } else {
61- logger .Info ("No Helm release secrets found while suspended, treating as version 0" )
62- }
63-
64- // Check if version is previous to just processed (latestVersion+1 == processedVersion)
65- // This is the ONLY condition when we unsuspend
66- shouldUnsuspend := false
67- if hr .Annotations != nil {
68- if processedVersionStr , exists := hr .Annotations [annotationLastProcessedVersion ]; exists {
69- processedVersion , err := strconv .Atoi (processedVersionStr )
70- if err == nil && latestVersion + 1 == processedVersion {
71- shouldUnsuspend = true
72- }
49+ // A HelmRelease that is being deleted must never stay suspended. helm-controller
50+ // v1.5.x skips the Helm uninstall and drops its finalizer for a suspended
51+ // HelmRelease (https://github.com/fluxcd/helm-controller/blob/v1.5.0/internal/controller/helmrelease_controller.go#L447-L475),
52+ // which orphans every resource the release owns. If flux-plunger owns the
53+ // suspension, release it so the uninstall proceeds; in every case do not begin
54+ // recovery on a terminating object.
55+ if ! hr .DeletionTimestamp .IsZero () {
56+ if hr .Spec .Suspend && ownsSuspension (hr ) {
57+ logger .Info ("HelmRelease is being deleted while suspended by flux-plunger, clearing suspension so uninstall can proceed" )
58+ if err := r .unsuspendHelmRelease (ctx , hr ); err != nil {
59+ // Requeue with backoff: leaving the suspension is exactly what
60+ // orphans the release, so retry rather than wait for another event.
61+ logger .Info ("Could not clear suspension on deleted HelmRelease, requeuing" , "error" , err .Error ())
62+ return ctrl.Result {}, err
7363 }
7464 }
65+ return ctrl.Result {}, nil
66+ }
7567
76- if shouldUnsuspend {
77- // Unsuspend the HelmRelease
78- logger .Info ("Secret was already deleted in previous run, removing suspend" , "latest" , latestVersion , "processed" , latestVersion + 1 )
79- if err := r .unsuspendHelmRelease (ctx , hr ); err != nil {
80- logger .Info ("Could not unsuspend HelmRelease, will retry on next reconcile" , "error" , err .Error ())
81- return ctrl.Result {}, nil
82- }
68+ // Check if HelmRelease is suspended
69+ if hr .Spec .Suspend {
70+ // Ownership is decided solely by the marker flux-plunger writes atomically
71+ // with the suspension. A suspension we own but see in a fresh reconcile means
72+ // recovery crashed mid-flight (default reconcile concurrency is one worker per
73+ // key, so it cannot be another in-flight run), so releasing it un-sticks the
74+ // release and lets error-recovery start over. A suspension without the marker
75+ // belongs to an operator or another controller and must be left untouched —
76+ // the processed-version annotation is deliberately NOT consulted here, because
77+ // it is never cleared and would misread a much later, unrelated operator
78+ // suspension as ours once the release's latest revision stops advancing.
79+ if ! ownsSuspension (hr ) {
80+ logger .Info ("HelmRelease is suspended by external process, skipping" )
8381 return ctrl.Result {}, nil
8482 }
8583
86- // If not previous to processed, skip all actions
87- logger .Info ("HelmRelease is suspended by external process, skipping" , "latest" , latestVersion )
84+ logger .Info ("Removing suspension owned by flux-plunger" )
85+ if err := r .unsuspendHelmRelease (ctx , hr ); err != nil {
86+ // Requeue with backoff so a stuck owned suspension is not left until the
87+ // next informer resync, symmetric with the deletion and recovery paths.
88+ logger .Info ("Could not unsuspend HelmRelease, requeuing" , "error" , err .Error ())
89+ return ctrl.Result {}, err
90+ }
8891 return ctrl.Result {}, nil
8992 }
9093
@@ -132,12 +135,37 @@ func (r *FluxPlunger) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Res
132135
133136 // Suspend the HelmRelease
134137 logger .Info ("Suspending HelmRelease" )
135- if err := r .suspendHelmRelease (ctx , hr ); err != nil {
138+ acquired , err := r .suspendHelmRelease (ctx , hr )
139+ if err != nil {
136140 // Optimistic lock conflicts are normal - FluxCD also updates HelmRelease
137141 // Don't return error, just log and let controller-runtime requeue on next update
138142 logger .Info ("Could not suspend HelmRelease, will retry on next reconcile" , "error" , err .Error ())
139143 return ctrl.Result {}, nil
140144 }
145+ if ! acquired {
146+ // The release was suspended by someone else, or started terminating, between
147+ // our read and our write, so flux-plunger does not own it. Do not delete its
148+ // storage Secret; let the next reconcile handle it (the suspended branch, or
149+ // the deletion branch which lets helm-controller uninstall).
150+ logger .Info ("HelmRelease was not acquired for recovery, deferring to next reconcile" )
151+ return ctrl.Result {}, nil
152+ }
153+
154+ // From here flux-plunger owns the suspension (acquired above). Guarantee it is
155+ // released before returning from any path below: a step that fails must never
156+ // leave the HelmRelease suspended, because a delete arriving on a suspended
157+ // release makes helm-controller skip the uninstall and orphan the release's
158+ // resources. The unsuspend is idempotent, so the happy path running it
159+ // explicitly and this deferred fallback are safe together.
160+ completed := false
161+ defer func () {
162+ if completed {
163+ return
164+ }
165+ if err := r .unsuspendHelmRelease (ctx , hr ); err != nil {
166+ logger .Info ("Could not clear suspension after failed recovery step, will retry on next reconcile" , "error" , err .Error ())
167+ }
168+ }()
141169
142170 // Delete the latest secret
143171 logger .Info ("Deleting latest Helm release secret" , "secret" , latestSecret .Name )
@@ -146,24 +174,44 @@ func (r *FluxPlunger) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Res
146174 return ctrl.Result {}, err
147175 }
148176
149- // Update annotation with processed version
177+ // Update annotation with processed version. This runs after the delete, so a
178+ // persistent failure here lets the next reconcile delete the following storage
179+ // revision too. That is the accepted cost of clearing the suspension on every
180+ // failed step: burning a revision is strictly better than staying suspended,
181+ // which is the orphan-on-delete hazard this recovery exists to avoid. Requeue
182+ // with backoff on failure, symmetric with the other steps: the deferred
183+ // unsuspend still clears the suspension, and the requeue guarantees a retry even
184+ // when the same transient cause also fails that deferred patch.
150185 logger .Info ("Updating annotation with processed version" , "version" , latestVersion )
151186 if err := r .updateProcessedVersionAnnotation (ctx , hr , latestVersion ); err != nil {
152- logger .Info ("Could not update annotation, will retry on next reconcile " , "error" , err .Error ())
153- return ctrl.Result {}, nil
187+ logger .Info ("Could not update annotation, requeuing " , "error" , err .Error ())
188+ return ctrl.Result {}, err
154189 }
155190
156- // Unsuspend the HelmRelease
191+ // Unsuspend the HelmRelease. Force a backoff requeue on failure, symmetric with
192+ // the deletion path: a persistently failing unsuspend (e.g. an admission webhook
193+ // rejecting the patch) would otherwise leave the release suspended-by-plunger
194+ // until the informer's periodic resync, far longer than necessary.
157195 logger .Info ("Unsuspending HelmRelease" )
158196 if err := r .unsuspendHelmRelease (ctx , hr ); err != nil {
159- logger .Info ("Could not unsuspend HelmRelease, will retry on next reconcile" , "error" , err .Error ())
160- return ctrl.Result {}, nil
197+ logger .Info ("Could not unsuspend HelmRelease, requeuing" , "error" , err .Error ())
198+ // The returned error requeues and retries; skip the redundant deferred attempt.
199+ completed = true
200+ return ctrl.Result {}, err
161201 }
162202
203+ completed = true
163204 logger .Info ("Successfully processed HelmRelease" , "version" , latestVersion )
164205 return ctrl.Result {}, nil
165206}
166207
208+ // ownsSuspension reports whether the current suspension was set by flux-plunger,
209+ // as opposed to an operator or another controller. Recovery and delete handling
210+ // must only clear a suspension flux-plunger owns.
211+ func ownsSuspension (hr * helmv2.HelmRelease ) bool {
212+ return hr .Annotations [annotationSuspendedByPlunger ] == "true"
213+ }
214+
167215// hasNoDeployedReleasesError checks if the HelmRelease has the specific error
168216func hasNoDeployedReleasesError (hr * helmv2.HelmRelease ) bool {
169217 for _ , condition := range hr .Status .Conditions {
@@ -240,27 +288,49 @@ func extractVersionNumber(secretName string) int {
240288 return num
241289}
242290
243- // suspendHelmRelease sets suspend to true on the HelmRelease
244- func (r * FluxPlunger ) suspendHelmRelease (ctx context.Context , hr * helmv2.HelmRelease ) error {
291+ // suspendHelmRelease sets suspend to true on the HelmRelease and marks the
292+ // suspension as owned by flux-plunger. It reports whether it acquired the
293+ // suspension: false means the release was already suspended by someone else, or
294+ // it started terminating after the top-of-Reconcile check, so the caller must
295+ // not proceed as if it owns it.
296+ func (r * FluxPlunger ) suspendHelmRelease (ctx context.Context , hr * helmv2.HelmRelease ) (bool , error ) {
245297 // Re-fetch the HelmRelease to get the latest state
246298 key := types.NamespacedName {Namespace : hr .Namespace , Name : hr .Name }
247299 latestHR := & helmv2.HelmRelease {}
248300 if err := r .Get (ctx , key , latestHR ); err != nil {
249- return fmt .Errorf ("failed to get latest HelmRelease: %w" , err )
301+ return false , fmt .Errorf ("failed to get latest HelmRelease: %w" , err )
250302 }
251303
252- // If already suspended, nothing to do
304+ // Never suspend a terminating release. A delete can arrive between the
305+ // top-of-Reconcile deletion check and this re-fetch, and suspending a release
306+ // helm-controller is about to uninstall is exactly what makes it skip the
307+ // uninstall and orphan the release's resources.
308+ if ! latestHR .DeletionTimestamp .IsZero () {
309+ return false , nil
310+ }
311+
312+ // If already suspended, do not take ownership of a suspension we did not set.
253313 if latestHR .Spec .Suspend {
254- return nil
314+ return false , nil
255315 }
256316
257317 patch := client .MergeFromWithOptions (latestHR .DeepCopy (), client.MergeFromWithOptimisticLock {})
258318 latestHR .Spec .Suspend = true
319+ if latestHR .Annotations == nil {
320+ latestHR .Annotations = make (map [string ]string )
321+ }
322+ latestHR .Annotations [annotationSuspendedByPlunger ] = "true"
259323
260- return r .Patch (ctx , latestHR , patch , client .FieldOwner (fieldManager ))
324+ if err := r .Patch (ctx , latestHR , patch , client .FieldOwner (fieldManager )); err != nil {
325+ return false , err
326+ }
327+ return true , nil
261328}
262329
263- // unsuspendHelmRelease sets suspend to false on the HelmRelease
330+ // unsuspendHelmRelease clears a flux-plunger-owned suspension and its marker.
331+ // Callers already gate on ownership; the re-fetched ownership check here is
332+ // defense-in-depth against a future caller that forgets to, so an operator's
333+ // suspension is never cleared by accident.
264334func (r * FluxPlunger ) unsuspendHelmRelease (ctx context.Context , hr * helmv2.HelmRelease ) error {
265335 // Re-fetch the HelmRelease to get the latest state
266336 key := types.NamespacedName {Namespace : hr .Namespace , Name : hr .Name }
@@ -269,13 +339,23 @@ func (r *FluxPlunger) unsuspendHelmRelease(ctx context.Context, hr *helmv2.HelmR
269339 return fmt .Errorf ("failed to get latest HelmRelease: %w" , err )
270340 }
271341
272- // If already unsuspended, nothing to do
342+ // If already unsuspended, clear any stale ownership marker and stop.
273343 if ! latestHR .Spec .Suspend {
344+ if _ , ok := latestHR .Annotations [annotationSuspendedByPlunger ]; ! ok {
345+ return nil
346+ }
347+ patch := client .MergeFromWithOptions (latestHR .DeepCopy (), client.MergeFromWithOptimisticLock {})
348+ delete (latestHR .Annotations , annotationSuspendedByPlunger )
349+ return r .Patch (ctx , latestHR , patch , client .FieldOwner (fieldManager ))
350+ }
351+
352+ if ! ownsSuspension (latestHR ) {
274353 return nil
275354 }
276355
277356 patch := client .MergeFromWithOptions (latestHR .DeepCopy (), client.MergeFromWithOptimisticLock {})
278357 latestHR .Spec .Suspend = false
358+ delete (latestHR .Annotations , annotationSuspendedByPlunger )
279359
280360 return r .Patch (ctx , latestHR , patch , client .FieldOwner (fieldManager ))
281361}
@@ -303,7 +383,8 @@ func (r *FluxPlunger) updateProcessedVersionAnnotation(ctx context.Context, hr *
303383func (r * FluxPlunger ) SetupWithManager (mgr ctrl.Manager ) error {
304384 // Watch HelmReleases that either:
305385 // 1. Have the specific error, OR
306- // 2. Are suspended with our annotation (to handle crash recovery)
386+ // 2. Are suspended with our ownership marker (crash recovery and a delete
387+ // arriving mid-recovery both need to wake a reconcile).
307388 pred := predicate .NewPredicateFuncs (func (obj client.Object ) bool {
308389 hr , ok := obj .(* helmv2.HelmRelease )
309390 if ! ok {
@@ -315,11 +396,9 @@ func (r *FluxPlunger) SetupWithManager(mgr ctrl.Manager) error {
315396 return true
316397 }
317398
318- // Also process suspended HelmReleases with our annotation (crash recovery)
319- if hr .Spec .Suspend && hr .Annotations != nil {
320- if _ , exists := hr .Annotations [annotationLastProcessedVersion ]; exists {
321- return true
322- }
399+ // Also process suspended HelmReleases flux-plunger owns.
400+ if hr .Spec .Suspend && ownsSuspension (hr ) {
401+ return true
323402 }
324403
325404 return false
0 commit comments