Skip to content

Commit bfb8c13

Browse files
committed
fix(fluxcd): keep Helm uninstall working during flux-plunger recovery
flux-plunger recovers a "has no deployed releases" HelmRelease by briefly suspending it, deleting the stale storage Secret and unsuspending. A delete arriving inside that window, or a failed step that leaves the release suspended, makes helm-controller v1.5.x skip the uninstall and drop its finalizer, orphaning the release's DataVolume, PVCs, RBAC and the storage Secret. Skip recovery on a terminating HelmRelease and release a suspension flux-plunger owns as soon as deletion is seen. Mark the suspension the plunger sets with a dedicated ownership annotation, written and cleared atomically with spec.suspend, so an operator's own suspension is never touched; the processed-version annotation is deliberately not consulted for ownership, because it is never cleared and a stale value would misread a much later, unrelated operator suspension as ours. Abort recovery without deleting the storage Secret when the release is suspended by someone else or starts terminating between the initial check and the suspend write. Guarantee the suspension is cleared after any failed step, and requeue with backoff at every step so a stuck suspension is not left until the next informer resync. Assisted-by: LLM Signed-off-by: Alexey Artamonov <[email protected]>
1 parent b99db14 commit bfb8c13

2 files changed

Lines changed: 611 additions & 59 deletions

File tree

‎internal/controller/fluxplunger/flux_plunger.go‎

Lines changed: 138 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,11 @@ import (
1919

2020
const (
2121
annotationLastProcessedVersion = "flux-plunger.cozystack.io/last-processed-version"
22+
// annotationSuspendedByPlunger marks a suspension that flux-plunger itself
23+
// set, so recovery can be told apart from a suspension an operator or another
24+
// controller owns. It is written atomically with spec.suspend=true and
25+
// removed atomically with spec.suspend=false.
26+
annotationSuspendedByPlunger = "flux-plunger.cozystack.io/suspended"
2227
errorMessageNoDeployedReleases = "has no deployed releases"
2328
fieldManager = "flux-client-side-apply"
2429
)
@@ -41,50 +46,48 @@ func (r *FluxPlunger) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Res
4146
return ctrl.Result{}, client.IgnoreNotFound(err)
4247
}
4348

44-
// Check if HelmRelease is suspended
45-
if hr.Spec.Suspend {
46-
logger.Info("HelmRelease is suspended, checking if we need to unsuspend")
47-
48-
// Get the list of Helm release secrets
49-
secrets, err := r.listHelmReleaseSecrets(ctx, hr.Namespace, hr.Name)
50-
if err != nil {
51-
logger.Error(err, "Failed to list Helm release secrets")
52-
return ctrl.Result{}, err
53-
}
54-
55-
// If no secrets, treat latest version as 0
56-
latestVersion := 0
57-
if len(secrets) > 0 {
58-
latestSecret := getLatestSecret(secrets)
59-
latestVersion = extractVersionNumber(latestSecret.Name)
60-
} else {
61-
logger.Info("No Helm release secrets found while suspended, treating as version 0")
62-
}
63-
64-
// Check if version is previous to just processed (latestVersion+1 == processedVersion)
65-
// This is the ONLY condition when we unsuspend
66-
shouldUnsuspend := false
67-
if hr.Annotations != nil {
68-
if processedVersionStr, exists := hr.Annotations[annotationLastProcessedVersion]; exists {
69-
processedVersion, err := strconv.Atoi(processedVersionStr)
70-
if err == nil && latestVersion+1 == processedVersion {
71-
shouldUnsuspend = true
72-
}
49+
// A HelmRelease that is being deleted must never stay suspended. helm-controller
50+
// v1.5.x skips the Helm uninstall and drops its finalizer for a suspended
51+
// HelmRelease (https://github.com/fluxcd/helm-controller/blob/v1.5.0/internal/controller/helmrelease_controller.go#L447-L475),
52+
// which orphans every resource the release owns. If flux-plunger owns the
53+
// suspension, release it so the uninstall proceeds; in every case do not begin
54+
// recovery on a terminating object.
55+
if !hr.DeletionTimestamp.IsZero() {
56+
if hr.Spec.Suspend && ownsSuspension(hr) {
57+
logger.Info("HelmRelease is being deleted while suspended by flux-plunger, clearing suspension so uninstall can proceed")
58+
if err := r.unsuspendHelmRelease(ctx, hr); err != nil {
59+
// Requeue with backoff: leaving the suspension is exactly what
60+
// orphans the release, so retry rather than wait for another event.
61+
logger.Info("Could not clear suspension on deleted HelmRelease, requeuing", "error", err.Error())
62+
return ctrl.Result{}, err
7363
}
7464
}
65+
return ctrl.Result{}, nil
66+
}
7567

76-
if shouldUnsuspend {
77-
// Unsuspend the HelmRelease
78-
logger.Info("Secret was already deleted in previous run, removing suspend", "latest", latestVersion, "processed", latestVersion+1)
79-
if err := r.unsuspendHelmRelease(ctx, hr); err != nil {
80-
logger.Info("Could not unsuspend HelmRelease, will retry on next reconcile", "error", err.Error())
81-
return ctrl.Result{}, nil
82-
}
68+
// Check if HelmRelease is suspended
69+
if hr.Spec.Suspend {
70+
// Ownership is decided solely by the marker flux-plunger writes atomically
71+
// with the suspension. A suspension we own but see in a fresh reconcile means
72+
// recovery crashed mid-flight (default reconcile concurrency is one worker per
73+
// key, so it cannot be another in-flight run), so releasing it un-sticks the
74+
// release and lets error-recovery start over. A suspension without the marker
75+
// belongs to an operator or another controller and must be left untouched —
76+
// the processed-version annotation is deliberately NOT consulted here, because
77+
// it is never cleared and would misread a much later, unrelated operator
78+
// suspension as ours once the release's latest revision stops advancing.
79+
if !ownsSuspension(hr) {
80+
logger.Info("HelmRelease is suspended by external process, skipping")
8381
return ctrl.Result{}, nil
8482
}
8583

86-
// If not previous to processed, skip all actions
87-
logger.Info("HelmRelease is suspended by external process, skipping", "latest", latestVersion)
84+
logger.Info("Removing suspension owned by flux-plunger")
85+
if err := r.unsuspendHelmRelease(ctx, hr); err != nil {
86+
// Requeue with backoff so a stuck owned suspension is not left until the
87+
// next informer resync, symmetric with the deletion and recovery paths.
88+
logger.Info("Could not unsuspend HelmRelease, requeuing", "error", err.Error())
89+
return ctrl.Result{}, err
90+
}
8891
return ctrl.Result{}, nil
8992
}
9093

@@ -132,12 +135,37 @@ func (r *FluxPlunger) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Res
132135

133136
// Suspend the HelmRelease
134137
logger.Info("Suspending HelmRelease")
135-
if err := r.suspendHelmRelease(ctx, hr); err != nil {
138+
acquired, err := r.suspendHelmRelease(ctx, hr)
139+
if err != nil {
136140
// Optimistic lock conflicts are normal - FluxCD also updates HelmRelease
137141
// Don't return error, just log and let controller-runtime requeue on next update
138142
logger.Info("Could not suspend HelmRelease, will retry on next reconcile", "error", err.Error())
139143
return ctrl.Result{}, nil
140144
}
145+
if !acquired {
146+
// The release was suspended by someone else, or started terminating, between
147+
// our read and our write, so flux-plunger does not own it. Do not delete its
148+
// storage Secret; let the next reconcile handle it (the suspended branch, or
149+
// the deletion branch which lets helm-controller uninstall).
150+
logger.Info("HelmRelease was not acquired for recovery, deferring to next reconcile")
151+
return ctrl.Result{}, nil
152+
}
153+
154+
// From here flux-plunger owns the suspension (acquired above). Guarantee it is
155+
// released before returning from any path below: a step that fails must never
156+
// leave the HelmRelease suspended, because a delete arriving on a suspended
157+
// release makes helm-controller skip the uninstall and orphan the release's
158+
// resources. The unsuspend is idempotent, so the happy path running it
159+
// explicitly and this deferred fallback are safe together.
160+
completed := false
161+
defer func() {
162+
if completed {
163+
return
164+
}
165+
if err := r.unsuspendHelmRelease(ctx, hr); err != nil {
166+
logger.Info("Could not clear suspension after failed recovery step, will retry on next reconcile", "error", err.Error())
167+
}
168+
}()
141169

142170
// Delete the latest secret
143171
logger.Info("Deleting latest Helm release secret", "secret", latestSecret.Name)
@@ -146,24 +174,44 @@ func (r *FluxPlunger) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Res
146174
return ctrl.Result{}, err
147175
}
148176

149-
// Update annotation with processed version
177+
// Update annotation with processed version. This runs after the delete, so a
178+
// persistent failure here lets the next reconcile delete the following storage
179+
// revision too. That is the accepted cost of clearing the suspension on every
180+
// failed step: burning a revision is strictly better than staying suspended,
181+
// which is the orphan-on-delete hazard this recovery exists to avoid. Requeue
182+
// with backoff on failure, symmetric with the other steps: the deferred
183+
// unsuspend still clears the suspension, and the requeue guarantees a retry even
184+
// when the same transient cause also fails that deferred patch.
150185
logger.Info("Updating annotation with processed version", "version", latestVersion)
151186
if err := r.updateProcessedVersionAnnotation(ctx, hr, latestVersion); err != nil {
152-
logger.Info("Could not update annotation, will retry on next reconcile", "error", err.Error())
153-
return ctrl.Result{}, nil
187+
logger.Info("Could not update annotation, requeuing", "error", err.Error())
188+
return ctrl.Result{}, err
154189
}
155190

156-
// Unsuspend the HelmRelease
191+
// Unsuspend the HelmRelease. Force a backoff requeue on failure, symmetric with
192+
// the deletion path: a persistently failing unsuspend (e.g. an admission webhook
193+
// rejecting the patch) would otherwise leave the release suspended-by-plunger
194+
// until the informer's periodic resync, far longer than necessary.
157195
logger.Info("Unsuspending HelmRelease")
158196
if err := r.unsuspendHelmRelease(ctx, hr); err != nil {
159-
logger.Info("Could not unsuspend HelmRelease, will retry on next reconcile", "error", err.Error())
160-
return ctrl.Result{}, nil
197+
logger.Info("Could not unsuspend HelmRelease, requeuing", "error", err.Error())
198+
// The returned error requeues and retries; skip the redundant deferred attempt.
199+
completed = true
200+
return ctrl.Result{}, err
161201
}
162202

203+
completed = true
163204
logger.Info("Successfully processed HelmRelease", "version", latestVersion)
164205
return ctrl.Result{}, nil
165206
}
166207

208+
// ownsSuspension reports whether the current suspension was set by flux-plunger,
209+
// as opposed to an operator or another controller. Recovery and delete handling
210+
// must only clear a suspension flux-plunger owns.
211+
func ownsSuspension(hr *helmv2.HelmRelease) bool {
212+
return hr.Annotations[annotationSuspendedByPlunger] == "true"
213+
}
214+
167215
// hasNoDeployedReleasesError checks if the HelmRelease has the specific error
168216
func hasNoDeployedReleasesError(hr *helmv2.HelmRelease) bool {
169217
for _, condition := range hr.Status.Conditions {
@@ -240,27 +288,49 @@ func extractVersionNumber(secretName string) int {
240288
return num
241289
}
242290

243-
// suspendHelmRelease sets suspend to true on the HelmRelease
244-
func (r *FluxPlunger) suspendHelmRelease(ctx context.Context, hr *helmv2.HelmRelease) error {
291+
// suspendHelmRelease sets suspend to true on the HelmRelease and marks the
292+
// suspension as owned by flux-plunger. It reports whether it acquired the
293+
// suspension: false means the release was already suspended by someone else, or
294+
// it started terminating after the top-of-Reconcile check, so the caller must
295+
// not proceed as if it owns it.
296+
func (r *FluxPlunger) suspendHelmRelease(ctx context.Context, hr *helmv2.HelmRelease) (bool, error) {
245297
// Re-fetch the HelmRelease to get the latest state
246298
key := types.NamespacedName{Namespace: hr.Namespace, Name: hr.Name}
247299
latestHR := &helmv2.HelmRelease{}
248300
if err := r.Get(ctx, key, latestHR); err != nil {
249-
return fmt.Errorf("failed to get latest HelmRelease: %w", err)
301+
return false, fmt.Errorf("failed to get latest HelmRelease: %w", err)
250302
}
251303

252-
// If already suspended, nothing to do
304+
// Never suspend a terminating release. A delete can arrive between the
305+
// top-of-Reconcile deletion check and this re-fetch, and suspending a release
306+
// helm-controller is about to uninstall is exactly what makes it skip the
307+
// uninstall and orphan the release's resources.
308+
if !latestHR.DeletionTimestamp.IsZero() {
309+
return false, nil
310+
}
311+
312+
// If already suspended, do not take ownership of a suspension we did not set.
253313
if latestHR.Spec.Suspend {
254-
return nil
314+
return false, nil
255315
}
256316

257317
patch := client.MergeFromWithOptions(latestHR.DeepCopy(), client.MergeFromWithOptimisticLock{})
258318
latestHR.Spec.Suspend = true
319+
if latestHR.Annotations == nil {
320+
latestHR.Annotations = make(map[string]string)
321+
}
322+
latestHR.Annotations[annotationSuspendedByPlunger] = "true"
259323

260-
return r.Patch(ctx, latestHR, patch, client.FieldOwner(fieldManager))
324+
if err := r.Patch(ctx, latestHR, patch, client.FieldOwner(fieldManager)); err != nil {
325+
return false, err
326+
}
327+
return true, nil
261328
}
262329

263-
// unsuspendHelmRelease sets suspend to false on the HelmRelease
330+
// unsuspendHelmRelease clears a flux-plunger-owned suspension and its marker.
331+
// Callers already gate on ownership; the re-fetched ownership check here is
332+
// defense-in-depth against a future caller that forgets to, so an operator's
333+
// suspension is never cleared by accident.
264334
func (r *FluxPlunger) unsuspendHelmRelease(ctx context.Context, hr *helmv2.HelmRelease) error {
265335
// Re-fetch the HelmRelease to get the latest state
266336
key := types.NamespacedName{Namespace: hr.Namespace, Name: hr.Name}
@@ -269,13 +339,23 @@ func (r *FluxPlunger) unsuspendHelmRelease(ctx context.Context, hr *helmv2.HelmR
269339
return fmt.Errorf("failed to get latest HelmRelease: %w", err)
270340
}
271341

272-
// If already unsuspended, nothing to do
342+
// If already unsuspended, clear any stale ownership marker and stop.
273343
if !latestHR.Spec.Suspend {
344+
if _, ok := latestHR.Annotations[annotationSuspendedByPlunger]; !ok {
345+
return nil
346+
}
347+
patch := client.MergeFromWithOptions(latestHR.DeepCopy(), client.MergeFromWithOptimisticLock{})
348+
delete(latestHR.Annotations, annotationSuspendedByPlunger)
349+
return r.Patch(ctx, latestHR, patch, client.FieldOwner(fieldManager))
350+
}
351+
352+
if !ownsSuspension(latestHR) {
274353
return nil
275354
}
276355

277356
patch := client.MergeFromWithOptions(latestHR.DeepCopy(), client.MergeFromWithOptimisticLock{})
278357
latestHR.Spec.Suspend = false
358+
delete(latestHR.Annotations, annotationSuspendedByPlunger)
279359

280360
return r.Patch(ctx, latestHR, patch, client.FieldOwner(fieldManager))
281361
}
@@ -303,7 +383,8 @@ func (r *FluxPlunger) updateProcessedVersionAnnotation(ctx context.Context, hr *
303383
func (r *FluxPlunger) SetupWithManager(mgr ctrl.Manager) error {
304384
// Watch HelmReleases that either:
305385
// 1. Have the specific error, OR
306-
// 2. Are suspended with our annotation (to handle crash recovery)
386+
// 2. Are suspended with our ownership marker (crash recovery and a delete
387+
// arriving mid-recovery both need to wake a reconcile).
307388
pred := predicate.NewPredicateFuncs(func(obj client.Object) bool {
308389
hr, ok := obj.(*helmv2.HelmRelease)
309390
if !ok {
@@ -315,11 +396,9 @@ func (r *FluxPlunger) SetupWithManager(mgr ctrl.Manager) error {
315396
return true
316397
}
317398

318-
// Also process suspended HelmReleases with our annotation (crash recovery)
319-
if hr.Spec.Suspend && hr.Annotations != nil {
320-
if _, exists := hr.Annotations[annotationLastProcessedVersion]; exists {
321-
return true
322-
}
399+
// Also process suspended HelmReleases flux-plunger owns.
400+
if hr.Spec.Suspend && ownsSuspension(hr) {
401+
return true
323402
}
324403

325404
return false

0 commit comments

Comments
 (0)