Skip to content

Commit 728f676

Browse files
fix(api): retry HelmRelease update on resourceVersion conflict
The aggregated API server translates app CR updates (Etcd, Postgres, ...) into a read-convert-update of the backing HelmRelease. Flux's helm-controller continuously writes the HelmRelease status, which shares the object's resourceVersion, so a CR update that lands while a prior reconcile is still in flight reads a resourceVersion that goes stale before the Update and the request fails with a 409 Conflict ("the object has been modified; please apply your changes to the latest version and try again"). The HelmRelease spec is fully derived from the Application the caller just applied, so a stale-resourceVersion conflict is never a real spec conflict here. Wrap the Update in retry.RetryOnConflict, refreshing the live resourceVersion on each conflict before retrying. The no-conflict path is unchanged, and this mirrors the existing RetryOnConflict usage in workloadmonitor_controller.go. Surfaced by the etcd.bats e2e test, which applies the same singleton Etcd resource three times back-to-back and intermittently failed the third apply with the 409. Assisted-By: Claude <[email protected]> Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]> Signed-off-by: Myasnikov Daniil <[email protected]> (cherry picked from commit 7641059)
1 parent 770ce36 commit 728f676

1 file changed

Lines changed: 22 additions & 2 deletions

File tree

  • pkg/registry/apps/application

‎pkg/registry/apps/application/rest.go‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ import (
4141
"k8s.io/apimachinery/pkg/watch"
4242
"k8s.io/apiserver/pkg/endpoints/request"
4343
"k8s.io/apiserver/pkg/registry/rest"
44+
"k8s.io/client-go/util/retry"
4445
"k8s.io/klog/v2"
4546
"sigs.k8s.io/controller-runtime/pkg/client"
4647

@@ -566,8 +567,27 @@ func (r *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObje
566567

567568
klog.V(6).Infof("Updating HelmRelease %s in namespace %s", helmRelease.Name, helmRelease.Namespace)
568569

569-
// Update the HelmRelease in Kubernetes
570-
err = r.c.Update(ctx, helmRelease, &client.UpdateOptions{Raw: &metav1.UpdateOptions{}})
570+
// Update the HelmRelease in Kubernetes.
571+
//
572+
// Flux's helm-controller (and other controllers) continuously write the
573+
// HelmRelease's status, which shares the object's resourceVersion. When a
574+
// caller updates an app CR while a prior reconcile is still in flight, the
575+
// resourceVersion read above goes stale and the Update is rejected with a
576+
// 409 Conflict. The HelmRelease spec is fully derived from the Application
577+
// the caller just applied, so a stale-resourceVersion conflict is never a
578+
// real spec conflict here: refresh the resourceVersion from the live object
579+
// and retry.
580+
err = retry.RetryOnConflict(retry.DefaultBackoff, func() error {
581+
updateErr := r.c.Update(ctx, helmRelease, &client.UpdateOptions{Raw: &metav1.UpdateOptions{}})
582+
if apierrors.IsConflict(updateErr) {
583+
cur := &helmv2.HelmRelease{}
584+
if getErr := r.c.Get(ctx, client.ObjectKey{Namespace: helmRelease.Namespace, Name: helmRelease.Name}, cur, &client.GetOptions{Raw: &metav1.GetOptions{}}); getErr != nil {
585+
return getErr
586+
}
587+
helmRelease.SetResourceVersion(cur.GetResourceVersion())
588+
}
589+
return updateErr
590+
})
571591
if err != nil {
572592
klog.Errorf("Failed to update HelmRelease %s: %v", helmRelease.Name, err)
573593
return nil, false, fmt.Errorf("failed to update HelmRelease: %v", err)

0 commit comments

Comments
 (0)