Skip to content

Commit 14b3d17

Browse files
GabriFedi97armrumnencia
authored
fix: register the cluster Healthy phase as the last reconciliation step (#10421)
Register the Cluster phase as Healthy as the last status mutation of a successful reconciliation loop. Previously the phase was set to Healthy mid-loop, before the post-reconcile plugin hooks ran. When a hook then returned an error, the error path set PhaseFailurePlugin, so the phase flipped between Healthy and FailurePlugin and the cluster never settled on a stable phase. Registering Healthy at the end means a loop that ends in a plugin error never reports Healthy. Healthy is still registered when a plugin only requests a requeue (for example the periodic status sync that requeues every 5s), so clusters with status-reporting plugins still reach Healthy. It is suppressed only when a plugin operation returns an error. Closes #8582 Signed-off-by: Gabriele Fedi <[email protected]> Signed-off-by: Armando Ruocco <[email protected]> Signed-off-by: Marco Nenciarini <[email protected]> Co-authored-by: Armando Ruocco <[email protected]> Co-authored-by: Marco Nenciarini <[email protected]>
1 parent f307dd6 commit 14b3d17

3 files changed

Lines changed: 191 additions & 16 deletions

File tree

‎internal/controller/cluster_controller.go‎

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -599,15 +599,9 @@ func (r *ClusterReconciler) reconcile(ctx context.Context, cluster *apiv1.Cluste
599599
return res, err
600600
}
601601

602-
// Calls post-reconcile hooks
603-
if hookResult := postReconcilePluginHooks(ctx, cluster, cluster); hookResult.Err != nil ||
604-
!hookResult.Result.IsZero() {
605-
contextLogger.Info("Post-reconcile hook stopped the reconciliation loop",
606-
"hookResult", hookResult)
607-
return hookResult.Result, hookResult.Err
608-
}
609-
610-
return setStatusPluginHook(ctx, r.Client, cnpgiClient.GetPluginClientFromContext(ctx), cluster)
602+
// Run plugin post-reconcile hooks, sync per-plugin statuses, and
603+
// register PhaseHealthy as the LAST status mutation. See #8582.
604+
return r.finalizeReconciliation(ctx, cnpgiClient.GetPluginClientFromContext(ctx), cluster)
611605
}
612606

613607
// evaluatePodReadinessGuards short-circuits the reconciliation loop with a
@@ -966,11 +960,6 @@ func (r *ClusterReconciler) reconcileResources(
966960
return ctrl.Result{RequeueAfter: 1 * time.Second}, ErrNextLoop
967961
}
968962

969-
// When everything is reconciled, update the status
970-
if err := r.RegisterPhase(ctx, cluster, apiv1.PhaseHealthy, ""); err != nil {
971-
return ctrl.Result{}, err
972-
}
973-
974963
r.cleanupCompletedJobs(ctx, resources.jobs)
975964

976965
return ctrl.Result{}, nil

‎internal/controller/plugins.go‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,3 +88,50 @@ func setStatusPluginHook(
8888
}
8989
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
9090
}
91+
92+
// finalizeReconciliation runs the post-reconcile plugin hooks, syncs the
93+
// per-plugin statuses, and registers PhaseHealthy as the very last status
94+
// mutation of the reconciliation loop.
95+
//
96+
// PhaseHealthy MUST be the final status update of a successful loop. Any
97+
// later phase patch (e.g. an error path that registers PhaseFailurePlugin)
98+
// would be overwritten by the error path on the next reconciliation,
99+
// oscillating Phase between Healthy and the error phase. See #8582.
100+
//
101+
// The result is propagated from the underlying plugin operations so any
102+
// requeue they request (notably the 5s polling from setStatusPluginHook
103+
// after a successful status patch) is honored alongside the Healthy
104+
// registration.
105+
func (r *ClusterReconciler) finalizeReconciliation(
106+
ctx context.Context,
107+
pluginClient cnpgiClient.Client,
108+
cluster *apiv1.Cluster,
109+
) (ctrl.Result, error) {
110+
contextLogger := log.FromContext(ctx)
111+
112+
hookResult := pluginClient.PostReconcile(ctx, cluster, cluster)
113+
if hookResult.Err != nil {
114+
contextLogger.Info("Post-reconcile hook returned an error",
115+
"hookResult", hookResult)
116+
return hookResult.Result, hookResult.Err
117+
}
118+
if !hookResult.Result.IsZero() {
119+
contextLogger.Info("Post-reconcile hook requested a requeue",
120+
"hookResult", hookResult)
121+
}
122+
123+
res := hookResult.Result
124+
if res.IsZero() {
125+
var err error
126+
res, err = setStatusPluginHook(ctx, r.Client, pluginClient, cluster)
127+
if err != nil {
128+
return res, err
129+
}
130+
}
131+
132+
if err := r.RegisterPhase(ctx, cluster, apiv1.PhaseHealthy, ""); err != nil {
133+
return ctrl.Result{}, err
134+
}
135+
136+
return res, nil
137+
}

‎internal/controller/plugins_test.go‎

Lines changed: 141 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,11 @@ package controller
2222
import (
2323
"context"
2424
"encoding/json"
25+
"errors"
26+
"time"
2527

2628
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
29+
ctrl "sigs.k8s.io/controller-runtime"
2730
k8client "sigs.k8s.io/controller-runtime/pkg/client"
2831
"sigs.k8s.io/controller-runtime/pkg/client/fake"
2932

@@ -37,14 +40,26 @@ import (
3740

3841
type fakePluginClient struct {
3942
pluginClient.Client
40-
setClusterStatus map[string]string
43+
setClusterStatus map[string]string
44+
setClusterStatusErr error
45+
setStatusInClusterCalled bool
46+
postReconcileResult pluginClient.ReconcilerHookResult
4147
}
4248

4349
func (f *fakePluginClient) SetStatusInCluster(
4450
_ context.Context,
4551
_ k8client.Object,
4652
) (map[string]string, error) {
47-
return f.setClusterStatus, nil
53+
f.setStatusInClusterCalled = true
54+
return f.setClusterStatus, f.setClusterStatusErr
55+
}
56+
57+
func (f *fakePluginClient) PostReconcile(
58+
_ context.Context,
59+
_ k8client.Object,
60+
_ k8client.Object,
61+
) pluginClient.ReconcilerHookResult {
62+
return f.postReconcileResult
4863
}
4964

5065
var _ = Describe("setStatusPluginHook", func() {
@@ -88,3 +103,127 @@ var _ = Describe("setStatusPluginHook", func() {
88103
Expect(cluster.Status.PluginStatus[0].Status).To(BeEquivalentTo(string(content)))
89104
})
90105
})
106+
107+
var _ = Describe("finalizeReconciliation", func() {
108+
const pluginName = "test1_plugin"
109+
const initialPhase = "InitialPhase"
110+
111+
var (
112+
cluster *apiv1.Cluster
113+
cli k8client.Client
114+
pluginCli *fakePluginClient
115+
reconciler *ClusterReconciler
116+
)
117+
118+
BeforeEach(func() {
119+
cluster = &apiv1.Cluster{
120+
ObjectMeta: metav1.ObjectMeta{
121+
Name: "test",
122+
Namespace: "test-suite",
123+
},
124+
Status: apiv1.ClusterStatus{
125+
Phase: initialPhase,
126+
PluginStatus: []apiv1.PluginStatus{
127+
{Name: pluginName},
128+
},
129+
},
130+
}
131+
cli = fake.NewClientBuilder().
132+
WithObjects(cluster).
133+
WithScheme(scheme.BuildWithAllKnownScheme()).
134+
WithStatusSubresource(&apiv1.Cluster{}).
135+
Build()
136+
137+
pluginCli = &fakePluginClient{}
138+
reconciler = &ClusterReconciler{
139+
Client: cli,
140+
}
141+
})
142+
143+
It("registers PhaseHealthy when no plugin sets a status", func(ctx SpecContext) {
144+
res, err := reconciler.finalizeReconciliation(ctx, pluginCli, cluster)
145+
146+
Expect(err).ToNot(HaveOccurred())
147+
Expect(res.IsZero()).To(BeTrue())
148+
149+
var fresh apiv1.Cluster
150+
Expect(cli.Get(ctx, k8client.ObjectKeyFromObject(cluster), &fresh)).To(Succeed())
151+
Expect(fresh.Status.Phase).To(Equal(apiv1.PhaseHealthy))
152+
})
153+
154+
It("registers PhaseHealthy and propagates the requeue when plugins patch their statuses", func(ctx SpecContext) {
155+
// setStatusPluginHook returns RequeueAfter=5s on every successful
156+
// status patch. That requeue must NOT short-circuit the PhaseHealthy
157+
// registration: clusters with status-reporting plugins (e.g.
158+
// barman-cloud) requeue every 5s in steady state and would otherwise
159+
// never reach Healthy.
160+
content, err := json.Marshal(map[string]string{"key": "value"})
161+
Expect(err).ToNot(HaveOccurred())
162+
pluginCli.setClusterStatus = map[string]string{pluginName: string(content)}
163+
164+
res, err := reconciler.finalizeReconciliation(ctx, pluginCli, cluster)
165+
166+
Expect(err).ToNot(HaveOccurred())
167+
Expect(res.RequeueAfter).To(Equal(5 * time.Second))
168+
169+
var fresh apiv1.Cluster
170+
Expect(cli.Get(ctx, k8client.ObjectKeyFromObject(cluster), &fresh)).To(Succeed())
171+
Expect(fresh.Status.Phase).To(Equal(apiv1.PhaseHealthy))
172+
Expect(fresh.Status.PluginStatus[0].Status).To(BeEquivalentTo(string(content)))
173+
})
174+
175+
It("does not register PhaseHealthy when post-reconcile returns an error", func(ctx SpecContext) {
176+
// Regression test for #8582: when a post-reconcile plugin hook
177+
// returns an error, the loop must not also mark the cluster
178+
// Healthy. Otherwise the next reconciliation overwrites the
179+
// PhaseFailurePlugin set by Reconcile()'s error path and the
180+
// cluster oscillates between Healthy and FailurePlugin.
181+
expectedErr := errors.New("plugin post-reconcile failure")
182+
pluginCli.postReconcileResult = pluginClient.ReconcilerHookResult{Err: expectedErr}
183+
184+
_, err := reconciler.finalizeReconciliation(ctx, pluginCli, cluster)
185+
186+
Expect(err).To(MatchError(expectedErr))
187+
// A post-reconcile error must short-circuit before the status sync.
188+
Expect(pluginCli.setStatusInClusterCalled).To(BeFalse())
189+
190+
var fresh apiv1.Cluster
191+
Expect(cli.Get(ctx, k8client.ObjectKeyFromObject(cluster), &fresh)).To(Succeed())
192+
Expect(fresh.Status.Phase).To(Equal(initialPhase))
193+
})
194+
195+
It("registers PhaseHealthy when post-reconcile requests a requeue without error", func(ctx SpecContext) {
196+
// A plugin requesting a requeue (no error) from PostReconcile must
197+
// still transition the cluster to Healthy, and the status sync is
198+
// skipped because the requeue already short-circuits the loop.
199+
pluginCli.postReconcileResult = pluginClient.ReconcilerHookResult{
200+
Result: ctrl.Result{RequeueAfter: 30 * time.Second},
201+
}
202+
203+
res, err := reconciler.finalizeReconciliation(ctx, pluginCli, cluster)
204+
205+
Expect(err).ToNot(HaveOccurred())
206+
Expect(res.RequeueAfter).To(Equal(30 * time.Second))
207+
Expect(pluginCli.setStatusInClusterCalled).To(BeFalse())
208+
209+
var fresh apiv1.Cluster
210+
Expect(cli.Get(ctx, k8client.ObjectKeyFromObject(cluster), &fresh)).To(Succeed())
211+
Expect(fresh.Status.Phase).To(Equal(apiv1.PhaseHealthy))
212+
})
213+
214+
It("does not register PhaseHealthy when SetStatusInCluster returns an error", func(ctx SpecContext) {
215+
expectedErr := errors.New("set status failure")
216+
pluginCli.setClusterStatusErr = expectedErr
217+
218+
_, err := reconciler.finalizeReconciliation(ctx, pluginCli, cluster)
219+
220+
// errors.Is to ensure the wrap chain is preserved (%w, not %v):
221+
// downstream callers depend on Unwrap to classify plugin errors via
222+
// cnpgiClient.ContainsPluginError.
223+
Expect(errors.Is(err, expectedErr)).To(BeTrue())
224+
225+
var fresh apiv1.Cluster
226+
Expect(cli.Get(ctx, k8client.ObjectKeyFromObject(cluster), &fresh)).To(Succeed())
227+
Expect(fresh.Status.Phase).To(Equal(initialPhase))
228+
})
229+
})

0 commit comments

Comments
 (0)