Skip to content

Commit 94b4d94

Browse files
mnenciaarmru
authored andcommitted
fix: re-evaluate applied objects after the cluster is demoted (#10871)
Once a Database, Publication, Subscription, or DatabaseRole was reconciled, the controller exited before the replica gate and never requeued, so a cluster demoted to a replica kept reporting the stale primary-side status forever. Re-check the replica condition inside the already-reconciled exit and watch the Cluster spec so the demotion is detected promptly. The recorded reconciliation is kept while the replica condition is reported, so the object retains ownership of the PostgreSQL entity it manages and a conflicting object created meanwhile cannot take it over once the cluster is promoted back; re-application is driven off the applied status instead. For DatabaseRole the same applies when an inline managed.roles entry takes over an already-applied role. This moves the Cluster fetch above the already-reconciled gate, so it overlaps with #10853 in the test files; whichever lands second needs a trivial rebase. Closes #10867 Assisted-by: Claude Signed-off-by: Marco Nenciarini <[email protected]> Signed-off-by: Armando Ruocco <[email protected]> Co-authored-by: Armando Ruocco <[email protected]> (cherry picked from commit 8183cf9)
1 parent 815580c commit 94b4d94

10 files changed

Lines changed: 570 additions & 15 deletions

‎api/v1/database_funcs.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,11 @@ func (db *Database) GetStatusMessage() string {
4848
return db.Status.Message
4949
}
5050

51+
// GetStatusApplied returns the applied status of the database
52+
func (db *Database) GetStatusApplied() *bool {
53+
return db.Status.Applied
54+
}
55+
5156
// GetClusterRef returns the cluster reference of the database
5257
func (db *Database) GetClusterRef() corev1.LocalObjectReference {
5358
return db.Spec.ClusterRef

‎api/v1/publication_funcs.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,11 @@ func (pub *Publication) GetStatusMessage() string {
4848
return pub.Status.Message
4949
}
5050

51+
// GetStatusApplied returns the applied status of the publication
52+
func (pub *Publication) GetStatusApplied() *bool {
53+
return pub.Status.Applied
54+
}
55+
5156
// GetClusterRef returns the cluster reference of the publication
5257
func (pub *Publication) GetClusterRef() corev1.LocalObjectReference {
5358
return pub.Spec.ClusterRef

‎api/v1/subscription_funcs.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,11 @@ func (sub *Subscription) GetStatusMessage() string {
4848
return sub.Status.Message
4949
}
5050

51+
// GetStatusApplied returns the applied status of the subscription
52+
func (sub *Subscription) GetStatusApplied() *bool {
53+
return sub.Status.Applied
54+
}
55+
5156
// GetClusterRef returns the cluster reference of the subscription
5257
func (sub *Subscription) GetClusterRef() corev1.LocalObjectReference {
5358
return sub.Spec.ClusterRef

‎internal/management/controller/common.go‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,13 @@ import (
3232
"github.com/cloudnative-pg/machinery/pkg/log"
3333
"github.com/jackc/pgx/v5"
3434
"github.com/lib/pq"
35+
corev1 "k8s.io/api/core/v1"
36+
apimeta "k8s.io/apimachinery/pkg/api/meta"
3537
"k8s.io/apimachinery/pkg/types"
3638
ctrl "sigs.k8s.io/controller-runtime"
3739
"sigs.k8s.io/controller-runtime/pkg/client"
40+
"sigs.k8s.io/controller-runtime/pkg/handler"
41+
"sigs.k8s.io/controller-runtime/pkg/reconcile"
3842

3943
apiv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1"
4044
)
@@ -82,6 +86,59 @@ func markAsUnknown(
8286
return cli.Status().Update(ctx, resource)
8387
}
8488

89+
type replicaTransitionAware interface {
90+
markableAsUnknown
91+
GetStatusApplied() *bool
92+
}
93+
94+
// handleReplicaRoleTransition decides what to do with a resource whose
95+
// current generation has already been reconciled, reacting to its cluster
96+
// moving in or out of the replica role.
97+
//
98+
// When the cluster is demoted, the designated primary reports the replica
99+
// condition on the resource, keeping the recorded reconciliation so the
100+
// resource retains the ownership of the Postgres object it manages. When the
101+
// cluster is promoted back (or the last evaluation didn't succeed), it asks
102+
// the caller to evaluate the resource again by returning proceed=true; the
103+
// regular reconciliation flow then decides which pod acts.
104+
func handleReplicaRoleTransition(
105+
ctx context.Context,
106+
cli client.Client,
107+
instance instanceInterface,
108+
cluster *apiv1.Cluster,
109+
resource replicaTransitionAware,
110+
requeueAfter time.Duration,
111+
) (result ctrl.Result, proceed bool, err error) {
112+
if !resource.GetDeletionTimestamp().IsZero() {
113+
return ctrl.Result{}, false, nil
114+
}
115+
116+
applied := resource.GetStatusApplied()
117+
118+
if !cluster.IsReplica() {
119+
proceed := applied == nil || !*applied
120+
return ctrl.Result{}, proceed, nil
121+
}
122+
123+
isDesignatedPrimary := cluster.Status.CurrentPrimary == instance.GetPodName()
124+
125+
if isDesignatedPrimary && applied != nil {
126+
if err := markAsUnknown(ctx, cli, resource, errClusterIsReplica); err != nil {
127+
return ctrl.Result{}, false, err
128+
}
129+
}
130+
131+
// Keep polling while the designated primary awaits the promotion, or
132+
// while the primary recorded in the cluster status is still settling
133+
// (e.g. a failover concurrent with the demotion): status-only updates
134+
// don't retrigger the cluster watch.
135+
if isDesignatedPrimary || applied != nil {
136+
return ctrl.Result{RequeueAfter: requeueAfter}, false, nil
137+
}
138+
139+
return ctrl.Result{}, false, nil
140+
}
141+
85142
type markableAsReady interface {
86143
client.Object
87144
SetAsReady()
@@ -97,6 +154,59 @@ func markAsReady(
97154
return cli.Status().Update(ctx, resource)
98155
}
99156

157+
// clusterScopedResource is a resource bound to a Cluster through a local
158+
// object reference.
159+
type clusterScopedResource interface {
160+
client.Object
161+
GetClusterRef() corev1.LocalObjectReference
162+
}
163+
164+
// mapClusterToManagedResources builds a watch mapper that enqueues every item
165+
// of the given list type targeting this instance's cluster, to react to
166+
// cluster spec changes (e.g. a demotion to replica).
167+
func mapClusterToManagedResources(
168+
instance instanceInterface,
169+
cli client.Client,
170+
newList func() client.ObjectList,
171+
) handler.MapFunc {
172+
return func(ctx context.Context, obj client.Object) []reconcile.Request {
173+
if obj.GetName() != instance.GetClusterName() ||
174+
obj.GetNamespace() != instance.GetNamespaceName() {
175+
return nil
176+
}
177+
178+
list := newList()
179+
if err := cli.List(ctx, list, client.InNamespace(obj.GetNamespace())); err != nil {
180+
log.FromContext(ctx).Error(err, "while listing resources to react to a cluster change",
181+
"kind", fmt.Sprintf("%T", list))
182+
return nil
183+
}
184+
185+
items, err := apimeta.ExtractList(list)
186+
if err != nil {
187+
log.FromContext(ctx).Error(err, "while extracting the resource list",
188+
"kind", fmt.Sprintf("%T", list))
189+
return nil
190+
}
191+
192+
requests := make([]reconcile.Request, 0, len(items))
193+
for _, item := range items {
194+
resource, ok := item.(clusterScopedResource)
195+
if !ok || resource.GetClusterRef().Name != obj.GetName() {
196+
continue
197+
}
198+
requests = append(requests, reconcile.Request{
199+
NamespacedName: types.NamespacedName{
200+
Namespace: resource.GetNamespace(),
201+
Name: resource.GetName(),
202+
},
203+
})
204+
}
205+
206+
return requests
207+
}
208+
}
209+
100210
func getClusterFromInstance(
101211
ctx context.Context,
102212
cli client.Client,

‎internal/management/controller/database_controller.go‎

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,11 @@ import (
2828
"github.com/cloudnative-pg/machinery/pkg/log"
2929
"k8s.io/apimachinery/pkg/runtime"
3030
ctrl "sigs.k8s.io/controller-runtime"
31+
"sigs.k8s.io/controller-runtime/pkg/builder"
3132
"sigs.k8s.io/controller-runtime/pkg/client"
33+
"sigs.k8s.io/controller-runtime/pkg/handler"
3234
"sigs.k8s.io/controller-runtime/pkg/manager"
35+
"sigs.k8s.io/controller-runtime/pkg/predicate"
3336

3437
apiv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1"
3538
"github.com/cloudnative-pg/cloudnative-pg/pkg/management/postgres"
@@ -115,17 +118,29 @@ func (r *DatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
115118
return ctrl.Result{}, nil
116119
}
117120

118-
// If everything is reconciled, we're done here
119-
if database.Generation == database.Status.ObservedGeneration {
120-
return ctrl.Result{}, nil
121-
}
122-
123121
// Fetch the Cluster from the cache
124122
cluster, err := r.GetCluster(ctx)
125123
if err != nil {
124+
// A reconciled database keeps its status: the cluster may be gone
125+
// or unreadable while it is being deleted.
126+
if database.Generation == database.Status.ObservedGeneration {
127+
return ctrl.Result{}, client.IgnoreNotFound(err)
128+
}
126129
return ctrl.Result{}, markAsFailed(ctx, r.Client, &database, fmt.Errorf("while fetching the cluster: %w", err))
127130
}
128131

132+
// If everything is reconciled, we're done here
133+
if database.Generation == database.Status.ObservedGeneration {
134+
// ...unless the cluster moved in or out of the replica role after
135+
// the database was applied: report the demotion on the status, and
136+
// evaluate the database again after the promotion.
137+
result, proceed, err := handleReplicaRoleTransition(
138+
ctx, r.Client, r.instance, cluster, &database, databaseReconciliationInterval)
139+
if err != nil || !proceed {
140+
return result, err
141+
}
142+
}
143+
129144
contextLogger.Info("Reconciling database")
130145
defer func() {
131146
contextLogger.Info("Reconciliation loop of database exited")
@@ -244,6 +259,13 @@ func NewDatabaseReconciler(
244259
func (r *DatabaseReconciler) SetupWithManager(mgr ctrl.Manager) error {
245260
return ctrl.NewControllerManagedBy(mgr).
246261
For(&apiv1.Database{}).
262+
Watches(
263+
&apiv1.Cluster{},
264+
handler.EnqueueRequestsFromMapFunc(mapClusterToManagedResources(
265+
r.instance, mgr.GetClient(),
266+
func() client.ObjectList { return &apiv1.DatabaseList{} })),
267+
builder.WithPredicates(predicate.GenerationChangedPredicate{}),
268+
).
247269
Named("instance-database").
248270
Complete(r)
249271
}

0 commit comments

Comments
 (0)