Skip to content

Commit 11b46c2

Browse files
ermakov-olegarmru
andauthored
fix(rollout): prevent supervised primary strategy from starving rollout slots (#9977)
The rollout manager uses a single global slot to coordinate Pod rollouts across all clusters. When a rollout is initiated, the slot is claimed and held for a configurable delay before another rollout can proceed. Previously, clusters using the supervised primary update strategy would claim the slot even though they only wait for user action (a manual switchover). This blocked the other clusters from performing their rollouts until the user intervened. This fix moves the supervised strategy check ensuring that supervised primaries never claim the slot. The slot is now only occupied when an actual rollout will proceed. Signed-off-by: ermakov-oleg <[email protected]> Signed-off-by: Armando Ruocco <[email protected]> Co-authored-by: Armando Ruocco <[email protected]>
1 parent ced9901 commit 11b46c2

4 files changed

Lines changed: 202 additions & 20 deletions

File tree

‎internal/controller/cluster_upgrade.go‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,20 @@ func (r *ClusterReconciler) rolloutRequiredInstances(
125125
return false, nil
126126
}
127127

128+
// If the primary update strategy is supervised, we should not consume a rollout slot.
129+
// The user must issue a switchover manually.
130+
if cluster.GetPrimaryUpdateStrategy() == apiv1.PrimaryUpdateStrategySupervised {
131+
contextLogger := log.FromContext(ctx)
132+
contextLogger.Info("Waiting for the user to request a switchover to complete the rolling update",
133+
"primaryPod", primaryPostgresqlStatus.Pod.Name,
134+
"reason", podRollout.reason)
135+
if err := r.RegisterPhase(ctx, cluster, apiv1.PhaseWaitingForUser,
136+
"User must issue a supervised switchover"); err != nil {
137+
return false, err
138+
}
139+
return true, nil
140+
}
141+
128142
managerResult := r.rolloutManager.CoordinateRollout(
129143
client.ObjectKeyFromObject(cluster),
130144
primaryPostgresqlStatus.Pod.Name)
@@ -154,19 +168,7 @@ func (r *ClusterReconciler) updatePrimaryPod(
154168
reason rolloutReason,
155169
) (bool, error) {
156170
contextLogger := log.FromContext(ctx)
157-
158-
// we need to check whether a manual switchover is required
159171
contextLogger = contextLogger.WithValues("primaryPod", primaryPod.Name)
160-
if cluster.GetPrimaryUpdateStrategy() == apiv1.PrimaryUpdateStrategySupervised {
161-
contextLogger.Info("Waiting for the user to request a switchover to complete the rolling update",
162-
"reason", reason)
163-
err := r.RegisterPhase(ctx, cluster, apiv1.PhaseWaitingForUser, "User must issue a supervised switchover")
164-
if err != nil {
165-
return false, err
166-
}
167-
168-
return true, nil
169-
}
170172

171173
if cluster.GetPrimaryUpdateMethod() == apiv1.PrimaryUpdateMethodRestart || forceRecreate {
172174
if inPlacePossible {

‎internal/controller/cluster_upgrade_test.go‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,16 +22,21 @@ package controller
2222
import (
2323
"context"
2424
"encoding/json"
25+
"time"
2526

2627
corev1 "k8s.io/api/core/v1"
2728
"k8s.io/apimachinery/pkg/api/resource"
2829
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
30+
"k8s.io/client-go/tools/record"
2931
k8client "sigs.k8s.io/controller-runtime/pkg/client"
32+
"sigs.k8s.io/controller-runtime/pkg/client/fake"
3033

3134
apiv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1"
3235
"github.com/cloudnative-pg/cloudnative-pg/internal/cnpi/plugin"
3336
pluginClient "github.com/cloudnative-pg/cloudnative-pg/internal/cnpi/plugin/client"
3437
"github.com/cloudnative-pg/cloudnative-pg/internal/configuration"
38+
rolloutManager "github.com/cloudnative-pg/cloudnative-pg/internal/controller/rollout"
39+
schemeBuilder "github.com/cloudnative-pg/cloudnative-pg/internal/scheme"
3540
"github.com/cloudnative-pg/cloudnative-pg/pkg/postgres"
3641
"github.com/cloudnative-pg/cloudnative-pg/pkg/specs"
3742
"github.com/cloudnative-pg/cloudnative-pg/pkg/utils"
@@ -919,3 +924,144 @@ var _ = Describe("checkPodSpec with plugins", Ordered, func() {
919924
"original and target PodSpec differ in containers: container postgres differs in environment"))
920925
})
921926
})
927+
928+
var _ = Describe("Supervised primary update strategy and rollout slots", func() {
929+
const namespace = "supervised-test"
930+
931+
var (
932+
reconciler *ClusterReconciler
933+
rm *rolloutManager.Manager
934+
k8sClient k8client.Client
935+
)
936+
937+
BeforeEach(func() {
938+
scheme := schemeBuilder.BuildWithAllKnownScheme()
939+
k8sClient = fake.NewClientBuilder().
940+
WithScheme(scheme).
941+
WithStatusSubresource(&apiv1.Cluster{}).
942+
Build()
943+
944+
// Create namespace
945+
ns := &corev1.Namespace{
946+
ObjectMeta: metav1.ObjectMeta{Name: namespace},
947+
}
948+
Expect(k8sClient.Create(context.Background(), ns)).To(Succeed())
949+
950+
// Rollout manager with a large cluster delay so we can detect slot consumption
951+
rm = rolloutManager.New(time.Hour, 0)
952+
953+
reconciler = &ClusterReconciler{
954+
Client: k8sClient,
955+
Scheme: scheme,
956+
Recorder: record.NewFakeRecorder(120),
957+
rolloutManager: rm,
958+
}
959+
960+
configuration.Current = configuration.NewConfiguration()
961+
})
962+
963+
// Helper to create a cluster in the fake client with the given primary update strategy
964+
createCluster := func(strategy apiv1.PrimaryUpdateStrategy) *apiv1.Cluster {
965+
cluster := &apiv1.Cluster{
966+
ObjectMeta: metav1.ObjectMeta{
967+
Name: "test-cluster",
968+
Namespace: namespace,
969+
},
970+
Spec: apiv1.ClusterSpec{
971+
Instances: 1,
972+
ImageName: "postgres:16.0",
973+
PrimaryUpdateStrategy: strategy,
974+
PrimaryUpdateMethod: apiv1.PrimaryUpdateMethodRestart,
975+
StorageConfiguration: apiv1.StorageConfiguration{Size: "1Gi"},
976+
},
977+
}
978+
cluster.SetDefaults()
979+
// Set status fields after SetDefaults() since it may clear them
980+
cluster.Status.CurrentPrimary = "test-cluster-1"
981+
cluster.Status.Image = "postgres:16.1"
982+
cluster.Status.Instances = 1
983+
Expect(k8sClient.Create(context.Background(), cluster)).To(Succeed())
984+
Expect(k8sClient.Status().Update(context.Background(), cluster)).To(Succeed())
985+
return cluster
986+
}
987+
988+
// Helper to build a pod status list where the primary needs rollout (image mismatch)
989+
buildPodListWithPrimaryNeedingRollout := func(cluster *apiv1.Cluster) *postgres.PostgresqlStatusList {
990+
pod := &corev1.Pod{
991+
ObjectMeta: metav1.ObjectMeta{
992+
Name: cluster.Status.CurrentPrimary,
993+
Namespace: cluster.Namespace,
994+
Annotations: map[string]string{
995+
utils.ClusterSerialAnnotationName: "1",
996+
},
997+
},
998+
Spec: corev1.PodSpec{
999+
Containers: []corev1.Container{
1000+
{
1001+
Name: "postgres",
1002+
Image: "postgres:16.0", // Different from cluster.Status.Image (16.1) -> triggers rollout
1003+
},
1004+
},
1005+
},
1006+
}
1007+
return &postgres.PostgresqlStatusList{
1008+
Items: []postgres.PostgresqlStatus{
1009+
{
1010+
Pod: pod,
1011+
IsPodReady: true,
1012+
ExecutableHash: "test_hash",
1013+
},
1014+
},
1015+
}
1016+
}
1017+
1018+
It("supervised cluster does NOT consume a rollout slot", func(ctx SpecContext) {
1019+
cluster := createCluster(apiv1.PrimaryUpdateStrategySupervised)
1020+
podList := buildPodListWithPrimaryNeedingRollout(cluster)
1021+
1022+
restarted, err := reconciler.rolloutRequiredInstances(ctx, cluster, podList)
1023+
Expect(err).ToNot(HaveOccurred())
1024+
Expect(restarted).To(BeTrue())
1025+
1026+
// Verify the rollout slot was NOT consumed: a second cluster should still be allowed
1027+
secondCluster := k8client.ObjectKey{Namespace: namespace, Name: "other-cluster"}
1028+
result := rm.CoordinateRollout(secondCluster, "other-pod")
1029+
Expect(result.RolloutAllowed).To(BeTrue(),
1030+
"supervised strategy should not consume the rollout slot")
1031+
})
1032+
1033+
It("supervised cluster returns true and sets PhaseWaitingForUser", func(ctx SpecContext) {
1034+
cluster := createCluster(apiv1.PrimaryUpdateStrategySupervised)
1035+
podList := buildPodListWithPrimaryNeedingRollout(cluster)
1036+
1037+
restarted, err := reconciler.rolloutRequiredInstances(ctx, cluster, podList)
1038+
Expect(err).ToNot(HaveOccurred())
1039+
Expect(restarted).To(BeTrue())
1040+
1041+
// Re-fetch the cluster to see the updated status
1042+
var updatedCluster apiv1.Cluster
1043+
Expect(k8sClient.Get(ctx,
1044+
k8client.ObjectKeyFromObject(cluster),
1045+
&updatedCluster)).To(Succeed())
1046+
Expect(updatedCluster.Status.Phase).To(Equal(apiv1.PhaseWaitingForUser))
1047+
})
1048+
1049+
It("unsupervised cluster DOES consume a rollout slot", func(ctx SpecContext) {
1050+
cluster := createCluster(apiv1.PrimaryUpdateStrategyUnsupervised)
1051+
podList := buildPodListWithPrimaryNeedingRollout(cluster)
1052+
1053+
// Create the pod in the fake client so upgradePod (Delete) can find it
1054+
pod := podList.Items[0].Pod.DeepCopy()
1055+
Expect(k8sClient.Create(ctx, pod)).To(Succeed())
1056+
1057+
restarted, err := reconciler.rolloutRequiredInstances(ctx, cluster, podList)
1058+
Expect(err).ToNot(HaveOccurred())
1059+
Expect(restarted).To(BeTrue())
1060+
1061+
// Verify the rollout slot WAS consumed: a second cluster should be blocked
1062+
secondCluster := k8client.ObjectKey{Namespace: namespace, Name: "other-cluster"}
1063+
result := rm.CoordinateRollout(secondCluster, "other-pod")
1064+
Expect(result.RolloutAllowed).To(BeFalse(),
1065+
"unsupervised strategy should consume the rollout slot")
1066+
})
1067+
})

‎internal/controller/rollout/doc.go‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,27 @@ SPDX-License-Identifier: Apache-2.0
1818
*/
1919

2020
// Package rollout contains the rollout manager, allowing
21-
// CloudNative-PG to spread Pod rollouts depending on
22-
// the passed configuration
21+
// CloudNativePG to spread Pod rollouts depending on
22+
// the passed configuration.
23+
//
24+
// The manager uses a single global slot to enforce that only one Pod
25+
// rollout proceeds at a time across all clusters managed by the
26+
// operator. When [Manager.CoordinateRollout] is called and the rollout
27+
// is allowed, the manager records the cluster, instance, and current
28+
// timestamp. Subsequent calls are denied until the configured delay
29+
// has elapsed: [Manager.clusterRolloutDelay] when the caller is a
30+
// different cluster, or [Manager.instanceRolloutDelay] when it is the
31+
// same cluster.
32+
//
33+
// Because the slot is global, callers that will not actually perform a
34+
// rollout must avoid calling [Manager.CoordinateRollout]. For example,
35+
// a cluster whose primary update strategy is "supervised" waits
36+
// indefinitely for a user-initiated switchover; if it claimed the slot,
37+
// it would block every other cluster from rolling out until the delay
38+
// expires, and would re-claim it on every reconciliation loop,
39+
// effectively starving other clusters.
40+
//
41+
// The delays are configured via the CLUSTERS_ROLLOUT_DELAY and
42+
// INSTANCES_ROLLOUT_DELAY operator environment variables (both
43+
// default to 0, meaning no delay).
2344
package rollout

‎internal/controller/rollout/rollout.go‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,13 @@ import (
2929
// The type of functions returning a moment in time
3030
type timeFunc func() time.Time
3131

32-
// Manager is the rollout manager. It is safe to use
33-
// concurrently
32+
// Manager is the rollout manager. It is safe to use concurrently.
33+
//
34+
// A single Manager instance is shared across all cluster reconciliations.
35+
// It tracks the most recent rollout in a single slot (lastCluster,
36+
// lastInstance, lastUpdate). This means only one rollout can be in
37+
// progress at a time: any call to [Manager.CoordinateRollout] that
38+
// arrives before the applicable delay has elapsed will be denied.
3439
type Manager struct {
3540
m sync.Mutex
3641

@@ -46,8 +51,10 @@ type Manager struct {
4651
// used by the unit tests to inject a fake time
4752
timeProvider timeFunc
4853

49-
// The following data is relative to the last
50-
// rollout
54+
// The following fields form the single global slot.
55+
// They record which cluster and instance last performed
56+
// a rollout, and when. All scheduling decisions are
57+
// derived from these three values.
5158
lastInstance string
5259
lastCluster client.ObjectKey
5360
lastUpdate time.Time
@@ -73,8 +80,14 @@ func New(clusterRolloutDelay, instancesRolloutDelay time.Duration) *Manager {
7380
}
7481
}
7582

76-
// CoordinateRollout is called to check whether this rollout is allowed or not
77-
// by the manager
83+
// CoordinateRollout checks whether a Pod rollout is allowed and, when
84+
// allowed, claims the global slot by recording the cluster, instance,
85+
// and current time.
86+
//
87+
// Callers must only invoke this method when they intend to actually
88+
// perform a rollout. Calling it without following through (e.g. for a
89+
// supervised primary that only waits for user action) would occupy the
90+
// slot and block other clusters from rolling out.
7891
func (manager *Manager) CoordinateRollout(
7992
cluster client.ObjectKey,
8093
instanceName string,

0 commit comments

Comments
 (0)