Skip to content

Commit 324709c

Browse files
armrumnencia
andauthored
fix(upgrade): break config uniformity deadlock during operator upgrades (#10342)
When upgrading from an operator version that includes cnpg.* parameters in the PostgreSQL configuration hash to one that excludes them (change introduced in #8868), pods running the old and new operator end up computing different hashes for the same effective configuration. The uniformity check in reconcilePods then blocks indefinitely, waiting for all pods to report the same hash, but that can never happen because the remaining pods still use the old algorithm. Neither the rolling update nor the in-place instance manager upgrade can run, since both live behind the blocked check. The fix introduces `isConfigNonUniformityFromUpgrade()`, which groups pods by their operator version and checks whether the hash split aligns exactly with the version split. If every pod running the same operator version agrees on its hash, but different versions disagree, the non-uniformity is structural and will never self-resolve. In that case we skip the wait and let `handleRollingUpdate` proceed. The operator version key combines both the bootstrap controller image (changes on rolling updates) and the executable hash (changes on in-place upgrades), so the deadlock is detected regardless of the upgrade strategy. When the non-uniformity is from normal config propagation (pods on the same operator version reporting different hashes), the function returns false and the existing wait-for-uniformity behavior is preserved. This keeps the fix scoped strictly to the operator upgrade deadlock scenario, with no change in behavior during normal operations. Closes: #10348 Assisted-by: Claude Signed-off-by: Armando Ruocco <[email protected]> Signed-off-by: Marco Nenciarini <[email protected]> Co-authored-by: Marco Nenciarini <[email protected]>
1 parent 145a109 commit 324709c

3 files changed

Lines changed: 169 additions & 4 deletions

File tree

‎internal/controller/cluster_controller.go‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1088,12 +1088,20 @@ func (r *ClusterReconciler) reconcilePods(
10881088
// proceed with a rolling update to upgrade the instance manager
10891089
// to a version that reports the configuration status.
10901090
// If all pods report their configuration, wait until all instances
1091-
// report the same configuration.
1091+
// report the same configuration, unless the non-uniformity is caused
1092+
// by an operator upgrade that changed the hash algorithm. In that case
1093+
// configuration hashes will never converge on their own; proceeding
1094+
// to the rolling update (or in-place upgrade) is the only way forward.
10921095
if uniform := report.IsUniform(); uniform != nil && !*uniform {
1093-
contextLogger.Debug(
1094-
"Waiting for all Pods to have the same PostgreSQL configuration",
1096+
if !isConfigNonUniformityFromUpgrade(instancesStatus) {
1097+
contextLogger.Debug(
1098+
"Waiting for all Pods to have the same PostgreSQL configuration",
1099+
"configurationReport", report)
1100+
return ctrl.Result{RequeueAfter: 1 * time.Second}, ErrNextLoop
1101+
}
1102+
contextLogger.Info(
1103+
"Configuration non-uniformity caused by operator upgrade, proceeding with upgrade",
10951104
"configurationReport", report)
1096-
return ctrl.Result{RequeueAfter: 1 * time.Second}, ErrNextLoop
10971105
}
10981106

10991107
return r.handleRollingUpdate(ctx, cluster, instancesStatus)

‎internal/controller/cluster_upgrade.go‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -336,6 +336,47 @@ func isInstanceNeedingRollout(
336336
return rollout{}
337337
}
338338

339+
// isConfigNonUniformityFromUpgrade returns true when config hash differences
340+
// across pods are caused by an operator upgrade (different hash algorithms)
341+
// rather than config propagation. It groups pods by operator version (using
342+
// both bootstrap image and executable hash) and checks whether each group
343+
// is internally uniform.
344+
func isConfigNonUniformityFromUpgrade(instancesStatus postgres.PostgresqlStatusList) bool {
345+
type versionKey struct {
346+
image string
347+
executableHash string
348+
}
349+
hashByVersion := make(map[versionKey]string, len(instancesStatus.Items))
350+
for i := range instancesStatus.Items {
351+
pod := instancesStatus.Items[i].Pod
352+
if pod == nil {
353+
continue
354+
}
355+
opImage, err := specs.GetBootstrapControllerImageName(*pod)
356+
if err != nil {
357+
continue
358+
}
359+
configHash := instancesStatus.Items[i].LoadedConfigurationHash
360+
if configHash == "" {
361+
continue
362+
}
363+
key := versionKey{
364+
image: opImage,
365+
executableHash: instancesStatus.Items[i].ExecutableHash,
366+
}
367+
if existing, ok := hashByVersion[key]; ok {
368+
if existing != configHash {
369+
// Same version, different hashes: config still propagating
370+
return false
371+
}
372+
} else {
373+
hashByVersion[key] = configHash
374+
}
375+
}
376+
377+
return len(hashByVersion) > 1
378+
}
379+
339380
// isPodNeedingRollout checks if a given cluster instance needs a rollout by comparing its current state
340381
// with its expected state defined inside the cluster struct.
341382
//

‎internal/controller/cluster_upgrade_test.go‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -820,6 +820,122 @@ var _ = Describe("Cluster upgrade with podSpec reconciliation disabled", func()
820820
})
821821
})
822822

823+
var _ = Describe("isConfigNonUniformityFromUpgrade", func() {
824+
const (
825+
oldOperatorImage = "ghcr.io/cloudnative-pg/cloudnative-pg:1.27.0"
826+
newOperatorImage = "ghcr.io/cloudnative-pg/cloudnative-pg:1.27.1"
827+
oldExecHash = "exec-old"
828+
newExecHash = "exec-new"
829+
)
830+
831+
makeStatus := func(
832+
name, image, execHash, configHash string,
833+
) postgres.PostgresqlStatus {
834+
return postgres.PostgresqlStatus{
835+
Pod: &corev1.Pod{
836+
ObjectMeta: metav1.ObjectMeta{Name: name},
837+
Spec: corev1.PodSpec{
838+
InitContainers: []corev1.Container{
839+
{
840+
Name: specs.BootstrapControllerContainerName,
841+
Image: image,
842+
},
843+
},
844+
},
845+
},
846+
ExecutableHash: execHash,
847+
LoadedConfigurationHash: configHash,
848+
}
849+
}
850+
851+
// Positive cases: deadlock detected, bypass should trigger
852+
853+
It("rolling update with uniform groups", func() {
854+
statusList := postgres.PostgresqlStatusList{
855+
Items: []postgres.PostgresqlStatus{
856+
makeStatus("pod-1", newOperatorImage, newExecHash, "hash-new"),
857+
makeStatus("pod-2", oldOperatorImage, oldExecHash, "hash-old"),
858+
makeStatus("pod-3", oldOperatorImage, oldExecHash, "hash-old"),
859+
},
860+
}
861+
Expect(isConfigNonUniformityFromUpgrade(statusList)).To(BeTrue())
862+
})
863+
864+
It("in-place upgrade with uniform groups", func() {
865+
// Same bootstrap image but different executable hashes from binary replacement
866+
statusList := postgres.PostgresqlStatusList{
867+
Items: []postgres.PostgresqlStatus{
868+
makeStatus("pod-1", oldOperatorImage, newExecHash, "hash-new"),
869+
makeStatus("pod-2", oldOperatorImage, oldExecHash, "hash-old"),
870+
makeStatus("pod-3", oldOperatorImage, oldExecHash, "hash-old"),
871+
},
872+
}
873+
Expect(isConfigNonUniformityFromUpgrade(statusList)).To(BeTrue())
874+
})
875+
876+
// Negative cases: not a deadlock, should keep waiting
877+
878+
It("same version, config still propagating", func() {
879+
statusList := postgres.PostgresqlStatusList{
880+
Items: []postgres.PostgresqlStatus{
881+
makeStatus("pod-1", oldOperatorImage, oldExecHash, "hash-a"),
882+
makeStatus("pod-2", oldOperatorImage, oldExecHash, "hash-b"),
883+
makeStatus("pod-3", oldOperatorImage, oldExecHash, "hash-a"),
884+
},
885+
}
886+
Expect(isConfigNonUniformityFromUpgrade(statusList)).To(BeFalse())
887+
})
888+
889+
It("config propagation within old-version group during upgrade", func() {
890+
statusList := postgres.PostgresqlStatusList{
891+
Items: []postgres.PostgresqlStatus{
892+
makeStatus("pod-1", newOperatorImage, newExecHash, "hash-new"),
893+
makeStatus("pod-2", oldOperatorImage, oldExecHash, "hash-old-a"),
894+
makeStatus("pod-3", oldOperatorImage, oldExecHash, "hash-old-b"),
895+
},
896+
}
897+
Expect(isConfigNonUniformityFromUpgrade(statusList)).To(BeFalse())
898+
})
899+
900+
It("config propagation within new-version group during upgrade", func() {
901+
statusList := postgres.PostgresqlStatusList{
902+
Items: []postgres.PostgresqlStatus{
903+
makeStatus("pod-1", newOperatorImage, newExecHash, "hash-new-stale"),
904+
makeStatus("pod-2", newOperatorImage, newExecHash, "hash-new-current"),
905+
makeStatus("pod-3", oldOperatorImage, oldExecHash, "hash-old"),
906+
},
907+
}
908+
Expect(isConfigNonUniformityFromUpgrade(statusList)).To(BeFalse())
909+
})
910+
911+
// Edge cases
912+
913+
It("empty status list", func() {
914+
statusList := postgres.PostgresqlStatusList{}
915+
Expect(isConfigNonUniformityFromUpgrade(statusList)).To(BeFalse())
916+
})
917+
918+
It("single pod", func() {
919+
statusList := postgres.PostgresqlStatusList{
920+
Items: []postgres.PostgresqlStatus{
921+
makeStatus("pod-1", oldOperatorImage, oldExecHash, "hash-a"),
922+
},
923+
}
924+
Expect(isConfigNonUniformityFromUpgrade(statusList)).To(BeFalse())
925+
})
926+
927+
It("nil Pod and empty config hash are skipped", func() {
928+
statusList := postgres.PostgresqlStatusList{
929+
Items: []postgres.PostgresqlStatus{
930+
{Pod: nil, LoadedConfigurationHash: "hash-a"},
931+
{LoadedConfigurationHash: ""},
932+
makeStatus("pod-3", oldOperatorImage, oldExecHash, "hash-a"),
933+
},
934+
}
935+
Expect(isConfigNonUniformityFromUpgrade(statusList)).To(BeFalse())
936+
})
937+
})
938+
823939
type fakePluginClientRollout struct {
824940
pluginClient.Client
825941
returnedPod *corev1.Pod

0 commit comments

Comments
 (0)