Skip to content

Commit 1c9d44e

Browse files
mnenciaarmru
authored andcommitted
fix(pvc-resize): avoid deadlock on simultaneous storage and resource change
When both storage.size and resources are changed in a single PATCH, the operator resizes PVCs and then immediately starts a rolling update that deletes a pod. The orphaned PVC was classified as "resizing" instead of "dangling", preventing the pod from being recreated and leaving the cluster stuck. Fix by always classifying podless resizing PVCs as "dangling" so a replacement pod is created. This is safe because volumes are usable during expansion and filesystem resize requires a mounted pod anyway. Also remove the now-unused isFileSystemResizePending helper, which was only called from the removed conditional branch in classifyPVC. Closes #9786 Signed-off-by: Marco Nenciarini <[email protected]>
1 parent fa5635c commit 1c9d44e

3 files changed

Lines changed: 33 additions & 53 deletions

File tree

‎pkg/reconciler/persistentvolumeclaim/resources.go‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -79,10 +79,6 @@ func isResizing(pvc corev1.PersistentVolumeClaim) bool {
7979
return hasPVCCondition(pvc, corev1.PersistentVolumeClaimResizing)
8080
}
8181

82-
func isFileSystemResizePending(pvc corev1.PersistentVolumeClaim) bool {
83-
return hasPVCCondition(pvc, corev1.PersistentVolumeClaimFileSystemResizePending)
84-
}
85-
8682
// BelongToInstance returns a boolean indicating if that given PVC belongs to an instance
8783
func BelongToInstance(cluster *apiv1.Cluster, instanceName, pvcName string) bool {
8884
expectedPVCs := getExpectedInstancePVCNamesFromCluster(cluster, instanceName)

‎pkg/reconciler/persistentvolumeclaim/resources_test.go‎

Lines changed: 22 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -109,43 +109,7 @@ var _ = Describe("PVCs used by instance", func() {
109109
})
110110
})
111111

112-
var _ = Describe("isFileSystemResizePending", func() {
113-
pvcWith := func(conditions ...corev1.PersistentVolumeClaimCondition) corev1.PersistentVolumeClaim {
114-
return corev1.PersistentVolumeClaim{
115-
Status: corev1.PersistentVolumeClaimStatus{Conditions: conditions},
116-
}
117-
}
118-
119-
It("returns false when no conditions are present", func() {
120-
Expect(isFileSystemResizePending(pvcWith())).To(BeFalse())
121-
})
122-
123-
It("returns false when only Resizing condition is present", func() {
124-
Expect(isFileSystemResizePending(pvcWith(
125-
corev1.PersistentVolumeClaimCondition{
126-
Type: corev1.PersistentVolumeClaimResizing, Status: corev1.ConditionTrue,
127-
},
128-
))).To(BeFalse())
129-
})
130-
131-
It("returns true when FileSystemResizePending condition is true", func() {
132-
Expect(isFileSystemResizePending(pvcWith(
133-
corev1.PersistentVolumeClaimCondition{
134-
Type: corev1.PersistentVolumeClaimFileSystemResizePending, Status: corev1.ConditionTrue,
135-
},
136-
))).To(BeTrue())
137-
})
138-
139-
It("returns false when FileSystemResizePending condition is false", func() {
140-
Expect(isFileSystemResizePending(pvcWith(
141-
corev1.PersistentVolumeClaimCondition{
142-
Type: corev1.PersistentVolumeClaimFileSystemResizePending, Status: corev1.ConditionFalse,
143-
},
144-
))).To(BeFalse())
145-
})
146-
})
147-
148-
var _ = Describe("PVC classification with FileSystemResizePending", func() {
112+
var _ = Describe("PVC classification with resizing PVCs", func() {
149113
clusterName := "myCluster"
150114
makeCluster := func() *apiv1.Cluster {
151115
return &apiv1.Cluster{
@@ -164,12 +128,31 @@ var _ = Describe("PVC classification with FileSystemResizePending", func() {
164128
Expect(cluster.Status.ResizingPVC).Should(BeEmpty())
165129
})
166130

167-
It("classifies resizing PVC without pod and without FileSystemResizePending as resizing", func(ctx SpecContext) {
131+
It("classifies resizing PVC without pod and without FileSystemResizePending as dangling", func(ctx SpecContext) {
168132
pvc := makePVC(clusterName, "1", "1", NewPgDataCalculator(), true)
169133
cluster := makeCluster()
170134
EnrichStatus(ctx, cluster, []corev1.Pod{}, []batchv1.Job{}, []corev1.PersistentVolumeClaim{pvc})
135+
Expect(cluster.Status.DanglingPVC).Should(Equal([]string{clusterName + "-1"}))
136+
Expect(cluster.Status.ResizingPVC).Should(BeEmpty())
137+
})
138+
139+
It("classifies resizing PVC as dangling when its pod was deleted during rolling update (#9786)", func(ctx SpecContext) {
140+
// Simulate simultaneous storage + resource change:
141+
// instance-1 has a running pod (primary), instance-2's pod was deleted
142+
// for a rolling update while both PVCs are resizing.
143+
pvc1 := makePVC(clusterName, "1", "1", NewPgDataCalculator(), true) // resizing, has pod
144+
pvc2 := makePVC(clusterName, "2", "2", NewPgDataCalculator(), true) // resizing, pod deleted
145+
cluster := makeCluster()
146+
EnrichStatus(
147+
ctx,
148+
cluster,
149+
[]corev1.Pod{makePod(clusterName, "1", specs.ClusterRoleLabelPrimary)},
150+
[]batchv1.Job{},
151+
[]corev1.PersistentVolumeClaim{pvc1, pvc2},
152+
)
171153
Expect(cluster.Status.ResizingPVC).Should(Equal([]string{clusterName + "-1"}))
172-
Expect(cluster.Status.DanglingPVC).Should(BeEmpty())
154+
Expect(cluster.Status.DanglingPVC).Should(Equal([]string{clusterName + "-2"}))
155+
Expect(cluster.Status.Instances).Should(BeEquivalentTo(2))
173156
})
174157
})
175158

‎pkg/reconciler/persistentvolumeclaim/status.go‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,8 @@ const (
6262
// List of PVCs that are being initialized (they have a corresponding Job but not a corresponding Pod)
6363
initializing status = "initializing"
6464

65-
// List of PVCs with resizing condition. Requires a pod restart.
65+
// List of PVCs with resizing condition that have a running pod.
66+
// Podless resizing PVCs are classified as dangling instead (see #9786).
6667
//
6768
// INFO: https://kubernetes.io/blog/2018/07/12/resizing-persistent-volumes-using-kubernetes/
6869
resizing status = "resizing"
@@ -269,16 +270,16 @@ func classifyPVC(
269270
return healthy
270271
}
271272

272-
// PVC is resizing without a pod
273+
// PVC is resizing without a pod — classify as dangling so a pod is
274+
// created. This is safe because volumes remain usable during
275+
// expansion, and filesystem resize actually requires a mounted pod
276+
// (kubelet performs it). Without this, a simultaneous storage +
277+
// resource change can leave the PVC orphaned as "resizing" while the
278+
// rolling-update-deleted pod is never recreated (see #9786).
273279
if isResizing(pvc) {
274-
// Filesystem resize requires a pod mount to complete; classify as
275-
// dangling so the reconciler creates one.
276-
if isFileSystemResizePending(pvc) {
277-
log.FromContext(ctx).Info("PVC filesystem resize pending without a pod, classifying as dangling",
278-
"pvc", pvc.Name)
279-
return dangling
280-
}
281-
return resizing
280+
log.FromContext(ctx).Info("PVC resizing without a pod, classifying as dangling",
281+
"pvc", pvc.Name)
282+
return dangling
282283
}
283284

284285
// PVC has a corresponding Job but not a corresponding Pod

0 commit comments

Comments
 (0)