Skip to content

Commit 0c0fa23

Browse files
jmealoarmrumnencia
authored
fix(pvc): prevent deadlock on filesystem resize without pod (#9981)
After a volume resize completes at the storage layer, the filesystem resize still needs a running pod to mount the volume. If the pod is missing, the instance was stuck indefinitely because no new pod was created. Now the operator detects this condition and recreates the pod to complete the resize. Closes #9980 Signed-off-by: Jeff Mealo <[email protected]> Signed-off-by: Armando Ruocco <[email protected]> Signed-off-by: Marco Nenciarini <[email protected]> Co-authored-by: Armando Ruocco <[email protected]> Co-authored-by: Marco Nenciarini <[email protected]>
1 parent b052263 commit 0c0fa23

3 files changed

Lines changed: 89 additions & 8 deletions

File tree

‎pkg/reconciler/persistentvolumeclaim/resources.go‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -65,17 +65,24 @@ external:
6565
return true
6666
}
6767

68-
// isResizing returns true if PersistentVolumeClaimResizing condition is present
69-
func isResizing(pvc corev1.PersistentVolumeClaim) bool {
68+
func hasPVCCondition(pvc corev1.PersistentVolumeClaim, condType corev1.PersistentVolumeClaimConditionType) bool {
7069
for _, condition := range pvc.Status.Conditions {
71-
if condition.Type == corev1.PersistentVolumeClaimResizing {
70+
if condition.Type == condType && condition.Status == corev1.ConditionTrue {
7271
return true
7372
}
7473
}
7574

7675
return false
7776
}
7877

78+
func isResizing(pvc corev1.PersistentVolumeClaim) bool {
79+
return hasPVCCondition(pvc, corev1.PersistentVolumeClaimResizing)
80+
}
81+
82+
func isFileSystemResizePending(pvc corev1.PersistentVolumeClaim) bool {
83+
return hasPVCCondition(pvc, corev1.PersistentVolumeClaimFileSystemResizePending)
84+
}
85+
7986
// BelongToInstance returns a boolean indicating if that given PVC belongs to an instance
8087
func BelongToInstance(cluster *apiv1.Cluster, instanceName, pvcName string) bool {
8188
expectedPVCs := getExpectedInstancePVCNamesFromCluster(cluster, instanceName)

‎pkg/reconciler/persistentvolumeclaim/resources_test.go‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,70 @@ var _ = Describe("PVCs used by instance", func() {
109109
})
110110
})
111111

112+
var _ = Describe("isFileSystemResizePending", func() {
113+
pvcWith := func(conditions ...corev1.PersistentVolumeClaimCondition) corev1.PersistentVolumeClaim {
114+
return corev1.PersistentVolumeClaim{
115+
Status: corev1.PersistentVolumeClaimStatus{Conditions: conditions},
116+
}
117+
}
118+
119+
It("returns false when no conditions are present", func() {
120+
Expect(isFileSystemResizePending(pvcWith())).To(BeFalse())
121+
})
122+
123+
It("returns false when only Resizing condition is present", func() {
124+
Expect(isFileSystemResizePending(pvcWith(
125+
corev1.PersistentVolumeClaimCondition{
126+
Type: corev1.PersistentVolumeClaimResizing, Status: corev1.ConditionTrue,
127+
},
128+
))).To(BeFalse())
129+
})
130+
131+
It("returns true when FileSystemResizePending condition is true", func() {
132+
Expect(isFileSystemResizePending(pvcWith(
133+
corev1.PersistentVolumeClaimCondition{
134+
Type: corev1.PersistentVolumeClaimFileSystemResizePending, Status: corev1.ConditionTrue,
135+
},
136+
))).To(BeTrue())
137+
})
138+
139+
It("returns false when FileSystemResizePending condition is false", func() {
140+
Expect(isFileSystemResizePending(pvcWith(
141+
corev1.PersistentVolumeClaimCondition{
142+
Type: corev1.PersistentVolumeClaimFileSystemResizePending, Status: corev1.ConditionFalse,
143+
},
144+
))).To(BeFalse())
145+
})
146+
})
147+
148+
var _ = Describe("PVC classification with FileSystemResizePending", func() {
149+
clusterName := "myCluster"
150+
makeCluster := func() *apiv1.Cluster {
151+
return &apiv1.Cluster{
152+
ObjectMeta: metav1.ObjectMeta{Name: clusterName},
153+
}
154+
}
155+
156+
It("classifies resizing PVC without pod and with FileSystemResizePending as dangling", func(ctx SpecContext) {
157+
pvc := makePVC(clusterName, "1", "1", NewPgDataCalculator(), true)
158+
pvc.Status.Conditions = append(pvc.Status.Conditions, corev1.PersistentVolumeClaimCondition{
159+
Type: corev1.PersistentVolumeClaimFileSystemResizePending, Status: corev1.ConditionTrue,
160+
})
161+
cluster := makeCluster()
162+
EnrichStatus(ctx, cluster, []corev1.Pod{}, []batchv1.Job{}, []corev1.PersistentVolumeClaim{pvc})
163+
Expect(cluster.Status.DanglingPVC).Should(Equal([]string{clusterName + "-1"}))
164+
Expect(cluster.Status.ResizingPVC).Should(BeEmpty())
165+
})
166+
167+
It("classifies resizing PVC without pod and without FileSystemResizePending as resizing", func(ctx SpecContext) {
168+
pvc := makePVC(clusterName, "1", "1", NewPgDataCalculator(), true)
169+
cluster := makeCluster()
170+
EnrichStatus(ctx, cluster, []corev1.Pod{}, []batchv1.Job{}, []corev1.PersistentVolumeClaim{pvc})
171+
Expect(cluster.Status.ResizingPVC).Should(Equal([]string{clusterName + "-1"}))
172+
Expect(cluster.Status.DanglingPVC).Should(BeEmpty())
173+
})
174+
})
175+
112176
var _ = Describe("instance with tablespace test", func() {
113177
clusterName := "cluster-tbs-pvc-instance"
114178
instanceName := clusterName + "-1"

‎pkg/reconciler/persistentvolumeclaim/status.go‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -261,16 +261,26 @@ func classifyPVC(
261261
return unusable
262262
}
263263

264-
// PVC is resizing
265-
if isResizing(pvc) {
266-
return resizing
267-
}
268-
269264
// PVC has a corresponding Pod
270265
if hasPod(pvc, podList) {
266+
if isResizing(pvc) {
267+
return resizing
268+
}
271269
return healthy
272270
}
273271

272+
// PVC is resizing without a pod
273+
if isResizing(pvc) {
274+
// Filesystem resize requires a pod mount to complete; classify as
275+
// dangling so the reconciler creates one.
276+
if isFileSystemResizePending(pvc) {
277+
log.FromContext(ctx).Info("PVC filesystem resize pending without a pod, classifying as dangling",
278+
"pvc", pvc.Name)
279+
return dangling
280+
}
281+
return resizing
282+
}
283+
274284
// PVC has a corresponding Job but not a corresponding Pod
275285
if hasJob(pvc, jobList) {
276286
return initializing

0 commit comments

Comments
 (0)