Skip to content

Commit 881706a

Browse files
mnenciaarmruleonardoce
committed
fix(pvc-resize): avoid deadlock on simultaneous storage and resource change (#10427)
When both storage.size and resources are changed in a single patch, the operator resizes PVCs and then immediately starts a rolling update that deletes a pod. The orphaned PVC was classified as "resizing" instead of "dangling", preventing the pod from being recreated and leaving the cluster stuck. Fix by always classifying podless resizing PVCs as "dangling" so a replacement pod is created. This is safe because volumes are usable during expansion and filesystem resize requires a mounted pod anyway. Also remove the now-unused isFileSystemResizePending helper, which was only called from the removed conditional branch in classifyPVC. Closes #9786 Signed-off-by: Marco Nenciarini <[email protected]> Signed-off-by: Armando Ruocco <[email protected]> Signed-off-by: Leonardo Cecchi <[email protected]> Co-authored-by: Armando Ruocco <[email protected]> Co-authored-by: Leonardo Cecchi <[email protected]> (cherry picked from commit 9147871)
1 parent 97561ae commit 881706a

5 files changed

Lines changed: 185 additions & 132 deletions

File tree

‎internal/controller/cluster_controller_test.go‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,59 @@ var _ = Describe("Updating target primary", func() {
273273
Expect(err).ToNot(HaveOccurred())
274274
})
275275
})
276+
277+
It("Issue #9786: findInstancePodToCreate selects podless resizing PVC classified as dangling by EnrichStatus",
278+
func(ctx SpecContext) {
279+
namespace := newFakeNamespace(env.client)
280+
cluster := newFakeCNPGCluster(env.client, namespace, func(cluster *apiv1.Cluster) {
281+
cluster.Spec.Instances = 2
282+
cluster.Status.LatestGeneratedNode = 2
283+
cluster.Status.ReadyInstances = 1
284+
})
285+
286+
By("creating cluster PVCs and marking them as resizing")
287+
pvcs := generateClusterPVC(env.client, cluster, persistentvolumeclaim.StatusReady)
288+
for i := range pvcs {
289+
pvcs[i].Status.Phase = corev1.ClaimBound
290+
pvcs[i].Status.Conditions = append(pvcs[i].Status.Conditions, corev1.PersistentVolumeClaimCondition{
291+
Type: corev1.PersistentVolumeClaimResizing,
292+
Status: corev1.ConditionTrue,
293+
})
294+
}
295+
296+
By("creating a pod for instance 1 only (instance 2's pod was deleted during rolling update)")
297+
pod1, err := specs.NewInstance(ctx, *cluster, 1, true)
298+
Expect(err).ToNot(HaveOccurred())
299+
cluster.SetInheritedDataAndOwnership(&pod1.ObjectMeta)
300+
Expect(env.client.Create(ctx, pod1)).To(Succeed())
301+
pod1.Status = corev1.PodStatus{
302+
Phase: corev1.PodRunning,
303+
Conditions: []corev1.PodCondition{
304+
{Type: corev1.ContainersReady, Status: corev1.ConditionTrue},
305+
},
306+
}
307+
308+
By("running EnrichStatus to classify PVCs")
309+
persistentvolumeclaim.EnrichStatus(ctx, cluster, []corev1.Pod{*pod1}, []batchv1.Job{}, pvcs)
310+
instance2Name := specs.GetInstanceName(cluster.Name, 2)
311+
Expect(cluster.Status.ResizingPVC).Should(HaveLen(1))
312+
Expect(cluster.Status.DanglingPVC).Should(Equal([]string{instance2Name}))
313+
314+
By("verifying findInstancePodToCreate selects the dangling instance for pod creation")
315+
statusList := postgres.PostgresqlStatusList{
316+
Items: []postgres.PostgresqlStatus{
317+
{
318+
IsPodReady: true,
319+
IsPrimary: true,
320+
Pod: pod1,
321+
},
322+
},
323+
}
324+
instanceToCreate, err := findInstancePodToCreate(ctx, cluster, statusList, pvcs)
325+
Expect(err).ToNot(HaveOccurred())
326+
Expect(instanceToCreate).ToNot(BeNil())
327+
Expect(instanceToCreate.Name).To(Equal(instance2Name))
328+
})
276329
})
277330

278331
var _ = Describe("isNodeUnschedulableOrBeingDrained", func() {

‎pkg/reconciler/persistentvolumeclaim/resources.go‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -79,10 +79,6 @@ func isResizing(pvc corev1.PersistentVolumeClaim) bool {
7979
return hasPVCCondition(pvc, corev1.PersistentVolumeClaimResizing)
8080
}
8181

82-
func isFileSystemResizePending(pvc corev1.PersistentVolumeClaim) bool {
83-
return hasPVCCondition(pvc, corev1.PersistentVolumeClaimFileSystemResizePending)
84-
}
85-
8682
// BelongToInstance returns a boolean indicating if that given PVC belongs to an instance
8783
func BelongToInstance(cluster *apiv1.Cluster, instanceName, pvcName string) bool {
8884
expectedPVCs := getExpectedInstancePVCNamesFromCluster(cluster, instanceName)

‎pkg/reconciler/persistentvolumeclaim/resources_test.go‎

Lines changed: 0 additions & 118 deletions
Original file line numberDiff line numberDiff line change
@@ -20,68 +20,14 @@ SPDX-License-Identifier: Apache-2.0
2020
package persistentvolumeclaim
2121

2222
import (
23-
batchv1 "k8s.io/api/batch/v1"
24-
corev1 "k8s.io/api/core/v1"
2523
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2624

2725
apiv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1"
28-
"github.com/cloudnative-pg/cloudnative-pg/pkg/specs"
2926

3027
. "github.com/onsi/ginkgo/v2"
3128
. "github.com/onsi/gomega"
3229
)
3330

34-
var _ = Describe("PVC detection", func() {
35-
It("will list PVCs with Jobs or Pods or which are Ready", func(ctx SpecContext) {
36-
clusterName := "myCluster"
37-
makeClusterPVC := func(serial string, isResizing bool) corev1.PersistentVolumeClaim {
38-
return makePVC(clusterName, serial, serial, NewPgDataCalculator(), isResizing)
39-
}
40-
pvcs := []corev1.PersistentVolumeClaim{
41-
makeClusterPVC("1", false), // has a Pod
42-
makeClusterPVC("2", false), // has a Job
43-
makeClusterPVC("3", true), // resizing
44-
makeClusterPVC("4", false), // dangling
45-
}
46-
cluster := &apiv1.Cluster{
47-
ObjectMeta: metav1.ObjectMeta{
48-
Name: clusterName,
49-
},
50-
}
51-
EnrichStatus(
52-
ctx,
53-
cluster,
54-
[]corev1.Pod{
55-
makePod(clusterName, "1", specs.ClusterRoleLabelPrimary),
56-
makePod(clusterName, "3", specs.ClusterRoleLabelReplica),
57-
},
58-
[]batchv1.Job{makeJob(clusterName, "2")},
59-
pvcs,
60-
)
61-
62-
Expect(cluster.Status.PVCCount).Should(BeEquivalentTo(4))
63-
Expect(cluster.Status.InstanceNames).Should(Equal([]string{
64-
clusterName + "-1",
65-
clusterName + "-2",
66-
clusterName + "-3",
67-
clusterName + "-4",
68-
}))
69-
Expect(cluster.Status.InitializingPVC).Should(Equal([]string{
70-
clusterName + "-2",
71-
}))
72-
Expect(cluster.Status.ResizingPVC).Should(Equal([]string{
73-
clusterName + "-3",
74-
}))
75-
Expect(cluster.Status.DanglingPVC).Should(Equal([]string{
76-
clusterName + "-4",
77-
}))
78-
Expect(cluster.Status.HealthyPVC).Should(Equal([]string{
79-
clusterName + "-1",
80-
}))
81-
Expect(cluster.Status.UnusablePVC).Should(BeEmpty())
82-
})
83-
})
84-
8531
var _ = Describe("PVCs used by instance", func() {
8632
clusterName := "cluster-pvc-instance"
8733
instanceName := clusterName + "-1"
@@ -109,70 +55,6 @@ var _ = Describe("PVCs used by instance", func() {
10955
})
11056
})
11157

112-
var _ = Describe("isFileSystemResizePending", func() {
113-
pvcWith := func(conditions ...corev1.PersistentVolumeClaimCondition) corev1.PersistentVolumeClaim {
114-
return corev1.PersistentVolumeClaim{
115-
Status: corev1.PersistentVolumeClaimStatus{Conditions: conditions},
116-
}
117-
}
118-
119-
It("returns false when no conditions are present", func() {
120-
Expect(isFileSystemResizePending(pvcWith())).To(BeFalse())
121-
})
122-
123-
It("returns false when only Resizing condition is present", func() {
124-
Expect(isFileSystemResizePending(pvcWith(
125-
corev1.PersistentVolumeClaimCondition{
126-
Type: corev1.PersistentVolumeClaimResizing, Status: corev1.ConditionTrue,
127-
},
128-
))).To(BeFalse())
129-
})
130-
131-
It("returns true when FileSystemResizePending condition is true", func() {
132-
Expect(isFileSystemResizePending(pvcWith(
133-
corev1.PersistentVolumeClaimCondition{
134-
Type: corev1.PersistentVolumeClaimFileSystemResizePending, Status: corev1.ConditionTrue,
135-
},
136-
))).To(BeTrue())
137-
})
138-
139-
It("returns false when FileSystemResizePending condition is false", func() {
140-
Expect(isFileSystemResizePending(pvcWith(
141-
corev1.PersistentVolumeClaimCondition{
142-
Type: corev1.PersistentVolumeClaimFileSystemResizePending, Status: corev1.ConditionFalse,
143-
},
144-
))).To(BeFalse())
145-
})
146-
})
147-
148-
var _ = Describe("PVC classification with FileSystemResizePending", func() {
149-
clusterName := "myCluster"
150-
makeCluster := func() *apiv1.Cluster {
151-
return &apiv1.Cluster{
152-
ObjectMeta: metav1.ObjectMeta{Name: clusterName},
153-
}
154-
}
155-
156-
It("classifies resizing PVC without pod and with FileSystemResizePending as dangling", func(ctx SpecContext) {
157-
pvc := makePVC(clusterName, "1", "1", NewPgDataCalculator(), true)
158-
pvc.Status.Conditions = append(pvc.Status.Conditions, corev1.PersistentVolumeClaimCondition{
159-
Type: corev1.PersistentVolumeClaimFileSystemResizePending, Status: corev1.ConditionTrue,
160-
})
161-
cluster := makeCluster()
162-
EnrichStatus(ctx, cluster, []corev1.Pod{}, []batchv1.Job{}, []corev1.PersistentVolumeClaim{pvc})
163-
Expect(cluster.Status.DanglingPVC).Should(Equal([]string{clusterName + "-1"}))
164-
Expect(cluster.Status.ResizingPVC).Should(BeEmpty())
165-
})
166-
167-
It("classifies resizing PVC without pod and without FileSystemResizePending as resizing", func(ctx SpecContext) {
168-
pvc := makePVC(clusterName, "1", "1", NewPgDataCalculator(), true)
169-
cluster := makeCluster()
170-
EnrichStatus(ctx, cluster, []corev1.Pod{}, []batchv1.Job{}, []corev1.PersistentVolumeClaim{pvc})
171-
Expect(cluster.Status.ResizingPVC).Should(Equal([]string{clusterName + "-1"}))
172-
Expect(cluster.Status.DanglingPVC).Should(BeEmpty())
173-
})
174-
})
175-
17658
var _ = Describe("instance with tablespace test", func() {
17759
clusterName := "cluster-tbs-pvc-instance"
17860
instanceName := clusterName + "-1"

‎pkg/reconciler/persistentvolumeclaim/status.go‎

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,8 @@ const (
6262
// List of PVCs that are being initialized (they have a corresponding Job but not a corresponding Pod)
6363
initializing status = "initializing"
6464

65-
// List of PVCs with resizing condition. Requires a pod restart.
65+
// List of PVCs with resizing condition that have a corresponding pod.
66+
// Podless resizing PVCs are classified as dangling instead (see #9786).
6667
//
6768
// INFO: https://kubernetes.io/blog/2018/07/12/resizing-persistent-volumes-using-kubernetes/
6869
resizing status = "resizing"
@@ -269,16 +270,21 @@ func classifyPVC(
269270
return healthy
270271
}
271272

272-
// PVC is resizing without a pod
273+
// A resizing PVC without a pod must be classified as dangling so the
274+
// reconciler recreates the pod. Two reasons:
275+
//
276+
// 1. Filesystem resize requires a mounted pod — kubelet performs it on mount.
277+
//
278+
// 2. Without a pod, the "resizing" classification is a dead end: a
279+
// simultaneous storage + resource change would leave the PVC orphaned
280+
// because the rolling-update-deleted pod is never recreated (see #9786).
281+
//
282+
// For online-expandable volumes reason 1 does not apply, but reason 2 still
283+
// does; classifying as dangling is always safe here.
273284
if isResizing(pvc) {
274-
// Filesystem resize requires a pod mount to complete; classify as
275-
// dangling so the reconciler creates one.
276-
if isFileSystemResizePending(pvc) {
277-
log.FromContext(ctx).Info("PVC filesystem resize pending without a pod, classifying as dangling",
278-
"pvc", pvc.Name)
279-
return dangling
280-
}
281-
return resizing
285+
log.FromContext(ctx).Info("PVC resizing without a pod, classifying as dangling",
286+
"pvc", pvc.Name)
287+
return dangling
282288
}
283289

284290
// PVC has a corresponding Job but not a corresponding Pod

‎pkg/reconciler/persistentvolumeclaim/status_test.go‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,128 @@ import (
3131

3232
apiv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1"
3333
"github.com/cloudnative-pg/cloudnative-pg/internal/scheme"
34+
"github.com/cloudnative-pg/cloudnative-pg/pkg/specs"
3435
"github.com/cloudnative-pg/cloudnative-pg/pkg/utils"
3536

3637
. "github.com/onsi/ginkgo/v2"
3738
. "github.com/onsi/gomega"
3839
)
3940

41+
var _ = Describe("PVC detection", func() {
42+
It("will list PVCs with Jobs or Pods or which are Ready", func(ctx SpecContext) {
43+
clusterName := "myCluster"
44+
makeClusterPVC := func(serial string, isResizing bool) corev1.PersistentVolumeClaim {
45+
return makePVC(clusterName, serial, serial, NewPgDataCalculator(), isResizing)
46+
}
47+
pvcs := []corev1.PersistentVolumeClaim{
48+
makeClusterPVC("1", false), // has a Pod
49+
makeClusterPVC("2", false), // has a Job
50+
makeClusterPVC("3", true), // resizing, has a Pod so stays "resizing" (not affected by #9786 fix)
51+
makeClusterPVC("4", false), // dangling
52+
}
53+
cluster := &apiv1.Cluster{
54+
ObjectMeta: metav1.ObjectMeta{
55+
Name: clusterName,
56+
},
57+
}
58+
EnrichStatus(
59+
ctx,
60+
cluster,
61+
[]corev1.Pod{
62+
makePod(clusterName, "1", specs.ClusterRoleLabelPrimary),
63+
makePod(clusterName, "3", specs.ClusterRoleLabelReplica),
64+
},
65+
[]batchv1.Job{makeJob(clusterName, "2")},
66+
pvcs,
67+
)
68+
69+
Expect(cluster.Status.PVCCount).Should(BeEquivalentTo(4))
70+
Expect(cluster.Status.InstanceNames).Should(Equal([]string{
71+
clusterName + "-1",
72+
clusterName + "-2",
73+
clusterName + "-3",
74+
clusterName + "-4",
75+
}))
76+
Expect(cluster.Status.InitializingPVC).Should(Equal([]string{
77+
clusterName + "-2",
78+
}))
79+
Expect(cluster.Status.ResizingPVC).Should(Equal([]string{
80+
clusterName + "-3",
81+
}))
82+
Expect(cluster.Status.DanglingPVC).Should(Equal([]string{
83+
clusterName + "-4",
84+
}))
85+
Expect(cluster.Status.HealthyPVC).Should(Equal([]string{
86+
clusterName + "-1",
87+
}))
88+
Expect(cluster.Status.UnusablePVC).Should(BeEmpty())
89+
})
90+
})
91+
92+
var _ = Describe("PVC classification with resizing PVCs", func() {
93+
clusterName := "myCluster"
94+
makeCluster := func() *apiv1.Cluster {
95+
return &apiv1.Cluster{
96+
ObjectMeta: metav1.ObjectMeta{Name: clusterName},
97+
}
98+
}
99+
100+
It("classifies resizing PVC without pod and with FileSystemResizePending as dangling", func(ctx SpecContext) {
101+
pvc := makePVC(clusterName, "1", "1", NewPgDataCalculator(), true)
102+
pvc.Status.Conditions = append(pvc.Status.Conditions, corev1.PersistentVolumeClaimCondition{
103+
Type: corev1.PersistentVolumeClaimFileSystemResizePending, Status: corev1.ConditionTrue,
104+
})
105+
cluster := makeCluster()
106+
EnrichStatus(ctx, cluster, []corev1.Pod{}, []batchv1.Job{}, []corev1.PersistentVolumeClaim{pvc})
107+
Expect(cluster.Status.DanglingPVC).Should(Equal([]string{clusterName + "-1"}))
108+
Expect(cluster.Status.ResizingPVC).Should(BeEmpty())
109+
})
110+
111+
It("classifies resizing PVC without pod and without FileSystemResizePending as dangling", func(ctx SpecContext) {
112+
pvc := makePVC(clusterName, "1", "1", NewPgDataCalculator(), true)
113+
cluster := makeCluster()
114+
EnrichStatus(ctx, cluster, []corev1.Pod{}, []batchv1.Job{}, []corev1.PersistentVolumeClaim{pvc})
115+
Expect(cluster.Status.DanglingPVC).Should(Equal([]string{clusterName + "-1"}))
116+
Expect(cluster.Status.ResizingPVC).Should(BeEmpty())
117+
})
118+
119+
// isResizing check takes precedence over hasJob: a resizing PVC with
120+
// a Job but no Pod is classified as dangling, not initializing.
121+
It("classifies resizing PVC with a job but no pod as dangling", func(ctx SpecContext) {
122+
pvc := makePVC(clusterName, "1", "1", NewPgDataCalculator(), true)
123+
cluster := makeCluster()
124+
EnrichStatus(
125+
ctx,
126+
cluster,
127+
[]corev1.Pod{},
128+
[]batchv1.Job{makeJob(clusterName, "1")},
129+
[]corev1.PersistentVolumeClaim{pvc},
130+
)
131+
Expect(cluster.Status.DanglingPVC).Should(Equal([]string{clusterName + "-1"}))
132+
Expect(cluster.Status.ResizingPVC).Should(BeEmpty())
133+
Expect(cluster.Status.InitializingPVC).Should(BeEmpty())
134+
})
135+
136+
It("classifies resizing PVC as dangling when pod was deleted during rollout (#9786)", func(ctx SpecContext) {
137+
// Simulate simultaneous storage + resource change:
138+
// instance-1 has a running pod (primary), instance-2's pod was deleted
139+
// for a rolling update while both PVCs are resizing.
140+
pvc1 := makePVC(clusterName, "1", "1", NewPgDataCalculator(), true) // resizing, has pod
141+
pvc2 := makePVC(clusterName, "2", "2", NewPgDataCalculator(), true) // resizing, pod deleted
142+
cluster := makeCluster()
143+
EnrichStatus(
144+
ctx,
145+
cluster,
146+
[]corev1.Pod{makePod(clusterName, "1", specs.ClusterRoleLabelPrimary)},
147+
[]batchv1.Job{},
148+
[]corev1.PersistentVolumeClaim{pvc1, pvc2},
149+
)
150+
Expect(cluster.Status.ResizingPVC).Should(Equal([]string{clusterName + "-1"}))
151+
Expect(cluster.Status.DanglingPVC).Should(Equal([]string{clusterName + "-2"}))
152+
Expect(cluster.Status.Instances).Should(BeEquivalentTo(2))
153+
})
154+
})
155+
40156
var _ = Describe("EnsureHealthyPVCsAnnotation", func() {
41157
var (
42158
ctx context.Context

0 commit comments

Comments
 (0)