Skip to content

Commit 07952d4

Browse files
jsilvelaarmru
authored andcommitted
feat: avoid fencing instance unnecessarily in volume snapshots (cloudnative-pg#2854)
Previously, the volume snapshot code could fence a node multiple times, leading to handling errors. The new implementation fences nodes only when necessary. Closes cloudnative-pg#2853 Signed-off-by: Jaime Silvela <[email protected]> Signed-off-by: Armando Ruocco <[email protected]> Co-authored-by: Armando Ruocco <[email protected]> Signed-off-by: Lennart Krauch <[email protected]>
1 parent 09cf31b commit 07952d4

3 files changed

Lines changed: 278 additions & 5 deletions

File tree

‎pkg/reconciler/backup/volumesnapshot/reconciler.go‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -123,10 +123,20 @@ func (se *Reconciler) Execute(
123123
targetPod *corev1.Pod,
124124
pvcs []corev1.PersistentVolumeClaim,
125125
) (*ctrl.Result, error) {
126+
if cluster.Spec.Backup == nil || cluster.Spec.Backup.VolumeSnapshot == nil {
127+
return nil, fmt.Errorf("cannot execute a VolumeSnapshot on a cluster without configuration")
128+
}
129+
126130
contextLogger := log.FromContext(ctx).WithValues("podName", targetPod.Name)
127131

132+
// Step 0: check if the snapshots have been created already
133+
volumeSnapshots, err := GetBackupVolumeSnapshots(ctx, se.cli, cluster.Namespace, backup.Name)
134+
if err != nil {
135+
return nil, err
136+
}
137+
128138
// Step 1: fencing
129-
if se.shouldFence {
139+
if len(volumeSnapshots) == 0 && se.shouldFence {
130140
contextLogger.Debug("Checking pre-requisites")
131141
if err := se.ensurePodIsFenced(ctx, cluster, backup, targetPod.Name); err != nil {
132142
return nil, err
@@ -138,10 +148,6 @@ func (se *Reconciler) Execute(
138148
}
139149

140150
// Step 2: create snapshot
141-
volumeSnapshots, err := GetBackupVolumeSnapshots(ctx, se.cli, cluster.Namespace, backup.Name)
142-
if err != nil {
143-
return nil, err
144-
}
145151
if len(volumeSnapshots) == 0 {
146152
// we execute the snapshots only if we don't find any
147153
if err := se.createSnapshotPVCGroupStep(ctx, cluster, pvcs, backup, targetPod); err != nil {
Lines changed: 238 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,238 @@
1+
/*
2+
Copyright The CloudNativePG Contributors
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package volumesnapshot
18+
19+
import (
20+
storagesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1"
21+
v1 "k8s.io/api/core/v1"
22+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
23+
"k8s.io/client-go/tools/record"
24+
"k8s.io/utils/ptr"
25+
"sigs.k8s.io/controller-runtime/pkg/client/fake"
26+
27+
apiv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1"
28+
"github.com/cloudnative-pg/cloudnative-pg/internal/scheme"
29+
"github.com/cloudnative-pg/cloudnative-pg/pkg/utils"
30+
31+
. "github.com/onsi/ginkgo/v2"
32+
. "github.com/onsi/gomega"
33+
)
34+
35+
var _ = Describe("Volumesnapshot reconciler", func() {
36+
const (
37+
namespace = "test-namespace"
38+
clusterName = "clusterName"
39+
backupName = "theBakcup"
40+
)
41+
var (
42+
cluster *apiv1.Cluster
43+
targetPod *v1.Pod
44+
pvcs []v1.PersistentVolumeClaim
45+
backup *apiv1.Backup
46+
)
47+
48+
BeforeEach(func() {
49+
cluster = &apiv1.Cluster{
50+
ObjectMeta: metav1.ObjectMeta{
51+
Namespace: namespace,
52+
Name: clusterName,
53+
},
54+
Spec: apiv1.ClusterSpec{
55+
Backup: &apiv1.BackupConfiguration{
56+
VolumeSnapshot: &apiv1.VolumeSnapshotConfiguration{
57+
ClassName: "csi-hostpath-snapclass",
58+
},
59+
},
60+
},
61+
}
62+
targetPod = &v1.Pod{
63+
ObjectMeta: metav1.ObjectMeta{
64+
Namespace: namespace,
65+
Name: clusterName + "-2",
66+
},
67+
}
68+
pvcs = []v1.PersistentVolumeClaim{
69+
{
70+
ObjectMeta: metav1.ObjectMeta{
71+
Name: clusterName + "-2",
72+
Namespace: namespace,
73+
},
74+
},
75+
{
76+
ObjectMeta: metav1.ObjectMeta{
77+
Name: clusterName + "-2-wal",
78+
Namespace: namespace,
79+
},
80+
},
81+
}
82+
backup = &apiv1.Backup{
83+
ObjectMeta: metav1.ObjectMeta{
84+
Namespace: namespace,
85+
Name: backupName,
86+
},
87+
}
88+
})
89+
90+
It("should fence the target pod when there are no volumesnapshots", func(ctx SpecContext) {
91+
mockClient := fake.NewClientBuilder().
92+
WithScheme(scheme.BuildWithAllKnownScheme()).
93+
WithObjects(cluster, targetPod).
94+
Build()
95+
fakeRecorder := record.NewFakeRecorder(3)
96+
97+
executor := NewExecutorBuilder(mockClient, fakeRecorder).
98+
FenceInstance(true).
99+
Build()
100+
101+
result, err := executor.Execute(ctx, cluster, backup, targetPod, pvcs)
102+
Expect(err).ToNot(HaveOccurred())
103+
// we should have requested creation of snapshots, and so we'd return to
104+
// wait for them to be ready
105+
Expect(result).ToNot(BeNil())
106+
hasEvent := false
107+
var events string
108+
go func() {
109+
evts := <-fakeRecorder.Events
110+
hasEvent = true
111+
events = evts
112+
}()
113+
Eventually(func() bool {
114+
return hasEvent
115+
}).Should(BeTrue())
116+
Eventually(func() string {
117+
return events
118+
}).Should(ContainSubstring("Requesting fencing for Pod " + targetPod.Name))
119+
})
120+
121+
It("should not fence the target pod when there are existing volumesnapshots", func(ctx SpecContext) {
122+
snapshots := storagesnapshotv1.VolumeSnapshotList{
123+
Items: []storagesnapshotv1.VolumeSnapshot{
124+
{
125+
ObjectMeta: metav1.ObjectMeta{
126+
Namespace: namespace,
127+
Name: clusterName + "-2",
128+
Labels: map[string]string{
129+
utils.BackupNameLabelName: backupName,
130+
},
131+
},
132+
},
133+
{
134+
ObjectMeta: metav1.ObjectMeta{
135+
Namespace: namespace,
136+
Name: clusterName + "-2-wal",
137+
Labels: map[string]string{
138+
utils.BackupNameLabelName: backupName,
139+
},
140+
},
141+
},
142+
},
143+
}
144+
145+
mockClient := fake.NewClientBuilder().
146+
WithScheme(scheme.BuildWithAllKnownScheme()).
147+
WithObjects(cluster, targetPod).
148+
WithLists(&snapshots).
149+
Build()
150+
fakeRecorder := record.NewFakeRecorder(3)
151+
152+
executor := NewExecutorBuilder(mockClient, fakeRecorder).
153+
FenceInstance(true).
154+
Build()
155+
156+
result, err := executor.Execute(ctx, cluster, backup, targetPod, pvcs)
157+
Expect(err).ToNot(HaveOccurred())
158+
// we should have found snapshots that are not ready, and so we'd return to
159+
// wait for them to be ready
160+
Expect(result).ToNot(BeNil())
161+
hasEvent := false
162+
var events string
163+
go func() {
164+
evts := <-fakeRecorder.Events
165+
hasEvent = true
166+
events = evts
167+
}()
168+
Consistently(func() bool {
169+
return hasEvent
170+
}).Should(BeFalse())
171+
Consistently(func() string {
172+
return events
173+
}).Should(BeEmpty())
174+
})
175+
176+
It("should unfence the target pod when the snapshots are ready", func(ctx SpecContext) {
177+
snapshots := storagesnapshotv1.VolumeSnapshotList{
178+
Items: []storagesnapshotv1.VolumeSnapshot{
179+
{
180+
ObjectMeta: metav1.ObjectMeta{
181+
Namespace: namespace,
182+
Name: clusterName + "-2",
183+
Labels: map[string]string{
184+
utils.BackupNameLabelName: backupName,
185+
},
186+
},
187+
Status: &storagesnapshotv1.VolumeSnapshotStatus{
188+
ReadyToUse: ptr.To(true),
189+
Error: nil,
190+
},
191+
},
192+
{
193+
ObjectMeta: metav1.ObjectMeta{
194+
Namespace: namespace,
195+
Name: clusterName + "-2-wal",
196+
Labels: map[string]string{
197+
utils.BackupNameLabelName: backupName,
198+
},
199+
},
200+
Status: &storagesnapshotv1.VolumeSnapshotStatus{
201+
ReadyToUse: ptr.To(true),
202+
Error: nil,
203+
},
204+
},
205+
},
206+
}
207+
208+
mockClient := fake.NewClientBuilder().
209+
WithScheme(scheme.BuildWithAllKnownScheme()).
210+
WithObjects(cluster, targetPod).
211+
WithLists(&snapshots).
212+
Build()
213+
fakeRecorder := record.NewFakeRecorder(3)
214+
215+
executor := NewExecutorBuilder(mockClient, fakeRecorder).
216+
FenceInstance(true).
217+
Build()
218+
219+
result, err := executor.Execute(ctx, cluster, backup, targetPod, pvcs)
220+
Expect(err).ToNot(HaveOccurred())
221+
// we should have found snapshots that are ready, and so the result
222+
// should be nil
223+
Expect(result).To(BeNil())
224+
hasEvent := false
225+
var events string
226+
go func() {
227+
evts := <-fakeRecorder.Events
228+
hasEvent = true
229+
events = evts
230+
}()
231+
Eventually(func() bool {
232+
return hasEvent
233+
}).Should(BeTrue())
234+
Eventually(func() string {
235+
return events
236+
}).Should(ContainSubstring("Un-fencing Pod " + targetPod.Name))
237+
})
238+
})
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/*
2+
Copyright The CloudNativePG Contributors
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package volumesnapshot
18+
19+
import (
20+
"testing"
21+
22+
. "github.com/onsi/ginkgo/v2"
23+
. "github.com/onsi/gomega"
24+
)
25+
26+
func TestReconciler(t *testing.T) {
27+
RegisterFailHandler(Fail)
28+
RunSpecs(t, "Volumesnapshot Reconciler Suite")
29+
}

0 commit comments

Comments
 (0)