Skip to content

Commit 11db475

Browse files
mnenciaarmru
andauthored
fix(bootstrap): recover lost first-primary initialization Job (#11039)
When createPrimaryInstance loses a status-patch optimistic-lock conflict after creating the data PVC but before creating the initialization Job, the orphan Pending PVC is counted as an instance, so the cluster looks past-bootstrap and the bootstrap gate is never re-entered. The PVC-state reconciler instead takes the reattach path and waits forever for a PVC that no Job will ever mark ready. Make the PVC-state reconciler self-healing: when the selected primary instance's PVC group is not ready and no initialization Job exists for it, recreate the bootstrap Job reusing the serial already assigned to the PVC, so no second instance is allocated. Closes #11036 Refs #4147 Signed-off-by: Marco Nenciarini <[email protected]> Signed-off-by: Armando Ruocco <[email protected]> Co-authored-by: Armando Ruocco <[email protected]>
1 parent e1f2947 commit 11db475

2 files changed

Lines changed: 284 additions & 66 deletions

File tree

‎internal/controller/cluster_controller_test.go‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ import (
3838
"github.com/cloudnative-pg/cloudnative-pg/pkg/postgres"
3939
"github.com/cloudnative-pg/cloudnative-pg/pkg/reconciler/persistentvolumeclaim"
4040
"github.com/cloudnative-pg/cloudnative-pg/pkg/specs"
41+
"github.com/cloudnative-pg/cloudnative-pg/pkg/utils"
4142

4243
. "github.com/onsi/ginkgo/v2"
4344
. "github.com/onsi/gomega"
@@ -183,6 +184,94 @@ var _ = Describe("ensureInstancesAreCreated reattachment while a PVC is terminat
183184
})
184185
})
185186

187+
var _ = Describe("ensureInstancesAreCreated recovers a lost first-primary bootstrap Job (#11036)", func() {
188+
var env *testingEnvironment
189+
var namespace string
190+
191+
BeforeEach(func() {
192+
env = buildTestEnvironment()
193+
namespace = newFakeNamespace(env.client)
194+
})
195+
196+
It("recreates the initdb Job reusing serial 1 when the primary PVC is stuck initializing", func(ctx SpecContext) {
197+
cluster := newFakeCNPGCluster(env.client, namespace, func(c *apiv1.Cluster) {
198+
c.Spec.Instances = 1
199+
})
200+
201+
// The first-primary bootstrap created the data PVC for serial 1 and patched
202+
// the cluster status (TargetPrimary set, Instances flipped to 1) but lost the
203+
// optimistic lock before creating the initdb Job: no Pod and no Job exist.
204+
primaryName := specs.GetInstanceName(cluster.Name, 1)
205+
cluster.Status.Instances = 1
206+
cluster.Status.ReadyInstances = 0
207+
cluster.Status.InstanceNames = []string{primaryName}
208+
cluster.Status.TargetPrimary = primaryName
209+
cluster.Status.DanglingPVC = []string{primaryName}
210+
211+
pvcGroup := newFakePVC(env.client, cluster, 1, persistentvolumeclaim.StatusInitializing)
212+
213+
resources := &managedResources{
214+
pvcs: corev1.PersistentVolumeClaimList{Items: pvcGroup},
215+
}
216+
statusList := postgres.PostgresqlStatusList{}
217+
218+
res, err := env.clusterReconciler.ensureInstancesAreCreated(ctx, cluster, resources, statusList)
219+
Expect(err).To(MatchError(ErrNextLoop))
220+
Expect(res.RequeueAfter).To(Equal(time.Second))
221+
222+
var jobs batchv1.JobList
223+
Expect(env.client.List(ctx, &jobs)).To(Succeed())
224+
Expect(jobs.Items).To(HaveLen(1), "the bootstrap Job for serial 1 should have been recreated")
225+
Expect(jobs.Items[0].Labels[utils.InstanceNameLabelName]).To(Equal(primaryName))
226+
227+
// No serial-2 instance or PVC must be created: the existing serial must be reused.
228+
var pvcs corev1.PersistentVolumeClaimList
229+
Expect(env.client.List(ctx, &pvcs)).To(Succeed())
230+
for _, pvc := range pvcs.Items {
231+
serial, serr := specs.GetNodeSerial(pvc.ObjectMeta)
232+
Expect(serr).ToNot(HaveOccurred())
233+
Expect(serial).To(Equal(1), "no PVC for a different serial should have been created")
234+
}
235+
236+
var pods corev1.PodList
237+
Expect(env.client.List(ctx, &pods)).To(Succeed())
238+
Expect(pods.Items).To(BeEmpty(), "no Pod should be created while the PVC is still initializing")
239+
})
240+
241+
It("waits instead of recreating the init Job when one already exists", func(ctx SpecContext) {
242+
cluster := newFakeCNPGCluster(env.client, namespace, func(c *apiv1.Cluster) {
243+
c.Spec.Instances = 1
244+
})
245+
246+
primaryName := specs.GetInstanceName(cluster.Name, 1)
247+
cluster.Status.Instances = 1
248+
cluster.Status.ReadyInstances = 0
249+
cluster.Status.InstanceNames = []string{primaryName}
250+
cluster.Status.TargetPrimary = primaryName
251+
cluster.Status.DanglingPVC = []string{primaryName}
252+
253+
// The initializing PVC already has its bootstrap Job: the recovery path
254+
// must defer to the regular wait, not create a second Job.
255+
pvcGroup := newFakePVC(env.client, cluster, 1, persistentvolumeclaim.StatusInitializing)
256+
existingJob := specs.CreatePrimaryJobViaInitdb(*cluster, 1)
257+
cluster.SetInheritedDataAndOwnership(&existingJob.ObjectMeta)
258+
259+
resources := &managedResources{
260+
pvcs: corev1.PersistentVolumeClaimList{Items: pvcGroup},
261+
jobs: batchv1.JobList{Items: []batchv1.Job{*existingJob}},
262+
}
263+
statusList := postgres.PostgresqlStatusList{}
264+
265+
res, err := env.clusterReconciler.ensureInstancesAreCreated(ctx, cluster, resources, statusList)
266+
Expect(err).To(MatchError(ErrNextLoop))
267+
Expect(res.RequeueAfter).To(Equal(time.Second))
268+
269+
var jobs batchv1.JobList
270+
Expect(env.client.List(ctx, &jobs)).To(Succeed())
271+
Expect(jobs.Items).To(BeEmpty(), "the pre-existing Job lives only in resources, none should be created")
272+
})
273+
})
274+
186275
var _ = Describe("Filtering cluster", func() {
187276
metrics := make(map[string]string, 1)
188277
metrics["a-secret"] = "test-version"

0 commit comments

Comments
 (0)