Skip to content

Commit 3326184

Browse files
mnenciaarmru
andauthored
fix(replica): preserve the demotion token on no-change reconciles (#11075)
`generateDemotionToken` returns an empty token to mean "stored one is already current", but `reconcileDemotionToken` patched that empty value back, wiping the stored token. It triggers when the reconcile is requeued between storing the token and cleaning up the transition metadata (e.g. the cleanup patch failing against a flaky webhook), leaving the switchover with an empty `Status.DemotionToken`. Skip the patch when the generated token is empty. Closes #11074 Signed-off-by: Marco Nenciarini <[email protected]> Signed-off-by: Armando Ruocco <[email protected]> Co-authored-by: Armando Ruocco <[email protected]>
1 parent 3b31bbb commit 3326184

3 files changed

Lines changed: 187 additions & 1 deletion

File tree

‎pkg/reconciler/replicaclusterswitch/reconciler.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,11 @@ func reconcileDemotionToken(
190190
return nil, err
191191
}
192192

193-
if cluster.Status.DemotionToken != demotionToken {
193+
// generateDemotionToken returns an empty token to signal that the stored
194+
// one is already up to date. Skip the patch in that case: writing the empty
195+
// value back would wipe a previously generated token if this reconcile is
196+
// requeued before the transition metadata is cleaned up.
197+
if demotionToken != "" && cluster.Status.DemotionToken != demotionToken {
194198
origCluster := cluster.DeepCopy()
195199
contextLogger.Info(
196200
"patching the demotionToken in the cluster status",
Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
/*
2+
Copyright © contributors to CloudNativePG, established as
3+
CloudNativePG a Series of LF Projects, LLC.
4+
5+
Licensed under the Apache License, Version 2.0 (the "License");
6+
you may not use this file except in compliance with the License.
7+
You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
17+
SPDX-License-Identifier: Apache-2.0
18+
*/
19+
20+
package replicaclusterswitch
21+
22+
import (
23+
"context"
24+
25+
corev1 "k8s.io/api/core/v1"
26+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
27+
"sigs.k8s.io/controller-runtime/pkg/client/fake"
28+
29+
apiv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1"
30+
schemeBuilder "github.com/cloudnative-pg/cloudnative-pg/internal/scheme"
31+
"github.com/cloudnative-pg/cloudnative-pg/pkg/management/postgres/webserver/client/remote"
32+
"github.com/cloudnative-pg/cloudnative-pg/pkg/postgres"
33+
"github.com/cloudnative-pg/cloudnative-pg/pkg/utils"
34+
35+
. "github.com/onsi/ginkgo/v2"
36+
. "github.com/onsi/gomega"
37+
)
38+
39+
// fakeControlData describes a cleanly shut down primary; its REDO WAL file is
40+
// referenced by archivedWALName below.
41+
const fakeControlData = `pg_control version number: 1002
42+
Catalog version number: 202201241
43+
Database cluster state: shut down
44+
Database system identifier: 12345678901234567890123456789012
45+
Latest checkpoint's TimeLineID: 3
46+
Latest checkpoint location: 0/3000FF0
47+
Latest checkpoint's REDO location: 0/3000CC0
48+
Latest checkpoint's REDO WAL file: 000000010000000000000003
49+
`
50+
51+
const archivedWALName = "000000010000000000000003"
52+
53+
// instanceClientMock is a minimal remote.InstanceClient implementation that
54+
// returns canned pg_controldata and partial-WAL-archive responses. The other
55+
// InstanceClient methods are inherited from the embedded nil interface and
56+
// panic if called.
57+
type instanceClientMock struct {
58+
remote.InstanceClient
59+
controlData string
60+
archivedWAL string
61+
archiveCalls int
62+
controlErr error
63+
}
64+
65+
func (m *instanceClientMock) GetPgControlDataFromInstance(
66+
_ context.Context,
67+
_ *corev1.Pod,
68+
) (string, error) {
69+
return m.controlData, m.controlErr
70+
}
71+
72+
func (m *instanceClientMock) ArchivePartialWAL(_ context.Context, _ *corev1.Pod) (string, error) {
73+
m.archiveCalls++
74+
return m.archivedWAL, nil
75+
}
76+
77+
var _ = Describe("reconcileDemotionToken", func() {
78+
const primaryPodName = "cluster-a-1"
79+
80+
// expectedToken is the token generateDemotionToken computes from fakeControlData.
81+
var expectedToken string
82+
BeforeEach(func() {
83+
var err error
84+
expectedToken, err = utils.ParsePgControldataOutput(fakeControlData).CreatePromotionToken()
85+
Expect(err).ToNot(HaveOccurred())
86+
Expect(expectedToken).ToNot(BeEmpty())
87+
})
88+
89+
buildCluster := func(storedToken string) *apiv1.Cluster {
90+
cluster := &apiv1.Cluster{
91+
ObjectMeta: metav1.ObjectMeta{Name: "cluster-a"},
92+
Spec: apiv1.ClusterSpec{
93+
ReplicaCluster: &apiv1.ReplicaClusterConfiguration{
94+
Primary: "cluster-b",
95+
Source: "cluster-b",
96+
},
97+
},
98+
Status: apiv1.ClusterStatus{
99+
CurrentPrimary: primaryPodName,
100+
DemotionToken: storedToken,
101+
},
102+
}
103+
return cluster
104+
}
105+
106+
instancesStatus := postgres.PostgresqlStatusList{
107+
Items: []postgres.PostgresqlStatus{
108+
{
109+
Pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: primaryPodName}},
110+
},
111+
},
112+
}
113+
114+
It("populates the demotion token when it is not set yet", func(ctx SpecContext) {
115+
cluster := buildCluster("")
116+
cli := fake.NewClientBuilder().
117+
WithScheme(schemeBuilder.BuildWithAllKnownScheme()).
118+
WithObjects(cluster).
119+
WithStatusSubresource(cluster).
120+
Build()
121+
instanceClient := &instanceClientMock{controlData: fakeControlData, archivedWAL: archivedWALName}
122+
123+
_, err := reconcileDemotionToken(ctx, cli, cluster, instanceClient, instancesStatus)
124+
Expect(err).ToNot(HaveOccurred())
125+
Expect(cluster.Status.DemotionToken).To(Equal(expectedToken))
126+
// A fresh token triggers exactly one partial WAL archive.
127+
Expect(instanceClient.archiveCalls).To(Equal(1))
128+
})
129+
130+
// Regression for #11074: when generateDemotionToken short-circuits with an
131+
// empty "no change" token, reconcileDemotionToken must not overwrite the
132+
// already-stored token. This happens whenever the reconcile is requeued
133+
// after the token was first written but before the transition metadata is
134+
// cleaned up (e.g. a failing webhook call on the cleanup patch).
135+
It("does not wipe an already-set demotion token on a no-change reconcile", func(ctx SpecContext) {
136+
cluster := buildCluster(expectedToken)
137+
cli := fake.NewClientBuilder().
138+
WithScheme(schemeBuilder.BuildWithAllKnownScheme()).
139+
WithObjects(cluster).
140+
WithStatusSubresource(cluster).
141+
Build()
142+
instanceClient := &instanceClientMock{controlData: fakeControlData, archivedWAL: archivedWALName}
143+
144+
_, err := reconcileDemotionToken(ctx, cli, cluster, instanceClient, instancesStatus)
145+
Expect(err).ToNot(HaveOccurred())
146+
Expect(cluster.Status.DemotionToken).To(Equal(expectedToken))
147+
// The partial WAL is archived only when a fresh token is generated.
148+
Expect(instanceClient.archiveCalls).To(BeZero())
149+
})
150+
})
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
Copyright © contributors to CloudNativePG, established as
3+
CloudNativePG a Series of LF Projects, LLC.
4+
5+
Licensed under the Apache License, Version 2.0 (the "License");
6+
you may not use this file except in compliance with the License.
7+
You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
17+
SPDX-License-Identifier: Apache-2.0
18+
*/
19+
20+
package replicaclusterswitch
21+
22+
import (
23+
"testing"
24+
25+
. "github.com/onsi/ginkgo/v2"
26+
. "github.com/onsi/gomega"
27+
)
28+
29+
func TestReplicaClusterSwitch(t *testing.T) {
30+
RegisterFailHandler(Fail)
31+
RunSpecs(t, "Replica cluster switch reconciler")
32+
}

0 commit comments

Comments
 (0)