Skip to content

Commit ea1d175

Browse files
mnenciaarmru
authored andcommitted
fix(major-upgrade): reset TimelineID to 1 after upgrade
After a major version upgrade, pg_upgrade creates a new database system with a new System ID and resets the timeline to 1. However, cluster.Status.TimelineID retained its old value (e.g., timeline 2), causing replicas to restore incompatible timeline history files from object storage. This resulted in PostgreSQL fatal errors: "requested timeline 2 is not a child of this server's history" "Latest checkpoint in file backup_label is at 0/13000080 on timeline 1, but in the history of the requested timeline, the server forked off from that timeline at 0/70226C8." The fix sets cluster.Status.TimelineID to 1 after major upgrade completion, ensuring validateTimelineHistoryFile() blocks incompatible timeline history files (e.g., 00000002.history). E2E tests verify this by performing a switchover to timeline 2 before the major upgrade, then confirming TimelineID is reset to 1 after upgrade completion. Closes #9764 Signed-off-by: Marco Nenciarini <[email protected]>
1 parent a8adc4c commit ea1d175

4 files changed

Lines changed: 219 additions & 0 deletions

File tree

‎pkg/reconciler/majorupgrade/reconciler.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,9 @@ func majorVersionUpgradeHandleCompletion(
237237
return nil, err
238238
}
239239

240+
// Reset timeline ID to 1 after major upgrade to match pg_upgrade behavior.
241+
// This prevents replicas from restoring incompatible timeline history files
242+
// from the pre-upgrade cluster in object storage.
240243
if err := status.PatchWithOptimisticLock(
241244
ctx,
242245
c,
@@ -245,6 +248,7 @@ func majorVersionUpgradeHandleCompletion(
245248
Image: jobImage,
246249
MajorVersion: requestedMajor,
247250
}),
251+
status.SetTimelineID(1),
248252
); err != nil {
249253
contextLogger.Error(err, "Unable to update cluster status after major upgrade completed.")
250254
return nil, err

‎pkg/resources/status/transactions.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,3 +76,10 @@ func SetPGDataImageInfo(imageInfo *apiv1.ImageInfo) Transaction {
7676
cluster.Status.PGDataImageInfo = imageInfo
7777
}
7878
}
79+
80+
// SetTimelineID is a transaction that sets the cluster timeline ID
81+
func SetTimelineID(timelineID int) Transaction {
82+
return func(cluster *apiv1.Cluster) {
83+
cluster.Status.TimelineID = timelineID
84+
}
85+
}
Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
/*
2+
Copyright © contributors to CloudNativePG, established as
3+
CloudNativePG a Series of LF Projects, LLC.
4+
5+
Licensed under the Apache License, Version 2.0 (the "License");
6+
you may not use this file except in compliance with the License.
7+
You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
17+
SPDX-License-Identifier: Apache-2.0
18+
*/
19+
20+
package status
21+
22+
import (
23+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
24+
25+
apiv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1"
26+
27+
. "github.com/onsi/ginkgo/v2"
28+
. "github.com/onsi/gomega"
29+
)
30+
31+
var _ = Describe("Status transactions", func() {
32+
Describe("SetClusterReadyCondition", func() {
33+
It("sets ready condition to true when phase is healthy", func() {
34+
cluster := &apiv1.Cluster{
35+
Status: apiv1.ClusterStatus{
36+
Phase: apiv1.PhaseHealthy,
37+
},
38+
}
39+
40+
SetClusterReadyCondition(cluster)
41+
42+
Expect(cluster.Status.Conditions).To(HaveLen(1))
43+
condition := cluster.Status.Conditions[0]
44+
Expect(condition.Type).To(Equal(string(apiv1.ConditionClusterReady)))
45+
Expect(condition.Status).To(Equal(metav1.ConditionTrue))
46+
Expect(condition.Reason).To(Equal(string(apiv1.ClusterReady)))
47+
Expect(condition.Message).To(Equal("Cluster is Ready"))
48+
})
49+
50+
It("sets ready condition to false when phase is not healthy", func() {
51+
cluster := &apiv1.Cluster{
52+
Status: apiv1.ClusterStatus{
53+
Phase: apiv1.PhaseMajorUpgrade,
54+
},
55+
}
56+
57+
SetClusterReadyCondition(cluster)
58+
59+
Expect(cluster.Status.Conditions).To(HaveLen(1))
60+
condition := cluster.Status.Conditions[0]
61+
Expect(condition.Type).To(Equal(string(apiv1.ConditionClusterReady)))
62+
Expect(condition.Status).To(Equal(metav1.ConditionFalse))
63+
Expect(condition.Reason).To(Equal(string(apiv1.ClusterIsNotReady)))
64+
Expect(condition.Message).To(Equal("Cluster Is Not Ready"))
65+
})
66+
67+
It("initializes conditions slice if nil", func() {
68+
cluster := &apiv1.Cluster{
69+
Status: apiv1.ClusterStatus{
70+
Phase: apiv1.PhaseHealthy,
71+
Conditions: nil,
72+
},
73+
}
74+
75+
SetClusterReadyCondition(cluster)
76+
77+
Expect(cluster.Status.Conditions).ToNot(BeNil())
78+
Expect(cluster.Status.Conditions).To(HaveLen(1))
79+
})
80+
})
81+
82+
Describe("SetPhase", func() {
83+
It("sets the cluster phase and reason", func() {
84+
cluster := &apiv1.Cluster{}
85+
86+
SetPhase(apiv1.PhaseMajorUpgrade, "Upgrading to version 17")(cluster)
87+
88+
Expect(cluster.Status.Phase).To(Equal(apiv1.PhaseMajorUpgrade))
89+
Expect(cluster.Status.PhaseReason).To(Equal("Upgrading to version 17"))
90+
})
91+
92+
It("overwrites existing phase and reason", func() {
93+
cluster := &apiv1.Cluster{
94+
Status: apiv1.ClusterStatus{
95+
Phase: apiv1.PhaseHealthy,
96+
PhaseReason: "All good",
97+
},
98+
}
99+
100+
SetPhase(apiv1.PhaseUpgrade, "Rolling update")(cluster)
101+
102+
Expect(cluster.Status.Phase).To(Equal(apiv1.PhaseUpgrade))
103+
Expect(cluster.Status.PhaseReason).To(Equal("Rolling update"))
104+
})
105+
})
106+
107+
Describe("SetImage", func() {
108+
It("sets the cluster image", func() {
109+
cluster := &apiv1.Cluster{}
110+
111+
SetImage("ghcr.io/cloudnative-pg/postgresql:17.2")(cluster)
112+
113+
Expect(cluster.Status.Image).To(Equal("ghcr.io/cloudnative-pg/postgresql:17.2"))
114+
})
115+
116+
It("overwrites existing image", func() {
117+
cluster := &apiv1.Cluster{
118+
Status: apiv1.ClusterStatus{
119+
Image: "ghcr.io/cloudnative-pg/postgresql:16.5",
120+
},
121+
}
122+
123+
SetImage("ghcr.io/cloudnative-pg/postgresql:17.2")(cluster)
124+
125+
Expect(cluster.Status.Image).To(Equal("ghcr.io/cloudnative-pg/postgresql:17.2"))
126+
})
127+
})
128+
129+
Describe("SetPGDataImageInfo", func() {
130+
It("sets the PGData image info", func() {
131+
cluster := &apiv1.Cluster{}
132+
imageInfo := &apiv1.ImageInfo{
133+
Image: "ghcr.io/cloudnative-pg/postgresql:17.2",
134+
MajorVersion: 17,
135+
}
136+
137+
SetPGDataImageInfo(imageInfo)(cluster)
138+
139+
Expect(cluster.Status.PGDataImageInfo).ToNot(BeNil())
140+
Expect(cluster.Status.PGDataImageInfo.Image).To(Equal("ghcr.io/cloudnative-pg/postgresql:17.2"))
141+
Expect(cluster.Status.PGDataImageInfo.MajorVersion).To(Equal(17))
142+
})
143+
144+
It("overwrites existing PGData image info", func() {
145+
cluster := &apiv1.Cluster{
146+
Status: apiv1.ClusterStatus{
147+
PGDataImageInfo: &apiv1.ImageInfo{
148+
Image: "ghcr.io/cloudnative-pg/postgresql:16.5",
149+
MajorVersion: 16,
150+
},
151+
},
152+
}
153+
newImageInfo := &apiv1.ImageInfo{
154+
Image: "ghcr.io/cloudnative-pg/postgresql:17.2",
155+
MajorVersion: 17,
156+
}
157+
158+
SetPGDataImageInfo(newImageInfo)(cluster)
159+
160+
Expect(cluster.Status.PGDataImageInfo.Image).To(Equal("ghcr.io/cloudnative-pg/postgresql:17.2"))
161+
Expect(cluster.Status.PGDataImageInfo.MajorVersion).To(Equal(17))
162+
})
163+
})
164+
165+
Describe("SetTimelineID", func() {
166+
It("sets the cluster timeline ID", func() {
167+
cluster := &apiv1.Cluster{
168+
Status: apiv1.ClusterStatus{
169+
TimelineID: 5,
170+
},
171+
}
172+
173+
SetTimelineID(10)(cluster)
174+
Expect(cluster.Status.TimelineID).To(Equal(10))
175+
})
176+
177+
It("resets timeline ID to 1 after major upgrade", func() {
178+
cluster := &apiv1.Cluster{
179+
Status: apiv1.ClusterStatus{
180+
TimelineID: 2,
181+
},
182+
}
183+
184+
SetTimelineID(1)(cluster)
185+
Expect(cluster.Status.TimelineID).To(Equal(1))
186+
})
187+
})
188+
})

‎tests/e2e/cluster_major_upgrade_test.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -326,6 +326,12 @@ var _ = Describe("Postgres Major Upgrade", Label(tests.LabelPostgresMajorUpgrade
326326
}
327327
}
328328

329+
verifyTimelineResetAfterUpgrade := func(ctx context.Context, client client.Client, cluster *apiv1.Cluster) {
330+
currentCluster, err := clusterutils.Get(ctx, client, cluster.Namespace, cluster.Name)
331+
Expect(err).ToNot(HaveOccurred())
332+
Expect(currentCluster.Status.TimelineID).To(Equal(1))
333+
}
334+
329335
BeforeEach(func() {
330336
if testLevelEnv.Depth < int(level) {
331337
Skip("Test depth is lower than the amount requested for this test")
@@ -353,6 +359,17 @@ var _ = Describe("Postgres Major Upgrade", Label(tests.LabelPostgresMajorUpgrade
353359
AssertClusterIsReady(cluster.Namespace, cluster.Name, testTimeouts[timeouts.ClusterIsReady],
354360
env)
355361

362+
By("Performing switchover to move to timeline 2")
363+
AssertSwitchover(cluster.Namespace, cluster.Name, env)
364+
365+
By("Verifying cluster is on timeline 2 before upgrade")
366+
Eventually(func(g Gomega) {
367+
currentCluster, err := clusterutils.Get(env.Ctx, env.Client, cluster.Namespace, cluster.Name)
368+
g.Expect(err).ToNot(HaveOccurred())
369+
g.Expect(currentCluster.Status.TimelineID).To(Equal(2))
370+
}).WithTimeout(time.Duration(testTimeouts[timeouts.NewPrimaryAfterSwitchover]) * time.Second).
371+
WithPolling(5 * time.Second).Should(Succeed())
372+
356373
By("Collecting the pods UUIDs")
357374
podList, err := clusterutils.ListPods(env.Ctx, env.Client, cluster.Name, cluster.Namespace)
358375
Expect(err).ToNot(HaveOccurred())
@@ -420,6 +437,9 @@ var _ = Describe("Postgres Major Upgrade", Label(tests.LabelPostgresMajorUpgrade
420437
By("Verifying the cluster is running the target version")
421438
verifyPostgresVersion(env, primary, oldStdOut, scenario.targetMajor)
422439

440+
By("Verifying timeline ID is reset to 1 after major upgrade")
441+
verifyTimelineResetAfterUpgrade(env.Ctx, env.Client, cluster)
442+
423443
// Expect temporary files to be deleted
424444
By("Checking no leftovers exist from the upgrade")
425445
verifyCleanupAfterUpgrade(env.Ctx, env.Client, primary)

0 commit comments

Comments
 (0)