Skip to content

Commit d149135

Browse files
dennispidunleonardocegbartolini
authored
fix(walrestore): allow history files when cluster timeline is unset (#10818)
During bootstrap recovery from an external archive, the recovering pod is not yet `CurrentPrimary` or `TargetPrimary` and `cluster.Status.TimelineID` is 0, because no primary has promoted and reported its timeline. The existing guard (`fileTimeline > clusterTimeline`) therefore blocks every `.history` file, causing PostgreSQL to stop recovery at the basebackup's timeline and silently drop transactions on newer timelines. Add an early-return exemption when `clusterTimeline == 0`: the split-brain scenario the guard protects against requires an active primary, which cannot exist when `TimelineID` is unset. Allowing history files in this state lets PostgreSQL walk forward across all timeline switches during bootstrap recovery. This PR also adds an e2e test that restores across a post-switchover timeline and verifies data written on the newer timeline is recovered. Closes #10442 Closes #10817 Assisted-by: Claude Opus 4.8 Signed-off-by: Dennis Pidun <[email protected]> Signed-off-by: Leonardo Cecchi <[email protected]> Signed-off-by: Gabriele Bartolini <[email protected]> Co-authored-by: Leonardo Cecchi <[email protected]> Co-authored-by: Gabriele Bartolini <[email protected]>
1 parent 0d2de08 commit d149135

3 files changed

Lines changed: 111 additions & 4 deletions

File tree

‎internal/cmd/manager/walrestore/cmd.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -466,6 +466,19 @@ func validateTimelineHistoryFile(ctx context.Context, walName string, cluster *a
466466
}
467467

468468
clusterTimeline := cluster.Status.TimelineID
469+
470+
// clusterTimeline == 0 means no primary has promoted yet and TimelineID has
471+
// never been written to the cluster status, as happens during bootstrap
472+
// recovery from an external archive. The split-brain scenario this guard
473+
// protects against requires an active primary, which cannot exist while
474+
// TimelineID is unset, so allowing all history files here is safe.
475+
if clusterTimeline == 0 {
476+
contextLog.Trace("Allowing timeline history file: cluster timeline not yet established",
477+
"walName", walName,
478+
"fileTimeline", fileTimeline)
479+
return nil
480+
}
481+
469482
if fileTimeline > clusterTimeline {
470483
contextLog.Warning("Refusing to restore future timeline history file",
471484
"walName", walName,

‎internal/cmd/manager/walrestore/cmd_test.go‎

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -213,15 +213,40 @@ var _ = Describe("validateTimelineHistoryFile", func() {
213213
Expect(err).To(Equal(barmanRestorer.ErrWALNotFound))
214214
})
215215

216-
It("should reject any timeline when cluster timeline is not yet set", func(ctx SpecContext) {
216+
It("should reject a future timeline history file for an established replica",
217+
func(ctx SpecContext) {
218+
cluster := &apiv1.Cluster{
219+
Status: apiv1.ClusterStatus{
220+
CurrentPrimary: "primary-pod",
221+
TargetPrimary: "primary-pod",
222+
TimelineID: 20,
223+
},
224+
}
225+
226+
err := validateTimelineHistoryFile(ctx, "00000015.history", cluster, "replica-pod")
227+
Expect(err).To(Equal(barmanRestorer.ErrWALNotFound))
228+
})
229+
230+
It("should allow any history file when cluster timeline is not yet established", func(ctx SpecContext) {
217231
cluster := &apiv1.Cluster{
218232
Status: apiv1.ClusterStatus{
219233
CurrentPrimary: "primary-pod",
220234
TimelineID: 0,
221235
},
222236
}
223237

224-
err := validateTimelineHistoryFile(ctx, "00000001.history", cluster, "replica-pod")
225-
Expect(err).To(Equal(barmanRestorer.ErrWALNotFound))
238+
err := validateTimelineHistoryFile(ctx, "00000014.history", cluster, "replica-pod")
239+
Expect(err).ToNot(HaveOccurred())
240+
})
241+
242+
It("should allow any history file when no primary has been elected yet (empty cluster status)", func(ctx SpecContext) {
243+
cluster := &apiv1.Cluster{
244+
Status: apiv1.ClusterStatus{
245+
TimelineID: 0,
246+
},
247+
}
248+
249+
err := validateTimelineHistoryFile(ctx, "00000014.history", cluster, "restore-pod")
250+
Expect(err).ToNot(HaveOccurred())
226251
})
227252
})

‎tests/e2e/backup_restore_objectstore_test.go‎

Lines changed: 70 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,12 @@ import (
2323
"fmt"
2424
"path/filepath"
2525

26+
apierrs "k8s.io/apimachinery/pkg/api/errors"
2627
"k8s.io/client-go/util/retry"
2728
ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
2829

2930
apiv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1"
31+
"github.com/cloudnative-pg/cloudnative-pg/pkg/utils"
3032
"github.com/cloudnative-pg/cloudnative-pg/tests"
3133
backupasserts "github.com/cloudnative-pg/cloudnative-pg/tests/internal/asserts/backup"
3234
clusterasserts "github.com/cloudnative-pg/cloudnative-pg/tests/internal/asserts/cluster"
@@ -43,6 +45,7 @@ import (
4345
"github.com/cloudnative-pg/cloudnative-pg/tests/utils/pods"
4446
"github.com/cloudnative-pg/cloudnative-pg/tests/utils/postgres"
4547
"github.com/cloudnative-pg/cloudnative-pg/tests/utils/secrets"
48+
storageutils "github.com/cloudnative-pg/cloudnative-pg/tests/utils/storage"
4649
"github.com/cloudnative-pg/cloudnative-pg/tests/utils/timeouts"
4750
"github.com/cloudnative-pg/cloudnative-pg/tests/utils/yaml"
4851

@@ -271,6 +274,31 @@ var _ = Describe("Object storage - Backup and restore", Label(tests.LabelBackupR
271274
Expect(err).ToNot(HaveOccurred())
272275
metricsasserts.AssertMetricsData(env, testTimeouts, namespace, targetDBOne, targetDBTwo, targetDBSecret, cluster)
273276

277+
By("deleting the first restored cluster and waiting for its resources to be removed", func() {
278+
firstRestoreClusterName, err := yaml.GetResourceNameFromYAML(env.Scheme, clusterRestoreSampleFile)
279+
Expect(err).ToNot(HaveOccurred())
280+
281+
err = resources.DeleteResourcesFromFile(env, namespace, clusterRestoreSampleFile)
282+
Expect(err).ToNot(HaveOccurred())
283+
284+
// The restored cluster is recreated later with the same name, so wait for
285+
// the cluster and its PVCs to be fully removed first. Otherwise the
286+
// recreated cluster could adopt stale PVCs instead of performing a fresh
287+
// restore from the object store, masking or breaking the assertions below.
288+
Eventually(func() bool {
289+
_, err := clusterutils.Get(env.Ctx, env.Client, namespace, firstRestoreClusterName)
290+
return apierrs.IsNotFound(err)
291+
}, testTimeouts[timeouts.ClusterIsReady]).Should(BeTrue())
292+
293+
Eventually(func(g Gomega) {
294+
pvcList, err := storageutils.GetPVCList(env.Ctx, env.Client, namespace)
295+
g.Expect(err).ToNot(HaveOccurred())
296+
for _, pvc := range pvcList.Items {
297+
g.Expect(pvc.Labels[utils.ClusterLabelName]).ToNot(Equal(firstRestoreClusterName))
298+
}
299+
}, testTimeouts[timeouts.ClusterIsReady]).Should(Succeed())
300+
})
301+
274302
previous := 0
275303
latestGZ := filepath.Join("*", clusterName, "*", "*.history.gz")
276304
By(fmt.Sprintf("checking the previous number of .history files in the object store, history file name is %v",
@@ -279,14 +307,55 @@ var _ = Describe("Object storage - Backup and restore", Label(tests.LabelBackupR
279307
Expect(err).ToNot(HaveOccurred())
280308
})
281309

282-
clusterasserts.AssertSwitchover(env, testTimeouts, namespace, clusterName)
310+
By("performing a switchover", func() {
311+
clusterasserts.AssertSwitchover(env, testTimeouts, namespace, clusterName)
312+
})
283313

284314
By("checking the number of .history after switchover", func() {
285315
Eventually(func() (int, error) {
286316
return objectstore.CountFiles(objectStoreEnv, latestGZ)
287317
}, 60).Should(BeNumerically(">", previous))
288318
})
289319

320+
const postSwitchoverTableName = "to_restore_post_switchover"
321+
322+
By("inserting data into source cluster after switchover", func() {
323+
postSwitchoverTableLocator := pgasserts.TableLocator{
324+
Namespace: namespace,
325+
ClusterName: clusterName,
326+
DatabaseName: postgres.AppDBName,
327+
TableName: postSwitchoverTableName,
328+
}
329+
pgasserts.AssertCreateTestData(env, postSwitchoverTableLocator)
330+
objectstoreasserts.AssertArchiveWalOnObjectStore(
331+
env, testTimeouts, objectStoreEnv, namespace, clusterName, clusterName)
332+
})
333+
334+
By("restoring cluster again and verifying data written after switchover is present", func() {
335+
resources.CreateResourceFromFile(env, namespace, clusterRestoreSampleFile)
336+
restoreClusterName, err := yaml.GetResourceNameFromYAML(env.Scheme, clusterRestoreSampleFile)
337+
Expect(err).ToNot(HaveOccurred())
338+
clusterasserts.AssertClusterIsReady(
339+
env, namespace, restoreClusterName, testTimeouts[timeouts.ClusterIsReadySlow],
340+
)
341+
342+
origTableLocator := pgasserts.TableLocator{
343+
Namespace: namespace,
344+
ClusterName: restoreClusterName,
345+
DatabaseName: postgres.AppDBName,
346+
TableName: tableName,
347+
}
348+
pgasserts.AssertDataExpectedCount(env, origTableLocator, 2)
349+
350+
postSwitchoverTableLocator := pgasserts.TableLocator{
351+
Namespace: namespace,
352+
ClusterName: restoreClusterName,
353+
DatabaseName: postgres.AppDBName,
354+
TableName: postSwitchoverTableName,
355+
}
356+
pgasserts.AssertDataExpectedCount(env, postSwitchoverTableLocator, 2)
357+
})
358+
290359
By("deleting the restored cluster", func() {
291360
err = resources.DeleteResourcesFromFile(env, namespace, clusterRestoreSampleFile)
292361
Expect(err).ToNot(HaveOccurred())

0 commit comments

Comments
 (0)