Skip to content

Commit ef73994

Browse files
mnenciaarmru
andauthored
fix(walrestore): prevent replicas from downloading future timeline history files (#9650)
Replicas can crash-loop when orphaned "future timeline" .history files exist in the WAL archive. This can occur during split-brain scenarios or other conditions where timeline history files are created for timelines that the cluster never officially adopts. This fix adds validation during WAL restore to prevent replicas from downloading timeline history files with timeline IDs greater than the cluster's current timeline. Primary instances retain full access. The validation works by: - Parsing timeline ID from .history filenames (e.g., 00000022.history) - Checking if the instance is a primary or replica - For replicas, rejecting files where fileTimeline > clusterTimeline - Returning "file not found" to PostgreSQL for rejected files This prevents PostgreSQL from ever seeing the problematic history file, allowing normal recovery to proceed. Combined with PR #9637 (auto-recovery via re-cloning), this provides complete coverage of timeline divergence scenarios. Closes #4188 Signed-off-by: Marco Nenciarini <[email protected]> Signed-off-by: Armando Ruocco <[email protected]> Co-authored-by: Armando Ruocco <[email protected]>
1 parent c9fc7f4 commit ef73994

8 files changed

Lines changed: 461 additions & 0 deletions

File tree

‎internal/cmd/manager/walrestore/cmd.go‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,10 @@ func run(ctx context.Context, pgData string, podName string, args []string) erro
206206
}
207207
} else {
208208
// This is not a regular WAL file, we fetch it directly
209+
if err := validateTimelineHistoryFile(ctx, walName, cluster, podName); err != nil {
210+
return err
211+
}
212+
209213
walFilesList = []string{walName}
210214
}
211215

@@ -433,3 +437,46 @@ func isStreamingAvailable(cluster *apiv1.Cluster, podName string) bool {
433437
// Primary, we do not replicate from nobody
434438
return false
435439
}
440+
441+
// validateTimelineHistoryFile prevents replicas from downloading timeline history files
442+
// for timelines ahead of the cluster's current timeline. Primaries can download any timeline.
443+
func validateTimelineHistoryFile(ctx context.Context, walName string, cluster *apiv1.Cluster, podName string) error {
444+
contextLog := log.FromContext(ctx)
445+
446+
if !strings.HasSuffix(walName, ".history") {
447+
return nil
448+
}
449+
450+
fileTimeline, err := postgres.ParseTimelineFromHistoryFilename(walName)
451+
if err != nil {
452+
contextLog.Warning("Could not parse timeline from history filename",
453+
"walName", walName,
454+
"error", err)
455+
return nil
456+
}
457+
458+
if cluster.Status.CurrentPrimary == podName || cluster.Status.TargetPrimary == podName {
459+
contextLog.Trace("Allowing timeline history file download for primary",
460+
"walName", walName,
461+
"fileTimeline", fileTimeline,
462+
"isPrimary", true)
463+
return nil
464+
}
465+
466+
clusterTimeline := cluster.Status.TimelineID
467+
if fileTimeline > clusterTimeline {
468+
contextLog.Warning("Refusing to restore future timeline history file",
469+
"walName", walName,
470+
"fileTimeline", fileTimeline,
471+
"clusterTimeline", clusterTimeline)
472+
473+
return barmanRestorer.ErrWALNotFound
474+
}
475+
476+
contextLog.Trace("Allowing timeline history file download for replica",
477+
"walName", walName,
478+
"fileTimeline", fileTimeline,
479+
"clusterTimeline", clusterTimeline)
480+
481+
return nil
482+
}

‎internal/cmd/manager/walrestore/cmd_test.go‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ SPDX-License-Identifier: Apache-2.0
2020
package walrestore
2121

2222
import (
23+
barmanRestorer "github.com/cloudnative-pg/barman-cloud/pkg/restorer"
2324
"k8s.io/utils/ptr"
2425

2526
apiv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1"
@@ -113,3 +114,114 @@ var _ = Describe("Function isStreamingAvailable", func() {
113114
Expect(isStreamingAvailable(&cluster, "primaryPod")).To(BeTrue())
114115
})
115116
})
117+
118+
var _ = Describe("validateTimelineHistoryFile", func() {
119+
It("should allow regular WAL files to pass through", func(ctx SpecContext) {
120+
cluster := &apiv1.Cluster{
121+
Status: apiv1.ClusterStatus{
122+
CurrentPrimary: "primary-pod",
123+
TimelineID: 5,
124+
},
125+
}
126+
127+
err := validateTimelineHistoryFile(ctx, "000000010000000000000001", cluster, "replica-pod")
128+
Expect(err).ToNot(HaveOccurred())
129+
})
130+
131+
It("should allow invalid history filenames to pass through", func(ctx SpecContext) {
132+
cluster := &apiv1.Cluster{
133+
Status: apiv1.ClusterStatus{
134+
CurrentPrimary: "primary-pod",
135+
TimelineID: 5,
136+
},
137+
}
138+
139+
err := validateTimelineHistoryFile(ctx, "invalid.history", cluster, "replica-pod")
140+
Expect(err).ToNot(HaveOccurred())
141+
})
142+
143+
It("should allow primary to download any timeline", func(ctx SpecContext) {
144+
cluster := &apiv1.Cluster{
145+
Status: apiv1.ClusterStatus{
146+
CurrentPrimary: "primary-pod",
147+
TimelineID: 5,
148+
},
149+
}
150+
151+
err := validateTimelineHistoryFile(ctx, "00000064.history", cluster, "primary-pod")
152+
Expect(err).ToNot(HaveOccurred())
153+
})
154+
155+
It("should allow target primary (being promoted) to download any timeline", func(ctx SpecContext) {
156+
cluster := &apiv1.Cluster{
157+
Status: apiv1.ClusterStatus{
158+
CurrentPrimary: "old-primary",
159+
TargetPrimary: "new-primary",
160+
TimelineID: 5,
161+
},
162+
}
163+
164+
err := validateTimelineHistoryFile(ctx, "00000064.history", cluster, "new-primary")
165+
Expect(err).ToNot(HaveOccurred())
166+
})
167+
168+
It("should allow replica to download current timeline", func(ctx SpecContext) {
169+
cluster := &apiv1.Cluster{
170+
Status: apiv1.ClusterStatus{
171+
CurrentPrimary: "primary-pod",
172+
TimelineID: 33,
173+
},
174+
}
175+
176+
err := validateTimelineHistoryFile(ctx, "00000021.history", cluster, "replica-pod")
177+
Expect(err).ToNot(HaveOccurred())
178+
})
179+
180+
It("should allow replica to download past timeline", func(ctx SpecContext) {
181+
cluster := &apiv1.Cluster{
182+
Status: apiv1.ClusterStatus{
183+
CurrentPrimary: "primary-pod",
184+
TimelineID: 33,
185+
},
186+
}
187+
188+
err := validateTimelineHistoryFile(ctx, "00000010.history", cluster, "replica-pod")
189+
Expect(err).ToNot(HaveOccurred())
190+
})
191+
192+
It("should reject future timeline for replica", func(ctx SpecContext) {
193+
cluster := &apiv1.Cluster{
194+
Status: apiv1.ClusterStatus{
195+
CurrentPrimary: "primary-pod",
196+
TimelineID: 33,
197+
},
198+
}
199+
200+
err := validateTimelineHistoryFile(ctx, "00000022.history", cluster, "replica-pod")
201+
Expect(err).To(Equal(barmanRestorer.ErrWALNotFound))
202+
})
203+
204+
It("should reject future timeline for replica with large timeline difference", func(ctx SpecContext) {
205+
cluster := &apiv1.Cluster{
206+
Status: apiv1.ClusterStatus{
207+
CurrentPrimary: "primary-pod",
208+
TimelineID: 5,
209+
},
210+
}
211+
212+
err := validateTimelineHistoryFile(ctx, "00000064.history", cluster, "replica-pod")
213+
Expect(err).To(Equal(barmanRestorer.ErrWALNotFound))
214+
})
215+
216+
It("should reject any timeline when cluster timeline is not yet set", func(ctx SpecContext) {
217+
cluster := &apiv1.Cluster{
218+
Status: apiv1.ClusterStatus{
219+
CurrentPrimary: "primary-pod",
220+
TimelineID: 0,
221+
},
222+
}
223+
224+
err := validateTimelineHistoryFile(ctx, "00000001.history", cluster, "replica-pod")
225+
Expect(err).To(Equal(barmanRestorer.ErrWALNotFound))
226+
})
227+
})

‎pkg/postgres/wal.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,9 @@ var (
7979

8080
// ErrorBadWALSegmentName is raised when parsing an invalid segment name
8181
ErrorBadWALSegmentName = errors.New("invalid WAL segment name")
82+
83+
// ErrorBadTimelineHistoryName is raised when parsing an invalid timeline history filename
84+
ErrorBadTimelineHistoryName = errors.New("invalid timeline history filename")
8285
)
8386

8487
// Segment contains the information inside a WAL segment name
@@ -100,6 +103,25 @@ func IsWALFile(name string) bool {
100103
return WALSegmentRe.MatchString(baseName)
101104
}
102105

106+
// ParseTimelineFromHistoryFilename extracts the timeline ID from a timeline history filename.
107+
// For example, "00000021.history" returns 33 (0x21 in hex).
108+
func ParseTimelineFromHistoryFilename(name string) (int, error) {
109+
baseName := path.Base(name)
110+
111+
// Timeline history files are exactly 16 characters: 8 hex digits + ".history" (8 chars)
112+
if len(baseName) < 16 || baseName[8:] != ".history" {
113+
return 0, ErrorBadTimelineHistoryName
114+
}
115+
116+
timelineHex := baseName[0:8]
117+
timeline, err := strconv.ParseInt(timelineHex, 16, 32)
118+
if err != nil {
119+
return 0, fmt.Errorf("%w: %v", ErrorBadTimelineHistoryName, err)
120+
}
121+
122+
return int(timeline), nil
123+
}
124+
103125
// SegmentFromName retrieves the timeline, log ID and segment ID
104126
// from the name of a xlog segment, and can also handle a full path
105127
// or a simple file name

‎pkg/postgres/wal_test.go‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -310,3 +310,79 @@ var _ = Describe("BuildWALPath", func() {
310310
})
311311
})
312312
})
313+
314+
var _ = Describe("Timeline history filename parsing", func() {
315+
It("can parse timeline from history filenames", func() {
316+
tests := []struct {
317+
name string
318+
expectedTimeline int
319+
expectError bool
320+
}{
321+
{
322+
name: "00000001.history",
323+
expectedTimeline: 1,
324+
expectError: false,
325+
},
326+
{
327+
name: "00000021.history", // 0x21 = 33 decimal
328+
expectedTimeline: 33,
329+
expectError: false,
330+
},
331+
{
332+
name: "0000002A.history", // 0x2A = 42 decimal
333+
expectedTimeline: 42,
334+
expectError: false,
335+
},
336+
{
337+
name: "000000FF.history", // 0xFF = 255 decimal
338+
expectedTimeline: 255,
339+
expectError: false,
340+
},
341+
{
342+
name: "0000000A.history", // 0x0A = 10 decimal
343+
expectedTimeline: 10,
344+
expectError: false,
345+
},
346+
{
347+
name: "/var/lib/postgresql/00000021.history", // with path
348+
expectedTimeline: 33,
349+
expectError: false,
350+
},
351+
// Error cases
352+
{
353+
name: "00000001", // missing .history extension
354+
expectError: true,
355+
},
356+
{
357+
name: "0000001.history", // wrong length (7 digits instead of 8)
358+
expectError: true,
359+
},
360+
{
361+
name: "000000001.history", // wrong length (9 digits instead of 8)
362+
expectError: true,
363+
},
364+
{
365+
name: "0000000X.history", // invalid hex character
366+
expectError: true,
367+
},
368+
{
369+
name: "000000010000000000000001", // regular WAL file, not history
370+
expectError: true,
371+
},
372+
{
373+
name: ".history", // no timeline digits
374+
expectError: true,
375+
},
376+
}
377+
378+
for _, test := range tests {
379+
timeline, err := ParseTimelineFromHistoryFilename(test.name)
380+
if test.expectError {
381+
Expect(err).To(HaveOccurred(), "Expected error for name: %s", test.name)
382+
} else {
383+
Expect(err).NotTo(HaveOccurred(), "Unexpected error for name: %s", test.name)
384+
Expect(timeline).To(Equal(test.expectedTimeline), "Timeline mismatch for name: %s", test.name)
385+
}
386+
}
387+
})
388+
})

0 commit comments

Comments
 (0)