Skip to content

Commit 2bd219f

Browse files
GabriFedi97armru
authored andcommitted
fix(backups): avoid resource leaks when concurrent backups race (#10747)
Previously, when multiple Backup objects existed concurrently, election among them was decided by alphabetical name order. A newer backup whose name sorted earlier could preempt one that had already started, leaving the displaced backup's replication slot and PostgreSQL session orphaned on the primary. Backups now run in strict creation-time order, so an already-executing backup is never preempted by a newer one — regardless of name. Name is kept only as a deterministic tie-breaker when timestamps match. Closes #10622 Signed-off-by: Gabriele Fedi <[email protected]> Signed-off-by: Armando Ruocco <[email protected]> Co-authored-by: Armando Ruocco <[email protected]> (cherry picked from commit 4fe7715)
1 parent 241c9d7 commit 2bd219f

2 files changed

Lines changed: 175 additions & 23 deletions

File tree

‎api/v1/backup_funcs.go‎

Lines changed: 33 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -134,17 +134,23 @@ func (backupStatus *BackupStatus) IsInProgress() bool {
134134
backupStatus.Phase == BackupPhaseRunning
135135
}
136136

137+
// IsExecuting check if a certain backup is being executed
138+
func (backupStatus *BackupStatus) IsExecuting() bool {
139+
return backupStatus.Phase == BackupPhaseStarted ||
140+
backupStatus.Phase == BackupPhaseRunning
141+
}
142+
137143
// GetPendingBackupNames returns the pending backup list
138144
func (list BackupList) GetPendingBackupNames() []string {
139145
// Retry the backup if another backup is running
140146
pendingBackups := make([]string, 0, len(list.Items))
141147
for _, concurrentBackup := range list.Items {
142-
if concurrentBackup.Status.IsDone() {
148+
if concurrentBackup.Status.IsDone() ||
149+
concurrentBackup.Status.IsExecuting() {
143150
continue
144151
}
145-
if !concurrentBackup.Status.IsInProgress() {
146-
pendingBackups = append(pendingBackups, concurrentBackup.Name)
147-
}
152+
153+
pendingBackups = append(pendingBackups, concurrentBackup.Name)
148154
}
149155

150156
return pendingBackups
@@ -153,34 +159,26 @@ func (list BackupList) GetPendingBackupNames() []string {
153159
// CanExecuteBackup control if we can start a reconciliation loop for a certain backup.
154160
//
155161
// A reconciliation loop can start if:
156-
// - there's no backup running, and if the first of the sorted list of backups
157-
// - the current backup is running and is the first running backup of the list
162+
// - the backup is currently being executed (Started or Running), or
163+
// - no other backup is being executed and the backup is the oldest pending one
164+
// (with backup name used as a tie-breaker for backups created at the same time).
158165
//
159-
// As a side effect, this function will sort the backup list
166+
// As a side effect, this function will sort the backup list by creation time.
160167
func (list *BackupList) CanExecuteBackup(backupName string) bool {
161-
var foundRunningBackup bool
162-
163-
list.SortByName()
168+
list.SortByCreationTimeAndName()
164169

165170
for _, concurrentBackup := range list.Items {
166-
if concurrentBackup.Status.IsInProgress() {
167-
if backupName == concurrentBackup.Name && !foundRunningBackup {
168-
return true
169-
}
170-
171-
foundRunningBackup = true
172-
if backupName != concurrentBackup.Name {
173-
return false
174-
}
171+
if concurrentBackup.Status.IsExecuting() {
172+
return backupName == concurrentBackup.Name
175173
}
176174
}
177175

178176
pendingBackups := list.GetPendingBackupNames()
179-
if len(pendingBackups) > 0 && pendingBackups[0] != backupName {
180-
return false
177+
if len(pendingBackups) > 0 && pendingBackups[0] == backupName {
178+
return true
181179
}
182180

183-
return true
181+
return false
184182
}
185183

186184
// SortByName sorts the backup items in alphabetical order
@@ -199,6 +197,19 @@ func (list *BackupList) SortByReverseCreationTime() {
199197
})
200198
}
201199

200+
// SortByCreationTimeAndName sorts the backup items in creation time order and,
201+
// in case of backups with the same creation time, in alphabetical order
202+
func (list *BackupList) SortByCreationTimeAndName() {
203+
sort.Slice(list.Items, func(i, j int) bool {
204+
ti := list.Items[i].CreationTimestamp.Time
205+
tj := list.Items[j].CreationTimestamp.Time
206+
if ti.Equal(tj) {
207+
return list.Items[i].Name < list.Items[j].Name
208+
}
209+
return ti.Before(tj)
210+
})
211+
}
212+
202213
// GetStatus gets the backup status
203214
func (backup *Backup) GetStatus() *BackupStatus {
204215
return &backup.Status

‎api/v1/backup_funcs_test.go‎

Lines changed: 142 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ var _ = Describe("BackupStatus structure", func() {
9393
Phase: BackupPhaseRunning,
9494
}
9595
Expect(b.IsInProgress()).To(BeTrue())
96+
Expect(b.IsExecuting()).To(BeTrue())
9697
Expect(b.IsDone()).To(BeFalse())
9798
})
9899
})
@@ -103,6 +104,7 @@ var _ = Describe("BackupStatus structure", func() {
103104
Phase: BackupPhasePending,
104105
}
105106
Expect(b.IsInProgress()).To(BeTrue())
107+
Expect(b.IsExecuting()).To(BeFalse())
106108
Expect(b.IsDone()).To(BeFalse())
107109
})
108110
})
@@ -113,6 +115,7 @@ var _ = Describe("BackupStatus structure", func() {
113115
Phase: BackupPhaseCompleted,
114116
}
115117
Expect(b.IsInProgress()).To(BeFalse())
118+
Expect(b.IsExecuting()).To(BeFalse())
116119
Expect(b.IsDone()).To(BeTrue())
117120
})
118121
})
@@ -123,6 +126,7 @@ var _ = Describe("BackupStatus structure", func() {
123126
Phase: BackupPhaseFailed,
124127
}
125128
Expect(b.IsInProgress()).To(BeFalse())
129+
Expect(b.IsExecuting()).To(BeFalse())
126130
Expect(b.IsDone()).To(BeTrue())
127131
})
128132
})
@@ -190,6 +194,70 @@ var _ = Describe("BackupList structure", func() {
190194
Expect(backupList.Items[2].Name).To(Equal("backup-ten-minutes"))
191195
})
192196

197+
It("can be sorted by creation time and name", func() {
198+
now := time.Now()
199+
backupList := BackupList{
200+
Items: []Backup{
201+
{
202+
ObjectMeta: metav1.ObjectMeta{
203+
Name: "backup-now",
204+
CreationTimestamp: metav1.NewTime(now),
205+
},
206+
},
207+
{
208+
ObjectMeta: metav1.ObjectMeta{
209+
Name: "backup-ten-minutes",
210+
CreationTimestamp: metav1.NewTime(now.Add(-10 * time.Minute)),
211+
},
212+
},
213+
{
214+
ObjectMeta: metav1.ObjectMeta{
215+
Name: "backup-five-minutes",
216+
CreationTimestamp: metav1.NewTime(now.Add(-5 * time.Minute)),
217+
},
218+
},
219+
},
220+
}
221+
backupList.SortByCreationTimeAndName()
222+
223+
Expect(backupList.Items).To(HaveLen(3))
224+
Expect(backupList.Items[0].Name).To(Equal("backup-ten-minutes"))
225+
Expect(backupList.Items[1].Name).To(Equal("backup-five-minutes"))
226+
Expect(backupList.Items[2].Name).To(Equal("backup-now"))
227+
})
228+
229+
It("breaks ties by name when creation times are equal", func() {
230+
now := time.Now()
231+
backupList := BackupList{
232+
Items: []Backup{
233+
{
234+
ObjectMeta: metav1.ObjectMeta{
235+
Name: "backup-c",
236+
CreationTimestamp: metav1.NewTime(now),
237+
},
238+
},
239+
{
240+
ObjectMeta: metav1.ObjectMeta{
241+
Name: "backup-a",
242+
CreationTimestamp: metav1.NewTime(now),
243+
},
244+
},
245+
{
246+
ObjectMeta: metav1.ObjectMeta{
247+
Name: "backup-b",
248+
CreationTimestamp: metav1.NewTime(now),
249+
},
250+
},
251+
},
252+
}
253+
backupList.SortByCreationTimeAndName()
254+
255+
Expect(backupList.Items).To(HaveLen(3))
256+
Expect(backupList.Items[0].Name).To(Equal("backup-a"))
257+
Expect(backupList.Items[1].Name).To(Equal("backup-b"))
258+
Expect(backupList.Items[2].Name).To(Equal("backup-c"))
259+
})
260+
193261
It("can isolate pending backups", func() {
194262
backupList := BackupList{
195263
Items: []Backup{
@@ -227,12 +295,20 @@ var _ = Describe("BackupList structure", func() {
227295
Phase: BackupPhaseFailed,
228296
},
229297
},
298+
{
299+
ObjectMeta: metav1.ObjectMeta{
300+
Name: "backup-7",
301+
},
302+
Status: BackupStatus{
303+
Phase: BackupPhasePending,
304+
},
305+
},
230306
},
231307
}
232308
backupList.SortByName()
233309

234310
pendingBackups := backupList.GetPendingBackupNames()
235-
Expect(pendingBackups).To(ConsistOf("backup-1", "backup-2"))
311+
Expect(pendingBackups).To(ConsistOf("backup-1", "backup-2", "backup-7"))
236312
})
237313
})
238314

@@ -366,6 +442,71 @@ var _ = Describe("backup_controller volumeSnapshot unit tests", func() {
366442
Expect(backupList.CanExecuteBackup("backup-3")).To(BeFalse())
367443
})
368444
})
445+
446+
When("a newer pending backup sorts alphabetically before an older started one", func() {
447+
// Regression guard for the preemption bug fixed in this change:
448+
// an alphabetically-earlier Pending backup must not preempt an
449+
// older Started backup that has already acquired resources.
450+
It("keeps the older started backup as the elected one", func() {
451+
now := time.Now()
452+
backupList := BackupList{
453+
Items: []Backup{
454+
{
455+
ObjectMeta: metav1.ObjectMeta{
456+
Name: "backup-a-newer",
457+
CreationTimestamp: metav1.NewTime(now),
458+
},
459+
Status: BackupStatus{
460+
Phase: BackupPhasePending,
461+
},
462+
},
463+
{
464+
ObjectMeta: metav1.ObjectMeta{
465+
Name: "backup-z-older",
466+
CreationTimestamp: metav1.NewTime(now.Add(-10 * time.Minute)),
467+
},
468+
Status: BackupStatus{
469+
Phase: BackupPhaseStarted,
470+
},
471+
},
472+
},
473+
}
474+
475+
Expect(backupList.CanExecuteBackup("backup-z-older")).To(BeTrue())
476+
Expect(backupList.CanExecuteBackup("backup-a-newer")).To(BeFalse())
477+
})
478+
})
479+
480+
When("the only pending backups have diverging name and creation order", func() {
481+
It("elects the oldest pending backup regardless of name", func() {
482+
now := time.Now()
483+
backupList := BackupList{
484+
Items: []Backup{
485+
{
486+
ObjectMeta: metav1.ObjectMeta{
487+
Name: "backup-a-newer",
488+
CreationTimestamp: metav1.NewTime(now),
489+
},
490+
Status: BackupStatus{
491+
Phase: BackupPhasePending,
492+
},
493+
},
494+
{
495+
ObjectMeta: metav1.ObjectMeta{
496+
Name: "backup-z-older",
497+
CreationTimestamp: metav1.NewTime(now.Add(-10 * time.Minute)),
498+
},
499+
Status: BackupStatus{
500+
Phase: BackupPhasePending,
501+
},
502+
},
503+
},
504+
}
505+
506+
Expect(backupList.CanExecuteBackup("backup-z-older")).To(BeTrue())
507+
Expect(backupList.CanExecuteBackup("backup-a-newer")).To(BeFalse())
508+
})
509+
})
369510
})
370511

371512
var _ = Describe("IsCompletedVolumeSnapshot", func() {

0 commit comments

Comments
 (0)