Skip to content

Commit 102d785

Browse files
armrumnencia
andauthored
fix(backup): use instance session ID to detect instance manager restarts (#9370)
This fix addresses an issue where backups would hang indefinitely during operator releases. When the operator and barman plugin were updated simultaneously, the backup goroutine running in the instance manager would be killed (due to the instance manager binary being replaced via exec()), but the operator would still think the backup was running because the container ID hadn't changed. The fix introduces a deterministic mechanism to detect instance manager restarts by tracking the SessionID Signed-off-by: Armando Ruocco <[email protected]> Signed-off-by: Marco Nenciarini <[email protected]> Co-authored-by: Marco Nenciarini <[email protected]>
1 parent 9bb8c16 commit 102d785

10 files changed

Lines changed: 188 additions & 40 deletions

File tree

‎api/v1/backup_funcs.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,11 +70,12 @@ func (backupStatus *BackupStatus) SetAsCompleted() {
7070
}
7171

7272
// SetAsStarted marks a certain backup as started
73-
func (backupStatus *BackupStatus) SetAsStarted(podName, containerID string, method BackupMethod) {
73+
func (backupStatus *BackupStatus) SetAsStarted(podName, containerID, sessionID string, method BackupMethod) {
7474
backupStatus.Phase = BackupPhaseStarted
7575
backupStatus.InstanceID = &InstanceID{
7676
PodName: podName,
7777
ContainerID: containerID,
78+
SessionID: sessionID,
7879
}
7980
backupStatus.Method = method
8081
backupStatus.ReconciliationStartedAt = ptr.To(metav1.Now())

‎api/v1/backup_funcs_test.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,13 @@ var _ = Describe("BackupStatus structure", func() {
5858
},
5959
}
6060

61-
status.SetAsStarted(pod.Name, pod.Status.ContainerStatuses[0].ContainerID, BackupMethodBarmanObjectStore)
61+
status.SetAsStarted(pod.Name, pod.Status.ContainerStatuses[0].ContainerID,
62+
"test-session-id", BackupMethodBarmanObjectStore)
6263
Expect(status.Phase).To(BeEquivalentTo(BackupPhaseStarted))
6364
Expect(status.InstanceID).ToNot(BeNil())
6465
Expect(status.InstanceID.PodName).To(Equal("cluster-example-1"))
6566
Expect(status.InstanceID.ContainerID).To(Equal("container-id"))
67+
Expect(status.InstanceID.SessionID).To(Equal("test-session-id"))
6668
Expect(status.IsDone()).To(BeFalse())
6769
Expect(status.ReconciliationStartedAt).ToNot(BeNil())
6870
})

‎api/v1/backup_types.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -319,6 +319,12 @@ type InstanceID struct {
319319
// The container ID
320320
// +optional
321321
ContainerID string `json:"ContainerID,omitempty"`
322+
// The instance manager session ID. This is a unique identifier generated at instance manager
323+
// startup and changes on every restart (including container reboots). Used to detect if
324+
// the instance manager was restarted during long-running operations like backups, which
325+
// would terminate any running backup process.
326+
// +optional
327+
SessionID string `json:"sessionID,omitempty"`
322328
}
323329

324330
// +genclient

‎config/crd/bases/postgresql.cnpg.io_backups.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -316,6 +316,13 @@ spec:
316316
podName:
317317
description: The pod name
318318
type: string
319+
sessionID:
320+
description: |-
321+
The instance manager session ID. This is a unique identifier generated at instance manager
322+
startup and changes on every restart (including container reboots). Used to detect if
323+
the instance manager was restarted during long-running operations like backups, which
324+
would terminate any running backup process.
325+
type: string
319326
type: object
320327
majorVersion:
321328
description: |-

‎docs/src/cloudnative-pg.v1.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1239,6 +1239,7 @@ _Appears in:_
12391239
| --- | --- | --- | --- | --- |
12401240
| `podName` _string_ | The pod name | | | |
12411241
| `ContainerID` _string_ | The container ID | | | |
1242+
| `sessionID` _string_ | The instance manager session ID. This is a unique identifier generated at instance manager<br />startup and changes on every restart (including container reboots). Used to detect if<br />the instance manager was restarted during long-running operations like backups, which<br />would terminate any running backup process. | | | |
12421243

12431244

12441245
#### InstanceReportedState

0 commit comments

Comments
 (0)