Skip to content

Commit 5e51c55

Browse files
authored
fix(backup): add reconciliation timestamps to backup status (cloudnative-pg#9351)
Add `reconciliationStartedAt` and `reconciliationTerminatedAt` fields to the BackupStatus to track when the operator started and terminated the backup reconciliation process. This separates the operator's reconciliation timing from the backup tool's execution timing (`startedAt`/`stoppedAt`), which allows for better monitoring of long-running backups. Previously, `startedAt` was not set until the backup completed, making it difficult to monitor backup progress. Now: - `reconciliationStartedAt`: Set when the operator begins processing the backup (in `SetAsStarted`) - `reconciliationTerminatedAt`: Set when the operator finishes processing (in `SetAsCompleted` or `SetAsFailed`) - `startedAt`/`stoppedAt`: Reserved for the backup tool's actual execution times For volume snapshot backups, `startedAt` and `stoppedAt` are now aligned with the reconciliation timestamps since the backup execution is managed by the operator. Fixes monitoring of long-running backups by providing accurate timing information throughout the backup lifecycle. Closes cloudnative-pg#9352 Signed-off-by: Armando Ruocco <[email protected]>
1 parent 09a1b6b commit 5e51c55

9 files changed

Lines changed: 74 additions & 10 deletions

File tree

‎.wordlist-en-custom.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1064,6 +1064,7 @@ managedservices
10641064
matchExpressions
10651065
matchLabels
10661066
mateusoliveira
1067+
maxClientConnections
10671068
maxParallel
10681069
maxStandbyNamesFromCluster
10691070
maxSyncReplicas
@@ -1287,6 +1288,8 @@ readthedocs
12871288
readyInstances
12881289
reconciler
12891290
reconciliationLoop
1291+
reconciliationStartedAt
1292+
reconciliationTerminatedAt
12901293
reconnection
12911294
recoverability
12921295
recoveredCluster

‎api/v1/backup_funcs.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@ func (backupStatus *BackupStatus) SetAsFailed(
5252
} else {
5353
backupStatus.Error = ""
5454
}
55+
56+
backupStatus.ReconciliationTerminatedAt = ptr.To(metav1.Now())
5557
}
5658

5759
// SetAsFinalizing marks a certain backup as finalizing
@@ -64,7 +66,7 @@ func (backupStatus *BackupStatus) SetAsFinalizing() {
6466
func (backupStatus *BackupStatus) SetAsCompleted() {
6567
backupStatus.Phase = BackupPhaseCompleted
6668
backupStatus.Error = ""
67-
backupStatus.StoppedAt = ptr.To(metav1.Now())
69+
backupStatus.ReconciliationTerminatedAt = ptr.To(metav1.Now())
6870
}
6971

7072
// SetAsStarted marks a certain backup as started
@@ -75,6 +77,7 @@ func (backupStatus *BackupStatus) SetAsStarted(podName, containerID string, meth
7577
ContainerID: containerID,
7678
}
7779
backupStatus.Method = method
80+
backupStatus.ReconciliationStartedAt = ptr.To(metav1.Now())
7881
}
7982

8083
// SetSnapshotElements sets the Snapshots field from a list of VolumeSnapshot

‎api/v1/backup_funcs_test.go‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ SPDX-License-Identifier: Apache-2.0
2020
package v1
2121

2222
import (
23+
"errors"
2324
"time"
2425

2526
volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
@@ -63,6 +64,35 @@ var _ = Describe("BackupStatus structure", func() {
6364
Expect(status.InstanceID.PodName).To(Equal("cluster-example-1"))
6465
Expect(status.InstanceID.ContainerID).To(Equal("container-id"))
6566
Expect(status.IsDone()).To(BeFalse())
67+
Expect(status.ReconciliationStartedAt).ToNot(BeNil())
68+
})
69+
70+
It("can be set as completed", func() {
71+
status := BackupStatus{}
72+
status.SetAsCompleted()
73+
Expect(status.Phase).To(BeEquivalentTo(BackupPhaseCompleted))
74+
Expect(status.Error).To(BeEmpty())
75+
Expect(status.ReconciliationTerminatedAt).ToNot(BeNil())
76+
Expect(status.IsDone()).To(BeTrue())
77+
})
78+
79+
It("can be set as failed with error", func() {
80+
status := BackupStatus{}
81+
testErr := errors.New("test error")
82+
status.SetAsFailed(testErr)
83+
Expect(status.Phase).To(BeEquivalentTo(BackupPhaseFailed))
84+
Expect(status.Error).To(Equal("test error"))
85+
Expect(status.ReconciliationTerminatedAt).ToNot(BeNil())
86+
Expect(status.IsDone()).To(BeTrue())
87+
})
88+
89+
It("can be set as failed without error", func() {
90+
status := BackupStatus{}
91+
status.SetAsFailed(nil)
92+
Expect(status.Phase).To(BeEquivalentTo(BackupPhaseFailed))
93+
Expect(status.Error).To(BeEmpty())
94+
Expect(status.ReconciliationTerminatedAt).ToNot(BeNil())
95+
Expect(status.IsDone()).To(BeTrue())
6696
})
6797

6898
It("can be set to contain a snapshot list", func() {

‎api/v1/backup_types.go‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -237,14 +237,22 @@ type BackupStatus struct {
237237
// +optional
238238
Phase BackupPhase `json:"phase,omitempty"`
239239

240-
// When the backup was started
240+
// When the backup execution was started by the backup tool
241241
// +optional
242242
StartedAt *metav1.Time `json:"startedAt,omitempty"`
243243

244-
// When the backup was terminated
244+
// When the backup execution was terminated by the backup tool
245245
// +optional
246246
StoppedAt *metav1.Time `json:"stoppedAt,omitempty"`
247247

248+
// When the backup process was started by the operator
249+
// +optional
250+
ReconciliationStartedAt *metav1.Time `json:"reconciliationStartedAt,omitempty"`
251+
252+
// When the reconciliation was terminated by the operator (either successfully or not)
253+
// +optional
254+
ReconciliationTerminatedAt *metav1.Time `json:"reconciliationTerminatedAt,omitempty"`
255+
248256
// The starting WAL
249257
// +optional
250258
BeginWal string `json:"beginWal,omitempty"`

‎api/v1/zz_generated.deepcopy.go‎

Lines changed: 8 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/crd/bases/postgresql.cnpg.io_backups.yaml‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -332,6 +332,15 @@ spec:
332332
type: string
333333
description: A map containing the plugin metadata
334334
type: object
335+
reconciliationStartedAt:
336+
description: When the backup process was started by the operator
337+
format: date-time
338+
type: string
339+
reconciliationTerminatedAt:
340+
description: When the reconciliation was terminated by the operator
341+
(either successfully or not)
342+
format: date-time
343+
type: string
335344
s3Credentials:
336345
description: The credentials to use to upload data to S3
337346
properties:
@@ -427,11 +436,12 @@ spec:
427436
type: array
428437
type: object
429438
startedAt:
430-
description: When the backup was started
439+
description: When the backup execution was started by the backup tool
431440
format: date-time
432441
type: string
433442
stoppedAt:
434-
description: When the backup was terminated
443+
description: When the backup execution was terminated by the backup
444+
tool
435445
format: date-time
436446
type: string
437447
tablespaceMapFile:

‎docs/src/cloudnative-pg.v1.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -275,8 +275,10 @@ _Appears in:_
275275
| `backupId` _string_ | The ID of the Barman backup | | | |
276276
| `backupName` _string_ | The Name of the Barman backup | | | |
277277
| `phase` _[BackupPhase](#backupphase)_ | The last backup status | | | |
278-
| `startedAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#time-v1-meta)_ | When the backup was started | | | |
279-
| `stoppedAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#time-v1-meta)_ | When the backup was terminated | | | |
278+
| `startedAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#time-v1-meta)_ | When the backup execution was started by the backup tool | | | |
279+
| `stoppedAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#time-v1-meta)_ | When the backup execution was terminated by the backup tool | | | |
280+
| `reconciliationStartedAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#time-v1-meta)_ | When the backup process was started by the operator | | | |
281+
| `reconciliationTerminatedAt` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#time-v1-meta)_ | When the reconciliation was terminated by the operator (either successfully or not) | | | |
280282
| `beginWal` _string_ | The starting WAL | | | |
281283
| `endWal` _string_ | The ending WAL | | | |
282284
| `beginLSN` _string_ | The starting xlog | | | |

‎internal/controller/backup_controller.go‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,13 +31,11 @@ import (
3131
volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
3232
corev1 "k8s.io/api/core/v1"
3333
apierrs "k8s.io/apimachinery/pkg/api/errors"
34-
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3534
"k8s.io/apimachinery/pkg/runtime"
3635
"k8s.io/client-go/discovery"
3736
"k8s.io/client-go/kubernetes"
3837
"k8s.io/client-go/tools/record"
3938
"k8s.io/client-go/util/retry"
40-
"k8s.io/utils/ptr"
4139
ctrl "sigs.k8s.io/controller-runtime"
4240
"sigs.k8s.io/controller-runtime/pkg/builder"
4341
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -567,7 +565,7 @@ func (r *BackupReconciler) reconcileSnapshotBackup(
567565
// given that we use only kubernetes resources we can use the backup name as ID
568566
backup.Status.BackupID = backup.Name
569567
backup.Status.BackupName = backup.Name
570-
backup.Status.StartedAt = ptr.To(metav1.Now())
568+
backup.Status.StartedAt = backup.Status.ReconciliationStartedAt.DeepCopy()
571569
if err := postgres.PatchBackupStatusAndRetry(ctx, r.Client, backup); err != nil {
572570
return nil, err
573571
}

‎pkg/reconciler/backup/volumesnapshot/reconciler.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -294,6 +294,8 @@ func (se *Reconciler) completeSnapshotBackupStep(
294294
) (*ctrl.Result, error) {
295295
contextLogger := log.FromContext(ctx)
296296
backup.Status.SetAsCompleted()
297+
backup.Status.StoppedAt = backup.Status.ReconciliationTerminatedAt.DeepCopy()
298+
297299
snapshots, err := getBackupVolumeSnapshots(ctx, se.cli, backup.Namespace, backup.Name)
298300
if err != nil {
299301
return nil, err

0 commit comments

Comments
 (0)