Skip to content

Commit e186842

Browse files
fix(backups): leave the base backup to the plugin once its cnpg.io/Backup is gone
plugin-barman-cloud's retention deletes a cnpg.io/Backup together with the base backup it describes, and the driver itself deletes one only with its artifact. An artifact that outlives its cnpg.io/Backup therefore names a base backup the catalog has lost. Pinning its id there purged the target and then failed the recovery with "no backup found with ID", on every retry, where the plugin's own choice would have recovered to the same end of the archive. The restore now reads the id from the live cnpg.io/Backup only, and when that is gone leaves the choice to the plugin and says so with a BaseBackupGone warning event. Recording the id on the artifact served only the case where pinning it is wrong, so the artifact no longer carries it. Assisted-by: LLM Signed-off-by: Mattia Eleuteri <[email protected]>
1 parent 45183b9 commit e186842

2 files changed

Lines changed: 73 additions & 103 deletions

File tree

‎internal/backupcontroller/cnpgstrategy_controller.go‎

Lines changed: 29 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,6 @@ const (
6464
cnpgEndpointURLKey = "cnpg.io/endpoint-url"
6565
cnpgClusterNameKey = "cnpg.io/cluster-name"
6666
cnpgS3SecretRefKey = "cnpg.io/s3-secret-ref"
67-
cnpgBackupIDKey = "cnpg.io/backup-id"
68-
cnpgBackupStoppedAtKey = "cnpg.io/backup-stopped-at"
6967

7068
// Polling cadence for the CNPG backup/restore lifecycle. Mirrors the
7169
// Velero strategy's defaults so behaviour is uniform across drivers.
@@ -603,14 +601,6 @@ func (r *BackupJobReconciler) createCNPGBackupArtifact(
603601
if rendered.BarmanObjectStore.S3Credentials != nil {
604602
driverMD[cnpgS3SecretRefKey] = rendered.BarmanObjectStore.S3Credentials.SecretRef.Name
605603
}
606-
// Recorded here because the cnpg.io/Backup does not outlive every
607-
// artifact, and a restore needs both to pin the backup it starts from.
608-
if cnpgBackup.Status.BackupID != "" {
609-
driverMD[cnpgBackupIDKey] = cnpgBackup.Status.BackupID
610-
}
611-
if cnpgBackup.Status.StoppedAt != nil && !cnpgBackup.Status.StoppedAt.IsZero() {
612-
driverMD[cnpgBackupStoppedAtKey] = cnpgBackup.Status.StoppedAt.UTC().Format(time.RFC3339)
613-
}
614604

615605
underlyingResources, err := marshalCNPGBackupSnapshot(sourceApp, resolved.Parameters)
616606
if err != nil {
@@ -874,14 +864,19 @@ func (r *RestoreJobReconciler) reconcileCNPGRestore(ctx context.Context, restore
874864
// reconciling, we patch + purge in peace, and only resume once
875865
// the Cluster + PVCs are fully gone so the next render lands
876866
// bootstrap.recovery on an empty namespace.
877-
sourceBackupID, sourceStoppedAt, err := r.cnpgBackupIdentity(ctx, backup)
867+
sourceCNPGBackup, err := r.cnpgSourceBackup(ctx, backup)
878868
if err != nil {
879869
return ctrl.Result{}, err
880870
}
881-
backupID := cnpgRestoreBackupID(sourceBackupID, sourceStoppedAt, options.RecoveryTime)
871+
if sourceCNPGBackup == nil {
872+
r.Recorder.Eventf(restoreJob, corev1.EventTypeWarning, "BaseBackupGone",
873+
"cnpg.io/Backup %s/%s no longer exists, so its base backup has likely been removed by retention; the barman-cloud plugin picks the base backup from the catalog",
874+
backup.Namespace, backup.Spec.DriverMetadata[cnpgBackupNameKey])
875+
}
876+
backupID := cnpgRestoreBackupID(sourceCNPGBackup, options.RecoveryTime)
882877
if backupID == "" {
883878
logger.Info("restore does not pin its base backup; the barman-cloud plugin picks one from the catalog",
884-
"backup", backup.Name, "backupID", sourceBackupID, "recoveryTime", options.RecoveryTime)
879+
"backup", backup.Name, "recoveryTime", options.RecoveryTime)
885880
}
886881
hrName := postgresAppPrefix + target.AppName
887882
if err := r.setCNPGRestoreHRSuspended(ctx, target.Namespace, hrName, true); err != nil {
@@ -1405,34 +1400,24 @@ func (r *RestoreJobReconciler) cnpgBackupWALArchived(ctx context.Context, backup
14051400
return true, fmt.Sprintf("cnpg.io/Backup %s/%s phase=completed, endWal=%q", backup.Namespace, backupName, cnpgBackup.Status.EndWal), nil
14061401
}
14071402

1408-
// cnpgBackupIdentity returns barman's backupId for the base backup behind a
1409-
// Backup artifact, and when that backup ended. Artifacts record both; older
1410-
// ones are resolved through their cnpg.io/Backup, and an artifact whose
1411-
// cnpg.io/Backup is gone yields neither.
1412-
func (r *RestoreJobReconciler) cnpgBackupIdentity(ctx context.Context, backup *backupsv1alpha1.Backup) (string, *time.Time, error) {
1413-
md := backup.Spec.DriverMetadata
1414-
if id := md[cnpgBackupIDKey]; id != "" {
1415-
if stopped, err := time.Parse(time.RFC3339, md[cnpgBackupStoppedAtKey]); err == nil {
1416-
return id, &stopped, nil
1417-
}
1418-
return id, nil, nil
1419-
}
1420-
name := md[cnpgBackupNameKey]
1403+
// cnpgSourceBackup returns the cnpg.io/Backup behind a Backup artifact, or
1404+
// nil once it is gone. The driver deletes it only together with its
1405+
// artifact, while plugin-barman-cloud's retention deletes it together with
1406+
// the base backup it describes (deleteBackupsNotInCatalog), so an artifact
1407+
// that outlives its cnpg.io/Backup names a base backup the catalog has lost.
1408+
func (r *RestoreJobReconciler) cnpgSourceBackup(ctx context.Context, backup *backupsv1alpha1.Backup) (*cnpgtypes.Backup, error) {
1409+
name := backup.Spec.DriverMetadata[cnpgBackupNameKey]
14211410
if name == "" {
1422-
return "", nil, nil
1411+
return nil, nil
14231412
}
14241413
cnpgBackup := &cnpgtypes.Backup{}
14251414
if err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: name}, cnpgBackup); err != nil {
14261415
if apierrors.IsNotFound(err) {
1427-
return "", nil, nil
1416+
return nil, nil
14281417
}
1429-
return "", nil, err
1430-
}
1431-
if cnpgBackup.Status.StoppedAt == nil || cnpgBackup.Status.StoppedAt.IsZero() {
1432-
return cnpgBackup.Status.BackupID, nil, nil
1418+
return nil, err
14331419
}
1434-
stopped := cnpgBackup.Status.StoppedAt.Time
1435-
return cnpgBackup.Status.BackupID, &stopped, nil
1420+
return cnpgBackup, nil
14361421
}
14371422

14381423
// cnpgRestoreBackupID decides whether a restore pins the base backup it
@@ -1447,26 +1432,31 @@ func (r *RestoreJobReconciler) cnpgBackupIdentity(ctx context.Context, backup *b
14471432
// several timelines either can start from another branch of the history
14481433
// and bring back other data than the backup asked for.
14491434
//
1435+
// A source that is gone (nil) is not pinned: its base backup is no longer
1436+
// in the catalog, and pinning it would fail the recovery after the target
1437+
// has been purged.
1438+
//
14501439
// A backup cannot serve a recoveryTime before it ended: recovery would stop
14511440
// before it is consistent. Such a request keeps the plugin's choice, which
14521441
// can find an older backup. stoppedAt is truncated to the second while the
14531442
// catalog compares barman's end time to the microsecond, so a backup is
14541443
// pinned only once recoveryTime is a full second past stoppedAt.
1455-
func cnpgRestoreBackupID(backupID string, stoppedAt *time.Time, recoveryTime string) string {
1456-
if backupID == "" {
1444+
func cnpgRestoreBackupID(source *cnpgtypes.Backup, recoveryTime string) string {
1445+
if source == nil || source.Status.BackupID == "" {
14571446
return ""
14581447
}
14591448
if recoveryTime == "" {
1460-
return backupID
1449+
return source.Status.BackupID
14611450
}
1462-
if stoppedAt == nil {
1451+
stoppedAt := source.Status.StoppedAt
1452+
if stoppedAt == nil || stoppedAt.IsZero() {
14631453
return ""
14641454
}
14651455
target, err := time.Parse(time.RFC3339Nano, recoveryTime)
14661456
if err != nil || target.Before(stoppedAt.Add(time.Second)) {
14671457
return ""
14681458
}
1469-
return backupID
1459+
return source.Status.BackupID
14701460
}
14711461

14721462
// cnpgClusterHealthy returns true once the named cnpg.io Cluster reports its

‎internal/backupcontroller/cnpgstrategy_controller_test.go‎

Lines changed: 44 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -2584,11 +2584,11 @@ func newCNPGStrategyTestClient(t *testing.T, objs ...client.Object) client.Clien
25842584
// the stream (no recoveryTime) or the newest one ending by recoveryTime, on
25852585
// any timeline, which is not necessarily the backup the RestoreJob names.
25862586
func TestCNPGRestoreBackupID(t *testing.T) {
2587-
stopped := time.Date(2026, 9, 21, 6, 27, 23, 0, time.UTC)
2587+
stopped := metav1.NewTime(time.Date(2026, 9, 21, 6, 27, 23, 0, time.UTC))
25882588
cases := []struct {
25892589
name string
25902590
backupID string
2591-
stoppedAt *time.Time
2591+
stoppedAt *metav1.Time
25922592
recoveryTime string
25932593
want string
25942594
}{
@@ -2604,11 +2604,17 @@ func TestCNPGRestoreBackupID(t *testing.T) {
26042604
}
26052605
for _, tc := range cases {
26062606
t.Run(tc.name, func(t *testing.T) {
2607-
if got := cnpgRestoreBackupID(tc.backupID, tc.stoppedAt, tc.recoveryTime); got != tc.want {
2607+
source := &cnpgtypes.Backup{Status: cnpgtypes.BackupStatus{BackupID: tc.backupID, StoppedAt: tc.stoppedAt}}
2608+
if got := cnpgRestoreBackupID(source, tc.recoveryTime); got != tc.want {
26082609
t.Errorf("cnpgRestoreBackupID(%q, %v, %q) = %q, want %q", tc.backupID, tc.stoppedAt, tc.recoveryTime, got, tc.want)
26092610
}
26102611
})
26112612
}
2613+
t.Run("a cnpg.io/Backup that is gone pins nothing", func(t *testing.T) {
2614+
if got := cnpgRestoreBackupID(nil, ""); got != "" {
2615+
t.Errorf("cnpgRestoreBackupID(nil, \"\") = %q, want none", got)
2616+
}
2617+
})
26122618
}
26132619

26142620
// TestBuildPostgresAppRestorePatch_ReplacesBackupID guards against a
@@ -2628,8 +2634,10 @@ func TestBuildPostgresAppRestorePatch_ReplacesBackupID(t *testing.T) {
26282634

26292635
// TestReconcileCNPGRestore_PinsTheRequestedBackup drives a restore through the
26302636
// purge step and reads back what the target Postgres app is patched with: the
2631-
// barman backupId of the backup the RestoreJob names, taken from the artifact,
2632-
// or from its cnpg.io/Backup for an artifact that predates recording it.
2637+
// barman backupId of the backup the RestoreJob names, read from its
2638+
// cnpg.io/Backup. Retention deletes that cnpg.io/Backup together with the base
2639+
// backup, and pinning an id the catalog has lost fails the recovery after the
2640+
// purge, so a cnpg.io/Backup that is gone leaves the choice to the plugin.
26332641
func TestReconcileCNPGRestore_PinsTheRequestedBackup(t *testing.T) {
26342642
const (
26352643
ns = "tenant"
@@ -2644,7 +2652,7 @@ func TestReconcileCNPGRestore_PinsTheRequestedBackup(t *testing.T) {
26442652
startedAt := metav1.NewTime(time.Now())
26452653
stoppedAt := metav1.NewTime(time.Date(2026, 9, 21, 6, 27, 23, 0, time.UTC))
26462654

2647-
mkBackupArtifact := func(t *testing.T, md map[string]string) *backupsv1alpha1.Backup {
2655+
mkBackupArtifact := func(t *testing.T) *backupsv1alpha1.Backup {
26482656
t.Helper()
26492657
snap, err := marshalCNPGBackupSnapshot(newPostgresApp(appName, ns), nil)
26502658
if err != nil {
@@ -2655,9 +2663,6 @@ func TestReconcileCNPGRestore_PinsTheRequestedBackup(t *testing.T) {
26552663
cnpgDestinationPathKey: "s3://bucket/" + appName + "/",
26562664
cnpgBackupNameKey: cnpgBkName,
26572665
}
2658-
for k, v := range md {
2659-
driverMD[k] = v
2660-
}
26612666
return &backupsv1alpha1.Backup{
26622667
ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bk"},
26632668
Spec: backupsv1alpha1.BackupSpec{
@@ -2698,10 +2703,16 @@ func TestReconcileCNPGRestore_PinsTheRequestedBackup(t *testing.T) {
26982703
}
26992704
return rj
27002705
}
2701-
restoredBootstrap := func(t *testing.T, backup *backupsv1alpha1.Backup, cnpgBackup *cnpgtypes.Backup, recoveryTime string) postgresapp.Bootstrap {
2706+
restoredBootstrap := func(t *testing.T, cnpgBackup *cnpgtypes.Backup, recoveryTime string) (postgresapp.Bootstrap, *record.FakeRecorder) {
27022707
t.Helper()
2703-
c := newCNPGStrategyTestClient(t, backup, mkRestoreJob(recoveryTime), strategy, cnpgBackup, newPostgresApp(appName, ns))
2704-
r := &RestoreJobReconciler{Client: c, Interface: dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t))}
2708+
backup := mkBackupArtifact(t)
2709+
objs := []client.Object{backup, mkRestoreJob(recoveryTime), strategy, newPostgresApp(appName, ns)}
2710+
if cnpgBackup != nil {
2711+
objs = append(objs, cnpgBackup)
2712+
}
2713+
c := newCNPGStrategyTestClient(t, objs...)
2714+
recorder := record.NewFakeRecorder(10)
2715+
r := &RestoreJobReconciler{Client: c, Interface: dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t)), Recorder: recorder}
27052716
rj := &backupsv1alpha1.RestoreJob{}
27062717
if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, rj); err != nil {
27072718
t.Fatalf("get seeded RestoreJob: %v", err)
@@ -2716,73 +2727,42 @@ func TestReconcileCNPGRestore_PinsTheRequestedBackup(t *testing.T) {
27162727
if !app.Spec.Bootstrap.Enabled {
27172728
t.Fatalf("the reconcile did not reach the restore patch")
27182729
}
2719-
return app.Spec.Bootstrap
2730+
return app.Spec.Bootstrap, recorder
27202731
}
27212732

27222733
t.Run("without recoveryTime the requested backup is pinned", func(t *testing.T) {
2723-
md := map[string]string{cnpgBackupIDKey: backupID, cnpgBackupStoppedAtKey: stoppedAt.UTC().Format(time.RFC3339)}
2724-
if got := restoredBootstrap(t, mkBackupArtifact(t, md), mkCNPGBackup(backupID), "").BackupID; got != backupID {
2725-
t.Errorf("bootstrap.backupID: got %q want %q", got, backupID)
2734+
if b, _ := restoredBootstrap(t, mkCNPGBackup(backupID), ""); b.BackupID != backupID {
2735+
t.Errorf("bootstrap.backupID: got %q want %q", b.BackupID, backupID)
27262736
}
27272737
})
27282738

27292739
t.Run("a recoveryTime after the backup ended pins it", func(t *testing.T) {
2730-
md := map[string]string{cnpgBackupIDKey: backupID, cnpgBackupStoppedAtKey: stoppedAt.UTC().Format(time.RFC3339)}
2731-
b := restoredBootstrap(t, mkBackupArtifact(t, md), mkCNPGBackup(backupID), "2026-09-22T10:00:00Z")
2740+
b, _ := restoredBootstrap(t, mkCNPGBackup(backupID), "2026-09-22T10:00:00Z")
27322741
if b.BackupID != backupID || b.RecoveryTime != "2026-09-22T10:00:00Z" {
27332742
t.Errorf("bootstrap: got backupID %q recoveryTime %q, want %q and the requested instant", b.BackupID, b.RecoveryTime, backupID)
27342743
}
27352744
})
27362745

27372746
t.Run("a recoveryTime before the backup ended leaves the choice to the plugin", func(t *testing.T) {
2738-
md := map[string]string{cnpgBackupIDKey: backupID, cnpgBackupStoppedAtKey: stoppedAt.UTC().Format(time.RFC3339)}
2739-
if got := restoredBootstrap(t, mkBackupArtifact(t, md), mkCNPGBackup(backupID), "2026-09-21T05:00:00Z").BackupID; got != "" {
2740-
t.Errorf("bootstrap.backupID: got %q, want none: the backup cannot serve an instant before its end", got)
2741-
}
2742-
})
2743-
2744-
t.Run("an artifact without the recorded id falls back to its cnpg.io/Backup", func(t *testing.T) {
2745-
if got := restoredBootstrap(t, mkBackupArtifact(t, nil), mkCNPGBackup(backupID), "").BackupID; got != backupID {
2746-
t.Errorf("bootstrap.backupID: got %q want %q", got, backupID)
2747+
if b, _ := restoredBootstrap(t, mkCNPGBackup(backupID), "2026-09-21T05:00:00Z"); b.BackupID != "" {
2748+
t.Errorf("bootstrap.backupID: got %q, want none: the backup cannot serve an instant before its end", b.BackupID)
27472749
}
27482750
})
2749-
}
2750-
2751-
// TestCreateCNPGBackupArtifact_RecordsBackupIdentity: a restore pins the
2752-
// backup it starts from by barman's backupId, and must be able to after the
2753-
// cnpg.io/Backup is gone, so the artifact carries it and the backup's end.
2754-
func TestCreateCNPGBackupArtifact_RecordsBackupIdentity(t *testing.T) {
2755-
apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup
2756-
strategyGroup := strategyv1alpha1.GroupVersion.Group
2757-
c := newCNPGStrategyTestClient(t)
2758-
r := &BackupJobReconciler{Client: c}
27592751

2760-
j := &backupsv1alpha1.BackupJob{
2761-
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"},
2762-
Spec: backupsv1alpha1.BackupJobSpec{
2763-
ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"},
2764-
},
2765-
}
2766-
resolved := &ResolvedBackupConfig{
2767-
StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "strat"},
2768-
}
2769-
stopped := metav1.NewTime(time.Date(2026, 9, 21, 6, 27, 23, 0, time.UTC))
2770-
cnpgBk := &cnpgtypes.Backup{
2771-
ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"},
2772-
Status: cnpgtypes.BackupStatus{BackupID: "20260921T062713", StoppedAt: &stopped},
2773-
}
2774-
rendered := &strategyv1alpha1.CNPGTemplate{
2775-
BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://b/"},
2776-
}
2777-
2778-
got, err := r.createCNPGBackupArtifact(context.Background(), j, resolved, cnpgBk, "postgres-pg", "postgres-pg", rendered, newPostgresApp("pg", "tenant"))
2779-
if err != nil {
2780-
t.Fatalf("createCNPGBackupArtifact: %v", err)
2781-
}
2782-
if id := got.Spec.DriverMetadata[cnpgBackupIDKey]; id != "20260921T062713" {
2783-
t.Errorf("driverMetadata[%s]: got %q", cnpgBackupIDKey, id)
2784-
}
2785-
if at := got.Spec.DriverMetadata[cnpgBackupStoppedAtKey]; at != "2026-09-21T06:27:23Z" {
2786-
t.Errorf("driverMetadata[%s]: got %q", cnpgBackupStoppedAtKey, at)
2752+
for _, recoveryTime := range []string{"", "2026-09-22T10:00:00Z"} {
2753+
t.Run("a cnpg.io/Backup removed by retention leaves the choice to the plugin, recoveryTime="+recoveryTime, func(t *testing.T) {
2754+
b, recorder := restoredBootstrap(t, nil, recoveryTime)
2755+
if b.BackupID != "" {
2756+
t.Errorf("bootstrap.backupID: got %q, want none: the catalog no longer holds that base backup", b.BackupID)
2757+
}
2758+
select {
2759+
case ev := <-recorder.Events:
2760+
if !strings.Contains(ev, "Warning BaseBackupGone") {
2761+
t.Errorf("event: got %q, want a BaseBackupGone warning", ev)
2762+
}
2763+
default:
2764+
t.Errorf("no event: a restore that cannot start from the backup it names must say so")
2765+
}
2766+
})
27872767
}
27882768
}

0 commit comments

Comments
 (0)