Skip to content

Commit 497dd6f

Browse files
fix(backups): start a Postgres restore from the backup it names
Without recoveryTarget.backupID the barman-cloud plugin chooses the base backup itself: the newest one in the catalog when no recoveryTime is set, and the newest one ending by recoveryTime otherwise, on any timeline since no targetTLI is passed. A RestoreJob naming an older Backup therefore started from another one, and on an archive holding sibling timelines - left by in-place restores that archived back under the source serverName - from another branch, bringing back that branch's data. The restore reads barman's backupId and the backup's stoppedAt from the live cnpg.io/Backup behind the Backup, and passes the id through the chart as bootstrap.recovery.recoveryTarget.backupID. The plugin gives backupID precedence over time-based selection, and CNPG does not count it among the mutually exclusive targets, so it combines with targetTime. Two cases keep the plugin's choice. A recoveryTime earlier than one second past stoppedAt: the named backup cannot serve an instant before it is consistent, and stoppedAt is truncated to the second while barman compares its end time to the microsecond. And a cnpg.io/Backup that is gone: plugin-barman-cloud's retention deletes it together with the base backup it describes, and the driver deletes one only with its artifact, so pinning its id would purge the target and then fail the recovery on every retry. That restore records a BaseBackupGone warning event instead. A re-restore into the same target clears a backupID left by a previous one. Assisted-by: LLM Signed-off-by: Mattia Eleuteri <[email protected]>
1 parent 5e32ef3 commit 497dd6f

13 files changed

Lines changed: 389 additions & 14 deletions

File tree

‎api/apps/v1alpha1/postgresql/types.go‎

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎internal/backupcontroller/cnpgstrategy_controller.go‎

Lines changed: 78 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -887,6 +887,20 @@ func (r *RestoreJobReconciler) reconcileCNPGRestore(ctx context.Context, restore
887887
// reconciling, we patch + purge in peace, and only resume once
888888
// the Cluster + PVCs are fully gone so the next render lands
889889
// bootstrap.recovery on an empty namespace.
890+
sourceCNPGBackup, err := r.cnpgSourceBackup(ctx, backup)
891+
if err != nil {
892+
return ctrl.Result{}, err
893+
}
894+
if sourceCNPGBackup == nil {
895+
r.Recorder.Eventf(restoreJob, corev1.EventTypeWarning, "BaseBackupGone",
896+
"cnpg.io/Backup %s/%s no longer exists, so its base backup has likely been removed by retention; the barman-cloud plugin picks the base backup from the catalog",
897+
backup.Namespace, backup.Spec.DriverMetadata[cnpgBackupNameKey])
898+
}
899+
backupID := cnpgRestoreBackupID(sourceCNPGBackup, options.RecoveryTime)
900+
if backupID == "" {
901+
logger.Info("restore does not pin its base backup; the barman-cloud plugin picks one from the catalog",
902+
"backup", backup.Name, "recoveryTime", options.RecoveryTime)
903+
}
890904
hrName := postgresAppPrefix + target.AppName
891905
if err := r.setCNPGRestoreHRSuspended(ctx, target.Namespace, hrName, true); err != nil {
892906
return ctrl.Result{}, err
@@ -900,7 +914,7 @@ func (r *RestoreJobReconciler) reconcileCNPGRestore(ctx context.Context, restore
900914
logger.Info("restored WAL-archive serverName collided with the recovery source",
901915
"cluster", clusterName, "serverName", newServerName)
902916
}
903-
if err := r.patchPostgresAppForRestore(ctx, targetApp, sourceServerName, newServerName, sourceDestinationPath, sourceEndpointURL, options.RecoveryTime, rendered.BarmanObjectStore.S3Credentials, rendered.BarmanObjectStore.EndpointCA, sourceDatabases, sourceUsers); err != nil {
917+
if err := r.patchPostgresAppForRestore(ctx, targetApp, sourceServerName, newServerName, sourceDestinationPath, sourceEndpointURL, options.RecoveryTime, backupID, rendered.BarmanObjectStore.S3Credentials, rendered.BarmanObjectStore.EndpointCA, sourceDatabases, sourceUsers); err != nil {
904918
// Resume HR before terminal failure so an operator deleting
905919
// the failed RestoreJob does not leave the HR stuck.
906920
_ = r.setCNPGRestoreHRSuspended(ctx, target.Namespace, hrName, false)
@@ -1206,13 +1220,13 @@ func (r *RestoreJobReconciler) resolveCNPGRestoreTarget(restoreJob *backupsv1alp
12061220
func (r *RestoreJobReconciler) patchPostgresAppForRestore(
12071221
ctx context.Context,
12081222
app *postgresapp.Postgres,
1209-
sourceServerName, newServerName, sourceDestinationPath, sourceEndpointURL, recoveryTime string,
1223+
sourceServerName, newServerName, sourceDestinationPath, sourceEndpointURL, recoveryTime, backupID string,
12101224
credsRef *strategyv1alpha1.S3CredentialsTemplate,
12111225
caRef *strategyv1alpha1.EndpointCARef,
12121226
sourceDatabases map[string]postgresapp.Database,
12131227
sourceUsers map[string]postgresapp.User,
12141228
) error {
1215-
patched := buildPostgresAppRestorePatch(app, sourceServerName, newServerName, sourceDestinationPath, sourceEndpointURL, recoveryTime, credsRef, caRef, sourceDatabases, sourceUsers)
1229+
patched := buildPostgresAppRestorePatch(app, sourceServerName, newServerName, sourceDestinationPath, sourceEndpointURL, recoveryTime, backupID, credsRef, caRef, sourceDatabases, sourceUsers)
12161230
return r.Patch(ctx, patched, client.MergeFrom(app), client.FieldOwner(cnpgFieldManager))
12171231
}
12181232

@@ -1268,7 +1282,7 @@ func restoredServerName(clusterName string, uid types.UID) string {
12681282
// anything not in spec) must see the source's exact map.
12691283
func buildPostgresAppRestorePatch(
12701284
app *postgresapp.Postgres,
1271-
sourceServerName, newServerName, sourceDestinationPath, sourceEndpointURL, recoveryTime string,
1285+
sourceServerName, newServerName, sourceDestinationPath, sourceEndpointURL, recoveryTime, backupID string,
12721286
credsRef *strategyv1alpha1.S3CredentialsTemplate,
12731287
caRef *strategyv1alpha1.EndpointCARef,
12741288
sourceDatabases map[string]postgresapp.Database,
@@ -1282,6 +1296,7 @@ func buildPostgresAppRestorePatch(
12821296
// in-place restore fails barman-cloud-check-wal-archive ("Expected empty archive").
12831297
patched.Spec.Bootstrap.NewServerName = newServerName
12841298
patched.Spec.Bootstrap.RecoveryTime = recoveryTime
1299+
patched.Spec.Bootstrap.BackupID = backupID
12851300

12861301
patched.Spec.Backup.DestinationPath = sourceDestinationPath
12871302
patched.Spec.Backup.EndpointURL = sourceEndpointURL
@@ -1541,6 +1556,65 @@ func (r *RestoreJobReconciler) cnpgBackupWALArchived(ctx context.Context, backup
15411556
return true, fmt.Sprintf("cnpg.io/Backup %s/%s phase=completed, endWal=%q", backup.Namespace, backupName, cnpgBackup.Status.EndWal), nil
15421557
}
15431558

1559+
// cnpgSourceBackup returns the cnpg.io/Backup behind a Backup artifact, or
1560+
// nil once it is gone. The driver deletes it only together with its
1561+
// artifact, while plugin-barman-cloud's retention deletes it together with
1562+
// the base backup it describes (deleteBackupsNotInCatalog), so an artifact
1563+
// that outlives its cnpg.io/Backup names a base backup the catalog has lost.
1564+
func (r *RestoreJobReconciler) cnpgSourceBackup(ctx context.Context, backup *backupsv1alpha1.Backup) (*cnpgtypes.Backup, error) {
1565+
name := backup.Spec.DriverMetadata[cnpgBackupNameKey]
1566+
if name == "" {
1567+
return nil, nil
1568+
}
1569+
cnpgBackup := &cnpgtypes.Backup{}
1570+
if err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: name}, cnpgBackup); err != nil {
1571+
if apierrors.IsNotFound(err) {
1572+
return nil, nil
1573+
}
1574+
return nil, err
1575+
}
1576+
return cnpgBackup, nil
1577+
}
1578+
1579+
// cnpgRestoreBackupID decides whether a restore pins the base backup it
1580+
// starts from, and returns the backupID to set in
1581+
// bootstrap.recovery.recoveryTarget, or "" to leave the choice to the
1582+
// barman-cloud plugin.
1583+
//
1584+
// Left to itself the plugin starts from the newest backup in the catalog
1585+
// when no recoveryTime is set, whichever backup the RestoreJob names, and
1586+
// otherwise from the newest one ending at or before recoveryTime, on any
1587+
// timeline (barman-cloud pkg/catalog FindBackupInfo). On a stream holding
1588+
// several timelines either can start from another branch of the history
1589+
// and bring back other data than the backup asked for.
1590+
//
1591+
// A source that is gone (nil) is not pinned: its base backup is no longer
1592+
// in the catalog, and pinning it would fail the recovery after the target
1593+
// has been purged.
1594+
//
1595+
// A backup cannot serve a recoveryTime before it ended: recovery would stop
1596+
// before it is consistent. Such a request keeps the plugin's choice, which
1597+
// can find an older backup. stoppedAt is truncated to the second while the
1598+
// catalog compares barman's end time to the microsecond, so a backup is
1599+
// pinned only once recoveryTime is a full second past stoppedAt.
1600+
func cnpgRestoreBackupID(source *cnpgtypes.Backup, recoveryTime string) string {
1601+
if source == nil || source.Status.BackupID == "" {
1602+
return ""
1603+
}
1604+
if recoveryTime == "" {
1605+
return source.Status.BackupID
1606+
}
1607+
stoppedAt := source.Status.StoppedAt
1608+
if stoppedAt == nil || stoppedAt.IsZero() {
1609+
return ""
1610+
}
1611+
target, err := time.Parse(time.RFC3339Nano, recoveryTime)
1612+
if err != nil || target.Before(stoppedAt.Add(time.Second)) {
1613+
return ""
1614+
}
1615+
return source.Status.BackupID
1616+
}
1617+
15441618
// cnpgClusterHealthy returns true once the named cnpg.io Cluster reports its
15451619
// healthy phase. Treats a missing Cluster as not-yet-healthy.
15461620
func (r *RestoreJobReconciler) cnpgClusterHealthy(ctx context.Context, namespace, clusterName string) (bool, error) {

0 commit comments

Comments
 (0)