Skip to content

Commit cf1b6ba

Browse files
committed
fix(backups/postgres): purge stale recovery cluster on repeat in-place restore
A repeat in-place PostgreSQL RestoreJob into an already-restored target silently no-opped: the destructive purge was skipped because the live Cluster still carried spec.bootstrap.recovery left over from the previous restore, and the re-purge guard mistook that for a freshly-recovered Cluster produced by this restore's own purge. The job then reported Succeeded while the PVC, disk, and data were never touched. Tell the two apart by the Cluster's creationTimestamp relative to the RestoreJob's StartedAt: only a recovery Cluster created at/after StartedAt is treated as this restore's own and skipped; one that predates StartedAt is leftover from an earlier completed restore and is purged so the restore re-bootstraps from the backup. Signed-off-by: Ivan Okhotnikov <[email protected]>
1 parent 215bd43 commit cf1b6ba

2 files changed

Lines changed: 179 additions & 46 deletions

File tree

‎internal/backupcontroller/cnpgstrategy_controller.go‎

Lines changed: 60 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -314,16 +314,46 @@ func cnpgBackupDeadlineExceeded(startedAt *metav1.Time) bool {
314314
// destructive step:
315315
// 1. The RestoreJob already records that we've purged
316316
// (restoreCondTargetPurged=True). Normal idempotent path.
317-
// 2. The live Cluster already has spec.bootstrap.recovery populated. This
318-
// only happens when an earlier reconcile purged successfully but the
317+
// 2. The live Cluster is a freshly-recovered one that THIS restore's own
318+
// purge + chart re-render produced (see cnpgClusterFreshlyRecovered).
319+
// This only happens when an earlier reconcile purged successfully but the
319320
// status-condition write failed; the chart has since re-rendered the
320321
// Cluster with our restore-shaped values. Re-purging here would delete
321322
// the Cluster CNPG is actively bootstrapping from S3.
322-
func cnpgPurgeNeeded(purgedCondition, liveClusterHasRecovery bool) bool {
323+
//
324+
// A live Cluster that carries spec.bootstrap.recovery but predates this
325+
// RestoreJob is NOT a skip signal: it is leftover from an earlier, already-
326+
// completed restore and still holds the old data. Skipping the purge on it
327+
// (the previous behaviour, which keyed only on "has recovery bootstrap") made
328+
// a repeat in-place restore a silent no-op - the job reported Succeeded while
329+
// the PVC, disk, and data were never touched.
330+
func cnpgPurgeNeeded(purgedCondition, liveClusterFreshlyRecovered bool) bool {
323331
if purgedCondition {
324332
return false
325333
}
326-
return !liveClusterHasRecovery
334+
return !liveClusterFreshlyRecovered
335+
}
336+
337+
// cnpgClusterFreshlyRecovered reports whether a live cnpg.io Cluster that
338+
// carries spec.bootstrap.recovery was produced by THIS RestoreJob's own purge
339+
// + chart re-render (its creationTimestamp is at or after the job's StartedAt)
340+
// rather than being left over from an earlier, already-completed restore.
341+
//
342+
// Only the former is safe to skip re-purging. A leftover recovery Cluster from
343+
// a prior restore predates StartedAt and still holds the prior data, so it must
344+
// be purged for the new restore to re-bootstrap from the backup. The fresh
345+
// Cluster in the status-write-race case is always created after StartedAt (the
346+
// job sets StartedAt on its first reconcile, long before it purges and the
347+
// chart re-renders), so the timestamp comparison cleanly separates the two.
348+
//
349+
// Returns false when freshness cannot be determined (no recovery bootstrap, or
350+
// a missing timestamp): the caller then proceeds to purge, which is the safe
351+
// default for any pre-existing, non-fresh Cluster.
352+
func cnpgClusterFreshlyRecovered(hasRecovery bool, clusterCreatedAt, restoreStartedAt *metav1.Time) bool {
353+
if !hasRecovery || clusterCreatedAt == nil || restoreStartedAt == nil {
354+
return false
355+
}
356+
return !clusterCreatedAt.Time.Before(restoreStartedAt.Time)
327357
}
328358

329359
// applyClusterBarmanObjectStore SSA-patches the live CNPG Cluster's
@@ -614,12 +644,18 @@ func (r *RestoreJobReconciler) reconcileCNPGRestore(ctx context.Context, restore
614644
// check the live Cluster for bootstrap.recovery: if present, the chart
615645
// has already re-rendered after a previous purge, and we must NOT delete
616646
// it again.
617-
hasRecovery, err := r.clusterHasRecoveryBootstrap(ctx, target.Namespace, clusterName)
647+
hasRecovery, clusterCreatedAt, err := r.recoveryBootstrapClusterState(ctx, target.Namespace, clusterName)
618648
if err != nil {
619649
return ctrl.Result{}, err
620650
}
621651
purgedCondition := apimeta.IsStatusConditionTrue(restoreJob.Status.Conditions, restoreCondTargetPurged)
622-
if cnpgPurgeNeeded(purgedCondition, hasRecovery) {
652+
// Only skip the destructive purge for a recovery Cluster THIS restore just
653+
// produced (created at/after StartedAt). A recovery Cluster left over from
654+
// an earlier completed restore predates StartedAt and still holds the old
655+
// data, so it must be purged - otherwise a repeat in-place restore silently
656+
// no-ops.
657+
freshlyRecovered := cnpgClusterFreshlyRecovered(hasRecovery, clusterCreatedAt, restoreJob.Status.StartedAt)
658+
if cnpgPurgeNeeded(purgedCondition, freshlyRecovered) {
623659
// Gate the destructive flow on the source cluster having shipped
624660
// the backup's required WALs to object storage. archive_command runs
625661
// on the source primary; once we delete the Cluster + PVCs, any
@@ -1126,11 +1162,15 @@ func (r *RestoreJobReconciler) cnpgClusterHealthy(ctx context.Context, namespace
11261162
return cluster.Status.Phase == cnpgClusterHealthyPhase, nil
11271163
}
11281164

1129-
// clusterHasRecoveryBootstrap returns true when the live cnpg.io Cluster's
1130-
// spec.bootstrap.recovery is populated - the signal that the chart has
1131-
// re-rendered with our restore-shaped values and the operator is using the
1132-
// recovery bootstrap path. Treats a missing Cluster as "not yet" rather
1133-
// than an error so the caller can keep polling while HelmRelease catches up.
1165+
// recoveryBootstrapClusterState fetches the live cnpg.io Cluster and reports
1166+
// whether its spec.bootstrap.recovery is populated - the signal that the chart
1167+
// has re-rendered with our restore-shaped values and the operator is using the
1168+
// recovery bootstrap path - together with the Cluster's creationTimestamp. The
1169+
// caller pairs the timestamp with the RestoreJob's StartedAt (via
1170+
// cnpgClusterFreshlyRecovered) to tell a Cluster this restore just produced
1171+
// apart from one left over by an earlier completed restore. Treats a missing
1172+
// Cluster as "not yet" rather than an error so the caller can keep polling
1173+
// while HelmRelease catches up.
11341174
//
11351175
// A Cluster carrying DeletionTimestamp is also treated as "not yet": that is
11361176
// the in-flight purge case, where r.Delete has fired but cnpg.io's
@@ -1143,18 +1183,22 @@ func (r *RestoreJobReconciler) cnpgClusterHealthy(ctx context.Context, namespace
11431183
// drops the change and the cluster ends up with the original initdb spec.
11441184
// Holding here forces the caller to requeue until the old CR is fully GC'd
11451185
// and the chart re-creates a fresh one.
1146-
func (r *RestoreJobReconciler) clusterHasRecoveryBootstrap(ctx context.Context, namespace, clusterName string) (bool, error) {
1186+
func (r *RestoreJobReconciler) recoveryBootstrapClusterState(ctx context.Context, namespace, clusterName string) (hasRecovery bool, createdAt *metav1.Time, err error) {
11471187
cluster := &cnpgtypes.Cluster{}
11481188
if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: clusterName}, cluster); err != nil {
11491189
if apierrors.IsNotFound(err) {
1150-
return false, nil
1190+
return false, nil, nil
11511191
}
1152-
return false, err
1192+
return false, nil, err
11531193
}
11541194
if !cluster.DeletionTimestamp.IsZero() {
1155-
return false, nil
1195+
return false, nil, nil
1196+
}
1197+
if cluster.Spec.Bootstrap == nil || cluster.Spec.Bootstrap.Recovery == nil {
1198+
return false, nil, nil
11561199
}
1157-
return cluster.Spec.Bootstrap != nil && cluster.Spec.Bootstrap.Recovery != nil, nil
1200+
created := cluster.CreationTimestamp
1201+
return true, &created, nil
11581202
}
11591203

11601204
// ---------------------------------------------------------------------------

‎internal/backupcontroller/cnpgstrategy_controller_test.go‎

Lines changed: 119 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -549,43 +549,122 @@ func TestBuildPostgresAppRestorePatch_NoSecretRefIsSkipped(t *testing.T) {
549549
// freshly-recovered Cluster on a status-update failure. The controller used
550550
// to rely solely on a Status condition: if the post-purge Status().Update
551551
// raced or failed, the next reconcile would re-purge the just-restored
552-
// Cluster, destroying recovery progress. Cross-checking the live Cluster's
553-
// bootstrap.recovery makes the second purge a no-op.
552+
// Cluster, destroying recovery progress. Cross-checking that the live
553+
// Cluster is a *freshly-recovered* one (bootstrap.recovery + created after
554+
// the job started) makes that second purge a no-op - while still purging a
555+
// stale recovery Cluster left over from an earlier completed restore, so a
556+
// repeat in-place restore is not a silent no-op.
554557
func TestCNPGPurgeNeeded(t *testing.T) {
555558
cases := []struct {
556-
name string
557-
purgedCondition bool
558-
hasRecovery bool
559-
want bool
559+
name string
560+
purgedCondition bool
561+
liveClusterFreshlyRecovered bool
562+
want bool
560563
}{
561564
{
562-
name: "fresh restore, old cluster still in place: purge",
563-
purgedCondition: false,
564-
hasRecovery: false,
565-
want: true,
565+
name: "fresh restore, old cluster still in place: purge",
566+
purgedCondition: false,
567+
liveClusterFreshlyRecovered: false,
568+
want: true,
566569
},
567570
{
568-
name: "purge already recorded: skip",
569-
purgedCondition: true,
570-
hasRecovery: false,
571-
want: false,
571+
name: "purge already recorded: skip",
572+
purgedCondition: true,
573+
liveClusterFreshlyRecovered: false,
574+
want: false,
572575
},
573576
{
574-
name: "live cluster already recovered (status write raced): skip - the bug fix",
575-
purgedCondition: false,
576-
hasRecovery: true,
577-
want: false,
577+
name: "live cluster freshly recovered by this restore (status write raced): skip",
578+
purgedCondition: false,
579+
liveClusterFreshlyRecovered: true,
580+
want: false,
578581
},
579582
{
580-
name: "both true: skip (post-purge steady state)",
581-
purgedCondition: true,
582-
hasRecovery: true,
583-
want: false,
583+
name: "stale recovery cluster from a previous completed restore: purge (repeat-restore fix)",
584+
purgedCondition: false,
585+
liveClusterFreshlyRecovered: false,
586+
want: true,
587+
},
588+
{
589+
name: "both true: skip (post-purge steady state)",
590+
purgedCondition: true,
591+
liveClusterFreshlyRecovered: true,
592+
want: false,
593+
},
594+
}
595+
for _, tc := range cases {
596+
t.Run(tc.name, func(t *testing.T) {
597+
got := cnpgPurgeNeeded(tc.purgedCondition, tc.liveClusterFreshlyRecovered)
598+
if got != tc.want {
599+
t.Errorf("got %v want %v", got, tc.want)
600+
}
601+
})
602+
}
603+
}
604+
605+
// TestCNPGClusterFreshlyRecovered locks in the signal that separates a
606+
// recovery Cluster THIS restore just produced from one left over by an
607+
// earlier completed restore. The repeat-in-place-restore bug was exactly a
608+
// stale recovery Cluster (created before StartedAt) being mistaken for a
609+
// freshly-recovered one and skipping the purge.
610+
func TestCNPGClusterFreshlyRecovered(t *testing.T) {
611+
started := metav1.NewTime(time.Now())
612+
after := metav1.NewTime(started.Add(time.Minute))
613+
before := metav1.NewTime(started.Add(-time.Hour))
614+
615+
cases := []struct {
616+
name string
617+
hasRecovery bool
618+
createdAt *metav1.Time
619+
startedAt *metav1.Time
620+
want bool
621+
}{
622+
{
623+
name: "no recovery bootstrap: not fresh",
624+
hasRecovery: false,
625+
createdAt: &after,
626+
startedAt: &started,
627+
want: false,
628+
},
629+
{
630+
name: "recovery cluster created after start (our own purge re-render): fresh",
631+
hasRecovery: true,
632+
createdAt: &after,
633+
startedAt: &started,
634+
want: true,
635+
},
636+
{
637+
name: "recovery cluster created before start (leftover from previous restore): not fresh",
638+
hasRecovery: true,
639+
createdAt: &before,
640+
startedAt: &started,
641+
want: false,
642+
},
643+
{
644+
name: "recovery cluster created exactly at start: fresh (boundary)",
645+
hasRecovery: true,
646+
createdAt: &started,
647+
startedAt: &started,
648+
want: true,
649+
},
650+
{
651+
name: "missing cluster creation timestamp: not fresh",
652+
hasRecovery: true,
653+
createdAt: nil,
654+
startedAt: &started,
655+
want: false,
656+
},
657+
{
658+
name: "missing job start timestamp: not fresh",
659+
hasRecovery: true,
660+
createdAt: &after,
661+
startedAt: nil,
662+
want: false,
584663
},
585664
}
586665
for _, tc := range cases {
587666
t.Run(tc.name, func(t *testing.T) {
588-
got := cnpgPurgeNeeded(tc.purgedCondition, tc.hasRecovery)
667+
got := cnpgClusterFreshlyRecovered(tc.hasRecovery, tc.createdAt, tc.startedAt)
589668
if got != tc.want {
590669
t.Errorf("got %v want %v", got, tc.want)
591670
}
@@ -898,7 +977,7 @@ func TestApplyClusterBarmanObjectStore_NotFoundOnMissingCluster(t *testing.T) {
898977
}
899978
}
900979

901-
// TestClusterHasRecoveryBootstrap_TerminatingCluster locks in the
980+
// TestRecoveryBootstrapClusterState_TerminatingCluster locks in the
902981
// DeletionTimestamp guard. Without it, a Cluster CR mid-deletion (after
903982
// purgeExistingCluster fired r.Delete but cnpg.io's finalizers haven't
904983
// drained yet) would get treated as "still has bootstrap" by the
@@ -907,7 +986,7 @@ func TestApplyClusterBarmanObjectStore_NotFoundOnMissingCluster(t *testing.T) {
907986
// might SSA-merge the chart's bootstrap.recovery onto the terminating
908987
// CR and cnpg-operator's bootstrap-immutability check would drop the
909988
// change, leaving the cluster on the original initdb spec.
910-
func TestClusterHasRecoveryBootstrap_TerminatingCluster(t *testing.T) {
989+
func TestRecoveryBootstrapClusterState_TerminatingCluster(t *testing.T) {
911990
now := metav1.Now()
912991
t.Run("terminating cluster reports not-yet-recovered", func(t *testing.T) {
913992
cluster := &cnpgtypes.Cluster{
@@ -925,18 +1004,22 @@ func TestClusterHasRecoveryBootstrap_TerminatingCluster(t *testing.T) {
9251004
}
9261005
c := newCNPGStrategyTestClient(t, cluster)
9271006
r := &RestoreJobReconciler{Client: c}
928-
got, err := r.clusterHasRecoveryBootstrap(context.Background(), "tenant", "postgres-app")
1007+
got, createdAt, err := r.recoveryBootstrapClusterState(context.Background(), "tenant", "postgres-app")
9291008
if err != nil {
9301009
t.Fatalf("unexpected error: %v", err)
9311010
}
9321011
if got {
9331012
t.Fatalf("expected false for terminating cluster, got true")
9341013
}
1014+
if createdAt != nil {
1015+
t.Fatalf("expected nil createdAt for terminating cluster, got %v", createdAt)
1016+
}
9351017
})
9361018

937-
t.Run("live recovery cluster reports recovered", func(t *testing.T) {
1019+
t.Run("live recovery cluster reports recovered with creation timestamp", func(t *testing.T) {
1020+
created := metav1.NewTime(time.Now().Add(-time.Hour))
9381021
cluster := &cnpgtypes.Cluster{
939-
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "postgres-app"},
1022+
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "postgres-app", CreationTimestamp: created},
9401023
Spec: cnpgtypes.ClusterSpec{
9411024
Bootstrap: &cnpgtypes.BootstrapConfiguration{
9421025
Recovery: &cnpgtypes.RecoverySource{Source: "pg-src"},
@@ -945,25 +1028,31 @@ func TestClusterHasRecoveryBootstrap_TerminatingCluster(t *testing.T) {
9451028
}
9461029
c := newCNPGStrategyTestClient(t, cluster)
9471030
r := &RestoreJobReconciler{Client: c}
948-
got, err := r.clusterHasRecoveryBootstrap(context.Background(), "tenant", "postgres-app")
1031+
got, createdAt, err := r.recoveryBootstrapClusterState(context.Background(), "tenant", "postgres-app")
9491032
if err != nil {
9501033
t.Fatalf("unexpected error: %v", err)
9511034
}
9521035
if !got {
9531036
t.Fatalf("expected true for live recovery cluster, got false")
9541037
}
1038+
if createdAt == nil {
1039+
t.Fatalf("expected non-nil createdAt for live recovery cluster")
1040+
}
9551041
})
9561042

9571043
t.Run("missing cluster reports not-yet", func(t *testing.T) {
9581044
c := newCNPGStrategyTestClient(t)
9591045
r := &RestoreJobReconciler{Client: c}
960-
got, err := r.clusterHasRecoveryBootstrap(context.Background(), "tenant", "postgres-app")
1046+
got, createdAt, err := r.recoveryBootstrapClusterState(context.Background(), "tenant", "postgres-app")
9611047
if err != nil {
9621048
t.Fatalf("unexpected error: %v", err)
9631049
}
9641050
if got {
9651051
t.Fatalf("expected false for missing cluster, got true")
9661052
}
1053+
if createdAt != nil {
1054+
t.Fatalf("expected nil createdAt for missing cluster, got %v", createdAt)
1055+
}
9671056
})
9681057
}
9691058

0 commit comments

Comments
 (0)