Skip to content

Commit 67bf9ca

Browse files
fix(postgres): read the live Cluster when deciding the WAL-archive prefix
applyClusterPluginBackup preserves the serverName already on the Cluster, so that a backup does not flip the archive prefix mid-stream. It read that value through the manager's cached client, and this controller registers For(&BackupJob{}) and nothing else, so a Cluster only reaches the cache through a lazily started informer with no resync tied to its own changes. An in-place RestoreJob deletes the Cluster and the chart re-renders it with bootstrap.newServerName seconds later. A read served during that window describes the object that was purged, so the source's serverName is written back over the freshly rendered one and the restored cluster archives onto the source's WAL prefix. barman-cloud-check-wal-archive then refuses to start with "Expected empty archive" and the RestoreJob ends on its deadline -- the failure newServerName was added to prevent. The stale read carries a stale UID too, and that UID owns the ObjectStore this function applies, so the store is garbage-collected as soon as it is written. Reading through the dynamic client keeps the preservation rule and makes the value it rests on truthful. Assisted-by: LLM Signed-off-by: Mattia Eleuteri <[email protected]>
1 parent 0a2054e commit 67bf9ca

2 files changed

Lines changed: 89 additions & 4 deletions

File tree

‎internal/backupcontroller/cnpgstrategy_controller.go‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2121
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
2222
"k8s.io/apimachinery/pkg/runtime"
23+
"k8s.io/apimachinery/pkg/runtime/schema"
2324
"k8s.io/apimachinery/pkg/types"
2425
ctrl "sigs.k8s.io/controller-runtime"
2526
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -455,6 +456,14 @@ func cnpgClusterFreshlyRecovered(hasRecovery bool, clusterCreatedAt, restoreStar
455456
return clusterCreatedAt.After(restoreStartedAt.Time)
456457
}
457458

459+
// cnpgClusterGVR addresses Clusters on the dynamic client, for reads that must
460+
// bypass the manager's cache.
461+
var cnpgClusterGVR = schema.GroupVersionResource{
462+
Group: cnpgtypes.GroupName,
463+
Version: cnpgtypes.Version,
464+
Resource: "clusters",
465+
}
466+
458467
// applyClusterPluginBackup wires the templated strategy onto the live CNPG
459468
// Cluster through the barman-cloud plugin: it SSA-applies an ObjectStore CR
460469
// carrying the S3/barman configuration and SSA-patches the Cluster's
@@ -478,10 +487,16 @@ func cnpgClusterFreshlyRecovered(hasRecovery bool, clusterCreatedAt, restoreStar
478487
// land under the old prefix while the base backup indexes under the new one,
479488
// and the eventual restore fails with "WAL not found".
480489
func (r *BackupJobReconciler) applyClusterPluginBackup(ctx context.Context, namespace, clusterName string, t *strategyv1alpha1.CNPGTemplate, serverName string) (string, error) {
490+
// Live read: the cached Cluster can predate a restore re-render, and both
491+
// the serverName and the UID below must come from the current object.
481492
existing := &cnpgtypes.Cluster{}
482-
if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: clusterName}, existing); err != nil {
493+
live, err := r.Resource(cnpgClusterGVR).Namespace(namespace).Get(ctx, clusterName, metav1.GetOptions{})
494+
if err != nil {
483495
return "", err
484496
}
497+
if err := runtime.DefaultUnstructuredConverter.FromUnstructured(live.Object, existing); err != nil {
498+
return "", fmt.Errorf("decode Cluster %s/%s: %w", namespace, clusterName, err)
499+
}
485500
if live := currentBarmanServerName(existing); live != "" {
486501
serverName = live
487502
}

‎internal/backupcontroller/cnpgstrategy_controller_test.go‎

Lines changed: 73 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1045,7 +1045,7 @@ func TestCreateCNPGBackupArtifact_AlreadyExistsReturnsExisting(t *testing.T) {
10451045
// would be rejected by the API server for missing required Cluster fields.
10461046
func TestApplyClusterPluginBackup_NotFoundOnMissingCluster(t *testing.T) {
10471047
c := newCNPGStrategyTestClient(t)
1048-
r := &BackupJobReconciler{Client: c}
1048+
r := &BackupJobReconciler{Client: c, Interface: cnpgDynamicFor(t)}
10491049
tmpl := &strategyv1alpha1.CNPGTemplate{
10501050
BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{
10511051
DestinationPath: "s3://bucket/x/",
@@ -1665,6 +1665,76 @@ func TestRecoveryTargetUnreachable_ClassifiesFromLogs(t *testing.T) {
16651665
})
16661666
}
16671667

1668+
// TestApplyClusterPluginBackup_ReadsTheLiveClusterNotTheCache seeds the cached
1669+
// and the live client with the Cluster before and after an in-place restore
1670+
// re-rendered it, and asserts the driver takes both the serverName and the
1671+
// ObjectStore owner UID from the live one.
1672+
func TestApplyClusterPluginBackup_ReadsTheLiveClusterNotTheCache(t *testing.T) {
1673+
archiver := true
1674+
purged := &cnpgtypes.Cluster{
1675+
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "postgres-app", UID: "uid-before-restore"},
1676+
}
1677+
purged.Spec.Plugins = []cnpgtypes.PluginConfiguration{{
1678+
Name: cnpgtypes.PluginName,
1679+
IsWALArchiver: &archiver,
1680+
Parameters: map[string]string{barmanObjectNameParam: "postgres-app", barmanServerNameParam: "postgres-app"},
1681+
}}
1682+
1683+
rebootstrapped := purged.DeepCopy()
1684+
rebootstrapped.UID = "uid-after-restore"
1685+
rebootstrapped.Spec.Plugins[0].Parameters[barmanServerNameParam] = "postgres-app-restore-0123456789abcdef"
1686+
1687+
c := newCNPGStrategyTestClient(t, purged)
1688+
r := &BackupJobReconciler{Client: c, Interface: cnpgDynamicFor(t, rebootstrapped)}
1689+
tmpl := &strategyv1alpha1.CNPGTemplate{
1690+
BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/x/"},
1691+
}
1692+
1693+
got, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "strategy-name")
1694+
if err != nil {
1695+
t.Fatalf("unexpected error: %v", err)
1696+
}
1697+
if want := "postgres-app-restore-0123456789abcdef"; got != want {
1698+
t.Fatalf("effective serverName = %q, want %q: the restored cluster would archive onto the source's prefix", got, want)
1699+
}
1700+
1701+
store := &cnpgtypes.ObjectStore{}
1702+
if err := c.Get(context.Background(), client.ObjectKey{Namespace: "tenant", Name: "postgres-app"}, store); err != nil {
1703+
t.Fatalf("get ObjectStore after apply: %v", err)
1704+
}
1705+
owners := store.GetOwnerReferences()
1706+
if len(owners) != 1 {
1707+
t.Fatalf("ObjectStore owner references = %v, want exactly the live Cluster", owners)
1708+
}
1709+
if got := string(owners[0].UID); got != "uid-after-restore" {
1710+
t.Fatalf("ObjectStore owned by UID %q, want the live Cluster's: it would be garbage-collected immediately", got)
1711+
}
1712+
}
1713+
1714+
// cnpgDynamicFor builds the dynamic client applyClusterPluginBackup reads
1715+
// Clusters through, seeded independently of the typed client.
1716+
func cnpgDynamicFor(t *testing.T, clusters ...*cnpgtypes.Cluster) dynamic.Interface {
1717+
t.Helper()
1718+
s := runtime.NewScheme()
1719+
s.AddKnownTypeWithName(cnpgtypes.GroupVersion.WithKind("Cluster"), &unstructured.Unstructured{})
1720+
s.AddKnownTypeWithName(cnpgtypes.GroupVersion.WithKind("ClusterList"), &unstructured.UnstructuredList{})
1721+
s.AddKnownTypeWithName(cnpgtypes.BarmanGroupVersion.WithKind("ObjectStore"), &unstructured.Unstructured{})
1722+
s.AddKnownTypeWithName(cnpgtypes.BarmanGroupVersion.WithKind("ObjectStoreList"), &unstructured.UnstructuredList{})
1723+
1724+
objs := make([]runtime.Object, 0, len(clusters))
1725+
for _, c := range clusters {
1726+
raw, err := runtime.DefaultUnstructuredConverter.ToUnstructured(c)
1727+
if err != nil {
1728+
t.Fatalf("converting Cluster %s: %v", c.Name, err)
1729+
}
1730+
u := &unstructured.Unstructured{Object: raw}
1731+
u.SetAPIVersion(cnpgtypes.GroupVersion.String())
1732+
u.SetKind("Cluster")
1733+
objs = append(objs, u)
1734+
}
1735+
return dynamicfake.NewSimpleDynamicClient(s, objs...)
1736+
}
1737+
16681738
// testCNPGScheme returns a runtime.Scheme that knows the unstructured
16691739
// HelmRelease GVK used by the dynamic-client tests above.
16701740
func testCNPGScheme(t *testing.T) *runtime.Scheme {
@@ -1723,7 +1793,7 @@ func TestApplyClusterPluginBackup_PreservesLiveServerName(t *testing.T) {
17231793
}
17241794
cluster.Spec.Plugins = tc.plugins
17251795
c := newCNPGStrategyTestClient(t, cluster)
1726-
r := &BackupJobReconciler{Client: c}
1796+
r := &BackupJobReconciler{Client: c, Interface: cnpgDynamicFor(t, cluster)}
17271797
tmpl := &strategyv1alpha1.CNPGTemplate{
17281798
BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/x/"},
17291799
}
@@ -1759,7 +1829,7 @@ func TestApplyClusterPluginBackup_PatchesExistingCluster(t *testing.T) {
17591829
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "postgres-app", UID: "cluster-uid-123"},
17601830
}
17611831
c := newCNPGStrategyTestClient(t, cluster)
1762-
r := &BackupJobReconciler{Client: c}
1832+
r := &BackupJobReconciler{Client: c, Interface: cnpgDynamicFor(t, cluster)}
17631833
tmpl := &strategyv1alpha1.CNPGTemplate{
17641834
BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{
17651835
DestinationPath: "s3://bucket/x/",

0 commit comments

Comments
 (0)