Skip to content

Commit d03fe86

Browse files
fix(postgres): refuse a physical restore across image flavors
A recovery replays the source's data directory, so it carries the same risk as switching the image of a live cluster: the postgresql and postgis images are built on different Debian releases, and the glibc change invalidates collatable indexes, while a postgis-to-postgresql move also loses the libraries behind every PostGIS object. The chart only guarded the live cluster, so a fresh release bootstrapped from a backup of the other flavor went through, and the guard's own advice was to restore a backup, which is that very move. The backup-controller now records the source flavor in the snapshot it stores on each Backup and fails a RestoreJob whose target runs another flavor, before the purge deletes the target's Cluster and PVCs. A snapshot without a flavor predates flavors and so comes from a postgresql cluster. For the chart-managed bootstrap, the chart compares against the recovery source's Cluster while it still exists. Both paths, and the live-cluster guard, point a flavor change to a logical dump. Assisted-by: LLM Signed-off-by: Mattia Eleuteri <[email protected]>
1 parent 3d67499 commit d03fe86

10 files changed

Lines changed: 329 additions & 21 deletions

File tree

‎api/apps/v1alpha1/postgresql/types.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎internal/backupcontroller/cnpgstrategy_controller.go‎

Lines changed: 35 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -771,13 +771,24 @@ func (r *RestoreJobReconciler) reconcileCNPGRestore(ctx context.Context, restore
771771
// time. Without this, restore-time rendering with parameters=nil
772772
// silently produced `<no value>` and patched the target Postgres app
773773
// with broken Secret references.
774-
sourceDatabases, sourceUsers, sourceParameters, err := unmarshalCNPGBackupSnapshot(backup)
774+
snapshot, err := unmarshalCNPGBackupSnapshot(backup)
775775
if err != nil {
776776
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf(
777777
"Backup %s/%s carries no usable source-spec snapshot in status.underlyingResources: %v "+
778778
"(re-take the backup with a controller version that persists source spec)",
779779
backup.Namespace, backup.Name, err))
780780
}
781+
sourceDatabases, sourceUsers, sourceParameters := snapshot.Databases, snapshot.Users, snapshot.Parameters
782+
783+
// Checked before the destructive purge below, which would otherwise wipe
784+
// the target only for the recovery to come up on the wrong image family.
785+
if src, dst := cnpgFlavor(snapshot.Flavor), cnpgFlavor(targetApp.Spec.Flavor); src != dst {
786+
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf(
787+
"Backup %s/%s was taken from a %s cluster and cannot be restored into %s/%s, which runs the %s flavor: "+
788+
"a physical restore must keep the image family. Restore into a Postgres app with flavor %s, "+
789+
"or move the data with a logical dump (pg_dump / pg_restore)",
790+
backup.Namespace, backup.Name, src, target.Namespace, target.AppName, dst, src))
791+
}
781792

782793
rendered, err := renderCNPGTemplate(strategy.Spec.Template, targetApp, sourceParameters)
783794
if err != nil {
@@ -1359,12 +1370,18 @@ func buildPostgresAppRestorePatch(
13591370
// names, prefixes, secret references), never cleartext credentials -
13601371
// credentials are always referenced through `secretRef`, not inlined, and
13611372
// the Secret content stays in etcd's encryption-at-rest store.
1373+
//
1374+
// Flavor records the source's image family. A physical backup only restores
1375+
// safely onto the same family: the two ship different glibc versions, which
1376+
// invalidates collatable indexes, and postgis objects need the postgis
1377+
// libraries.
13621378
type cnpgBackupSnapshot struct {
13631379
Kind string `json:"kind"`
13641380
APIVersion string `json:"apiVersion"`
13651381
Databases map[string]postgresapp.Database `json:"databases,omitempty"`
13661382
Users map[string]postgresapp.User `json:"users,omitempty"`
13671383
Parameters map[string]string `json:"parameters,omitempty"`
1384+
Flavor string `json:"flavor,omitempty"`
13681385
}
13691386

13701387
const cnpgBackupSnapshotKind = "CNPGBackupSnapshot"
@@ -1387,6 +1404,7 @@ func marshalCNPGBackupSnapshot(app *postgresapp.Postgres, parameters map[string]
13871404
Databases: app.Spec.Databases,
13881405
Users: app.Spec.Users,
13891406
Parameters: parameters,
1407+
Flavor: app.Spec.Flavor,
13901408
}
13911409
raw, err := json.Marshal(snap)
13921410
if err != nil {
@@ -1405,19 +1423,29 @@ func marshalCNPGBackupSnapshot(app *postgresapp.Postgres, parameters map[string]
14051423
// `{{ .Parameters.foo }}` references stores nil here, and a Backup taken
14061424
// by a controller version that pre-dates parameter persistence has no
14071425
// parameters block at all). The caller must treat a nil/empty map the
1408-
// same as "no parameter overrides" rather than failing.
1409-
func unmarshalCNPGBackupSnapshot(backup *backupsv1alpha1.Backup) (map[string]postgresapp.Database, map[string]postgresapp.User, map[string]string, error) {
1426+
// same as "no parameter overrides" rather than failing. Flavor is empty on
1427+
// a Backup taken before flavors existed, when every cluster ran postgresql;
1428+
// compare it through cnpgFlavor.
1429+
func unmarshalCNPGBackupSnapshot(backup *backupsv1alpha1.Backup) (*cnpgBackupSnapshot, error) {
14101430
if backup == nil || backup.Status.UnderlyingResources == nil || len(backup.Status.UnderlyingResources.Raw) == 0 {
1411-
return nil, nil, nil, fmt.Errorf("status.underlyingResources is empty")
1431+
return nil, fmt.Errorf("status.underlyingResources is empty")
14121432
}
14131433
var snap cnpgBackupSnapshot
14141434
if err := json.Unmarshal(backup.Status.UnderlyingResources.Raw, &snap); err != nil {
1415-
return nil, nil, nil, fmt.Errorf("decode snapshot: %w", err)
1435+
return nil, fmt.Errorf("decode snapshot: %w", err)
14161436
}
14171437
if snap.Kind != cnpgBackupSnapshotKind {
1418-
return nil, nil, nil, fmt.Errorf("unexpected snapshot kind %q (want %q)", snap.Kind, cnpgBackupSnapshotKind)
1438+
return nil, fmt.Errorf("unexpected snapshot kind %q (want %q)", snap.Kind, cnpgBackupSnapshotKind)
1439+
}
1440+
return &snap, nil
1441+
}
1442+
1443+
// cnpgFlavor resolves an unset flavor to the chart default.
1444+
func cnpgFlavor(flavor string) string {
1445+
if flavor == "" {
1446+
return "postgresql"
14191447
}
1420-
return snap.Databases, snap.Users, snap.Parameters, nil
1448+
return flavor
14211449
}
14221450

14231451
// purgeExistingCluster deletes the live cnpg.io Cluster and its PVCs so the

‎internal/backupcontroller/cnpgstrategy_controller_test.go‎

Lines changed: 144 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -230,10 +230,11 @@ func TestRenderCNPGTemplate_RoundTripsParametersThroughSnapshot(t *testing.T) {
230230
t.Fatalf("marshal snapshot: %v", err)
231231
}
232232
bk := &backupsv1alpha1.Backup{Status: backupsv1alpha1.BackupStatus{UnderlyingResources: raw}}
233-
_, _, gotParams, err := unmarshalCNPGBackupSnapshot(bk)
233+
snap, err := unmarshalCNPGBackupSnapshot(bk)
234234
if err != nil {
235235
t.Fatalf("unmarshal snapshot: %v", err)
236236
}
237+
gotParams := snap.Parameters
237238

238239
// Restore-time render with retrieved parameters: same values out.
239240
rendRestore, err := renderCNPGTemplate(tmpl, app, gotParams)
@@ -888,10 +889,11 @@ func TestMarshalUnmarshalCNPGBackupSnapshot(t *testing.T) {
888889
}
889890

890891
bk := &backupsv1alpha1.Backup{Status: backupsv1alpha1.BackupStatus{UnderlyingResources: raw}}
891-
dbs, users, params, err := unmarshalCNPGBackupSnapshot(bk)
892+
snap, err := unmarshalCNPGBackupSnapshot(bk)
892893
if err != nil {
893894
t.Fatalf("unmarshal: %v", err)
894895
}
896+
dbs, users, params := snap.Databases, snap.Users, snap.Parameters
895897
if dbs["app"].Extensions[0] != "pgcrypto" {
896898
t.Errorf("databases round-trip mismatch: %#v", dbs)
897899
}
@@ -916,12 +918,17 @@ func TestMarshalUnmarshalCNPGBackupSnapshot(t *testing.T) {
916918
func TestMarshalUnmarshalCNPGBackupSnapshot_BackwardCompat(t *testing.T) {
917919
legacyRaw := []byte(`{"kind":"CNPGBackupSnapshot","apiVersion":"backups.cozystack.io/v1alpha1","databases":{"app":{}},"users":{"app":{}}}`)
918920
bk := &backupsv1alpha1.Backup{Status: backupsv1alpha1.BackupStatus{UnderlyingResources: &runtime.RawExtension{Raw: legacyRaw}}}
919-
_, _, params, err := unmarshalCNPGBackupSnapshot(bk)
921+
snap, err := unmarshalCNPGBackupSnapshot(bk)
920922
if err != nil {
921923
t.Fatalf("unmarshal legacy snapshot: %v", err)
922924
}
923-
if len(params) != 0 {
924-
t.Errorf("legacy snapshot must yield empty parameters, got %#v", params)
925+
if len(snap.Parameters) != 0 {
926+
t.Errorf("legacy snapshot must yield empty parameters, got %#v", snap.Parameters)
927+
}
928+
// Flavors did not exist when this snapshot was written: every cluster
929+
// ran postgresql, and the restore flavor check must read it that way.
930+
if got := cnpgFlavor(snap.Flavor); got != "postgresql" {
931+
t.Errorf("legacy snapshot must resolve to the postgresql flavor, got %q", got)
925932
}
926933
}
927934

@@ -942,7 +949,7 @@ func TestUnmarshalCNPGBackupSnapshot_RejectsMissingOrWrongKind(t *testing.T) {
942949
for _, tc := range cases {
943950
t.Run(tc.name, func(t *testing.T) {
944951
bk := &backupsv1alpha1.Backup{Status: backupsv1alpha1.BackupStatus{UnderlyingResources: tc.ur}}
945-
_, _, _, err := unmarshalCNPGBackupSnapshot(bk)
952+
_, err := unmarshalCNPGBackupSnapshot(bk)
946953
if err == nil {
947954
t.Fatalf("expected error, got nil")
948955
}
@@ -2216,6 +2223,137 @@ func TestReconcileCNPGRestore_RepeatInPlacePurgesStaleRecoveryCluster(t *testing
22162223
})
22172224
}
22182225

2226+
// TestReconcileCNPGRestore_RefusesCrossFlavorRestore: a physical backup only
2227+
// restores onto the image family it was taken from. The postgresql and
2228+
// postgis images ship different glibc versions, and postgis objects need the
2229+
// postgis libraries, so a cross-flavor RestoreJob must fail before the purge
2230+
// deletes the target's Cluster and PVCs. A snapshot without a flavor predates
2231+
// flavors and therefore comes from a postgresql cluster.
2232+
func TestReconcileCNPGRestore_RefusesCrossFlavorRestore(t *testing.T) {
2233+
const (
2234+
ns = "tenant"
2235+
appName = "app"
2236+
clusterName = "postgres-app"
2237+
cnpgBkName = "cnpgbk"
2238+
)
2239+
apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup
2240+
strategyGroup := strategyv1alpha1.GroupVersion.Group
2241+
ctx := context.Background()
2242+
startedAt := metav1.NewTime(time.Now())
2243+
2244+
strategy := &strategyv1alpha1.CNPG{
2245+
ObjectMeta: metav1.ObjectMeta{Name: "cnpg-strategy"},
2246+
Spec: strategyv1alpha1.CNPGSpec{
2247+
Template: strategyv1alpha1.CNPGTemplate{
2248+
BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/"},
2249+
},
2250+
},
2251+
}
2252+
cnpgBackup := &cnpgtypes.Backup{
2253+
ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: cnpgBkName},
2254+
Spec: cnpgtypes.BackupSpec{Cluster: cnpgtypes.ClusterReference{Name: clusterName}},
2255+
Status: cnpgtypes.BackupStatus{Phase: cnpgBackupPhaseComplete, EndWal: "000000010000000000000003"},
2256+
}
2257+
2258+
cases := []struct {
2259+
name string
2260+
sourceFlavor string
2261+
targetFlavor string
2262+
wantFailed bool
2263+
wantInMessage string
2264+
}{
2265+
{name: "postgis backup into a postgresql app", sourceFlavor: "postgis", targetFlavor: "", wantFailed: true, wantInMessage: "taken from a postgis cluster"},
2266+
{name: "pre-flavor backup into a postgis app", sourceFlavor: "", targetFlavor: "postgis", wantFailed: true, wantInMessage: "taken from a postgresql cluster"},
2267+
{name: "postgis backup into a postgis app", sourceFlavor: "postgis", targetFlavor: "postgis", wantFailed: false},
2268+
}
2269+
for _, tc := range cases {
2270+
t.Run(tc.name, func(t *testing.T) {
2271+
source := newPostgresApp(appName, ns)
2272+
source.Spec.Flavor = tc.sourceFlavor
2273+
snap, err := marshalCNPGBackupSnapshot(source, nil)
2274+
if err != nil {
2275+
t.Fatalf("marshal snapshot: %v", err)
2276+
}
2277+
backup := &backupsv1alpha1.Backup{
2278+
ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bk"},
2279+
Spec: backupsv1alpha1.BackupSpec{
2280+
ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: appName},
2281+
StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "cnpg-strategy"},
2282+
DriverMetadata: map[string]string{
2283+
cnpgServerNameKey: appName,
2284+
cnpgDestinationPathKey: "s3://bucket/" + appName + "/",
2285+
cnpgBackupNameKey: cnpgBkName,
2286+
},
2287+
},
2288+
Status: backupsv1alpha1.BackupStatus{UnderlyingResources: snap},
2289+
}
2290+
target := newPostgresApp(appName, ns)
2291+
target.Spec.Flavor = tc.targetFlavor
2292+
sa := startedAt
2293+
restoreJob := &backupsv1alpha1.RestoreJob{
2294+
ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "rj"},
2295+
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}},
2296+
Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &sa, Phase: backupsv1alpha1.RestoreJobPhaseRunning},
2297+
}
2298+
liveCluster := &cnpgtypes.Cluster{
2299+
ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: clusterName, CreationTimestamp: metav1.NewTime(startedAt.Add(-time.Hour))},
2300+
}
2301+
pvc := &corev1.PersistentVolumeClaim{
2302+
ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: clusterName + "-1", Labels: map[string]string{cnpgClusterLabel: clusterName}},
2303+
}
2304+
c := newCNPGStrategyTestClient(t, backup, restoreJob, strategy, cnpgBackup, target, liveCluster, pvc)
2305+
r := &RestoreJobReconciler{Client: c, Interface: dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t)), Recorder: record.NewFakeRecorder(10)}
2306+
2307+
rj := &backupsv1alpha1.RestoreJob{}
2308+
if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, rj); err != nil {
2309+
t.Fatalf("get seeded RestoreJob: %v", err)
2310+
}
2311+
if _, err := r.reconcileCNPGRestore(ctx, rj, backup); err != nil {
2312+
t.Fatalf("reconcileCNPGRestore: %v", err)
2313+
}
2314+
2315+
got := &backupsv1alpha1.RestoreJob{}
2316+
if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, got); err != nil {
2317+
t.Fatalf("get RestoreJob: %v", err)
2318+
}
2319+
clusterErr := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: clusterName}, &cnpgtypes.Cluster{})
2320+
app, err := r.getPostgresApp(ctx, ns, appName)
2321+
if err != nil {
2322+
t.Fatalf("get target app: %v", err)
2323+
}
2324+
2325+
if !tc.wantFailed {
2326+
if got.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed {
2327+
t.Fatalf("same-flavor restore must proceed, got Failed: %s", got.Status.Message)
2328+
}
2329+
if !apierrors.IsNotFound(clusterErr) {
2330+
t.Fatalf("same-flavor restore must purge the target Cluster, got err=%v", clusterErr)
2331+
}
2332+
return
2333+
}
2334+
if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
2335+
t.Fatalf("cross-flavor restore must fail, got phase %q", got.Status.Phase)
2336+
}
2337+
if !strings.Contains(got.Status.Message, tc.wantInMessage) || !strings.Contains(got.Status.Message, "pg_dump") {
2338+
t.Errorf("failure message must name the source flavor and point to a logical dump, got %q", got.Status.Message)
2339+
}
2340+
if clusterErr != nil {
2341+
t.Fatalf("cross-flavor restore must leave the target Cluster in place, got err=%v", clusterErr)
2342+
}
2343+
pvcs := &corev1.PersistentVolumeClaimList{}
2344+
if err := c.List(ctx, pvcs, client.InNamespace(ns), client.MatchingLabels{cnpgClusterLabel: clusterName}); err != nil {
2345+
t.Fatalf("list PVCs: %v", err)
2346+
}
2347+
if len(pvcs.Items) != 1 {
2348+
t.Fatalf("cross-flavor restore must leave the target PVCs in place, have %d", len(pvcs.Items))
2349+
}
2350+
if app.Spec.Bootstrap.Enabled {
2351+
t.Fatalf("cross-flavor restore must not patch the target app for recovery")
2352+
}
2353+
})
2354+
}
2355+
}
2356+
22192357
// TestReconcileCNPGRestore_HealthyClusterSucceeds drives the reconcile to its
22202358
// success branch: with the target already purged (TargetPurged=True) and the
22212359
// recovery Cluster reporting healthy, the RestoreJob must reach phase

‎internal/backupcontroller/postgresapp/types.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
// Package postgresapp declares the typed shape of the apps.cozystack.io/v1alpha1
44
// Postgres CR that the CNPG backup driver reads and patches. It carries only
5-
// the fields the driver touches - bootstrap, backup, databases, users -
5+
// the fields the driver touches - bootstrap, backup, databases, users, flavor -
66
// because the live CR has many more fields (resources, quorum, postgresql
77
// parameters, ...) that we never look at and that round-tripping through
88
// this partial schema would otherwise silently drop.
@@ -68,6 +68,9 @@ type PostgresSpec struct {
6868
Backup Backup `json:"backup,omitempty"`
6969
Databases map[string]Database `json:"databases,omitempty"`
7070
Users map[string]User `json:"users,omitempty"`
71+
// Flavor is read, never patched: a restore must land on the image family
72+
// the backup was taken from. Empty means the chart default, postgresql.
73+
Flavor string `json:"flavor,omitempty"`
7174
}
7275

7376
type Bootstrap struct {

0 commit comments

Comments
 (0)