Skip to content

Commit ad105a0

Browse files
committed
fix(backups): trust only the endpoint CA where the backup writer does
With an endpoint CA the probe trusted the system roots plus that CA. barman-cloud gets endpointCA as AWS_CA_BUNDLE, which replaces the default trust store, so a CNPG probe could accept an endpoint the backup itself would have refused. Let the connection say whether the CA adds to the system roots or replaces them, and replace them for CNPG. mariadb-operator appends its CA to the system pool, so MariaDB keeps that. Pin in reconcile tests that a Create the apiserver rejects still fails the BackupJob, so only the read-back of an existing Backup is retried. Assisted-by: LLM Signed-off-by: Andrey Kolkov <[email protected]>
1 parent 73b5cf4 commit ad105a0

5 files changed

Lines changed: 175 additions & 13 deletions

File tree

‎internal/backupcontroller/artifact_size.go‎

Lines changed: 27 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,11 @@ type s3Connection struct {
4747
AccessKey string
4848
SecretKey string
4949
SessionToken string
50-
// CAPEM is trusted on top of the system pool when set.
51-
CAPEM []byte
50+
// CAPEM is an endpoint CA bundle. It is trusted on top of the system
51+
// roots, or instead of them when CAOnly is set: the probe must trust
52+
// what the writer of the backup trusted, no more.
53+
CAPEM []byte
54+
CAOnly bool
5255
}
5356

5457
func (r *BackupJobReconciler) s3Lister(conn s3Connection) (s3ObjectLister, error) {
@@ -74,16 +77,9 @@ func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) {
7477
if err != nil {
7578
return nil, err
7679
}
77-
pool := tr.TLSClientConfig.RootCAs
78-
if pool == nil {
79-
if pool, err = x509.SystemCertPool(); err != nil {
80-
pool = x509.NewCertPool()
81-
}
82-
}
83-
if !pool.AppendCertsFromPEM(conn.CAPEM) {
84-
return nil, errors.New("no PEM certificates in the endpoint CA bundle")
80+
if tr.TLSClientConfig.RootCAs, err = endpointTrustPool(conn, tr.TLSClientConfig.RootCAs); err != nil {
81+
return nil, err
8582
}
86-
tr.TLSClientConfig.RootCAs = pool
8783
opts.Transport = tr
8884
}
8985
// minio.New refuses an endpoint with a path prefix, which it cannot
@@ -95,6 +91,26 @@ func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) {
9591
return &minioObjectLister{client: client}, nil
9692
}
9793

94+
// endpointTrustPool returns the roots to verify the endpoint against: the
95+
// CA bundle on top of system, or the CA bundle alone with CAOnly. A nil
96+
// system means the platform's roots.
97+
func endpointTrustPool(conn s3Connection, system *x509.CertPool) (*x509.CertPool, error) {
98+
pool := system
99+
switch {
100+
case conn.CAOnly:
101+
pool = x509.NewCertPool()
102+
case pool == nil:
103+
var err error
104+
if pool, err = x509.SystemCertPool(); err != nil {
105+
pool = x509.NewCertPool()
106+
}
107+
}
108+
if !pool.AppendCertsFromPEM(conn.CAPEM) {
109+
return nil, errors.New("no PEM certificates in the endpoint CA bundle")
110+
}
111+
return pool, nil
112+
}
113+
98114
func (l *minioObjectLister) ListObjects(ctx context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) {
99115
return collectObjects(ctx, l.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: recursive}))
100116
}

‎internal/backupcontroller/artifact_size_test.go‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,17 @@ package backupcontroller
33

44
import (
55
"context"
6+
"crypto/ecdsa"
7+
"crypto/elliptic"
8+
"crypto/rand"
9+
"crypto/x509"
10+
"crypto/x509/pkix"
11+
"encoding/pem"
612
"errors"
713
"fmt"
14+
"math/big"
15+
"net/http"
16+
"net/http/httptest"
817
"testing"
918
"time"
1019

@@ -223,3 +232,56 @@ func TestFailBackupArtifact(t *testing.T) {
223232
}
224233
})
225234
}
235+
236+
// TestEndpointTrustPool_CAOnly tells apart "the CA on top of the system
237+
// roots" and "the CA alone". The system pool holds the endpoint's
238+
// certificate, and the configured CA is an unrelated one, so the endpoint
239+
// must stop being trusted only when CAOnly is set.
240+
func TestEndpointTrustPool_CAOnly(t *testing.T) {
241+
endpoint := httptest.NewTLSServer(http.NotFoundHandler())
242+
defer endpoint.Close()
243+
system := x509.NewCertPool()
244+
system.AddCert(endpoint.Certificate())
245+
246+
// Every httptest server shares one certificate, so the unrelated CA is
247+
// minted here.
248+
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
249+
if err != nil {
250+
t.Fatal(err)
251+
}
252+
tmpl := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "unrelated CA"}, NotBefore: time.Now(), NotAfter: time.Now().Add(time.Hour), IsCA: true, BasicConstraintsValid: true}
253+
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
254+
if err != nil {
255+
t.Fatal(err)
256+
}
257+
unrelatedCA := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
258+
259+
verifies := func(caOnly bool) bool {
260+
pool, err := endpointTrustPool(s3Connection{CAPEM: unrelatedCA, CAOnly: caOnly}, system.Clone())
261+
if err != nil {
262+
t.Fatalf("endpointTrustPool: %v", err)
263+
}
264+
_, err = endpoint.Certificate().Verify(x509.VerifyOptions{Roots: pool})
265+
return err == nil
266+
}
267+
if !verifies(false) {
268+
t.Error("CA on top of the system roots: the endpoint was not trusted")
269+
}
270+
if verifies(true) {
271+
t.Error("CA alone: the endpoint was trusted through the system roots")
272+
}
273+
}
274+
275+
// forbidBackupCreate wraps a fake client whose Create of a Backup is
276+
// rejected, the way an admission webhook rejects one.
277+
func forbidBackupCreate(t *testing.T, c client.Client) client.Client {
278+
t.Helper()
279+
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
280+
Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error {
281+
if _, ok := obj.(*backupsv1alpha1.Backup); ok {
282+
return apierrors.NewForbidden(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), obj.GetName(), errors.New("denied by policy"))
283+
}
284+
return cl.Create(ctx, obj, opts...)
285+
},
286+
})
287+
}

‎internal/backupcontroller/cnpg_artifact_size.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,9 @@ func (r *BackupJobReconciler) cnpgS3Connection(ctx context.Context, namespace st
105105
if err != nil {
106106
return conn, err
107107
}
108-
conn.CAPEM = []byte(pem)
108+
// barman-cloud gets endpointCA as AWS_CA_BUNDLE, which replaces the
109+
// default trust store rather than adding to it.
110+
conn.CAPEM, conn.CAOnly = []byte(pem), true
109111
}
110112
return conn, nil
111113
}

‎internal/backupcontroller/cnpg_artifact_size_test.go‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ func TestCNPGArtifactSizeBytes(t *testing.T) {
126126
if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", store, "srv", "20261021T100000"); got != 5 {
127127
t.Errorf("size = %d, want 5", got)
128128
}
129-
if l.conn.AccessKey != "ID2" || l.conn.SecretKey != "KEY2" || string(l.conn.CAPEM) != "PEM" {
129+
if l.conn.AccessKey != "ID2" || l.conn.SecretKey != "KEY2" || string(l.conn.CAPEM) != "PEM" || !l.conn.CAOnly {
130130
t.Errorf("connection = %+v", l.conn)
131131
}
132132
})
@@ -417,3 +417,37 @@ func TestCreateCNPGBackupArtifact_FailedExistenceCheckSkipsProbe(t *testing.T) {
417417
t.Errorf("S3 probed %d times after the existence check failed", l.calls)
418418
}
419419
}
420+
421+
// TestReconcileCNPG_RejectedCreateFailsTheJob pins that only the read-back
422+
// is retried: a Create the apiserver rejects still fails the BackupJob.
423+
func TestReconcileCNPG_RejectedCreateFailsTheJob(t *testing.T) {
424+
f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Minute)),
425+
newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)),
426+
newRolloutInstancePod(time.Now().Add(-time.Hour)))
427+
_, backups := f.pass()
428+
if len(backups) != 1 {
429+
t.Fatalf("got %d cnpg.io/Backups after the first pass, want 1", len(backups))
430+
}
431+
ctx := context.Background()
432+
completed := backups[0].DeepCopy()
433+
completed.Status.Phase = cnpgBackupPhaseComplete
434+
if err := f.c.Update(ctx, completed); err != nil {
435+
t.Fatalf("complete cnpg.io/Backup: %v", err)
436+
}
437+
f.r.Client = forbidBackupCreate(t, f.c)
438+
439+
j := &backupsv1alpha1.BackupJob{}
440+
if err := f.c.Get(ctx, client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil {
441+
t.Fatalf("get BackupJob: %v", err)
442+
}
443+
if _, err := f.r.reconcileCNPG(ctx, j, f.resolved); err != nil {
444+
t.Fatalf("reconcileCNPG err = %v, want the BackupJob failed rather than retried", err)
445+
}
446+
persisted := &backupsv1alpha1.BackupJob{}
447+
if err := f.c.Get(ctx, client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, persisted); err != nil {
448+
t.Fatalf("get BackupJob: %v", err)
449+
}
450+
if persisted.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
451+
t.Errorf("phase = %q, want Failed for a rejected Create", persisted.Status.Phase)
452+
}
453+
}

‎internal/backupcontroller/mariadb_artifact_size_test.go‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -427,3 +427,51 @@ func TestReconcileMariaDB_UnreadableBackupIsRetried(t *testing.T) {
427427
t.Error("the BackupJob was failed although its Backup exists")
428428
}
429429
}
430+
431+
// TestReconcileMariaDB_RejectedCreateFailsTheJob pins that only the
432+
// read-back is retried: a Create the apiserver rejects still fails the
433+
// BackupJob.
434+
func TestReconcileMariaDB_RejectedCreateFailsTheJob(t *testing.T) {
435+
strategyGroup := strategyv1alpha1.GroupVersion.Group
436+
now := metav1.Now()
437+
bj := newMariaDBBackupJob("bj-rejected", "tenant")
438+
bj.Status.StartedAt = &now
439+
bj.Status.Phase = backupsv1alpha1.BackupJobPhaseRunning
440+
strategy := &strategyv1alpha1.MariaDB{
441+
ObjectMeta: metav1.ObjectMeta{Name: "mdb-strategy"},
442+
Spec: strategyv1alpha1.MariaDBSpec{Template: *newRenderedMariaDBTemplate()},
443+
}
444+
operatorMariaDB := &mariadbtypes.MariaDB{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: mariadbNameForApp("mariadb-src")}}
445+
operatorBackup := &mariadbtypes.Backup{
446+
ObjectMeta: metav1.ObjectMeta{
447+
Namespace: "tenant", Name: "op-bk",
448+
Labels: map[string]string{
449+
backupsv1alpha1.OwningJobNameLabel: bj.Name,
450+
backupsv1alpha1.OwningJobNamespaceLabel: bj.Namespace,
451+
},
452+
},
453+
Status: mariadbtypes.BackupStatus{Conditions: []metav1.Condition{{
454+
Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionTrue, LastTransitionTime: now,
455+
}}},
456+
}
457+
store := newMariaDBStrategyTestClient(t, bj, strategy, newMariaDBApp("mariadb-src", "tenant"), operatorMariaDB, operatorBackup)
458+
r := &BackupJobReconciler{
459+
Client: forbidBackupCreate(t, store),
460+
Scheme: store.Scheme(),
461+
Recorder: record.NewFakeRecorder(10),
462+
}
463+
resolved := &ResolvedBackupConfig{StrategyRef: corev1.TypedLocalObjectReference{
464+
APIGroup: &strategyGroup, Kind: strategyv1alpha1.MariaDBStrategyKind, Name: strategy.Name,
465+
}}
466+
467+
if _, err := r.reconcileMariaDB(context.Background(), bj, resolved); err != nil {
468+
t.Fatalf("reconcileMariaDB err = %v, want the BackupJob failed rather than retried", err)
469+
}
470+
persisted := &backupsv1alpha1.BackupJob{}
471+
if err := store.Get(context.Background(), client.ObjectKeyFromObject(bj), persisted); err != nil {
472+
t.Fatalf("get BackupJob: %v", err)
473+
}
474+
if persisted.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
475+
t.Errorf("phase = %q, want Failed for a rejected Create", persisted.Status.Phase)
476+
}
477+
}

0 commit comments

Comments
 (0)