Skip to content

Commit 43bed8f

Browse files
committed
fix(backups): keep the Redis probe verifying TLS unless every container skips
The probe read S3_INSECURE_SKIP_VERIFY from any container of the backup Job, so a custom strategy whose sidecar sets it while the uploading container verifies made the probe skip verification the upload did not. The probe cannot tell which container uploaded, so it now skips only when every container sets the variable to true, the last entry winning as in the kubelet. An s3-ca volume of a source other than a Secret yields no CA the probe can read; it is now treated like a CA Secret that is gone, so it never turns into skip-verify. Tests pin the guards that decide the probe's trust: the Redis CA volume mounted next to the S3 one, a decoy variable set to true, a CA that did not sign the endpoint, and the AlreadyExists read-back past the cache. Assisted-by: LLM Signed-off-by: Andrey Kolkov <[email protected]>
1 parent e03986e commit 43bed8f

2 files changed

Lines changed: 117 additions & 18 deletions

File tree

‎internal/backupcontroller/redis_artifact_size.go‎

Lines changed: 39 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -86,26 +86,31 @@ const (
8686
)
8787

8888
// redisS3Trust mirrors how the default strategy script verifies the S3
89-
// endpoint: it trusts ca.crt, then tls.crt, from the Secret mounted as the
90-
// s3-ca volume, and only when neither exists does it honour
91-
// S3_INSECURE_SKIP_VERIFY=true. Both are read from the pod spec the backup
92-
// Job ran with, but the Secret is read now, so a CA Secret the pod mounts
93-
// and that is gone by now never lets the probe skip verification: the Job
94-
// may have verified against it. Without skip-verify a missing CA Secret
95-
// leaves the system roots, as in the script, which mounts it optionally.
89+
// endpoint: it trusts ca.crt, then tls.crt, from the s3-ca volume, and only
90+
// when neither exists does it honour S3_INSECURE_SKIP_VERIFY=true. Both are
91+
// read from the pod spec the backup Job ran with, but the Secret is read
92+
// now, so an s3-ca volume that yields no CA now - a Secret that is gone or
93+
// lacks the keys, or a source other than a Secret - never lets the probe
94+
// skip verification: the Job may have verified against it. Without
95+
// skip-verify such a volume leaves the system roots, as in the script,
96+
// which mounts it optionally.
9697
func (r *BackupJobReconciler) redisS3Trust(ctx context.Context, namespace string, podSpec *corev1.PodSpec) ([]byte, bool, error) {
9798
if podSpec == nil {
9899
return nil, false, nil
99100
}
100-
caMissing := false
101+
caUnknown := false
101102
for _, v := range podSpec.Volumes {
102-
if v.Name != redisS3CAVolume || v.Secret == nil {
103+
if v.Name != redisS3CAVolume {
103104
continue
104105
}
106+
if v.Secret == nil {
107+
caUnknown = true
108+
break
109+
}
105110
secret := &corev1.Secret{}
106111
err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: v.Secret.SecretName}, secret)
107112
if apierrors.IsNotFound(err) {
108-
caMissing = true
113+
caUnknown = true
109114
break
110115
}
111116
if err != nil {
@@ -116,19 +121,35 @@ func (r *BackupJobReconciler) redisS3Trust(ctx context.Context, namespace string
116121
return pem, false, nil
117122
}
118123
}
119-
caMissing = true
124+
caUnknown = true
125+
}
126+
if !redisSkipsVerify(podSpec.Containers) {
127+
return nil, false, nil
120128
}
121-
for _, c := range podSpec.Containers {
129+
if caUnknown {
130+
return nil, false, fmt.Errorf("the %s volume yields no CA now; not skipping TLS verification in its place", redisS3CAVolume)
131+
}
132+
return nil, true, nil
133+
}
134+
135+
// redisSkipsVerify reports whether every container sets
136+
// S3_INSECURE_SKIP_VERIFY=true: the probe cannot tell which container
137+
// uploaded, so one that verified is enough to keep verification on. When a
138+
// container lists the variable twice, the last entry wins, as in the
139+
// kubelet.
140+
func redisSkipsVerify(containers []corev1.Container) bool {
141+
for _, c := range containers {
142+
value := ""
122143
for _, e := range c.Env {
123-
if e.Name == redisS3InsecureSkipEnv && e.Value == "true" {
124-
if caMissing {
125-
return nil, false, fmt.Errorf("the %s volume yields no CA now; not skipping TLS verification in its place", redisS3CAVolume)
126-
}
127-
return nil, true, nil
144+
if e.Name == redisS3InsecureSkipEnv {
145+
value = e.Value
128146
}
129147
}
148+
if value != "true" {
149+
return false
150+
}
130151
}
131-
return nil, false, nil
152+
return len(containers) > 0
132153
}
133154

134155
// secretKeyForFile returns the Secret key a secret volume projects to file:

‎internal/backupcontroller/redis_artifact_size_test.go‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"net/http/httptest"
1010
"strings"
1111
"testing"
12+
"time"
1213

1314
batchv1 "k8s.io/api/batch/v1"
1415
corev1 "k8s.io/api/core/v1"
@@ -98,6 +99,11 @@ func TestRedisArtifact(t *testing.T) {
9899
// covers, in one pass, the trust settings the backup Job used, request
99100
// signing and Content-Length parsing.
100101
func TestRedisArtifact_MinioClient(t *testing.T) {
102+
// minio-go retries a TLS failure with backoff; a refusal needs no more
103+
// than the first attempt.
104+
saved := artifactSizeProbeTimeout
105+
artifactSizeProbeTimeout = time.Second
106+
defer func() { artifactSizeProbeTimeout = saved }()
101107
var gotMethod, gotPath, gotAuth string
102108
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
103109
gotMethod, gotPath, gotAuth = req.Method, req.URL.Path, req.Header.Get("Authorization")
@@ -136,6 +142,14 @@ func TestRedisArtifact_MinioClient(t *testing.T) {
136142
if got := r.redisArtifact(context.Background(), "tenant", redisTestKey, redisPodSpec("", "true")); got == nil || got.SizeBytes != 4096 {
137143
t.Errorf("with S3_INSECURE_SKIP_VERIFY=true: artifact = %#v, want SizeBytes 4096", got)
138144
}
145+
146+
ca.Data["ca.crt"] = unrelatedCAPEM(t)
147+
if err := c.Update(context.Background(), ca); err != nil {
148+
t.Fatalf("update CA Secret: %v", err)
149+
}
150+
if got := r.redisArtifact(context.Background(), "tenant", redisTestKey, redisPodSpec("s3-endpoint-ca", "true")); got != nil {
151+
t.Errorf("with a CA that did not sign the endpoint and S3_INSECURE_SKIP_VERIFY=true: artifact = %#v, want nil", got)
152+
}
139153
}
140154

141155
// redisPodSpec builds the parts of the backup Job's pod spec that
@@ -155,6 +169,22 @@ func redisPodSpec(caSecret, skipVerify string) *corev1.PodSpec {
155169
return spec
156170
}
157171

172+
func withVolume(spec *corev1.PodSpec, name, secret string) *corev1.PodSpec {
173+
spec.Volumes = append(spec.Volumes, corev1.Volume{
174+
Name: name,
175+
VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: secret}},
176+
})
177+
return spec
178+
}
179+
180+
func configMapS3CA(spec *corev1.PodSpec) *corev1.PodSpec {
181+
spec.Volumes = append(spec.Volumes, corev1.Volume{
182+
Name: "s3-ca",
183+
VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: "s3-endpoint-ca"}}},
184+
})
185+
return spec
186+
}
187+
158188
func TestRedisS3Trust(t *testing.T) {
159189
caSecret := func(data map[string]string) *corev1.Secret {
160190
s := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "s3-endpoint-ca"}, Data: map[string][]byte{}}
@@ -199,7 +229,44 @@ func TestRedisS3Trust(t *testing.T) {
199229
spec: redisPodSpec("", "false"),
200230
},
201231
"no pod spec": {},
232+
"the Redis CA volume is not the S3 CA": {
233+
spec: withVolume(redisPodSpec("", "true"), "redis-ca", "redis-tls"),
234+
objs: []client.Object{caSecret(map[string]string{"ca.crt": "CA"}),
235+
&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "redis-tls"}, Data: map[string][]byte{"ca.crt": []byte("REDIS")}}},
236+
wantSkip: true,
237+
},
238+
"the S3 CA next to the Redis CA": {
239+
spec: withVolume(withVolume(&corev1.PodSpec{Containers: redisPodSpec("", "").Containers}, "redis-ca", "redis-tls"), "s3-ca", "s3-endpoint-ca"),
240+
objs: []client.Object{caSecret(map[string]string{"ca.crt": "CA"}),
241+
&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "redis-tls"}, Data: map[string][]byte{"ca.crt": []byte("REDIS")}}},
242+
wantCA: "CA",
243+
},
244+
"another variable set to true is not skip-verify": {
245+
spec: &corev1.PodSpec{Containers: []corev1.Container{{Name: "backup", Env: []corev1.EnvVar{{Name: "REDIS_TLS", Value: "true"}}}}},
246+
},
247+
"skip-verify only when every container skips": {
248+
spec: &corev1.PodSpec{Containers: []corev1.Container{
249+
{Name: "upload", Env: []corev1.EnvVar{{Name: "S3_INSECURE_SKIP_VERIFY", Value: "false"}}},
250+
{Name: "sidecar", Env: []corev1.EnvVar{{Name: "S3_INSECURE_SKIP_VERIFY", Value: "true"}}},
251+
}},
252+
},
253+
"the last entry of a repeated variable wins": {
254+
spec: &corev1.PodSpec{Containers: []corev1.Container{{Name: "backup", Env: []corev1.EnvVar{
255+
{Name: "S3_INSECURE_SKIP_VERIFY", Value: "true"},
256+
{Name: "S3_INSECURE_SKIP_VERIFY", Value: "false"},
257+
}}}},
258+
},
259+
"an s3-ca volume of another source leaves the system roots": {
260+
spec: configMapS3CA(redisPodSpec("", "")),
261+
},
202262
}
263+
t.Run("an s3-ca volume of another source never turns into skip-verify", func(t *testing.T) {
264+
r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t)}
265+
ca, skip, err := r.redisS3Trust(context.Background(), "tenant", configMapS3CA(redisPodSpec("", "true")))
266+
if err == nil || skip || ca != nil {
267+
t.Errorf("trust = (%q, %v, %v), want an error and no skip-verify", ca, skip, err)
268+
}
269+
})
203270
t.Run("a mounted CA Secret without CA keys never turns into skip-verify", func(t *testing.T) {
204271
r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t, caSecret(map[string]string{"other": "x"}))}
205272
ca, skip, err := r.redisS3Trust(context.Background(), "tenant", redisPodSpec("s3-endpoint-ca", "true"))
@@ -355,6 +422,17 @@ func TestCreateRedisBackupArtifact_RetryCheck(t *testing.T) {
355422
}
356423
})
357424

425+
t.Run("a Create collision is resolved from the API server past the cache", func(t *testing.T) {
426+
backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-1"}}
427+
store := newMariaDBStrategyTestClient(t, j, backup, projectedCredsSecret(nil))
428+
r := redisReconciler(t, &fakeS3Lister{statSize: 777})
429+
r.Client, r.APIReader = staleCache(t, store), notFoundOnce(t, store)
430+
got, err := r.createRedisBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, redisTestKey, nil)
431+
if err != nil || got == nil || got.Name != "bj-1" {
432+
t.Fatalf("createRedisBackupArtifact = %v, %v; want the Backup that already exists", got, err)
433+
}
434+
})
435+
358436
t.Run("an existing Backup that cannot be read back is retried", func(t *testing.T) {
359437
backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-1"}}
360438
store := newMariaDBStrategyTestClient(t, j, backup, projectedCredsSecret(nil))

0 commit comments

Comments
 (0)