Skip to content

Commit c2f1d03

Browse files
committed
fix(apps): preserve shared wildcard secrets during cleanup
A shared wildcard Secret can use the same name as an old per-app key. Uninstall must preserve that Secret while still reclaiming keys from earlier TLS-enabled revisions. Signed-off-by: Yan Bondarenko <[email protected]> Assisted-by: LLM
1 parent cb1891c commit c2f1d03

8 files changed

Lines changed: 186 additions & 1 deletion

File tree

‎hack/app-cleanup-hooks_test.bats‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -867,3 +867,75 @@ line_of() {
867867
fi
868868
rm -rf "$tmp"
869869
}
870+
871+
@test "harbor and bucket preserve a colliding wildcard key and still clean their solvers" {
872+
for app in harbor bucket; do
873+
tmp=$(mktemp -d)
874+
write_fake_kubectl "$tmp"
875+
if [ "$app" = harbor ]; then
876+
suffix=ingress-tls job=harbor-test-cleanup
877+
else
878+
suffix=ui-tls job=bucket-test-cleanup-acme
879+
fi
880+
key="$app-test-$suffix"
881+
render_hook "packages/apps/$app" "$app-test" tenant-root "$job" "$tmp/hook.sh" \
882+
--set '_namespace.host=example.org' --set "_cluster.wildcard-secret-name=$key"
883+
884+
export KLOG="$tmp/calls" CHALLENGE_LIST='old-challenge' SOLVER_LIST='old-solver'
885+
export EXISTING="$key old-challenge old-solver"
886+
PATH="$tmp:$PATH" sh "$tmp/hook.sh"
887+
if grep -Eq 'delete (certificate|secret) ' "$tmp/calls"; then
888+
echo "FAIL: $app attempted to delete the shared wildcard key"
889+
cat "$tmp/calls"
890+
false
891+
fi
892+
grep -q 'delete challenge .*old-challenge' "$tmp/calls"
893+
grep -q 'delete ingress .*old-solver' "$tmp/calls"
894+
rm -rf "$tmp"
895+
done
896+
}
897+
898+
@test "harbor and bucket still remove old app keys with a different wildcard name" {
899+
for app in harbor bucket; do
900+
tmp=$(mktemp -d)
901+
write_fake_kubectl "$tmp"
902+
if [ "$app" = harbor ]; then
903+
suffix=ingress-tls job=harbor-test-cleanup
904+
else
905+
suffix=ui-tls job=bucket-test-cleanup-acme
906+
fi
907+
key="$app-test-$suffix"
908+
render_hook "packages/apps/$app" "$app-test" tenant-root "$job" "$tmp/hook.sh" \
909+
--set '_namespace.host=example.org' --set '_cluster.wildcard-secret-name=shared-tls'
910+
911+
export KLOG="$tmp/calls" CHALLENGE_LIST='' SOLVER_LIST=''
912+
export EXISTING="$key shared-tls"
913+
PATH="$tmp:$PATH" sh "$tmp/hook.sh"
914+
grep -q "delete certificate .*$key" "$tmp/calls"
915+
grep -q "delete secret .*$key" "$tmp/calls"
916+
if grep -q 'shared-tls' "$tmp/calls"; then
917+
echo "FAIL: $app touched the shared wildcard key"
918+
cat "$tmp/calls"
919+
false
920+
fi
921+
rm -rf "$tmp"
922+
done
923+
}
924+
925+
@test "the Secret helper removes app keys while preserving a colliding wildcard key" {
926+
tmp=$(mktemp -d)
927+
write_fake_kubectl "$tmp"
928+
render_hook "$QDRANT_CHART" myapp tenant-root myapp-secrets-cleanup "$tmp/hook.sh" \
929+
--set '_namespace.host=example.org' --set 'tls.enabled=false' \
930+
--set '_cluster.wildcard-secret-name=myapp-tls'
931+
932+
export KLOG="$tmp/calls" EXISTING='myapp-ca myapp-tls'
933+
PATH="$tmp:$PATH" sh "$tmp/hook.sh"
934+
grep -q 'delete secret .*myapp-ca' "$tmp/calls"
935+
if grep -q 'myapp-tls' "$tmp/calls"; then
936+
echo 'FAIL: the helper touched the shared wildcard key'
937+
cat "$tmp/calls"
938+
false
939+
fi
940+
rm -rf "$tmp"
941+
}

‎packages/apps/bucket/templates/hooks/cleanup-acme.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
{{- $sharedTLS := eq (printf "%s-ui-tls" .Release.Name) (index (.Values._cluster | default dict) "wildcard-secret-name" | default "") -}}
12
---
23
apiVersion: batch/v1
34
kind: Job
@@ -76,6 +77,7 @@ spec:
7677
# left behind keeps the private key of the bucket host, so that
7778
# one fails the hook at the end and the Job keeps its log.
7879
key_left=0
80+
{{- if not $sharedTLS }}
7981
if kubectl delete certificate {{ .Release.Name }}-ui-tls -n {{ .Release.Namespace }} --ignore-not-found --wait=false; then
8082
# cert-manager writes the Secret without an owner reference, so
8183
# it outlives the Certificate together with the private key of
@@ -92,6 +94,7 @@ spec:
9294
echo "ERROR: cert-manager cleanup failed for {{ .Release.Name }}, so Secret {{ .Release.Name }}-ui-tls is left as well and orphaned ACME resources may remain" >&2
9395
key_left=1
9496
fi
97+
{{- end }}
9598
# Orders are owned by the Certificate and are removed by the cascade
9699
# above; Challenges carry no labels and are reaped by the
97100
# host-scoped .spec.dnsName sweep below. There is therefore no
@@ -159,6 +162,7 @@ metadata:
159162
"helm.sh/hook-weight": "5"
160163
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
161164
rules:
165+
{{- if not $sharedTLS }}
162166
- apiGroups: ["cert-manager.io"]
163167
resources: ["certificates"]
164168
verbs: ["get", "list", "delete"]
@@ -168,6 +172,7 @@ rules:
168172
resources: ["secrets"]
169173
resourceNames: ["{{ .Release.Name }}-ui-tls"]
170174
verbs: ["get", "delete"]
175+
{{- end }}
171176
- apiGroups: ["acme.cert-manager.io"]
172177
resources: ["orders", "challenges"]
173178
verbs: ["get", "list", "delete"]

‎packages/apps/bucket/tests/cleanup_acme_test.yaml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,3 +202,33 @@ tests:
202202
- equal:
203203
path: subjects[0].name
204204
value: my-bucket-cleanup-acme
205+
206+
- it: preserves the shared wildcard Certificate and Secret
207+
release:
208+
name: bucket-test
209+
set:
210+
_cluster.wildcard-secret-name: bucket-test-ui-tls
211+
documentSelector:
212+
path: kind
213+
value: Job
214+
asserts:
215+
- notMatchRegex:
216+
path: spec.template.spec.containers[0].command[2]
217+
pattern: 'kubectl delete (certificate|secret) '
218+
219+
- it: grants no Secret deletion when the TLS name is shared
220+
release:
221+
name: bucket-test
222+
set:
223+
_cluster.wildcard-secret-name: bucket-test-ui-tls
224+
documentSelector:
225+
path: kind
226+
value: Role
227+
asserts:
228+
- notContains:
229+
path: rules
230+
content:
231+
apiGroups: [""]
232+
resources: ["secrets"]
233+
resourceNames: ["bucket-test-ui-tls"]
234+
verbs: ["get", "delete"]

‎packages/apps/harbor/templates/hooks/cleanup.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
{{- $sharedTLS := eq (printf "%s-ingress-tls" .Release.Name) (index (.Values._cluster | default dict) "wildcard-secret-name" | default "") -}}
12
{{- $harborHost := .Values.host | default (printf "%s.%s" .Release.Name .Values._namespace.host) }}
23
---
34
apiVersion: batch/v1
@@ -109,6 +110,7 @@ spec:
109110
-l release={{ .Release.Name }}-system --ignore-not-found --wait=true --timeout=120s \
110111
|| fail "failed to delete PVCs matching release={{ .Release.Name }}-system in {{ .Release.Namespace }}; orphaned PVCs may remain"
111112
113+
{{- if not $sharedTLS }}
112114
# Remove the cert-manager Certificate owned by this release's
113115
# Ingress so an in-flight http01 challenge does not leave a
114116
# dangling cm-acme-http-solver-* Pod/Service/Ingress behind.
@@ -138,6 +140,7 @@ spec:
138140
fail "failed to delete Certificate {{ .Release.Namespace }}/{{ .Release.Name }}-ingress-tls, so its Secret is left as well and orphaned ACME resources may remain"
139141
fi
140142

143+
{{- end }}
141144
# Belt-and-braces sweep for ACME http01 solver objects that
142145
# cert-manager may have orphaned (e.g. the Certificate was
143146
# already gone, so deleting it above did not cascade). We scope
@@ -215,6 +218,7 @@ rules:
215218
- apiGroups: [""]
216219
resources: ["persistentvolumeclaims"]
217220
verbs: ["get", "list", "watch", "delete"]
221+
{{- if not $sharedTLS }}
218222
- apiGroups: ["cert-manager.io"]
219223
resources: ["certificates"]
220224
verbs: ["get", "list", "watch", "delete"]
@@ -224,6 +228,7 @@ rules:
224228
resources: ["secrets"]
225229
resourceNames: ["{{ .Release.Name }}-ingress-tls"]
226230
verbs: ["get", "delete"]
231+
{{- end }}
227232
# Only Challenges are touched directly (the host-scoped sweep lists and
228233
# deletes them); Orders are cascade-deleted by cert-manager itself when the
229234
# Certificate is removed, so the hook needs no RBAC for them.

‎packages/apps/harbor/tests/cleanup_hook_test.yaml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -322,3 +322,33 @@ tests:
322322
- equal:
323323
path: subjects[0].name
324324
value: harbor-test-cleanup
325+
326+
- it: preserves the shared wildcard Certificate and Secret
327+
release:
328+
name: harbor-test
329+
set:
330+
_cluster.wildcard-secret-name: harbor-test-ingress-tls
331+
documentSelector:
332+
path: kind
333+
value: Job
334+
asserts:
335+
- notMatchRegex:
336+
path: spec.template.spec.containers[0].command[2]
337+
pattern: 'kubectl delete (certificate|secret) '
338+
339+
- it: grants no Secret deletion when the TLS name is shared
340+
release:
341+
name: harbor-test
342+
set:
343+
_cluster.wildcard-secret-name: harbor-test-ingress-tls
344+
documentSelector:
345+
path: kind
346+
value: Role
347+
asserts:
348+
- notContains:
349+
path: rules
350+
content:
351+
apiGroups: [""]
352+
resources: ["secrets"]
353+
resourceNames: ["harbor-test-ingress-tls"]
354+
verbs: ["get", "delete"]

‎packages/extra/bootbox/tests/cleanup_secrets_test.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,3 +88,12 @@ tests:
8888
- hasDocuments:
8989
count: 1
9090
filterAware: true
91+
92+
- it: preserves the shared wildcard Secret when it uses the historical app name
93+
templates:
94+
- templates/hooks/cleanup-secrets.yaml
95+
set:
96+
_cluster.wildcard-secret-name: bootbox-tls
97+
asserts:
98+
- hasDocuments:
99+
count: 0

‎packages/library/cozy-lib/templates/_hooks.tpl‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ Usage from a template of the app chart, with certificates optional.
1818
"secrets" (list (printf "%s-ca" .Release.Name) (printf "%s-tls" .Release.Name))) }}
1919
2020
secrets lists every Secret name the chart can create, whatever its current
21-
values. The hook renders unconditionally and deletes with --ignore-not-found,
21+
values. The hook deletes with --ignore-not-found,
2222
so a release that had TLS on earlier and off at uninstall is cleaned as well.
2323
Delete is granted by explicit resourceNames rather than by a label selector.
2424
cert-manager stamps its Secrets only with the cluster-wide
@@ -65,6 +65,12 @@ a failure there leaves the Secrets in place.
6565
{{- fail (printf "cozy-lib.hooks.deleteSecrets: every entry of certificates must be a Kubernetes resource name, got %q" .) -}}
6666
{{- end -}}
6767
{{- end -}}
68+
{{- /* The configured wildcard Secret belongs to the cluster, even if its
69+
name matches a key this app would otherwise create. */ -}}
70+
{{- $wildcardSecret := index ($root.Values._cluster | default dict) "wildcard-secret-name" | default "" -}}
71+
{{- $secrets = without $secrets $wildcardSecret -}}
72+
{{- $certificates = without $certificates $wildcardSecret -}}
73+
{{- if $secrets -}}
6874
{{- /* A Job name goes into the job-name label of its pods, which takes at
6975
most 63 characters, while the apps API admits release names of up to
7076
53. Cutting the suffix keeps the whole release name, so the name stays
@@ -208,3 +214,4 @@ spec:
208214
- name: tmp
209215
emptyDir: {}
210216
{{- end -}}
217+
{{- end -}}

‎packages/tests/cozy-lib-tests/tests/hooks_deletesecrets_test.yaml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,3 +248,30 @@ tests:
248248
- equal:
249249
path: rules[0].resourceNames
250250
value: ["myapp.ca-tls", "myapp.tls"]
251+
252+
- it: excludes a shared wildcard Secret and Certificate while cleaning other keys
253+
set:
254+
_cluster.wildcard-secret-name: myapp-tls
255+
deleteSecrets:
256+
secrets: [myapp-ca, myapp-tls]
257+
certificates: [myapp-ca, myapp-tls]
258+
documentSelector:
259+
path: kind
260+
value: Role
261+
asserts:
262+
- equal:
263+
path: rules[0].resourceNames
264+
value: [myapp-ca]
265+
- equal:
266+
path: rules[1].resourceNames
267+
value: [myapp-ca]
268+
269+
- it: omits all hook objects when only the shared wildcard Secret remains
270+
set:
271+
_cluster.wildcard-secret-name: myapp-tls
272+
deleteSecrets:
273+
secrets: [myapp-tls]
274+
certificates: [myapp-tls]
275+
asserts:
276+
- hasDocuments:
277+
count: 0

0 commit comments

Comments
 (0)