Skip to content

Commit 4425d3a

Browse files
scooby87scooby87
committed
fix(bucket): clean up orphaned cert-manager ACME solver resources on delete (#3060)
cert-manager ACME http01-solver Ingress/Pod/Service objects were left orphaned after a Bucket app CR was deleted (a stuck challenge leaves them without a live owner to cascade-delete). Add a post-delete hook that deletes the release Certificate/Order/Challenge and sweeps leftover solver objects by the acme.cert-manager.io/http01-solver label. RBAC verified against the vendored cert-manager CRDs. The cleanup Job runs non-root with a read-only root fs and dropped capabilities; failures are logged but never block teardown. Signed-off-by: Алексей Артамонов <[email protected]>
1 parent 374cac1 commit 4425d3a

3 files changed

Lines changed: 299 additions & 0 deletions

File tree

‎packages/apps/bucket/Makefile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
include ../../../hack/package.mk
22

3+
.PHONY: test
4+
test:
5+
helm unittest .
6+
37
generate:
48
cozyvalues-gen -m 'bucket' -v values.yaml -s values.schema.json -r README.md -g ../../../api/apps/v1alpha1/bucket/types.go
59
../../../hack/update-crd.sh

‎packages/apps/bucket/templates/hooks/cleanup-acme.yaml‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
---
2+
apiVersion: batch/v1
3+
kind: Job
4+
metadata:
5+
name: {{ .Release.Name }}-cleanup-acme
6+
labels:
7+
app.kubernetes.io/instance: {{ .Release.Name }}
8+
annotations:
9+
"helm.sh/hook": post-delete
10+
"helm.sh/hook-weight": "10"
11+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
12+
spec:
13+
template:
14+
metadata:
15+
labels:
16+
app.kubernetes.io/instance: {{ .Release.Name }}
17+
policy.cozystack.io/allow-to-apiserver: "true"
18+
spec:
19+
serviceAccountName: {{ .Release.Name }}-cleanup-acme
20+
restartPolicy: Never
21+
securityContext:
22+
runAsNonRoot: true
23+
runAsUser: 65534
24+
runAsGroup: 65534
25+
seccompProfile:
26+
type: RuntimeDefault
27+
containers:
28+
- name: cleanup
29+
image: docker.io/clastix/kubectl:v1.32
30+
securityContext:
31+
allowPrivilegeEscalation: false
32+
readOnlyRootFilesystem: true
33+
capabilities:
34+
drop: ["ALL"]
35+
resources:
36+
requests:
37+
cpu: 10m
38+
memory: 64Mi
39+
limits:
40+
cpu: 200m
41+
memory: 128Mi
42+
env:
43+
- name: HOME
44+
value: /tmp
45+
command:
46+
- /bin/sh
47+
- -c
48+
- |
49+
echo "Cleaning up orphaned cert-manager ACME resources for {{ .Release.Name }}..."
50+
# This release's TLS host. The bucket-system chart renders its
51+
# Ingress / Certificate for "<bucketName>.<_namespace.host>", and
52+
# bucketName is set to .Release.Name in the HelmRelease. cert-manager
53+
# stamps this exact value onto its Challenge .spec.dnsName and onto
54+
# the orphaned solver Ingress .spec.rules[].host, so the host is the
55+
# one reliable per-release identifier for ACME solver objects (they
56+
# carry no app/release label of their own).
57+
HOST="{{ .Release.Name }}.{{ .Values._namespace.host }}"
58+
# Delete the per-bucket Certificate/Order/Challenge. cert-manager
59+
# then cascade-removes the cm-acme-http-solver-* solver objects it
60+
# owns. The release's TLS Certificate is named after the secret it
61+
# populates ({{ .Release.Name }}-ui-tls).
62+
kubectl delete certificate {{ .Release.Name }}-ui-tls -n {{ .Release.Namespace }} --ignore-not-found \
63+
|| echo "WARNING: cert-manager cleanup failed for {{ .Release.Name }}; orphaned ACME resources may remain" >&2
64+
kubectl delete certificate,order,challenge -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} --ignore-not-found \
65+
|| echo "WARNING: cert-manager cleanup failed for {{ .Release.Name }}; orphaned ACME resources may remain" >&2
66+
# Release-scoped solver sweep. We deliberately do NOT delete
67+
# ingress/service/pod objects namespace-wide by the
68+
# acme.cert-manager.io/http01-solver=true label: that label is set
69+
# on EVERY app's solver objects in this shared tenant namespace, so
70+
# a blanket delete would tear down sibling apps' in-flight ACME
71+
# challenges. Instead we scope by HOST, which is unique to this
72+
# release, so we only ever touch our own solver objects.
73+
#
74+
# Delete Challenges for our host (cascades the solver
75+
# Ingress/Service/Pod they own).
76+
for ch in $(kubectl get challenge -n {{ .Release.Namespace }} -o jsonpath="{range .items[?(@.spec.dnsName=='$HOST')]}{.metadata.name} {end}" 2>/dev/null); do
77+
kubectl delete challenge -n {{ .Release.Namespace }} "$ch" --ignore-not-found \
78+
|| echo "WARNING: failed to delete challenge $ch for {{ .Release.Name }}; orphaned ACME resources may remain" >&2
79+
done
80+
# Delete any orphaned solver Ingress for our host (no live owner
81+
# left to cascade it). Its solver Service/Pod are owned by a
82+
# Challenge and are cascade-deleted with it; truly owner-less
83+
# solver Service/Pod with no matching Challenge are not expected
84+
# and are not swept namespace-wide.
85+
for ing in $(kubectl get ingress -n {{ .Release.Namespace }} -l acme.cert-manager.io/http01-solver=true -o jsonpath="{range .items[?(@.spec.rules[0].host=='$HOST')]}{.metadata.name} {end}" 2>/dev/null); do
86+
kubectl delete ingress -n {{ .Release.Namespace }} "$ing" --ignore-not-found \
87+
|| echo "WARNING: failed to delete solver ingress $ing for {{ .Release.Name }}; orphaned ACME resources may remain" >&2
88+
done
89+
echo "ACME cleanup complete."
90+
volumeMounts:
91+
- name: tmp
92+
mountPath: /tmp
93+
volumes:
94+
- name: tmp
95+
emptyDir: {}
96+
---
97+
apiVersion: v1
98+
kind: ServiceAccount
99+
metadata:
100+
name: {{ .Release.Name }}-cleanup-acme
101+
labels:
102+
app.kubernetes.io/instance: {{ .Release.Name }}
103+
annotations:
104+
"helm.sh/hook": post-delete
105+
"helm.sh/hook-weight": "0"
106+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
107+
---
108+
apiVersion: rbac.authorization.k8s.io/v1
109+
kind: Role
110+
metadata:
111+
name: {{ .Release.Name }}-cleanup-acme
112+
labels:
113+
app.kubernetes.io/instance: {{ .Release.Name }}
114+
annotations:
115+
"helm.sh/hook": post-delete
116+
"helm.sh/hook-weight": "5"
117+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
118+
rules:
119+
- apiGroups: ["cert-manager.io"]
120+
resources: ["certificates"]
121+
verbs: ["get", "list", "delete"]
122+
- apiGroups: ["acme.cert-manager.io"]
123+
resources: ["orders", "challenges"]
124+
verbs: ["get", "list", "delete"]
125+
# Needed to list+delete this release's orphaned http01 solver Ingress objects
126+
# by host. Services/Pods are intentionally omitted: the solver Service/Pod are
127+
# owned by their Challenge and cascade-delete with it, so we never delete them
128+
# directly (and thus never sweep them namespace-wide).
129+
- apiGroups: ["networking.k8s.io"]
130+
resources: ["ingresses"]
131+
verbs: ["get", "list", "delete"]
132+
---
133+
apiVersion: rbac.authorization.k8s.io/v1
134+
kind: RoleBinding
135+
metadata:
136+
name: {{ .Release.Name }}-cleanup-acme
137+
labels:
138+
app.kubernetes.io/instance: {{ .Release.Name }}
139+
annotations:
140+
"helm.sh/hook": post-delete
141+
"helm.sh/hook-weight": "5"
142+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
143+
roleRef:
144+
apiGroup: rbac.authorization.k8s.io
145+
kind: Role
146+
name: {{ .Release.Name }}-cleanup-acme
147+
subjects:
148+
- kind: ServiceAccount
149+
name: {{ .Release.Name }}-cleanup-acme

‎packages/apps/bucket/tests/cleanup_acme_test.yaml‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
suite: bucket post-delete ACME cleanup hook
2+
templates:
3+
- templates/hooks/cleanup-acme.yaml
4+
5+
release:
6+
name: my-bucket
7+
namespace: tenant-root
8+
9+
set:
10+
_namespace:
11+
host: example.org
12+
13+
tests:
14+
- it: renders a post-delete cleanup Job with the expected name and SA
15+
documentIndex: 0
16+
asserts:
17+
- equal:
18+
path: kind
19+
value: Job
20+
- equal:
21+
path: metadata.name
22+
value: my-bucket-cleanup-acme
23+
- equal:
24+
path: metadata.annotations["helm.sh/hook"]
25+
value: post-delete
26+
- equal:
27+
path: spec.template.spec.serviceAccountName
28+
value: my-bucket-cleanup-acme
29+
- equal:
30+
path: spec.template.spec.restartPolicy
31+
value: Never
32+
- equal:
33+
path: spec.template.metadata.labels["policy.cozystack.io/allow-to-apiserver"]
34+
value: "true"
35+
36+
- it: deletes release-scoped cert-manager resources with a host-scoped solver sweep
37+
documentIndex: 0
38+
asserts:
39+
- matchRegex:
40+
path: spec.template.spec.containers[0].command[2]
41+
pattern: "kubectl delete certificate my-bucket-ui-tls -n tenant-root"
42+
- matchRegex:
43+
path: spec.template.spec.containers[0].command[2]
44+
pattern: "kubectl delete certificate,order,challenge -n tenant-root -l app.kubernetes.io/instance=my-bucket"
45+
# Host derived from <release>.<_namespace.host> == my-bucket.example.org
46+
- matchRegex:
47+
path: spec.template.spec.containers[0].command[2]
48+
pattern: 'HOST="my-bucket\.example\.org"'
49+
# Host-scoped challenge sweep filtering on .spec.dnsName == our host
50+
- matchRegex:
51+
path: spec.template.spec.containers[0].command[2]
52+
pattern: "kubectl get challenge -n tenant-root -o jsonpath=.*@.spec.dnsName=='\\$HOST'"
53+
- matchRegex:
54+
path: spec.template.spec.containers[0].command[2]
55+
pattern: 'kubectl delete challenge -n tenant-root "\$ch" --ignore-not-found'
56+
# Host-scoped orphaned solver Ingress sweep filtering on rules[0].host
57+
- matchRegex:
58+
path: spec.template.spec.containers[0].command[2]
59+
pattern: "kubectl get ingress -n tenant-root -l acme.cert-manager.io/http01-solver=true -o jsonpath=.*@.spec.rules\\[0\\].host=='\\$HOST'"
60+
- matchRegex:
61+
path: spec.template.spec.containers[0].command[2]
62+
pattern: 'kubectl delete ingress -n tenant-root "\$ing" --ignore-not-found'
63+
# NO namespace-wide blanket solver sweep
64+
- notMatchRegex:
65+
path: spec.template.spec.containers[0].command[2]
66+
pattern: "kubectl delete ingress,svc,pod"
67+
- notMatchRegex:
68+
path: spec.template.spec.containers[0].command[2]
69+
pattern: "delete .*-l acme.cert-manager.io/http01-solver=true"
70+
71+
- it: sets resource requests and limits on the cleanup container
72+
documentIndex: 0
73+
asserts:
74+
- equal:
75+
path: spec.template.spec.containers[0].resources.requests.cpu
76+
value: 10m
77+
- equal:
78+
path: spec.template.spec.containers[0].resources.requests.memory
79+
value: 64Mi
80+
- equal:
81+
path: spec.template.spec.containers[0].resources.limits.cpu
82+
value: 200m
83+
- equal:
84+
path: spec.template.spec.containers[0].resources.limits.memory
85+
value: 128Mi
86+
87+
- it: creates a ServiceAccount for the cleanup job
88+
documentIndex: 1
89+
asserts:
90+
- equal:
91+
path: kind
92+
value: ServiceAccount
93+
- equal:
94+
path: metadata.name
95+
value: my-bucket-cleanup-acme
96+
97+
- it: grants RBAC limited to cert-manager and acme resources
98+
documentIndex: 2
99+
asserts:
100+
- equal:
101+
path: kind
102+
value: Role
103+
- contains:
104+
path: rules
105+
content:
106+
apiGroups: ["cert-manager.io"]
107+
resources: ["certificates"]
108+
verbs: ["get", "list", "delete"]
109+
- contains:
110+
path: rules
111+
content:
112+
apiGroups: ["acme.cert-manager.io"]
113+
resources: ["orders", "challenges"]
114+
verbs: ["get", "list", "delete"]
115+
# Ingresses re-added: we list+delete this release's orphaned solver
116+
# Ingress objects by host.
117+
- contains:
118+
path: rules
119+
content:
120+
apiGroups: ["networking.k8s.io"]
121+
resources: ["ingresses"]
122+
verbs: ["get", "list", "delete"]
123+
# Services/Pods stay OUT: solver Service/Pod cascade-delete with their
124+
# owning Challenge, so the hook never deletes them directly.
125+
- notContains:
126+
path: rules
127+
content:
128+
apiGroups: [""]
129+
resources: ["services", "pods"]
130+
verbs: ["get", "list", "delete"]
131+
- lengthEqual:
132+
path: rules
133+
count: 3
134+
135+
- it: binds the Role to the cleanup ServiceAccount
136+
documentIndex: 3
137+
asserts:
138+
- equal:
139+
path: kind
140+
value: RoleBinding
141+
- equal:
142+
path: roleRef.name
143+
value: my-bucket-cleanup-acme
144+
- equal:
145+
path: subjects[0].name
146+
value: my-bucket-cleanup-acme

0 commit comments

Comments
 (0)