Skip to content

Commit d113273

Browse files
committed
fix(harbor): clean up orphaned jobservice PVC and ACME solver resources on delete (#3061)
The jobservice PVC (resourcePolicy: keep) and cert-manager ACME http01-solver objects were left orphaned after a Harbor app CR was deleted. Add a post-delete hook that deletes the release PVCs (selecting by app.kubernetes.io/instance=<release>-system, the label the upstream subchart stamps) and the release Certificate/Order/Challenge plus the http01-solver sweep. A hook is required because resourcePolicy: keep blocks normal reclaim. The cleanup Job runs non-root with a read-only root fs and dropped capabilities; failures are logged but never block teardown. Signed-off-by: Алексей Артамонов <[email protected]>
1 parent 374cac1 commit d113273

3 files changed

Lines changed: 318 additions & 0 deletions

File tree

‎packages/apps/harbor/Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ NAME=harbor
22

33
include ../../../hack/package.mk
44

5+
.PHONY: test
56
test:
67
helm unittest .
78

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
{{- $harborHost := .Values.host | default (printf "%s.%s" .Release.Name .Values._namespace.host) }}
2+
---
3+
apiVersion: batch/v1
4+
kind: Job
5+
metadata:
6+
name: {{ .Release.Name }}-cleanup
7+
labels:
8+
app.kubernetes.io/instance: {{ .Release.Name }}
9+
annotations:
10+
"helm.sh/hook": post-delete
11+
"helm.sh/hook-weight": "10"
12+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
13+
spec:
14+
template:
15+
metadata:
16+
labels:
17+
app.kubernetes.io/instance: {{ .Release.Name }}
18+
policy.cozystack.io/allow-to-apiserver: "true"
19+
spec:
20+
serviceAccountName: {{ .Release.Name }}-cleanup
21+
restartPolicy: Never
22+
securityContext:
23+
runAsNonRoot: true
24+
runAsUser: 65534
25+
runAsGroup: 65534
26+
seccompProfile:
27+
type: RuntimeDefault
28+
containers:
29+
- name: cleanup
30+
image: docker.io/clastix/kubectl:v1.32
31+
securityContext:
32+
allowPrivilegeEscalation: false
33+
readOnlyRootFilesystem: true
34+
capabilities:
35+
drop: ["ALL"]
36+
resources:
37+
requests:
38+
cpu: 10m
39+
memory: 64Mi
40+
limits:
41+
cpu: 200m
42+
memory: 128Mi
43+
env:
44+
- name: HOME
45+
value: /tmp
46+
command:
47+
- /bin/sh
48+
- -c
49+
- |
50+
set -u
51+
# Harbor's jobservice PVC is annotated with
52+
# helm.sh/resource-policy: keep by the upstream subchart
53+
# (persistence.resourcePolicy: keep), so Helm/Flux deliberately
54+
# leaves it behind on uninstall. The subchart runs as the
55+
# "{{ .Release.Name }}-system" HelmRelease, so its PVCs carry
56+
# app.kubernetes.io/instance={{ .Release.Name }}-system.
57+
echo "Deleting orphaned PVCs for {{ .Release.Name }}-system..."
58+
kubectl delete pvc -n {{ .Release.Namespace }} \
59+
-l app.kubernetes.io/instance={{ .Release.Name }}-system --ignore-not-found \
60+
|| echo "WARNING: PVC cleanup failed for {{ .Release.Name }}; orphaned PVCs may remain" >&2
61+
62+
# Remove cert-manager ACME objects owned by this release's
63+
# Ingress so an in-flight http01 challenge does not leave a
64+
# dangling cm-acme-http-solver-* Pod/Service/Ingress behind.
65+
# Deleting the release's Certificate (deterministic name
66+
# "{{ .Release.Name }}-ingress-tls") triggers cert-manager to
67+
# cascade-delete the Orders, Challenges and solver
68+
# Pod/Service/Ingress objects it owns for this release.
69+
echo "Deleting ACME objects for {{ .Release.Name }}-ingress..."
70+
kubectl delete certificate,order,challenge -n {{ .Release.Namespace }} \
71+
{{ .Release.Name }}-ingress-tls --ignore-not-found \
72+
|| echo "WARNING: cert-manager cleanup failed for {{ .Release.Name }}; orphaned ACME resources may remain" >&2
73+
74+
# Belt-and-braces sweep for ACME http01 solver objects that
75+
# cert-manager may have orphaned (e.g. the Certificate was
76+
# already gone, so deleting it above did not cascade). We scope
77+
# STRICTLY to THIS release's TLS host ("$HOST") instead of a
78+
# namespace-wide acme.cert-manager.io/http01-solver=true label
79+
# sweep: a Cozystack tenant namespace can host many apps and
80+
# cert-manager sets that label on every app's solver objects, so
81+
# a blanket sweep would delete sibling apps' in-flight ACME
82+
# challenge solvers. Filtering Challenges by .spec.dnsName and
83+
# solver Ingresses by .spec.rules[0].host keeps us to objects
84+
# issued for this release's single Harbor host only.
85+
HOST="{{ $harborHost }}"
86+
echo "Sweeping orphaned ACME http01 solvers for host $HOST..."
87+
for ch in $(kubectl get challenge -n {{ .Release.Namespace }} \
88+
-o jsonpath="{range .items[?(@.spec.dnsName=='$HOST')]}{.metadata.name} {end}" 2>/dev/null); do
89+
kubectl delete challenge -n {{ .Release.Namespace }} "$ch" --ignore-not-found \
90+
|| echo "WARNING: failed to delete challenge $ch for {{ .Release.Name }}; orphaned solver may remain" >&2
91+
done
92+
for ing in $(kubectl get ingress -n {{ .Release.Namespace }} \
93+
-l acme.cert-manager.io/http01-solver=true \
94+
-o jsonpath="{range .items[?(@.spec.rules[0].host=='$HOST')]}{.metadata.name} {end}" 2>/dev/null); do
95+
kubectl delete ingress -n {{ .Release.Namespace }} "$ing" --ignore-not-found \
96+
|| echo "WARNING: failed to delete solver ingress $ing for {{ .Release.Name }}; orphaned solver may remain" >&2
97+
done
98+
99+
echo "Cleanup complete."
100+
volumeMounts:
101+
- name: tmp
102+
mountPath: /tmp
103+
volumes:
104+
- name: tmp
105+
emptyDir: {}
106+
---
107+
apiVersion: v1
108+
kind: ServiceAccount
109+
metadata:
110+
name: {{ .Release.Name }}-cleanup
111+
labels:
112+
app.kubernetes.io/instance: {{ .Release.Name }}
113+
annotations:
114+
"helm.sh/hook": post-delete
115+
helm.sh/hook-weight: "0"
116+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
117+
---
118+
apiVersion: rbac.authorization.k8s.io/v1
119+
kind: Role
120+
metadata:
121+
name: {{ .Release.Name }}-cleanup
122+
labels:
123+
app.kubernetes.io/instance: {{ .Release.Name }}
124+
annotations:
125+
"helm.sh/hook": post-delete
126+
"helm.sh/hook-weight": "5"
127+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
128+
rules:
129+
- apiGroups: [""]
130+
resources: ["persistentvolumeclaims"]
131+
verbs: ["get", "list", "delete"]
132+
- apiGroups: ["cert-manager.io"]
133+
resources: ["certificates"]
134+
verbs: ["get", "list", "delete"]
135+
- apiGroups: ["acme.cert-manager.io"]
136+
resources: ["orders", "challenges"]
137+
verbs: ["get", "list", "delete"]
138+
# We list ACME http01 solver Ingresses (filtered by this release's host)
139+
# and delete the orphaned ones by name.
140+
- apiGroups: ["networking.k8s.io"]
141+
resources: ["ingresses"]
142+
verbs: ["get", "list", "delete"]
143+
---
144+
apiVersion: rbac.authorization.k8s.io/v1
145+
kind: RoleBinding
146+
metadata:
147+
name: {{ .Release.Name }}-cleanup
148+
labels:
149+
app.kubernetes.io/instance: {{ .Release.Name }}
150+
annotations:
151+
"helm.sh/hook": post-delete
152+
helm.sh/hook-weight: "5"
153+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
154+
roleRef:
155+
apiGroup: rbac.authorization.k8s.io
156+
kind: Role
157+
name: {{ .Release.Name }}-cleanup
158+
subjects:
159+
- kind: ServiceAccount
160+
name: {{ .Release.Name }}-cleanup
Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
suite: harbor post-delete cleanup hook
2+
templates:
3+
- templates/hooks/cleanup.yaml
4+
5+
release:
6+
name: harbor-test
7+
namespace: tenant-root
8+
9+
# The cleanup hook scopes its ACME solver sweep to this release's TLS host,
10+
# which defaults to "<release>.<_namespace.host>".
11+
set:
12+
_namespace:
13+
host: example.org
14+
15+
tests:
16+
- it: renders the cleanup Job as a post-delete hook
17+
documentIndex: 0
18+
asserts:
19+
- equal:
20+
path: kind
21+
value: Job
22+
- equal:
23+
path: metadata.name
24+
value: harbor-test-cleanup
25+
- equal:
26+
path: metadata.annotations["helm.sh/hook"]
27+
value: post-delete
28+
- equal:
29+
path: spec.template.spec.serviceAccountName
30+
value: harbor-test-cleanup
31+
32+
- it: targets the jobservice PVC label (release-system instance) and ACME objects
33+
documentIndex: 0
34+
asserts:
35+
# The upstream Harbor subchart runs as the "<release>-system"
36+
# HelmRelease, so its kept jobservice PVC is labeled
37+
# app.kubernetes.io/instance=<release>-system, not <release>.
38+
- matchRegex:
39+
path: spec.template.spec.containers[0].command[2]
40+
pattern: app.kubernetes.io/instance=harbor-test-system
41+
- matchRegex:
42+
path: spec.template.spec.containers[0].command[2]
43+
pattern: harbor-test-ingress-tls
44+
45+
- it: scopes the ACME solver sweep to this release's TLS host
46+
documentIndex: 0
47+
asserts:
48+
# The solver sweep must be scoped to this release's host, not a
49+
# namespace-wide label sweep, so sibling apps' in-flight challenges in
50+
# the shared tenant namespace are never touched.
51+
- matchRegex:
52+
path: spec.template.spec.containers[0].command[2]
53+
pattern: HOST="harbor-test\.example\.org"
54+
# Challenges are filtered by .spec.dnsName == $HOST.
55+
- matchRegex:
56+
path: spec.template.spec.containers[0].command[2]
57+
pattern: \@\.spec\.dnsName==.\$HOST
58+
# Solver Ingresses are filtered by .spec.rules[0].host == $HOST.
59+
- matchRegex:
60+
path: spec.template.spec.containers[0].command[2]
61+
pattern: \@\.spec\.rules\[0\]\.host==.\$HOST
62+
# There must be NO namespace-wide blanket delete that targets every
63+
# object carrying the http01-solver label. The label may still appear
64+
# in the scoped `kubectl get ingress -l ...` selector, but it must
65+
# never be used in a `kubectl delete` against the whole namespace.
66+
- notMatchRegex:
67+
path: spec.template.spec.containers[0].command[2]
68+
pattern: kubectl delete[^\n]*-l acme.cert-manager.io/http01-solver=true
69+
70+
- it: sets resource requests and limits on the cleanup container
71+
documentIndex: 0
72+
asserts:
73+
- equal:
74+
path: spec.template.spec.containers[0].resources.requests.cpu
75+
value: 10m
76+
- equal:
77+
path: spec.template.spec.containers[0].resources.requests.memory
78+
value: 64Mi
79+
- equal:
80+
path: spec.template.spec.containers[0].resources.limits.cpu
81+
value: 200m
82+
- equal:
83+
path: spec.template.spec.containers[0].resources.limits.memory
84+
value: 128Mi
85+
86+
- it: creates a ServiceAccount for the cleanup job
87+
documentIndex: 1
88+
asserts:
89+
- equal:
90+
path: kind
91+
value: ServiceAccount
92+
- equal:
93+
path: metadata.name
94+
value: harbor-test-cleanup
95+
96+
- it: grants RBAC for PVCs and cert-manager ACME resources
97+
documentIndex: 2
98+
asserts:
99+
- equal:
100+
path: kind
101+
value: Role
102+
- equal:
103+
path: metadata.name
104+
value: harbor-test-cleanup
105+
- contains:
106+
path: rules
107+
content:
108+
apiGroups: [""]
109+
resources: ["persistentvolumeclaims"]
110+
verbs: ["get", "list", "delete"]
111+
- contains:
112+
path: rules
113+
content:
114+
apiGroups: ["cert-manager.io"]
115+
resources: ["certificates"]
116+
verbs: ["get", "list", "delete"]
117+
- contains:
118+
path: rules
119+
content:
120+
apiGroups: ["acme.cert-manager.io"]
121+
resources: ["orders", "challenges"]
122+
verbs: ["get", "list", "delete"]
123+
# We list+delete orphaned http01 solver Ingresses by name, so we need
124+
# get/list/delete on networking.k8s.io ingresses.
125+
- contains:
126+
path: rules
127+
content:
128+
apiGroups: ["networking.k8s.io"]
129+
resources: ["ingresses"]
130+
verbs: ["get", "list", "delete"]
131+
# The hook must not grant services/pods: solver Pods/Services cascade
132+
# from the Challenge/Certificate and are never deleted directly.
133+
- notContains:
134+
path: rules
135+
content:
136+
apiGroups: [""]
137+
resources: ["services"]
138+
verbs: ["get", "list", "delete"]
139+
- notContains:
140+
path: rules
141+
content:
142+
apiGroups: [""]
143+
resources: ["pods"]
144+
verbs: ["get", "list", "delete"]
145+
146+
- it: binds the Role to the cleanup ServiceAccount
147+
documentIndex: 3
148+
asserts:
149+
- equal:
150+
path: kind
151+
value: RoleBinding
152+
- equal:
153+
path: roleRef.name
154+
value: harbor-test-cleanup
155+
- equal:
156+
path: subjects[0].name
157+
value: harbor-test-cleanup

0 commit comments

Comments
 (0)