Skip to content

Commit 63db8ca

Browse files
lllamnyplllamnyp
committed
[kubernetes] Helm hooks for cleanup
## What this PR does When deleting a Kubernetes, some resources may linger post deletion because of a race to remove HelmReleases deployed inside the tenant cluster and the removal of the cluster and its controlplane itself. This patch modifies the existing pre-delete hook to remove those helmreleases instead of simply suspending them. Similarly, datavolumes may also remain. These are now delete with a post-delete hook. ### Release note ```release-note [kubernetes] Use Helm hooks to clean up HelmReleases deployed in tenant clusters and DataVolumes backing the tenant clusters' PVCs when deleting a tenant Kubernetes. ``` Signed-off-by: Timofei Larkin <[email protected]>
1 parent 4278692 commit 63db8ca

2 files changed

Lines changed: 98 additions & 21 deletions

File tree

‎packages/apps/kubernetes/templates/csi/delete.yaml‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
---
2+
apiVersion: batch/v1
3+
kind: Job
4+
metadata:
5+
annotations:
6+
"helm.sh/hook": post-delete
7+
"helm.sh/hook-weight": "10"
8+
"helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation,hook-failed
9+
name: {{ .Release.Name }}-datavolume-cleanup
10+
spec:
11+
template:
12+
spec:
13+
serviceAccountName: {{ .Release.Name }}-datavolume-cleanup
14+
restartPolicy: Never
15+
tolerations:
16+
- key: CriticalAddonsOnly
17+
operator: Exists
18+
- key: node-role.kubernetes.io/control-plane
19+
operator: Exists
20+
effect: "NoSchedule"
21+
containers:
22+
- name: kubectl
23+
image: docker.io/clastix/kubectl:v1.32
24+
command:
25+
- /bin/sh
26+
- -c
27+
- kubectl -n {{ .Release.Namespace }} delete datavolumes
28+
-l "cluster.x-k8s.io/cluster-name={{ .Release.Name }}"
29+
--ignore-not-found=true
30+
31+
32+
---
33+
apiVersion: v1
34+
kind: ServiceAccount
35+
metadata:
36+
name: {{ .Release.Name }}-datavolume-cleanup
37+
annotations:
38+
helm.sh/hook: post-delete
39+
helm.sh/hook-delete-policy: before-hook-creation,hook-failed,hook-succeeded
40+
helm.sh/hook-weight: "0"
41+
---
42+
apiVersion: rbac.authorization.k8s.io/v1
43+
kind: Role
44+
metadata:
45+
annotations:
46+
"helm.sh/hook": post-delete
47+
"helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation,hook-failed
48+
"helm.sh/hook-weight": "5"
49+
name: {{ .Release.Name }}-datavolume-cleanup
50+
rules:
51+
- apiGroups:
52+
- "cdi.kubevirt.io"
53+
resources:
54+
- datavolumes
55+
verbs:
56+
- get
57+
- list
58+
- delete
59+
---
60+
apiVersion: rbac.authorization.k8s.io/v1
61+
kind: RoleBinding
62+
metadata:
63+
annotations:
64+
"helm.sh/hook": post-delete
65+
"helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation,hook-failed
66+
"helm.sh/hook-weight": "5"
67+
name: {{ .Release.Name }}-datavolume-cleanup
68+
roleRef:
69+
apiGroup: rbac.authorization.k8s.io
70+
kind: Role
71+
name: {{ .Release.Name }}-datavolume-cleanup
72+
subjects:
73+
- kind: ServiceAccount
74+
name: {{ .Release.Name }}-datavolume-cleanup
75+
namespace: {{ .Release.Namespace }}
76+

‎packages/apps/kubernetes/templates/helmreleases/delete.yaml‎

Lines changed: 22 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -24,34 +24,34 @@ spec:
2424
command:
2525
- /bin/sh
2626
- -c
27-
- |
28-
kubectl
29-
--namespace={{ .Release.Namespace }}
30-
patch
31-
helmrelease
32-
{{ .Release.Name }}-cilium
33-
{{ .Release.Name }}-gateway-api-crds
34-
{{ .Release.Name }}-csi
35-
{{ .Release.Name }}-cert-manager
36-
{{ .Release.Name }}-cert-manager-crds
37-
{{ .Release.Name }}-vertical-pod-autoscaler
38-
{{ .Release.Name }}-vertical-pod-autoscaler-crds
39-
{{ .Release.Name }}-ingress-nginx
40-
{{ .Release.Name }}-fluxcd-operator
41-
{{ .Release.Name }}-fluxcd
42-
{{ .Release.Name }}-gpu-operator
43-
{{ .Release.Name }}-velero
44-
{{ .Release.Name }}-coredns
45-
-p '{"spec": {"suspend": true}}'
46-
--type=merge --field-manager=flux-client-side-apply || true
27+
- >-
28+
kubectl
29+
--namespace={{ .Release.Namespace }}
30+
patch
31+
helmrelease
32+
{{ .Release.Name }}-cilium
33+
{{ .Release.Name }}-gateway-api-crds
34+
{{ .Release.Name }}-csi
35+
{{ .Release.Name }}-cert-manager
36+
{{ .Release.Name }}-cert-manager-crds
37+
{{ .Release.Name }}-vertical-pod-autoscaler
38+
{{ .Release.Name }}-vertical-pod-autoscaler-crds
39+
{{ .Release.Name }}-ingress-nginx
40+
{{ .Release.Name }}-fluxcd-operator
41+
{{ .Release.Name }}-fluxcd
42+
{{ .Release.Name }}-gpu-operator
43+
{{ .Release.Name }}-velero
44+
{{ .Release.Name }}-coredns
45+
-p '{"spec": {"suspend": true}}'
46+
--type=merge --field-manager=flux-client-side-apply || true
4747
---
4848
apiVersion: v1
4949
kind: ServiceAccount
5050
metadata:
5151
name: {{ .Release.Name }}-flux-teardown
5252
annotations:
5353
helm.sh/hook: pre-delete
54-
helm.sh/hook-delete-policy: before-hook-creation,hook-failed
54+
helm.sh/hook-delete-policy: before-hook-creation,hook-failed,hook-succeeded
5555
helm.sh/hook-weight: "0"
5656
---
5757
apiVersion: rbac.authorization.k8s.io/v1
@@ -75,6 +75,7 @@ rules:
7575
- {{ .Release.Name }}-csi
7676
- {{ .Release.Name }}-cert-manager
7777
- {{ .Release.Name }}-cert-manager-crds
78+
- {{ .Release.Name }}-gateway-api-crds
7879
- {{ .Release.Name }}-vertical-pod-autoscaler
7980
- {{ .Release.Name }}-vertical-pod-autoscaler-crds
8081
- {{ .Release.Name }}-ingress-nginx

0 commit comments

Comments
 (0)