Skip to content

Commit f886f26

Browse files
committed
fix(apps): bound the wait the cleanup hooks now perform
With watch granted, kubectl delete waits for the claims to be gone, and its default for that wait is a week. A claim held by a pod that will not stop, or a stuck detach, would hold the post-delete hook until helm-controller's 300s uninstall budget ran out and leave the HelmRelease stuck uninstalling. The PVC deletes in the clickhouse, harbor, mariadb and qdrant hooks now wait 120s at most and report the claims still terminating by name, and each Job carries activeDeadlineSeconds 270 with a 10s termination grace period, the numbers packages/apps/tenant/templates/cleanup-job.yaml documents against that budget, so a kubectl that never returns cannot hold the hook either. The mariadb Role no longer grants watch on the named Secrets: they carry no finalizer, so the delete never reaches the watch, and a resourceNames rule does not authorize a collection watch anyway. Signed-off-by: Yan Bondarenko <[email protected]> Assisted-by: LLM
1 parent 711fcc1 commit f886f26

8 files changed

Lines changed: 88 additions & 6 deletions

File tree

‎packages/apps/clickhouse/templates/hooks/cleanup-pvc.yaml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ metadata:
1313
# hook-succeeded so they are reclaimed after a successful uninstall.
1414
"helm.sh/hook-delete-policy": before-hook-creation
1515
spec:
16+
activeDeadlineSeconds: 270
1617
template:
1718
metadata:
1819
labels:
@@ -21,6 +22,7 @@ spec:
2122
spec:
2223
serviceAccountName: "{{ .Release.Name }}-cleanup"
2324
restartPolicy: Never
25+
terminationGracePeriodSeconds: 10
2426
securityContext:
2527
runAsNonRoot: true
2628
runAsUser: 65534
@@ -51,7 +53,7 @@ spec:
5153
- |
5254
set -eu
5355
echo "Deleting orphaned PVCs for {{ .Release.Name }}..."
54-
if ! kubectl delete pvc -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} --ignore-not-found; then
56+
if ! kubectl delete pvc -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} --ignore-not-found --wait=true --timeout=120s; then
5557
echo "ERROR: PVC cleanup failed for {{ .Release.Name }}; orphaned PVCs may remain" >&2
5658
exit 1
5759
fi

‎packages/apps/clickhouse/tests/cleanup_pvc_test.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,24 @@ tests:
6969
- watch
7070
- delete
7171

72+
- it: "bounds the cleanup Job so a claim that will not go cannot hang the uninstall"
73+
documentSelector:
74+
path: kind
75+
value: Job
76+
asserts:
77+
- equal:
78+
path: spec.activeDeadlineSeconds
79+
value: 270
80+
- equal:
81+
path: spec.template.spec.terminationGracePeriodSeconds
82+
value: 10
83+
- matchRegex:
84+
path: spec.template.spec.containers[0].command[2]
85+
pattern: 'kubectl delete pvc -n tenant-test -l app\.kubernetes\.io/instance=ch-test --ignore-not-found --wait=true --timeout=120s'
86+
- notMatchRegex:
87+
path: spec.template.spec.containers[0].command[2]
88+
pattern: '--request-timeout='
89+
7290
- it: "binds the cleanup ServiceAccount to the cleanup Role"
7391
documentSelector:
7492
path: kind

‎packages/apps/harbor/templates/hooks/cleanup.yaml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ spec:
2323
# timeout instead of the hook's own error. Load-bearing only since the script
2424
# gained a non-zero exit; before that the script could not report one.
2525
backoffLimit: 1
26+
activeDeadlineSeconds: 270
2627
template:
2728
metadata:
2829
labels:
@@ -31,6 +32,7 @@ spec:
3132
spec:
3233
serviceAccountName: {{ .Release.Name }}-cleanup
3334
restartPolicy: Never
35+
terminationGracePeriodSeconds: 10
3436
securityContext:
3537
runAsNonRoot: true
3638
runAsUser: 65534
@@ -104,7 +106,7 @@ spec:
104106
# this selector matches only those two kept PVCs.
105107
echo "Deleting orphaned PVCs for {{ .Release.Name }}-system..."
106108
kubectl delete pvc -n {{ .Release.Namespace }} \
107-
-l release={{ .Release.Name }}-system --ignore-not-found \
109+
-l release={{ .Release.Name }}-system --ignore-not-found --wait=true --timeout=120s \
108110
|| fail "failed to delete PVCs matching release={{ .Release.Name }}-system in {{ .Release.Namespace }}; orphaned PVCs may remain"
109111
110112
# Remove the cert-manager Certificate owned by this release's

‎packages/apps/harbor/tests/cleanup_hook_test.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,24 @@ tests:
103103
path: spec.template.spec.containers[0].command[2]
104104
pattern: kubectl delete[^\n]*-l acme.cert-manager.io/http01-solver=true
105105

106+
- it: bounds the cleanup Job so a claim that will not go cannot hang the uninstall
107+
documentSelector:
108+
path: kind
109+
value: Job
110+
asserts:
111+
- equal:
112+
path: spec.activeDeadlineSeconds
113+
value: 270
114+
- equal:
115+
path: spec.template.spec.terminationGracePeriodSeconds
116+
value: 10
117+
- matchRegex:
118+
path: spec.template.spec.containers[0].command[2]
119+
pattern: '-l release=harbor-test-system --ignore-not-found --wait=true --timeout=120s'
120+
- notMatchRegex:
121+
path: spec.template.spec.containers[0].command[2]
122+
pattern: '--request-timeout='
123+
106124
- it: pins the cleanup image by digest and routes it through the images registry
107125
documentSelector:
108126
path: kind

‎packages/apps/mariadb/templates/hooks/cleanup-pvc.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ metadata:
1010
"helm.sh/hook-weight": "10"
1111
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
1212
spec:
13+
activeDeadlineSeconds: 270
1314
template:
1415
metadata:
1516
labels:
@@ -18,6 +19,7 @@ spec:
1819
spec:
1920
serviceAccountName: {{ .Release.Name }}-cleanup
2021
restartPolicy: Never
22+
terminationGracePeriodSeconds: 10
2123
securityContext:
2224
runAsNonRoot: true
2325
runAsUser: 65534
@@ -58,7 +60,7 @@ spec:
5860
# errors behind `|| echo ...`.
5961
cleanup_failed=0
6062
echo "Deleting orphaned PVCs for {{ .Release.Name }}..."
61-
if kubectl delete pvc -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} --ignore-not-found; then
63+
if kubectl delete pvc -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} --ignore-not-found --wait=true --timeout=120s; then
6264
echo "PVC cleanup complete."
6365
else
6466
echo "WARNING: PVC cleanup failed for {{ .Release.Name }}; orphaned PVCs may remain" >&2
@@ -132,7 +134,7 @@ rules:
132134
- {{ .Release.Name }}-repl-password
133135
- {{ .Release.Name }}.ca-tls
134136
- {{ .Release.Name }}.tls
135-
verbs: ["get", "watch", "delete"]
137+
verbs: ["get", "delete"]
136138
---
137139
apiVersion: rbac.authorization.k8s.io/v1
138140
kind: RoleBinding

‎packages/apps/mariadb/tests/cleanup_hook_test.yaml‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,24 @@ tests:
5959
path: spec.template.spec.containers[0].command[2]
6060
pattern: 'exit "\$\{cleanup_failed\}"'
6161

62+
- it: "bounds the cleanup Job so a claim that will not go cannot hang the uninstall"
63+
documentSelector:
64+
path: kind
65+
value: Job
66+
asserts:
67+
- equal:
68+
path: spec.activeDeadlineSeconds
69+
value: 270
70+
- equal:
71+
path: spec.template.spec.terminationGracePeriodSeconds
72+
value: 10
73+
- matchRegex:
74+
path: spec.template.spec.containers[0].command[2]
75+
pattern: 'kubectl delete pvc -n tenant-test -l app\.kubernetes\.io/instance=mariadb-test --ignore-not-found --wait=true --timeout=120s'
76+
- notMatchRegex:
77+
path: spec.template.spec.containers[0].command[2]
78+
pattern: '--request-timeout='
79+
6280
- it: "cleanup Role grants get/list/watch/delete on persistentvolumeclaims"
6381
documentSelector:
6482
path: kind
@@ -86,7 +104,7 @@ tests:
86104
- mariadb-test-repl-password
87105
- mariadb-test.ca-tls
88106
- mariadb-test.tls
89-
verbs: ["get", "watch", "delete"]
107+
verbs: ["get", "delete"]
90108
# Lock the exact rule set (PVC rule + the single scoped secrets rule).
91109
# Without this, a broader unscoped `secrets` rule could be added and the
92110
# `contains` checks above would still pass.

‎packages/apps/qdrant/templates/hooks/cleanup-pvc.yaml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ spec:
2222
# timeout instead of the hook's own error. Load-bearing only since the script
2323
# gained a non-zero exit; before that the script could not report one.
2424
backoffLimit: 1
25+
activeDeadlineSeconds: 270
2526
template:
2627
metadata:
2728
labels:
@@ -30,6 +31,7 @@ spec:
3031
spec:
3132
serviceAccountName: {{ .Release.Name }}-qdrant-cleanup
3233
restartPolicy: Never
34+
terminationGracePeriodSeconds: 10
3335
securityContext:
3436
runAsNonRoot: true
3537
runAsUser: 65534
@@ -104,7 +106,7 @@ spec:
104106
if [ -z "$pvcs" ]; then
105107
echo "No data PVCs found for release {{ .Release.Name }}; nothing to delete."
106108
else
107-
echo "$pvcs" | xargs kubectl delete -n {{ .Release.Namespace }} --ignore-not-found \
109+
echo "$pvcs" | xargs kubectl delete -n {{ .Release.Namespace }} --ignore-not-found --wait=true --timeout=120s \
108110
|| fail "failed to delete data PVCs of {{ .Release.Name }} in {{ .Release.Namespace }} ($(echo "$pvcs" | tr '\n' ' ')); orphaned PVCs may remain"
109111
fi
110112
else

‎packages/apps/qdrant/tests/cleanup_pvc_test.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,26 @@ tests:
6565
path: spec.template.spec.containers[0].command[2]
6666
pattern: 'grep "\^persistentvolumeclaim/qdrant-storage-qdrant-test-"'
6767

68+
- it: "bounds the cleanup Job so a claim that will not go cannot hang the uninstall"
69+
templates:
70+
- templates/hooks/cleanup-pvc.yaml
71+
documentSelector:
72+
path: kind
73+
value: Job
74+
asserts:
75+
- equal:
76+
path: spec.activeDeadlineSeconds
77+
value: 270
78+
- equal:
79+
path: spec.template.spec.terminationGracePeriodSeconds
80+
value: 10
81+
- matchRegex:
82+
path: spec.template.spec.containers[0].command[2]
83+
pattern: 'xargs kubectl delete -n tenant-test --ignore-not-found --wait=true --timeout=120s'
84+
- notMatchRegex:
85+
path: spec.template.spec.containers[0].command[2]
86+
pattern: '--request-timeout='
87+
6888
- it: "creates qdrant-scoped RBAC for the cleanup hook"
6989
templates:
7090
- templates/hooks/cleanup-pvc.yaml

0 commit comments

Comments
 (0)