Skip to content

Commit ab70cce

Browse files
committed
fix(kafka): delete the release's StrimziPodSets on uninstall
On a cluster upgraded from Strimzi 0.x, deleting a Kafka application leaves its brokers running. kube-controller-manager's garbage collector first saw the StrimziPodSets under v1beta2 and never updates that record, so once v1beta2 is no longer served it retries them forever instead of deleting them, until kube-controller-manager restarts (kubernetes/kubernetes#102641). The pre-delete hook now deletes the release's StrimziPodSets after its node pools, so uninstall no longer depends on garbage collection for the brokers. An absent CRD is not a failure; any other error fails the hook like the pool delete does, because Helm never runs it for the release again. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <[email protected]>
1 parent 19bc440 commit ab70cce

3 files changed

Lines changed: 155 additions & 3 deletions

File tree

‎hack/kafka-pre-delete-hook.bats‎

Lines changed: 108 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,11 @@
3232
# the hook skips keeps its brokers running after the release is gone;
3333
# * a KafkaNodePool CRD that is absent exits 0, and any other failure of
3434
# that delete exits non-zero, so the uninstall is retried rather than
35-
# finishing with the pools still in place.
35+
# finishing with the pools still in place;
36+
# * the release's StrimziPodSets are deleted after the pools, with the same
37+
# exit codes. Garbage collection does not take them on a cluster upgraded
38+
# from Strimzi 0.x until kube-controller-manager restarts
39+
# (kubernetes/kubernetes#102641), so the hook is what stops the brokers.
3640
#
3741
# Run from the repo root with make bats-unit-tests. Direct status checks
3842
# retain compatibility with cozytest, which has no Bats run/setup helpers.
@@ -65,7 +69,8 @@ render_hook() {
6569
# The KafkaNodePool delete answers from POOL_OUT / POOL_RC instead, so a test
6670
# can fail one call without failing the other. POOL_RC defaults to a clean
6771
# delete, because the topic-path tests above are about the topic call; every
68-
# pool-path test sets it explicitly.
72+
# pool-path test sets it explicitly. The StrimziPodSet delete answers from
73+
# SPS_OUT / SPS_RC the same way.
6974
# No line below is a bare column-0 `}`, so cozytest.sh's parser leaves it alone.
7075
write_fake_kubectl() {
7176
cat > "$1/kubectl" <<'KEOF'
@@ -75,6 +80,9 @@ case "$*" in
7580
*kafkanodepools*)
7681
printf '%s\n' "${POOL_OUT:-kafkanodepool.kafka.strimzi.io \"b-0\" deleted}"
7782
exit "${POOL_RC:-0}" ;;
83+
*strimzipodsets*)
84+
printf '%s\n' "${SPS_OUT:-strimzipodset.core.strimzi.io \"kafka-test-b-0\" deleted}"
85+
exit "${SPS_RC:-0}" ;;
7886
esac
7987
if [ -z "${DELETE_RC:-}" ]; then
8088
echo "fake kubectl: DELETE_RC unset; the test would assert nothing" >&2
@@ -284,6 +292,11 @@ KEOF
284292
cat "$tmp/calls"
285293
false
286294
fi
295+
if ! grep -q -- "delete strimzipodsets.core.strimzi.io -l strimzi.io/cluster=kafka-test " "$tmp/calls"; then
296+
echo "FAIL: the StrimziPodSets were skipped after the topic wait timed out"
297+
cat "$tmp/calls"
298+
false
299+
fi
287300
rm -rf "$tmp"
288301
}
289302

@@ -342,3 +355,96 @@ KEOF
342355
fi
343356
rm -rf "$tmp"
344357
}
358+
359+
@test "kafka pre-delete hook deletes this release's StrimziPodSets after its node pools" {
360+
tmp=$(mktemp -d)
361+
write_fake_kubectl "$tmp"
362+
render_hook "$KAFKA_CHART" kafka-test tenant-test kafka-test-pre-delete "$tmp/hook.sh"
363+
364+
export KLOG="$tmp/calls"
365+
export DELETE_RC=0
366+
export POOL_RC=0
367+
export SPS_RC=0
368+
rc=0
369+
PATH="$tmp:$PATH" sh "$tmp/hook.sh" > "$tmp/out" 2> "$tmp/err" || rc=$?
370+
if [ "$rc" -ne 0 ]; then
371+
echo "FAIL: hook exited $rc on a clean delete"
372+
cat "$tmp/out" "$tmp/err"
373+
false
374+
fi
375+
# Anchored like the selectors above.
376+
sps_line=$(grep -n -- "delete strimzipodsets.core.strimzi.io -l strimzi.io/cluster=kafka-test " "$tmp/calls" | cut -d: -f1)
377+
pool_line=$(grep -n -- "delete kafkanodepools.kafka.strimzi.io -l strimzi.io/cluster=kafka-test " "$tmp/calls" | cut -d: -f1)
378+
if [ -z "$sps_line" ] || [ -z "$pool_line" ]; then
379+
echo "FAIL: expected one release-scoped delete of each kind"
380+
cat "$tmp/calls"
381+
false
382+
fi
383+
# Deleting the sets while their pools exist lets the operator recreate them.
384+
if [ "$sps_line" -le "$pool_line" ]; then
385+
echo "FAIL: the StrimziPodSets were deleted before the node pools"
386+
cat "$tmp/calls"
387+
false
388+
fi
389+
if ! grep -q -- "strimzipodsets.* --wait=false" "$tmp/calls"; then
390+
echo "FAIL: the StrimziPodSet delete waits, which spends the Job's deadline"
391+
cat "$tmp/calls"
392+
false
393+
fi
394+
rm -rf "$tmp"
395+
}
396+
397+
@test "kafka pre-delete hook exits 0 when the StrimziPodSet CRD is absent" {
398+
for msg in 'error: the server doesn'"'"'t have a resource type "strimzipodsets"' \
399+
'error: no matches for kind "StrimziPodSet" in version "core.strimzi.io/v1"'; do
400+
tmp=$(mktemp -d)
401+
write_fake_kubectl "$tmp"
402+
render_hook "$KAFKA_CHART" kafka-test tenant-test kafka-test-pre-delete "$tmp/hook.sh"
403+
404+
export KLOG="$tmp/calls"
405+
export DELETE_RC=0
406+
export POOL_RC=0
407+
export SPS_RC=1
408+
export SPS_OUT="$msg"
409+
rc=0
410+
PATH="$tmp:$PATH" sh "$tmp/hook.sh" > "$tmp/out" 2> "$tmp/err" || rc=$?
411+
if [ "$rc" -ne 0 ]; then
412+
echo "FAIL: hook exited $rc on an absent StrimziPodSet CRD, wedging the uninstall"
413+
cat "$tmp/out" "$tmp/err"
414+
false
415+
fi
416+
if ! grep -q 'StrimziPodSet CRD absent' "$tmp/out"; then
417+
echo "FAIL: the absent CRD was not reported as such"
418+
cat "$tmp/out"
419+
false
420+
fi
421+
rm -rf "$tmp"
422+
done
423+
}
424+
425+
@test "kafka pre-delete hook exits non-zero when the StrimziPodSet delete fails" {
426+
# A set left here keeps its brokers running after the release is gone, and
427+
# Helm never runs this hook for the release again.
428+
tmp=$(mktemp -d)
429+
write_fake_kubectl "$tmp"
430+
render_hook "$KAFKA_CHART" kafka-test tenant-test kafka-test-pre-delete "$tmp/hook.sh"
431+
432+
export KLOG="$tmp/calls"
433+
export DELETE_RC=0
434+
export POOL_RC=0
435+
export SPS_RC=1
436+
export SPS_OUT='Error from server (Forbidden): strimzipodsets.core.strimzi.io is forbidden'
437+
rc=0
438+
PATH="$tmp:$PATH" sh "$tmp/hook.sh" > "$tmp/out" 2> "$tmp/err" || rc=$?
439+
if [ "$rc" -eq 0 ]; then
440+
echo "FAIL: hook exited 0 on a Forbidden, so the uninstall would finish and leave the brokers"
441+
cat "$tmp/out" "$tmp/err"
442+
false
443+
fi
444+
if ! grep -q '^ERROR: deleting StrimziPodSets failed' "$tmp/err"; then
445+
echo "FAIL: the failure was not reported as an error"
446+
cat "$tmp/err"
447+
false
448+
fi
449+
rm -rf "$tmp"
450+
}

‎packages/apps/kafka/templates/hooks/delete.yaml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,29 @@ spec:
126126
else
127127
printf '%s\n' "${out}"
128128
fi
129+
130+
# Garbage collection should take the StrimziPodSets with their
131+
# pools, but on a cluster upgraded from Strimzi 0.x it does not
132+
# until kube-controller-manager restarts: its collector still
133+
# tracks them under v1beta2, which is no longer served, and
134+
# retries them forever (kubernetes/kubernetes#102641). Deleting
135+
# them here does not depend on that. Fails like the pool call,
136+
# for the same reason.
137+
echo "Deleting StrimziPodSets for {{ .Release.Name }}"
138+
if ! out=$(kubectl -n {{ .Release.Namespace }} delete strimzipodsets.core.strimzi.io \
139+
-l 'strimzi.io/cluster={{ .Release.Name }}' \
140+
--ignore-not-found=true \
141+
--wait=false 2>&1); then
142+
printf '%s\n' "${out}"
143+
case "${out}" in
144+
*"the server doesn't have a resource type"*|*"no matches for kind"*)
145+
echo "StrimziPodSet CRD absent; nothing to delete" ;;
146+
*)
147+
echo "ERROR: deleting StrimziPodSets failed: ${out}" >&2; exit 1 ;;
148+
esac
149+
else
150+
printf '%s\n' "${out}"
151+
fi
129152
volumeMounts:
130153
- name: tmp
131154
mountPath: /tmp
@@ -166,6 +189,9 @@ rules:
166189
- apiGroups: ["kafka.strimzi.io"]
167190
resources: ["kafkanodepools"]
168191
verbs: ["list", "delete"]
192+
- apiGroups: ["core.strimzi.io"]
193+
resources: ["strimzipodsets"]
194+
verbs: ["list", "delete"]
169195
---
170196
apiVersion: rbac.authorization.k8s.io/v1
171197
kind: RoleBinding

‎packages/apps/kafka/tests/delete_test.yaml‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,27 @@ tests:
154154
documentIndex: 2
155155
- lengthEqual:
156156
path: rules
157-
count: 2
157+
count: 3
158+
documentIndex: 2
159+
160+
- it: deletes release-scoped StrimziPodSets after the KafkaNodePools
161+
asserts:
162+
# After an upgrade from Strimzi 0.x, kube-controller-manager's garbage
163+
# collector still tracks each StrimziPodSet under v1beta2, a version no
164+
# longer served, and never deletes it when its pool goes
165+
# (kubernetes/kubernetes#102641). The brokers keep running until
166+
# kube-controller-manager restarts, so the hook deletes the sets itself.
167+
# Same selector and no wait, as for the pools.
168+
- matchRegex:
169+
path: spec.template.spec.containers[0].command[2]
170+
pattern: "(?s)delete kafkanodepools\\.kafka\\.strimzi\\.io .*delete strimzipodsets\\.core\\.strimzi\\.io \\\\\\s+-l 'strimzi\\.io/cluster=kafka-test' \\\\\\s+--ignore-not-found=true \\\\\\s+--wait=false"
171+
documentIndex: 0
172+
- contains:
173+
path: rules
174+
content:
175+
apiGroups: ["core.strimzi.io"]
176+
resources: ["strimzipodsets"]
177+
verbs: ["list", "delete"]
158178
documentIndex: 2
159179

160180
- it: prefixes the hook image with the cluster images registry when set

0 commit comments

Comments
 (0)