Skip to content

Commit 4f35af6

Browse files
fix(monitoring-agents): let the release uninstall when the VictoriaMetrics operator is gone first
Disabling the Monitoring Agents addon of a tenant cluster removes this release, the VictoriaMetrics operator and the VPA CRDs together, and Flux deletes them in parallel without honouring dependsOn. When the operator goes first, nothing clears apps.victoriametrics.com/finalizer on the VMAgent and the other VictoriaMetrics objects this chart created, so they stay Terminating and the release sits in UninstallFailed for good. Clearing those finalizers has to happen inside the uninstall and must never be what blocks it, and a live operator still has to get the first chance to finalize its objects. Only objects of this release may be touched, and release names are unique only within a namespace, so the release name alone does not identify them. kubectl loads in-cluster credentials only while the client config is otherwise default, which a --request-timeout breaks. A VPA recorded in the release manifest while its CRD is not served cannot be mapped at uninstall either. Assisted-by: LLM Signed-off-by: Mattia Eleuteri <[email protected]>
1 parent 048c029 commit 4f35af6

5 files changed

Lines changed: 638 additions & 0 deletions

File tree

Lines changed: 312 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,312 @@
1+
#!/usr/bin/env bats
2+
# Behavioural tests for the monitoring-agents pre-delete hook:
3+
#
4+
# packages/system/monitoring-agents/templates/vm-cleanup-hook.yaml
5+
#
6+
# The script is extracted from the rendered chart and RUN against a fake
7+
# kubectl that keeps the VictoriaMetrics objects in a state file, with fake
8+
# `date` and `sleep` so the 60s grace period passes in an instant. The
9+
# helm-unittest suite in packages/system/monitoring-agents/tests/
10+
# uninstall_test.yaml matches the script text instead, and stayed green when the
11+
# ownership condition was removed, when the finalizer patch was made a no-op and
12+
# when the polling condition was inverted. Each of those turns this file red.
13+
#
14+
# What is pinned here:
15+
# * only objects whose Helm annotations name this release, by name and
16+
# namespace both, are deleted or patched: an object of another release, of
17+
# a same-named release in another namespace, or one Helm does not own, is
18+
# left alone;
19+
# * with the operator gone, the hook waits before touching finalizers, then
20+
# clears them, and nothing of this release is left behind;
21+
# * with the operator alive, it finalizes the objects itself and the hook
22+
# never patches a finalizer;
23+
# * a kind whose CRD is gone, or a release with nothing left, exits 0, since a
24+
# failing pre-delete hook would itself block the uninstall.
25+
#
26+
# Run from the repo root with make bats-unit-tests. Direct status checks
27+
# retain compatibility with cozytest, which has no Bats run/setup helpers.
28+
# Inline cleanup preserves Bats' TAP handler on failed assertions.
29+
30+
load test_helper
31+
32+
MA_CHART=packages/system/monitoring-agents
33+
MA_RELEASE=monitoring-agents
34+
MA_NAMESPACE=cozy-monitoring
35+
36+
# render_hook <job-name> <out>
37+
# Extract the named Job's shell script from the rendered chart into <out>. The
38+
# non-empty check must be an early `return 1`: hack/cozytest.sh rewrites every
39+
# line matching ^}$ into `return 0` plus `}`, helpers included, so a check left
40+
# in last-command position has its status discarded.
41+
render_hook() {
42+
helm template "$MA_RELEASE" "$MA_CHART" --namespace "$MA_NAMESPACE" \
43+
| yq eval "select(.kind == \"Job\" and .metadata.name == \"$1\") | .spec.template.spec.containers[0].command[2]" - > "$2"
44+
[ -s "$2" ] || return 1
45+
}
46+
47+
# write_fakes <dir>
48+
# Emit fake `kubectl`, `date` and `sleep` into <dir>.
49+
#
50+
# kubectl reads and writes $STATE, one object per line as
51+
# kind|release-name|release-namespace|namespace|name, where the two release
52+
# fields are the Helm annotations (empty when Helm does not own the object).
53+
# `get` prints this kind's objects in the hook's jsonpath format, or fails for a
54+
# kind listed in $MISSING_KINDS as kubectl does when the CRD is gone. `delete`
55+
# removes the object only when $OPERATOR_LIVE is 1, standing for an operator
56+
# that finalizes it; otherwise the object stays, Terminating. `patch` removes it
57+
# only when the payload actually clears the finalizers. Every call goes to $KLOG.
58+
#
59+
# date advances 1s per call from $CLOCK, so the 60s wait ends after sixty
60+
# of polls, and records each time it returns in $KLOG; sleep records itself,
61+
# and with $SLEEP_ADVANCES=1 also moves $CLOCK by its argument. The log thus
62+
# shows how much fake time passed before each patch.
63+
# No line below is a bare column-0 `}`, so cozytest.sh's parser leaves it alone.
64+
write_fakes() {
65+
cat > "$1/kubectl" <<'KEOF'
66+
#!/bin/sh
67+
echo "kubectl $*" >> "$KLOG"
68+
drop() {
69+
awk -F'|' -v k="$1" -v ns="$2" -v n="$3" '!($1 == k && $4 == ns && $5 == n)' "$STATE" > "$STATE.new"
70+
mv "$STATE.new" "$STATE"; }
71+
case "$1" in
72+
get)
73+
kind=${2%%.*}
74+
case " ${MISSING_KINDS:-} " in
75+
*" $kind "*)
76+
echo "error: the server doesn't have a resource type \"$kind\"" >&2
77+
exit 1 ;;
78+
esac
79+
awk -F'|' -v k="$kind" '$1 == k { print $2 "|" $3 "|" $4 "|" $5 }' "$STATE"
80+
exit 0 ;;
81+
-n)
82+
ns=$2 verb=$3 kind=${4%%.*} name=$5
83+
case "$verb" in
84+
delete)
85+
[ "${OPERATOR_LIVE:-0}" = 1 ] && drop "$kind" "$ns" "$name"
86+
echo "$kind \"$name\" deleted"
87+
exit 0 ;;
88+
patch)
89+
case "$*" in
90+
*'"finalizers":null'*) drop "$kind" "$ns" "$name" ;;
91+
esac
92+
echo "$kind \"$name\" patched"
93+
exit 0 ;;
94+
esac ;;
95+
esac
96+
echo "fake kubectl: unexpected call: $*" >&2
97+
exit 64
98+
KEOF
99+
cat > "$1/date" <<'KEOF'
100+
#!/bin/sh
101+
c=$(cat "$CLOCK" 2>/dev/null || echo 0)
102+
echo $((c + 1)) > "$CLOCK"
103+
echo "date $((1000000 + c))" >> "$KLOG"
104+
echo $((1000000 + c))
105+
KEOF
106+
cat > "$1/sleep" <<'KEOF'
107+
#!/bin/sh
108+
echo "sleep $*" >> "$KLOG"
109+
if [ "${SLEEP_ADVANCES:-0}" = 1 ]; then
110+
c=$(cat "$CLOCK" 2>/dev/null || echo 0)
111+
echo $((c + $1)) > "$CLOCK"
112+
fi
113+
KEOF
114+
chmod +x "$1/kubectl" "$1/date" "$1/sleep"
115+
}
116+
117+
# write_state <file>
118+
# Three objects of this release in two namespaces; a VMRule of another release;
119+
# a VMRule whose release name matches but whose release namespace does not, so
120+
# only the pair of annotations tells it apart; and a VMServiceScrape that
121+
# carries no Helm annotations.
122+
write_state() {
123+
cat > "$1" <<EOF
124+
vmagents|$MA_RELEASE|$MA_NAMESPACE|$MA_NAMESPACE|vmagent
125+
vmrules|$MA_RELEASE|$MA_NAMESPACE|$MA_NAMESPACE|alerts-node
126+
vmservicescrapes|$MA_RELEASE|$MA_NAMESPACE|cozy-victoria-metrics-operator|operator
127+
vmrules|other|$MA_NAMESPACE|$MA_NAMESPACE|foreign-rule
128+
vmrules|$MA_RELEASE|other-namespace|$MA_NAMESPACE|foreign-ns-rule
129+
vmservicescrapes|||$MA_NAMESPACE|unmanaged-scrape
130+
EOF
131+
}
132+
133+
# run_hook <dir>
134+
# Run the rendered hook with the fakes first on PATH; its status lands in $rc.
135+
run_hook() {
136+
export KLOG="$1/calls" STATE="$1/state" CLOCK="$1/clock"
137+
: > "$KLOG"
138+
rc=0
139+
PATH="$1:$PATH" sh "$1/hook.sh" > "$1/out" 2> "$1/err" || rc=$?
140+
}
141+
142+
@test "a render that produces no script fails instead of passing as a no-op" {
143+
# Guards the guard: every test below runs whatever render_hook produced, so an
144+
# empty render would satisfy the exit-0 assertions without executing a line of
145+
# the hook.
146+
tmp=$(mktemp -d)
147+
if render_hook no-such-job "$tmp/hook.sh"; then
148+
echo "FAIL: render_hook reported success for a Job name the chart never emits"
149+
false
150+
fi
151+
rm -rf "$tmp"
152+
}
153+
154+
@test "vm cleanup hook clears the finalizers of this release's objects only, after the grace period" {
155+
tmp=$(mktemp -d)
156+
write_fakes "$tmp"
157+
write_state "$tmp/state"
158+
render_hook vm-objects-cleanup "$tmp/hook.sh"
159+
export OPERATOR_LIVE=0 MISSING_KINDS=""
160+
run_hook "$tmp"
161+
162+
if [ "$rc" -ne 0 ]; then
163+
echo "FAIL: hook exited $rc"
164+
cat "$tmp/out" "$tmp/err"
165+
false
166+
fi
167+
for obj in "vmagents.operator.victoriametrics.com vmagent" \
168+
"vmrules.operator.victoriametrics.com alerts-node" \
169+
"vmservicescrapes.operator.victoriametrics.com operator"; do
170+
if ! grep -q -- " delete $obj --wait=false" "$tmp/calls"; then
171+
echo "FAIL: $obj was not deleted"
172+
cat "$tmp/calls"
173+
false
174+
fi
175+
done
176+
# The objects of this release must be gone, which the fake only does for a
177+
# patch that really clears the finalizers.
178+
if grep -q "^[a-z]*|$MA_RELEASE|$MA_NAMESPACE|" "$tmp/state"; then
179+
echo "FAIL: objects of this release are still there, Terminating"
180+
cat "$tmp/state" "$tmp/calls"
181+
false
182+
fi
183+
# Another release's object, one of a same-named release in another
184+
# namespace, and an object Helm does not own are never touched.
185+
if grep -E -q -- ' (delete|patch) [a-z.]+ (foreign-rule|foreign-ns-rule|unmanaged-scrape)( |$)' "$tmp/calls"; then
186+
echo "FAIL: the hook touched an object this release does not own"
187+
cat "$tmp/calls"
188+
false
189+
fi
190+
if [ "$(grep -c '|' "$tmp/state")" -ne 3 ]; then
191+
echo "FAIL: the three objects this release does not own should remain"
192+
cat "$tmp/state"
193+
false
194+
fi
195+
# The operator gets its chance first: the hook polls and sleeps before the
196+
# first patch, and does not stop waiting before the 60s are up.
197+
first_patch=$(grep -n ' patch ' "$tmp/calls" | head -1 | cut -d: -f1)
198+
if [ -z "$first_patch" ]; then
199+
echo "FAIL: no finalizer was cleared"
200+
cat "$tmp/calls"
201+
false
202+
fi
203+
# The first date call sets the deadline; the last one before the first patch
204+
# is the time the hook gave up waiting.
205+
start=$(grep '^date ' "$tmp/calls" | head -1 | cut -d' ' -f2)
206+
waited_until=$(head -n "$first_patch" "$tmp/calls" | grep '^date ' | tail -1 | cut -d' ' -f2)
207+
if [ -z "$start" ] || [ $((waited_until - start)) -lt 60 ]; then
208+
echo "FAIL: finalizers cleared after $((waited_until - start))s; the 60s grace period was cut short"
209+
cat "$tmp/calls"
210+
false
211+
fi
212+
rm -rf "$tmp"
213+
}
214+
215+
@test "vm cleanup hook clears the finalizers before its Job's deadline" {
216+
# The grace-period test above keeps sleep free so a shortened wait shows; here
217+
# sleep costs its argument, so a poll interval long enough to overshoot the
218+
# Job's activeDeadlineSeconds fails instead of passing.
219+
tmp=$(mktemp -d)
220+
write_fakes "$tmp"
221+
write_state "$tmp/state"
222+
render_hook vm-objects-cleanup "$tmp/hook.sh"
223+
job_deadline=$(helm template "$MA_RELEASE" "$MA_CHART" --namespace "$MA_NAMESPACE" \
224+
| yq eval 'select(.kind == "Job" and .metadata.name == "vm-objects-cleanup") | .spec.activeDeadlineSeconds' -)
225+
export OPERATOR_LIVE=0 MISSING_KINDS="" SLEEP_ADVANCES=1
226+
run_hook "$tmp"
227+
unset SLEEP_ADVANCES
228+
229+
if [ "$rc" -ne 0 ]; then
230+
echo "FAIL: hook exited $rc"
231+
cat "$tmp/out" "$tmp/err"
232+
false
233+
fi
234+
first_patch=$(grep -n ' patch ' "$tmp/calls" | head -1 | cut -d: -f1)
235+
start=$(grep '^date ' "$tmp/calls" | head -1 | cut -d' ' -f2)
236+
waited_until=$(head -n "${first_patch:-0}" "$tmp/calls" | grep '^date ' | tail -1 | cut -d' ' -f2)
237+
if [ -z "$first_patch" ] || [ -z "$job_deadline" ] || [ $((waited_until - start)) -ge "$job_deadline" ]; then
238+
echo "FAIL: first finalizer patch at $((waited_until - start))s, Job deadline ${job_deadline}s"
239+
cat "$tmp/calls"
240+
false
241+
fi
242+
rm -rf "$tmp"
243+
}
244+
245+
@test "vm cleanup hook leaves the finalizers to a live operator" {
246+
tmp=$(mktemp -d)
247+
write_fakes "$tmp"
248+
write_state "$tmp/state"
249+
render_hook vm-objects-cleanup "$tmp/hook.sh"
250+
export OPERATOR_LIVE=1 MISSING_KINDS=""
251+
run_hook "$tmp"
252+
253+
if [ "$rc" -ne 0 ]; then
254+
echo "FAIL: hook exited $rc"
255+
cat "$tmp/out" "$tmp/err"
256+
false
257+
fi
258+
if ! grep -q ' delete vmagents.operator.victoriametrics.com vmagent ' "$tmp/calls"; then
259+
echo "FAIL: the VMAgent was not deleted"
260+
cat "$tmp/calls"
261+
false
262+
fi
263+
if grep -q ' patch ' "$tmp/calls"; then
264+
echo "FAIL: the hook cleared finalizers the live operator was handling"
265+
cat "$tmp/calls"
266+
false
267+
fi
268+
rm -rf "$tmp"
269+
}
270+
271+
@test "vm cleanup hook exits 0 and cleans the other kinds when a CRD is gone" {
272+
tmp=$(mktemp -d)
273+
write_fakes "$tmp"
274+
write_state "$tmp/state"
275+
render_hook vm-objects-cleanup "$tmp/hook.sh"
276+
export OPERATOR_LIVE=0 MISSING_KINDS="vmrules vmpodscrapes vmnodescrapes"
277+
run_hook "$tmp"
278+
279+
if [ "$rc" -ne 0 ]; then
280+
echo "FAIL: hook exited $rc with a CRD gone, which would block the uninstall"
281+
cat "$tmp/out" "$tmp/err"
282+
false
283+
fi
284+
if grep -q "^vmagents|$MA_RELEASE|" "$tmp/state"; then
285+
echo "FAIL: the VMAgent was left Terminating because another kind's CRD was gone"
286+
cat "$tmp/state" "$tmp/calls"
287+
false
288+
fi
289+
rm -rf "$tmp"
290+
}
291+
292+
@test "vm cleanup hook exits 0 without deleting anything when the release owns nothing" {
293+
tmp=$(mktemp -d)
294+
write_fakes "$tmp"
295+
printf 'vmrules|other|%s|%s|foreign-rule\nvmrules|%s|other-namespace|%s|foreign-ns-rule\n' \
296+
"$MA_NAMESPACE" "$MA_NAMESPACE" "$MA_RELEASE" "$MA_NAMESPACE" > "$tmp/state"
297+
render_hook vm-objects-cleanup "$tmp/hook.sh"
298+
export OPERATOR_LIVE=0 MISSING_KINDS=""
299+
run_hook "$tmp"
300+
301+
if [ "$rc" -ne 0 ]; then
302+
echo "FAIL: hook exited $rc with nothing to clean"
303+
cat "$tmp/out" "$tmp/err"
304+
false
305+
fi
306+
if grep -E -q ' (delete|patch) ' "$tmp/calls"; then
307+
echo "FAIL: the hook deleted or patched with nothing of this release left"
308+
cat "$tmp/calls"
309+
false
310+
fi
311+
rm -rf "$tmp"
312+
}

0 commit comments

Comments
 (0)