|
| 1 | +#!/usr/bin/env bats |
| 2 | +# Behavioural tests for the monitoring-agents pre-delete hook: |
| 3 | +# |
| 4 | +# packages/system/monitoring-agents/templates/vm-cleanup-hook.yaml |
| 5 | +# |
| 6 | +# The script is extracted from the rendered chart and RUN against a fake |
| 7 | +# kubectl that keeps the VictoriaMetrics objects in a state file, with fake |
| 8 | +# `date` and `sleep` so the 60s grace period passes in an instant. The |
| 9 | +# helm-unittest suite in packages/system/monitoring-agents/tests/ |
| 10 | +# uninstall_test.yaml matches the script text instead, and stayed green when the |
| 11 | +# ownership condition was removed, when the finalizer patch was made a no-op and |
| 12 | +# when the polling condition was inverted. Each of those turns this file red. |
| 13 | +# |
| 14 | +# What is pinned here: |
| 15 | +# * only objects whose Helm annotations name this release, by name and |
| 16 | +# namespace both, are deleted or patched: an object of another release, of |
| 17 | +# a same-named release in another namespace, or one Helm does not own, is |
| 18 | +# left alone; |
| 19 | +# * with the operator gone, the hook waits before touching finalizers, then |
| 20 | +# clears them, and nothing of this release is left behind; |
| 21 | +# * with the operator alive, it finalizes the objects itself and the hook |
| 22 | +# never patches a finalizer; |
| 23 | +# * a kind whose CRD is gone, or a release with nothing left, exits 0, since a |
| 24 | +# failing pre-delete hook would itself block the uninstall. |
| 25 | +# |
| 26 | +# Run from the repo root with make bats-unit-tests. Direct status checks |
| 27 | +# retain compatibility with cozytest, which has no Bats run/setup helpers. |
| 28 | +# Inline cleanup preserves Bats' TAP handler on failed assertions. |
| 29 | + |
| 30 | +load test_helper |
| 31 | + |
| 32 | +MA_CHART=packages/system/monitoring-agents |
| 33 | +MA_RELEASE=monitoring-agents |
| 34 | +MA_NAMESPACE=cozy-monitoring |
| 35 | + |
| 36 | +# render_hook <job-name> <out> |
| 37 | +# Extract the named Job's shell script from the rendered chart into <out>. The |
| 38 | +# non-empty check must be an early `return 1`: hack/cozytest.sh rewrites every |
| 39 | +# line matching ^}$ into `return 0` plus `}`, helpers included, so a check left |
| 40 | +# in last-command position has its status discarded. |
| 41 | +render_hook() { |
| 42 | + helm template "$MA_RELEASE" "$MA_CHART" --namespace "$MA_NAMESPACE" \ |
| 43 | + | yq eval "select(.kind == \"Job\" and .metadata.name == \"$1\") | .spec.template.spec.containers[0].command[2]" - > "$2" |
| 44 | + [ -s "$2" ] || return 1 |
| 45 | +} |
| 46 | + |
| 47 | +# write_fakes <dir> |
| 48 | +# Emit fake `kubectl`, `date` and `sleep` into <dir>. |
| 49 | +# |
| 50 | +# kubectl reads and writes $STATE, one object per line as |
| 51 | +# kind|release-name|release-namespace|namespace|name, where the two release |
| 52 | +# fields are the Helm annotations (empty when Helm does not own the object). |
| 53 | +# `get` prints this kind's objects in the hook's jsonpath format, or fails for a |
| 54 | +# kind listed in $MISSING_KINDS as kubectl does when the CRD is gone. `delete` |
| 55 | +# removes the object only when $OPERATOR_LIVE is 1, standing for an operator |
| 56 | +# that finalizes it; otherwise the object stays, Terminating. `patch` removes it |
| 57 | +# only when the payload actually clears the finalizers. Every call goes to $KLOG. |
| 58 | +# |
| 59 | +# date advances 1s per call from $CLOCK, so the 60s wait ends after sixty |
| 60 | +# of polls, and records each time it returns in $KLOG; sleep records itself, |
| 61 | +# and with $SLEEP_ADVANCES=1 also moves $CLOCK by its argument. The log thus |
| 62 | +# shows how much fake time passed before each patch. |
| 63 | +# No line below is a bare column-0 `}`, so cozytest.sh's parser leaves it alone. |
| 64 | +write_fakes() { |
| 65 | + cat > "$1/kubectl" <<'KEOF' |
| 66 | +#!/bin/sh |
| 67 | +echo "kubectl $*" >> "$KLOG" |
| 68 | +drop() { |
| 69 | + awk -F'|' -v k="$1" -v ns="$2" -v n="$3" '!($1 == k && $4 == ns && $5 == n)' "$STATE" > "$STATE.new" |
| 70 | + mv "$STATE.new" "$STATE"; } |
| 71 | +case "$1" in |
| 72 | + get) |
| 73 | + kind=${2%%.*} |
| 74 | + case " ${MISSING_KINDS:-} " in |
| 75 | + *" $kind "*) |
| 76 | + echo "error: the server doesn't have a resource type \"$kind\"" >&2 |
| 77 | + exit 1 ;; |
| 78 | + esac |
| 79 | + awk -F'|' -v k="$kind" '$1 == k { print $2 "|" $3 "|" $4 "|" $5 }' "$STATE" |
| 80 | + exit 0 ;; |
| 81 | + -n) |
| 82 | + ns=$2 verb=$3 kind=${4%%.*} name=$5 |
| 83 | + case "$verb" in |
| 84 | + delete) |
| 85 | + [ "${OPERATOR_LIVE:-0}" = 1 ] && drop "$kind" "$ns" "$name" |
| 86 | + echo "$kind \"$name\" deleted" |
| 87 | + exit 0 ;; |
| 88 | + patch) |
| 89 | + case "$*" in |
| 90 | + *'"finalizers":null'*) drop "$kind" "$ns" "$name" ;; |
| 91 | + esac |
| 92 | + echo "$kind \"$name\" patched" |
| 93 | + exit 0 ;; |
| 94 | + esac ;; |
| 95 | +esac |
| 96 | +echo "fake kubectl: unexpected call: $*" >&2 |
| 97 | +exit 64 |
| 98 | +KEOF |
| 99 | + cat > "$1/date" <<'KEOF' |
| 100 | +#!/bin/sh |
| 101 | +c=$(cat "$CLOCK" 2>/dev/null || echo 0) |
| 102 | +echo $((c + 1)) > "$CLOCK" |
| 103 | +echo "date $((1000000 + c))" >> "$KLOG" |
| 104 | +echo $((1000000 + c)) |
| 105 | +KEOF |
| 106 | + cat > "$1/sleep" <<'KEOF' |
| 107 | +#!/bin/sh |
| 108 | +echo "sleep $*" >> "$KLOG" |
| 109 | +if [ "${SLEEP_ADVANCES:-0}" = 1 ]; then |
| 110 | + c=$(cat "$CLOCK" 2>/dev/null || echo 0) |
| 111 | + echo $((c + $1)) > "$CLOCK" |
| 112 | +fi |
| 113 | +KEOF |
| 114 | + chmod +x "$1/kubectl" "$1/date" "$1/sleep" |
| 115 | +} |
| 116 | + |
| 117 | +# write_state <file> |
| 118 | +# Three objects of this release in two namespaces; a VMRule of another release; |
| 119 | +# a VMRule whose release name matches but whose release namespace does not, so |
| 120 | +# only the pair of annotations tells it apart; and a VMServiceScrape that |
| 121 | +# carries no Helm annotations. |
| 122 | +write_state() { |
| 123 | + cat > "$1" <<EOF |
| 124 | +vmagents|$MA_RELEASE|$MA_NAMESPACE|$MA_NAMESPACE|vmagent |
| 125 | +vmrules|$MA_RELEASE|$MA_NAMESPACE|$MA_NAMESPACE|alerts-node |
| 126 | +vmservicescrapes|$MA_RELEASE|$MA_NAMESPACE|cozy-victoria-metrics-operator|operator |
| 127 | +vmrules|other|$MA_NAMESPACE|$MA_NAMESPACE|foreign-rule |
| 128 | +vmrules|$MA_RELEASE|other-namespace|$MA_NAMESPACE|foreign-ns-rule |
| 129 | +vmservicescrapes|||$MA_NAMESPACE|unmanaged-scrape |
| 130 | +EOF |
| 131 | +} |
| 132 | + |
| 133 | +# run_hook <dir> |
| 134 | +# Run the rendered hook with the fakes first on PATH; its status lands in $rc. |
| 135 | +run_hook() { |
| 136 | + export KLOG="$1/calls" STATE="$1/state" CLOCK="$1/clock" |
| 137 | + : > "$KLOG" |
| 138 | + rc=0 |
| 139 | + PATH="$1:$PATH" sh "$1/hook.sh" > "$1/out" 2> "$1/err" || rc=$? |
| 140 | +} |
| 141 | + |
| 142 | +@test "a render that produces no script fails instead of passing as a no-op" { |
| 143 | + # Guards the guard: every test below runs whatever render_hook produced, so an |
| 144 | + # empty render would satisfy the exit-0 assertions without executing a line of |
| 145 | + # the hook. |
| 146 | + tmp=$(mktemp -d) |
| 147 | + if render_hook no-such-job "$tmp/hook.sh"; then |
| 148 | + echo "FAIL: render_hook reported success for a Job name the chart never emits" |
| 149 | + false |
| 150 | + fi |
| 151 | + rm -rf "$tmp" |
| 152 | +} |
| 153 | + |
| 154 | +@test "vm cleanup hook clears the finalizers of this release's objects only, after the grace period" { |
| 155 | + tmp=$(mktemp -d) |
| 156 | + write_fakes "$tmp" |
| 157 | + write_state "$tmp/state" |
| 158 | + render_hook vm-objects-cleanup "$tmp/hook.sh" |
| 159 | + export OPERATOR_LIVE=0 MISSING_KINDS="" |
| 160 | + run_hook "$tmp" |
| 161 | + |
| 162 | + if [ "$rc" -ne 0 ]; then |
| 163 | + echo "FAIL: hook exited $rc" |
| 164 | + cat "$tmp/out" "$tmp/err" |
| 165 | + false |
| 166 | + fi |
| 167 | + for obj in "vmagents.operator.victoriametrics.com vmagent" \ |
| 168 | + "vmrules.operator.victoriametrics.com alerts-node" \ |
| 169 | + "vmservicescrapes.operator.victoriametrics.com operator"; do |
| 170 | + if ! grep -q -- " delete $obj --wait=false" "$tmp/calls"; then |
| 171 | + echo "FAIL: $obj was not deleted" |
| 172 | + cat "$tmp/calls" |
| 173 | + false |
| 174 | + fi |
| 175 | + done |
| 176 | + # The objects of this release must be gone, which the fake only does for a |
| 177 | + # patch that really clears the finalizers. |
| 178 | + if grep -q "^[a-z]*|$MA_RELEASE|$MA_NAMESPACE|" "$tmp/state"; then |
| 179 | + echo "FAIL: objects of this release are still there, Terminating" |
| 180 | + cat "$tmp/state" "$tmp/calls" |
| 181 | + false |
| 182 | + fi |
| 183 | + # Another release's object, one of a same-named release in another |
| 184 | + # namespace, and an object Helm does not own are never touched. |
| 185 | + if grep -E -q -- ' (delete|patch) [a-z.]+ (foreign-rule|foreign-ns-rule|unmanaged-scrape)( |$)' "$tmp/calls"; then |
| 186 | + echo "FAIL: the hook touched an object this release does not own" |
| 187 | + cat "$tmp/calls" |
| 188 | + false |
| 189 | + fi |
| 190 | + if [ "$(grep -c '|' "$tmp/state")" -ne 3 ]; then |
| 191 | + echo "FAIL: the three objects this release does not own should remain" |
| 192 | + cat "$tmp/state" |
| 193 | + false |
| 194 | + fi |
| 195 | + # The operator gets its chance first: the hook polls and sleeps before the |
| 196 | + # first patch, and does not stop waiting before the 60s are up. |
| 197 | + first_patch=$(grep -n ' patch ' "$tmp/calls" | head -1 | cut -d: -f1) |
| 198 | + if [ -z "$first_patch" ]; then |
| 199 | + echo "FAIL: no finalizer was cleared" |
| 200 | + cat "$tmp/calls" |
| 201 | + false |
| 202 | + fi |
| 203 | + # The first date call sets the deadline; the last one before the first patch |
| 204 | + # is the time the hook gave up waiting. |
| 205 | + start=$(grep '^date ' "$tmp/calls" | head -1 | cut -d' ' -f2) |
| 206 | + waited_until=$(head -n "$first_patch" "$tmp/calls" | grep '^date ' | tail -1 | cut -d' ' -f2) |
| 207 | + if [ -z "$start" ] || [ $((waited_until - start)) -lt 60 ]; then |
| 208 | + echo "FAIL: finalizers cleared after $((waited_until - start))s; the 60s grace period was cut short" |
| 209 | + cat "$tmp/calls" |
| 210 | + false |
| 211 | + fi |
| 212 | + rm -rf "$tmp" |
| 213 | +} |
| 214 | + |
| 215 | +@test "vm cleanup hook clears the finalizers before its Job's deadline" { |
| 216 | + # The grace-period test above keeps sleep free so a shortened wait shows; here |
| 217 | + # sleep costs its argument, so a poll interval long enough to overshoot the |
| 218 | + # Job's activeDeadlineSeconds fails instead of passing. |
| 219 | + tmp=$(mktemp -d) |
| 220 | + write_fakes "$tmp" |
| 221 | + write_state "$tmp/state" |
| 222 | + render_hook vm-objects-cleanup "$tmp/hook.sh" |
| 223 | + job_deadline=$(helm template "$MA_RELEASE" "$MA_CHART" --namespace "$MA_NAMESPACE" \ |
| 224 | + | yq eval 'select(.kind == "Job" and .metadata.name == "vm-objects-cleanup") | .spec.activeDeadlineSeconds' -) |
| 225 | + export OPERATOR_LIVE=0 MISSING_KINDS="" SLEEP_ADVANCES=1 |
| 226 | + run_hook "$tmp" |
| 227 | + unset SLEEP_ADVANCES |
| 228 | + |
| 229 | + if [ "$rc" -ne 0 ]; then |
| 230 | + echo "FAIL: hook exited $rc" |
| 231 | + cat "$tmp/out" "$tmp/err" |
| 232 | + false |
| 233 | + fi |
| 234 | + first_patch=$(grep -n ' patch ' "$tmp/calls" | head -1 | cut -d: -f1) |
| 235 | + start=$(grep '^date ' "$tmp/calls" | head -1 | cut -d' ' -f2) |
| 236 | + waited_until=$(head -n "${first_patch:-0}" "$tmp/calls" | grep '^date ' | tail -1 | cut -d' ' -f2) |
| 237 | + if [ -z "$first_patch" ] || [ -z "$job_deadline" ] || [ $((waited_until - start)) -ge "$job_deadline" ]; then |
| 238 | + echo "FAIL: first finalizer patch at $((waited_until - start))s, Job deadline ${job_deadline}s" |
| 239 | + cat "$tmp/calls" |
| 240 | + false |
| 241 | + fi |
| 242 | + rm -rf "$tmp" |
| 243 | +} |
| 244 | + |
| 245 | +@test "vm cleanup hook leaves the finalizers to a live operator" { |
| 246 | + tmp=$(mktemp -d) |
| 247 | + write_fakes "$tmp" |
| 248 | + write_state "$tmp/state" |
| 249 | + render_hook vm-objects-cleanup "$tmp/hook.sh" |
| 250 | + export OPERATOR_LIVE=1 MISSING_KINDS="" |
| 251 | + run_hook "$tmp" |
| 252 | + |
| 253 | + if [ "$rc" -ne 0 ]; then |
| 254 | + echo "FAIL: hook exited $rc" |
| 255 | + cat "$tmp/out" "$tmp/err" |
| 256 | + false |
| 257 | + fi |
| 258 | + if ! grep -q ' delete vmagents.operator.victoriametrics.com vmagent ' "$tmp/calls"; then |
| 259 | + echo "FAIL: the VMAgent was not deleted" |
| 260 | + cat "$tmp/calls" |
| 261 | + false |
| 262 | + fi |
| 263 | + if grep -q ' patch ' "$tmp/calls"; then |
| 264 | + echo "FAIL: the hook cleared finalizers the live operator was handling" |
| 265 | + cat "$tmp/calls" |
| 266 | + false |
| 267 | + fi |
| 268 | + rm -rf "$tmp" |
| 269 | +} |
| 270 | + |
| 271 | +@test "vm cleanup hook exits 0 and cleans the other kinds when a CRD is gone" { |
| 272 | + tmp=$(mktemp -d) |
| 273 | + write_fakes "$tmp" |
| 274 | + write_state "$tmp/state" |
| 275 | + render_hook vm-objects-cleanup "$tmp/hook.sh" |
| 276 | + export OPERATOR_LIVE=0 MISSING_KINDS="vmrules vmpodscrapes vmnodescrapes" |
| 277 | + run_hook "$tmp" |
| 278 | + |
| 279 | + if [ "$rc" -ne 0 ]; then |
| 280 | + echo "FAIL: hook exited $rc with a CRD gone, which would block the uninstall" |
| 281 | + cat "$tmp/out" "$tmp/err" |
| 282 | + false |
| 283 | + fi |
| 284 | + if grep -q "^vmagents|$MA_RELEASE|" "$tmp/state"; then |
| 285 | + echo "FAIL: the VMAgent was left Terminating because another kind's CRD was gone" |
| 286 | + cat "$tmp/state" "$tmp/calls" |
| 287 | + false |
| 288 | + fi |
| 289 | + rm -rf "$tmp" |
| 290 | +} |
| 291 | + |
| 292 | +@test "vm cleanup hook exits 0 without deleting anything when the release owns nothing" { |
| 293 | + tmp=$(mktemp -d) |
| 294 | + write_fakes "$tmp" |
| 295 | + printf 'vmrules|other|%s|%s|foreign-rule\nvmrules|%s|other-namespace|%s|foreign-ns-rule\n' \ |
| 296 | + "$MA_NAMESPACE" "$MA_NAMESPACE" "$MA_RELEASE" "$MA_NAMESPACE" > "$tmp/state" |
| 297 | + render_hook vm-objects-cleanup "$tmp/hook.sh" |
| 298 | + export OPERATOR_LIVE=0 MISSING_KINDS="" |
| 299 | + run_hook "$tmp" |
| 300 | + |
| 301 | + if [ "$rc" -ne 0 ]; then |
| 302 | + echo "FAIL: hook exited $rc with nothing to clean" |
| 303 | + cat "$tmp/out" "$tmp/err" |
| 304 | + false |
| 305 | + fi |
| 306 | + if grep -E -q ' (delete|patch) ' "$tmp/calls"; then |
| 307 | + echo "FAIL: the hook deleted or patched with nothing of this release left" |
| 308 | + cat "$tmp/calls" |
| 309 | + false |
| 310 | + fi |
| 311 | + rm -rf "$tmp" |
| 312 | +} |
0 commit comments