Skip to content

Commit cac1520

Browse files
themoriartithemoriarti
committed
fix(e2e): read the KubernetesNodes pool release in the HelmRelease guard
cozy_guard_all_helmreleases reads the parent and the kubernetes-<test>- addons. The worker pool is kubernetes-nodes-<test>-md0, an Application HelmRelease the apiserver stamps with RetryOnFailure like the parent, on the node-join path through the talos-reconcile Job it renders. Its name does not start with the addon prefix, so a failed action on it was never read, on the passing path or in the failure-path report. Read it on both paths. The pool name is derived from the parent, the way the addon prefix is, so the two cannot drift. The failure-path report checks that the release exists first: a run that failed between the parent and the pool apply has none, and that is a fact about where the run stopped rather than a failed read. Closes #4737 Signed-off-by: Marian Koreniuk <[email protected]> Assisted-by: LLM
1 parent aed7fa8 commit cac1520

2 files changed

Lines changed: 195 additions & 9 deletions

File tree

‎hack/e2e-chainsaw/_lib/run-kubernetes.sh‎

Lines changed: 34 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5230,9 +5230,10 @@ _cozy_guard_kubectl() {
52305230
cozy_guard_helmrelease() {
52315231
local _ns="$1"
52325232
local _hr="$2"
5233-
# Non-empty on a release whose install strategy the cluster is known to set,
5234-
# which is the parent Application HelmRelease and nothing else here. See the
5235-
# branch that reads it for why the check cannot be applied to the addons.
5233+
# Non-empty on a release whose install strategy the cluster is known to set:
5234+
# the parent Application HelmRelease and the worker pool release, which is an
5235+
# Application HelmRelease too. See the branch that reads it for why the check
5236+
# cannot be applied to the addons.
52365237
local _strategy_required="${3:-}"
52375238
local _probe _strategy _history _ready _read_rc
52385239
_cozy_guard_read_bounds
@@ -5437,16 +5438,32 @@ cozy_guard_addon_helmreleases() {
54375438
# keeps the parent's own verdict, which an early return past the addons would
54385439
# drop.
54395440
#
5441+
# The worker pool release of parent kubernetes-<test>. The pool is a
5442+
# KubernetesNodes named <test>-md0 (see the apply in the run), and the
5443+
# aggregated apiserver names its HelmRelease kubernetes-nodes-<test>-md0. That
5444+
# name does not start with the addon prefix kubernetes-<test>-, so the addon
5445+
# listing never selects it, and it is read on its own. Derived from the parent
5446+
# rather than passed in, for the same reason as the addon prefix below.
5447+
cozy_pool_helmrelease_name() {
5448+
local _parent="$1"
5449+
printf 'kubernetes-nodes-%s-md0\n' "${_parent#kubernetes-}"
5450+
}
5451+
54405452
# The addon prefix is derived here rather than passed in, so the parent name and
54415453
# the prefix that selects its children cannot drift apart.
54425454
cozy_guard_all_helmreleases() {
54435455
local _ns="$1"
54445456
local _parent="$2"
54455457
local _failed=0
5446-
# The parent is asked to prove its install strategy read back non-empty; the
5447-
# addons are not, because most of them legitimately set none. See the branch
5448-
# in cozy_guard_helmrelease that reads this argument.
5458+
# The parent and the pool release are asked to prove their install strategy
5459+
# read back non-empty: both are Application HelmReleases, which the apiserver
5460+
# stamps. The addons are not, because most of them legitimately set none. See
5461+
# the branch in cozy_guard_helmrelease that reads this argument.
54495462
cozy_guard_helmrelease "$_ns" "${_parent}" required || _failed=1
5463+
# The pool release sits on the node-join path, through the talos-reconcile
5464+
# Job it renders, so a failed action on it is as much a finding as one on the
5465+
# parent. By the time this runs the pool has scaled, so the release exists.
5466+
cozy_guard_helmrelease "$_ns" "$(cozy_pool_helmrelease_name "${_parent}")" required || _failed=1
54505467
cozy_guard_addon_helmreleases "$_ns" "${_parent}-" || _failed=1
54515468
# These releases are now classified, and the same flag the failure-path report
54525469
# uses records it. The two are not alternatives: this call runs before the
@@ -5515,7 +5532,7 @@ cozy_report_helmrelease_remediation() {
55155532
local _ns="$1"
55165533
local _parent="$2"
55175534
local _label="${3:-}"
5518-
local _addon_hrs _hr _deadline _budget
5535+
local _addon_hrs _hr _pool _deadline _budget
55195536
[ "${_COZY_GUARD_REPORT_DONE}" -eq 0 ] || return 0
55205537
# An empty parent name means the run failed before the release was created, so
55215538
# there is no history anywhere to read yet.
@@ -5541,6 +5558,16 @@ cozy_report_helmrelease_remediation() {
55415558
cozy_diag_phase_has_time "HelmRelease remediation footprint of ${_parent}" || return 0
55425559
echo "=== ${_label}tenant HelmRelease remediation footprint (management cluster, ns ${_ns}) — classification only, this does not decide the run ==="
55435560
cozy_guard_helmrelease "$_ns" "${_parent}" required || true
5561+
# The pool release is created by a later apply than the parent, so a run that
5562+
# failed between the two has none to read. Its absence is then a fact about
5563+
# where the run stopped, not a failed read, and is reported as such.
5564+
_pool=$(cozy_pool_helmrelease_name "${_parent}")
5565+
cozy_diag_phase_has_time "HelmRelease remediation footprint of ${_pool}" || return 0
5566+
if _cozy_guard_kubectl get hr -n "$_ns" "${_pool}" -o name >/dev/null 2>&1; then
5567+
cozy_guard_helmrelease "$_ns" "${_pool}" required || true
5568+
else
5569+
echo "» ${_pool} does not exist - the run ended before the worker pool was applied, or its HelmRelease was never created; nothing to read"
5570+
fi
55445571
cozy_diag_phase_has_time "the addon HelmRelease listing of ${_parent}" || return 0
55455572
if [ "$(date +%s)" -ge "${_deadline}" ]; then
55465573
echo "=== HelmRelease remediation footprint: stopped after ${_budget}s; the addon releases of ${_parent} were not listed, so nothing was observed about any of them ===" >&2

‎hack/run-kubernetes-remediation_test.bats‎

Lines changed: 161 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/usr/bin/env bats
22
# -----------------------------------------------------------------------------
3-
# Unit tests for cozy_guard_addon_helmreleases in
3+
# Unit tests for cozy_guard_all_helmreleases and cozy_guard_addon_helmreleases in
44
# hack/e2e-chainsaw/_lib/run-kubernetes.sh
55
#
66
# cozy_guard_helmrelease judges one release; cozy_guard_addon_helmreleases runs
@@ -32,7 +32,8 @@ load test_helper
3232
# created. The stub answers the shapes the guard issues: the namespace listing
3333
# (from ALL_HR), one release's install strategy and Ready condition together
3434
# (<name>.strategy and <name>.ready, joined the way the guard's jsonpath joins
35-
# them), and its history statuses (<name>.history). A missing fixture file
35+
# them), its history statuses (<name>.history), and whether it exists at all
36+
# (`-o name`, which an ABSENT marker turns into NotFound). A missing fixture file
3637
# stands for a field the object does not carry, which is what kubectl prints for
3738
# an absent jsonpath - nothing.
3839
#
@@ -96,6 +97,7 @@ for a in "$@"; do
9697
describe) mode=describe ;;
9798
*'status.history'*) mode=history ;;
9899
*'install.strategy.name'*) mode=probe ;;
100+
name) mode=exists ;;
99101
kubernetes-*) name=$a ;;
100102
esac
101103
done
@@ -145,6 +147,15 @@ case "$mode" in
145147
"$(cat "$FIXTURES/$name.strategy" 2>/dev/null || true)" \
146148
"$(cat "$FIXTURES/$name.ready" 2>/dev/null || true)"
147149
;;
150+
exists)
151+
# `get hr <name> -o name`: NotFound is a non-zero exit with the message on
152+
# stderr, which an ABSENT marker reproduces; anything else exists.
153+
if [ -f "$FIXTURES/$name.ABSENT" ]; then
154+
echo "Error from server (NotFound): helmreleases.helm.toolkit.fluxcd.io \"$name\" not found" >&2
155+
exit 1
156+
fi
157+
echo "helmrelease.helm.toolkit.fluxcd.io/$name"
158+
;;
148159
describe) echo "stub describe of $name" ;;
149160
esac
150161
exit 0
@@ -252,6 +263,154 @@ STUB
252263
rm -rf "$tmp"
253264
}
254265

266+
@test "the pool release is read by the full guard, and its footprint fails the run" {
267+
# The worker pool is kubernetes-nodes-<test>-md0, an Application HelmRelease
268+
# the apiserver stamps like the parent, on the node-join path through the
269+
# talos-reconcile Job it renders. Its name does not start with the addon
270+
# prefix kubernetes-<test>-, so the addon listing never selects it, and
271+
# before this read a failed action on it was invisible to the guard on every
272+
# path. Parent and addon are clean here; the pool alone carries the finding.
273+
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
274+
tmp=$(mktemp -d)
275+
cozy_make_kubectl_stub "$tmp"
276+
export FIXTURES="$tmp/fixtures"
277+
printf 'kubernetes-test-latest-version-coredns\n' > "$FIXTURES/ALL_HR"
278+
printf 'RetryOnFailure' > "$FIXTURES/kubernetes-test-latest-version.strategy"
279+
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version.history"
280+
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version-coredns.history"
281+
printf 'RetryOnFailure' > "$FIXTURES/kubernetes-nodes-test-latest-version-md0.strategy"
282+
printf 'failed\ndeployed\n' > "$FIXTURES/kubernetes-nodes-test-latest-version-md0.history"
283+
284+
rc=0
285+
out=$(PATH="$tmp/bin:$PATH" cozy_guard_all_helmreleases tenant-test kubernetes-test-latest-version 2>&1) || rc=$?
286+
if [ "$rc" -eq 0 ]; then
287+
echo "expected the pool release footprint to fail the run, got: $out" >&2
288+
exit 1
289+
fi
290+
if ! printf '%s\n' "$out" | grep -q -F 'HelmRelease kubernetes-nodes-test-latest-version-md0 carries a failed or uninstalled Snapshot, and its install strategy is RetryOnFailure'; then
291+
echo "expected the pool release to be named in the verdict, got: $out" >&2
292+
exit 1
293+
fi
294+
if ! grep -q -F 'kubernetes-nodes-test-latest-version-md0' "$FIXTURES/CALLS"; then
295+
echo "expected a read of the pool release, calls were: $(cat "$FIXTURES/CALLS")" >&2
296+
exit 1
297+
fi
298+
299+
rm -rf "$tmp"
300+
}
301+
302+
@test "a clean pool release leaves the full guard green" {
303+
# The pin for the passing path: with every release clean the composite
304+
# returns zero, so the pool read above is a read and not an unconditional
305+
# verdict. The pool name is derived from the parent, the way the addon
306+
# prefix is, which this fixture spells out.
307+
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
308+
tmp=$(mktemp -d)
309+
cozy_make_kubectl_stub "$tmp"
310+
export FIXTURES="$tmp/fixtures"
311+
printf 'kubernetes-test-latest-version-coredns\n' > "$FIXTURES/ALL_HR"
312+
printf 'RetryOnFailure' > "$FIXTURES/kubernetes-test-latest-version.strategy"
313+
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version.history"
314+
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version-coredns.history"
315+
printf 'RetryOnFailure' > "$FIXTURES/kubernetes-nodes-test-latest-version-md0.strategy"
316+
printf 'deployed\n' > "$FIXTURES/kubernetes-nodes-test-latest-version-md0.history"
317+
318+
PATH="$tmp/bin:$PATH" cozy_guard_all_helmreleases tenant-test kubernetes-test-latest-version
319+
[ "$(cozy_pool_helmrelease_name kubernetes-test-latest-version)" = kubernetes-nodes-test-latest-version-md0 ]
320+
321+
rm -rf "$tmp"
322+
}
323+
324+
@test "the pool release is held to a stamped install strategy like the parent" {
325+
# The pool is an Application HelmRelease, so an empty strategy there is the
326+
# read and never the object, the same as on the parent. Without this the
327+
# jsonpath rename the parent check exists for would leave the pool's
328+
# footprint on the lenient branch while the parent's check still passed.
329+
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
330+
tmp=$(mktemp -d)
331+
cozy_make_kubectl_stub "$tmp"
332+
export FIXTURES="$tmp/fixtures"
333+
printf 'kubernetes-test-latest-version-coredns\n' > "$FIXTURES/ALL_HR"
334+
printf 'RetryOnFailure' > "$FIXTURES/kubernetes-test-latest-version.strategy"
335+
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version.history"
336+
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version-coredns.history"
337+
printf 'True' > "$FIXTURES/kubernetes-nodes-test-latest-version-md0.ready"
338+
printf 'deployed\n' > "$FIXTURES/kubernetes-nodes-test-latest-version-md0.history"
339+
340+
rc=0
341+
out=$(PATH="$tmp/bin:$PATH" cozy_guard_all_helmreleases tenant-test kubernetes-test-latest-version 2>&1) || rc=$?
342+
if [ "$rc" -eq 0 ]; then
343+
echo "expected an empty install strategy on the pool release to fail the run, got: $out" >&2
344+
exit 1
345+
fi
346+
if ! printf '%s\n' "$out" | grep -q -F 'spec.install.strategy.name of kubernetes-nodes-test-latest-version-md0 came back empty'; then
347+
echo "expected the failure to name the pool release, got: $out" >&2
348+
exit 1
349+
fi
350+
351+
rm -rf "$tmp"
352+
}
353+
354+
@test "the failure-path report classifies the pool release too" {
355+
# Same reading as the passing path, from the exit handler: a pool release
356+
# that failed an action is named, and the report still returns zero.
357+
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
358+
tmp=$(mktemp -d)
359+
cozy_make_kubectl_stub "$tmp"
360+
export FIXTURES="$tmp/fixtures"
361+
printf 'kubernetes-test-latest-version-coredns\n' > "$FIXTURES/ALL_HR"
362+
printf 'RetryOnFailure' > "$FIXTURES/kubernetes-test-latest-version.strategy"
363+
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version.history"
364+
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version-coredns.history"
365+
printf 'RetryOnFailure' > "$FIXTURES/kubernetes-nodes-test-latest-version-md0.strategy"
366+
printf 'failed\ndeployed\n' > "$FIXTURES/kubernetes-nodes-test-latest-version-md0.history"
367+
368+
rc=0
369+
out=$(PATH="$tmp/bin:$PATH" cozy_report_helmrelease_remediation tenant-test kubernetes-test-latest-version 2>&1) || rc=$?
370+
if [ "$rc" -ne 0 ]; then
371+
echo "expected the report to return zero, got rc=$rc: $out" >&2
372+
exit 1
373+
fi
374+
if ! printf '%s\n' "$out" | grep -q -F 'HelmRelease kubernetes-nodes-test-latest-version-md0 carries a failed or uninstalled Snapshot'; then
375+
echo "expected the pool release footprint to be classified, got: $out" >&2
376+
exit 1
377+
fi
378+
379+
rm -rf "$tmp"
380+
}
381+
382+
@test "the failure-path report says when the pool release was never created" {
383+
# The pool is applied after the parent, so a run that failed between the two
384+
# has no pool release. That is a fact about where the run stopped, and the
385+
# report says so instead of reporting a failed read of a missing object.
386+
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
387+
tmp=$(mktemp -d)
388+
cozy_make_kubectl_stub "$tmp"
389+
export FIXTURES="$tmp/fixtures"
390+
printf 'kubernetes-test-latest-version-coredns\n' > "$FIXTURES/ALL_HR"
391+
printf 'RetryOnFailure' > "$FIXTURES/kubernetes-test-latest-version.strategy"
392+
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version.history"
393+
printf 'deployed\n' > "$FIXTURES/kubernetes-test-latest-version-coredns.history"
394+
: > "$FIXTURES/kubernetes-nodes-test-latest-version-md0.ABSENT"
395+
396+
rc=0
397+
out=$(PATH="$tmp/bin:$PATH" cozy_report_helmrelease_remediation tenant-test kubernetes-test-latest-version 2>&1) || rc=$?
398+
if [ "$rc" -ne 0 ]; then
399+
echo "expected the report to return zero, got rc=$rc: $out" >&2
400+
exit 1
401+
fi
402+
if ! printf '%s\n' "$out" | grep -q -F 'kubernetes-nodes-test-latest-version-md0 does not exist'; then
403+
echo "expected the report to say the pool release does not exist, got: $out" >&2
404+
exit 1
405+
fi
406+
if printf '%s\n' "$out" | grep -q -F 'Reading .spec.install.strategy.name and the Ready condition of kubernetes-nodes-test-latest-version-md0 failed'; then
407+
echo "expected no failed read of the absent pool release, got: $out" >&2
408+
exit 1
409+
fi
410+
411+
rm -rf "$tmp"
412+
}
413+
255414
@test "single-release guard fails a RetryOnFailure release that failed and recovered" {
256415
# The entry point the parent kubernetes-<test> release goes through, and the
257416
# shape the rule exists for. The apiserver stamps RetryOnFailure on every

0 commit comments

Comments
 (0)