Skip to content

Commit f0b99da

Browse files
committed
test(kubernetes): make kubectl_wait_retry set -e safe
The chainsaw scripts source run-kubernetes.sh under `set -eu`, where the wrapper's bare `_out=$(kubectl wait ... 2>&1)` on a failing wait aborts the whole sourcing script at the assignment -- before `_rc=$?`, before the retry loop, before anything prints, with the error text captured into the substitution and lost. Every `kubectl wait` failure therefore killed the suite instantly and silently, and the transient-retry machinery the wrapper exists for (etcd leader flap et al.) has never been reachable. This is how the kubernetes-latest suite died in CI with an empty stderr: the wait it was executing failed, and the script simply stopped. Guard the assignment (`_rc=0; _out=$(...) || _rc=$?`) so the wrapper's own logic decides: allowlisted transient errors retry with a visible notice, everything else prints and propagates, exactly the documented contract. New unit suite executes the wrapper in a `sh -eu` subshell against a fake kubectl, the same way chainsaw runs it: a transient etcd error must be retried with the script surviving (against the previous lib this exits 1 after one call with empty output), and NotFound must not be retried, must fail, and must be printed. Assisted-By: Claude <[email protected]> Signed-off-by: Timofei Larkin <[email protected]>
1 parent 847c83a commit f0b99da

3 files changed

Lines changed: 94 additions & 2 deletions

File tree

‎hack/e2e-chainsaw/_lib/run-kubernetes.sh‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,19 @@
2121
# This wrapper retries a small number of times against a curated allowlist
2222
# of transient server-side signatures. It does NOT swallow legitimate
2323
# timeouts (`--timeout=... expired`) or NotFound; those still surface.
24+
#
25+
# The capture guards the assignment with `|| _rc=$?`: the chainsaw scripts run
26+
# under `set -eu`, where a bare `_out=$(kubectl wait ...)` on a failing wait
27+
# aborts the WHOLE sourcing script right there -- before the retry loop, before
28+
# anything prints (stderr is captured into _out and dies with the shell). That
29+
# exact shape killed the kubernetes-latest suite silently the moment a wait
30+
# failed, with the transient-retry machinery never once reachable.
2431
kubectl_wait_retry() {
2532
local _attempts=3
2633
local _i _out _rc
2734
for _i in $(seq 1 "${_attempts}"); do
28-
_out=$(kubectl wait "$@" 2>&1)
29-
_rc=$?
35+
_rc=0
36+
_out=$(kubectl wait "$@" 2>&1) || _rc=$?
3037
if [ "${_rc}" = 0 ]; then
3138
printf '%s\n' "${_out}"
3239
return 0

‎hack/kubectl-wait-retry_test.bats‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
#!/usr/bin/env bats
2+
# -----------------------------------------------------------------------------
3+
# Unit tests for kubectl_wait_retry in hack/e2e-chainsaw/_lib/run-kubernetes.sh
4+
#
5+
# The wrapper exists to retry `kubectl wait` against a curated allowlist of
6+
# transient server errors (etcd leader flap et al.) while letting real failures
7+
# (NotFound, --timeout expired) surface. The chainsaw scripts source the lib
8+
# under `set -eu`, where a bare `_out=$(kubectl wait ...)` on a failing wait
9+
# aborts the whole script before the retry loop runs -- silently, since stderr
10+
# is captured into the substitution. These tests execute the wrapper in a
11+
# `sh -eu` subshell, exactly as chainsaw runs it, against a fake kubectl.
12+
#
13+
# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its
14+
# own line; there is no bats `run`/`$status`. Assertions are direct shell tests.
15+
# Run with: hack/cozytest.sh hack/kubectl-wait-retry_test.bats
16+
# -----------------------------------------------------------------------------
17+
18+
FAKEBIN="$PWD/hack/testdata/kubectl-wait-retry"
19+
20+
prep() {
21+
chmod +x "$FAKEBIN/kubectl"
22+
WORK=$(mktemp -d)
23+
export FAKE_CMDLOG="$WORK/cmdlog"
24+
: > "$FAKE_CMDLOG"
25+
}
26+
27+
@test "a transient etcd error is retried under set -eu instead of silently killing the script" {
28+
prep
29+
export FAKE_MODE=transient-once
30+
rc=0
31+
out=$(PATH="$FAKEBIN:$PATH" sh -euc '
32+
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
33+
kubectl_wait_retry deploy x -n tenant-test --for=condition=available
34+
echo SCRIPT-REACHED-END
35+
' 2>&1) || rc=$?
36+
echo "$out"
37+
# The wrapper retried past the transient error and the sourcing script lived.
38+
[ "$rc" -eq 0 ]
39+
echo "$out" | grep -q 'retrying'
40+
echo "$out" | grep -q 'SCRIPT-REACHED-END'
41+
[ "$(grep -c . "$FAKE_CMDLOG")" -eq 2 ]
42+
rm -rf "$WORK"
43+
}
44+
45+
@test "NotFound is not retried, fails the script, and its message is printed (fail-loud preserved)" {
46+
prep
47+
export FAKE_MODE=notfound
48+
rc=0
49+
out=$(PATH="$FAKEBIN:$PATH" sh -euc '
50+
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
51+
kubectl_wait_retry deploy x -n tenant-test --for=condition=available
52+
echo SCRIPT-REACHED-END
53+
' 2>&1) || rc=$?
54+
echo "$out"
55+
# A real failure still aborts, is not retried, and the error is visible.
56+
[ "$rc" -ne 0 ]
57+
echo "$out" | grep -q 'NotFound'
58+
! echo "$out" | grep -q 'SCRIPT-REACHED-END'
59+
[ "$(grep -c . "$FAKE_CMDLOG")" -eq 1 ]
60+
rm -rf "$WORK"
61+
}
Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/bin/sh
2+
# Fake kubectl for hack/kubectl-wait-retry_test.bats. Mocks only `kubectl wait`.
3+
# Env:
4+
# FAKE_CMDLOG path to append one line per invocation (also the call counter)
5+
# FAKE_MODE transient-once: first call fails with an etcd leader-change
6+
# message (the wrapper's retry allowlist), later calls succeed
7+
# notfound: every call fails NotFound (must NOT be retried)
8+
echo "wait $*" >> "$FAKE_CMDLOG"
9+
calls=$(grep -c . "$FAKE_CMDLOG")
10+
case "${FAKE_MODE:-}" in
11+
transient-once)
12+
if [ "$calls" -le 1 ]; then
13+
echo "Error from server: etcdserver: leader changed" >&2
14+
exit 1
15+
fi
16+
echo "deployment.apps/x condition met"
17+
exit 0 ;;
18+
notfound)
19+
echo 'Error from server (NotFound): deployments.apps "x" not found' >&2
20+
exit 1 ;;
21+
*)
22+
echo "condition met"
23+
exit 0 ;;
24+
esac

0 commit comments

Comments
 (0)