Skip to content

Commit cc731e3

Browse files
committed
test(e2e): keep redis-tls failures legible under the op ceiling
The CA-projection step caps each kubectl at 30s, but more capped reads run than its 5m op can cover, so a degraded apiserver could spend the ceiling and leave nothing but "signal: killed". The reads now share a step budget held under the op, and a read that hits its bound or finds the budget spent says which read it was. Two exec checks reported the 15s redis-cli cap for any failure, sending a reader after a slow redis when the exec was refused or the container was missing. They now report the exit status and what the exec printed. The step also runs under bash with pipefail: under dash a failed Secret read took base64's zero status and surfaced as a wrong source name. The private-key check reads the projection already fetched, since a second read that failed made it pass. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <[email protected]>
1 parent a057d73 commit cc731e3

2 files changed

Lines changed: 258 additions & 22 deletions

File tree

‎hack/chainsaw-op-budgets.bats‎

Lines changed: 207 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,12 @@
1010
# grace included, is readable from the text. kubectl's own `--timeout` is not
1111
# counted: it bounds only the wait after the delete request, not discovery or
1212
# the request itself. Only those two spellings are summed: a wait written any
13-
# other way, a `kubectl wait` or a sleep loop, is invisible to the sum.
13+
# other way, a `kubectl wait` or a sleep loop, is invisible to the sum. The
14+
# redis-tls step cannot be summed that way, because some of its
15+
# reads sit in loops and one loop polls, so it carries a step budget that
16+
# every kubectl call is clamped to, and the guard holds that budget under the
17+
# op. Its failure messages are checked by running the step's script against a
18+
# stubbed kubectl.
1419
#
1520
# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its
1621
# own line; a `}` at column zero gets `return 0` inserted ahead of it, so the
@@ -21,10 +26,15 @@
2126

2227
GATEWAY_SUITE="hack/e2e-chainsaw/gateway/chainsaw-test.yaml"
2328
GATEWAY_STEP="parent-child-and-route-drive-listener-set"
29+
REDIS_SUITE="hack/e2e-chainsaw/redis/chainsaw-test.yaml"
30+
REDIS_STEP="verify-tenant-ca-projection"
2431

2532
# Seconds the gateway op keeps above its summed waits for what carries no cap
2633
# of its own: three applies and one label read.
2734
GATEWAY_UNCAPPED=60
35+
# Seconds the redis op keeps above its step budget: the kill grace on the last
36+
# clamped read, one 5s poll sleep, and the openssl and shell work between reads.
37+
REDIS_AFTER_BUDGET=30
2838

2939
# Print the lines of the first `- script:` op inside step $2 of file $1, with
3040
# the op's own indentation, from `- script:` up to the next op or step.
@@ -53,6 +63,15 @@ op_timeout_seconds() {
5363
'
5464
}
5565

66+
# Print the op's shell, or `sh`, the Chainsaw default, when it names none.
67+
op_shell() {
68+
sh_name=$(printf '%s\n' "$1" | awk '
69+
/^[[:space:]]*shell:[[:space:]]*[^[:space:]]+[[:space:]]*$/ { sub(/^[[:space:]]*shell:[[:space:]]*/, ""); sub(/[[:space:]]*$/, ""); print; exit }
70+
/^[[:space:]]*content:/ { exit }
71+
')
72+
printf '%s\n' "${sh_name:-sh}"
73+
}
74+
5675
# Print the op's script, dedented, without the content key.
5776
op_content() {
5877
printf '%s\n' "$1" | awk '
@@ -66,6 +85,81 @@ op_content() {
6685
'
6786
}
6887

88+
# Write a kubectl stub into directory $1 that answers the redis-tls step's
89+
# reads as a healthy cluster would. STUB_FAIL, when set, is a pattern matched
90+
# against the arguments: a matching call prints STUB_ERR to stderr and exits
91+
# STUB_RC instead, after printing STUB_OUT to stdout when that is set, from the
92+
# STUB_FAIL_FROM-th matching call on (default the first). STUB_PROJ_KEY=1 puts
93+
# PEM private key material into the projection. STUB_SLEEP_ON is a pattern
94+
# whose matching calls sleep STUB_SLEEP seconds first; the stub execs sleep,
95+
# so a timeout that kills it leaves no child holding the output pipe.
96+
write_redis_stubs() {
97+
cat > "$1/kubectl" <<'STUB'
98+
#!/bin/sh
99+
args="$*"
100+
if [ -n "${STUB_SLEEP_ON:-}" ]; then
101+
case "$args" in
102+
$STUB_SLEEP_ON) exec sleep "${STUB_SLEEP:-60}" ;;
103+
esac
104+
fi
105+
if [ -n "${STUB_FAIL:-}" ]; then
106+
case "$args" in
107+
$STUB_FAIL)
108+
echo x >> "$(dirname "$0")/fail-matches"
109+
if [ "$(wc -l < "$(dirname "$0")/fail-matches")" -ge "${STUB_FAIL_FROM:-1}" ]; then
110+
[ -z "${STUB_OUT:-}" ] || echo "$STUB_OUT"
111+
echo "${STUB_ERR:-error: stubbed failure}" >&2; exit "${STUB_RC:-1}"
112+
fi ;;
113+
esac
114+
fi
115+
cert_b64=$(printf 'CERT' | base64)
116+
proj_b64=$cert_b64
117+
if [ "${STUB_PROJ_KEY:-}" = 1 ]; then
118+
proj_b64=$(printf 'CERT\n-----BEGIN PRIVATE KEY-----\nKEY\n' | base64 | tr -d '\n')
119+
fi
120+
case "$args" in
121+
*"get secret redis-secure.tenant-ca -o go-template"*) printf 'ca.crt\n' ;;
122+
*"get secret redis-secure.ca-cert -o go-template"*) printf 'ca.crt\n' ;;
123+
*"get secret redis-secure.ca-tls -o go-template"*) printf 'ca.crt\ntls.crt\ntls.key\n' ;;
124+
*"get secret redis-secure.ca-tls -o jsonpath"*) printf '%s' "$cert_b64" ;;
125+
*"get secret redis-secure.tenant-ca -o jsonpath"*) printf '%s' "$proj_b64" ;;
126+
*"get tenantsecret redis-secure.tenant-ca -o jsonpath"*) printf '%s' "$(printf -- '-----BEGIN CERTIFICATE-----' | base64)" ;;
127+
*"get tenantsecret "*) echo "Error from server (NotFound): tenantsecrets \"x\" not found" >&2; exit 1 ;;
128+
*"exec rfr-redis-secure-0 -c redis -- cat /tls/ca.crt"*) printf 'CERT\n' ;;
129+
*"exec rfr-redis-secure-0 -c redis -- timeout 15 redis-cli --tls"*) printf 'PONG\n' ;;
130+
*"exec rfr-redis-secure-0 -c redis -- timeout 15 redis-cli -h"*) printf 'I/O error\n'; exit 1 ;;
131+
*"exec rfr-redis-secure-0 -c redis -- sh -c"*) printf 'role:master\n' ;;
132+
*"exec rfr-redis-secure-1 -c redis -- sh -c"*) printf 'role:slave\nmaster_link_status:up\n' ;;
133+
*"get pod -l app.kubernetes.io/component=sentinel"*) printf 'rfs-redis-secure-0' ;;
134+
*"exec rfs-redis-secure-0 -c sentinel"*) printf '10.0.0.1\n6379\n' ;;
135+
*) echo "kubectl stub: unexpected call: $args" >&2; exit 99 ;;
136+
esac
137+
STUB
138+
cat > "$1/openssl" <<'STUB'
139+
#!/bin/sh
140+
cat >/dev/null
141+
echo "sha256 Fingerprint=AA"
142+
STUB
143+
chmod +x "$1/kubectl" "$1/openssl"
144+
}
145+
146+
# Run the redis-tls step's script, as its op's shell would, against the stubs
147+
# in $1. Prints the script's combined output and then `rc=<status>`. RS_BUDGET,
148+
# when set, replaces the step's budget so the clamp can be driven in seconds.
149+
run_redis_step() {
150+
rs_op=$(op_lines "$REDIS_SUITE" "$REDIS_STEP")
151+
rs_shell=$(op_shell "$rs_op")
152+
op_content "$rs_op" > "$1/step.sh"
153+
if [ -n "${RS_BUDGET:-}" ]; then
154+
sed "s/^step_budget=[0-9]*\$/step_budget=${RS_BUDGET}/" "$1/step.sh" > "$1/step.budget"
155+
if cmp -s "$1/step.sh" "$1/step.budget"; then echo "no step_budget line to override"; echo "rc=97"; return; fi
156+
mv "$1/step.budget" "$1/step.sh"
157+
fi
158+
rs_rc=0
159+
rs_out=$(cd "$1" && PATH="$1:$PATH" NAMESPACE=tenant-test STUB_FAIL="${STUB_FAIL:-}" STUB_FAIL_FROM="${STUB_FAIL_FROM:-1}" STUB_RC="${STUB_RC:-1}" STUB_ERR="${STUB_ERR:-}" STUB_OUT="${STUB_OUT:-}" STUB_SLEEP_ON="${STUB_SLEEP_ON:-}" STUB_SLEEP="${STUB_SLEEP:-}" STUB_PROJ_KEY="${STUB_PROJ_KEY:-}" "$rs_shell" "$1/step.sh" 2>&1) || rs_rc=$?
160+
printf '%s\nrc=%s\n' "$rs_out" "$rs_rc"
161+
}
162+
69163
@test "the gateway listener-set op outlasts the waits inside it" {
70164
gw_op=$(op_lines "$GATEWAY_SUITE" "$GATEWAY_STEP")
71165
[ -n "$gw_op" ] || { echo "no script op found in step $GATEWAY_STEP of $GATEWAY_SUITE" >&2; exit 1; }
@@ -129,3 +223,115 @@ op_content() {
129223
done
130224
rm -rf "$gw_dir"
131225
}
226+
227+
@test "the redis-tls projection op outlasts the step budget its reads are clamped to" {
228+
rt_op=$(op_lines "$REDIS_SUITE" "$REDIS_STEP")
229+
[ -n "$rt_op" ] || { echo "no script op found in step $REDIS_STEP of $REDIS_SUITE" >&2; exit 1; }
230+
rt_timeout=$(op_timeout_seconds "$rt_op")
231+
case "$rt_timeout" in ''|*[!0-9]*) echo "the $REDIS_STEP op states no timeout in m or s (got '${rt_timeout}')" >&2; exit 1 ;; esac
232+
rt_body=$(op_content "$rt_op")
233+
rt_budget=$(printf '%s\n' "$rt_body" | sed -n 's/^step_budget=\([0-9][0-9]*\)$/\1/p')
234+
[ -n "$rt_budget" ] || {
235+
echo "$REDIS_STEP sets no step_budget, so nothing bounds the sum of its reads below the ${rt_timeout}s op" >&2
236+
echo "its kubectl call sites alone come to $(printf '%s\n' "$rt_body" | grep -cE '(^|[^[:alnum:]_])k -n ')" >&2
237+
exit 1
238+
}
239+
[ $(( rt_timeout - rt_budget )) -ge "$REDIS_AFTER_BUDGET" ] || {
240+
echo "the $REDIS_STEP op allows ${rt_timeout}s and the step budget is ${rt_budget}s" >&2
241+
echo "it must keep ${REDIS_AFTER_BUDGET}s above the budget for the last read's kill grace and the work between reads" >&2
242+
exit 1
243+
}
244+
# The budget bounds only the reads that go through k(). A bare kubectl
245+
# outside it would add an uncapped read the arithmetic above never sees.
246+
rt_bare=$(printf '%s\n' "$rt_body" | grep -vE '^[[:space:]]*#' | grep -nE '(^|[^[:alnum:]_-])kubectl([[:space:]]|$)' | grep -v '"\$k_left" kubectl' || true)
247+
[ -z "$rt_bare" ] || { echo "kubectl calls in $REDIS_STEP that bypass the clamped k():" >&2; printf '%s\n' "$rt_bare" >&2; exit 1; }
248+
}
249+
250+
@test "the redis-tls projection script passes against a healthy stub" {
251+
# The positive control for the two failure tests below: without it, a
252+
# stub that broke the script somewhere else would make them pass.
253+
rt_dir=$(mktemp -d)
254+
write_redis_stubs "$rt_dir"
255+
rt_out=$(run_redis_step "$rt_dir")
256+
case "$rt_out" in *"
257+
rc=0") ;; *) printf '%s\n' "$rt_out" >&2; echo "the step failed against a healthy stub" >&2; exit 1 ;; esac
258+
rm -rf "$rt_dir"
259+
}
260+
261+
@test "a failed replication exec reports its own error, not the 15s cap" {
262+
rt_dir=$(mktemp -d)
263+
write_redis_stubs "$rt_dir"
264+
rt_out=$(STUB_FAIL='*exec rfr-redis-secure-1 -c redis -- sh -c*' STUB_RC=1 \
265+
STUB_ERR='Error from server (Forbidden): pods "rfr-redis-secure-1" is forbidden' run_redis_step "$rt_dir")
266+
case "$rt_out" in *"
267+
rc=0") printf '%s\n' "$rt_out" >&2; echo "the step passed with a forbidden exec" >&2; exit 1 ;; esac
268+
case "$rt_out" in *"15s cap"*) printf '%s\n' "$rt_out" >&2; echo "a forbidden exec was reported as the 15s cap firing" >&2; exit 1 ;; esac
269+
case "$rt_out" in *"rfr-redis-secure-1"*"exit 1"*"Forbidden"*) ;; *) printf '%s\n' "$rt_out" >&2; echo "the message does not carry the exit status and what the exec printed" >&2; exit 1 ;; esac
270+
rm -rf "$rt_dir"
271+
}
272+
273+
@test "a failed sentinel exec reports its own error, not the 15s cap" {
274+
rt_dir=$(mktemp -d)
275+
write_redis_stubs "$rt_dir"
276+
rt_out=$(STUB_FAIL='*exec rfs-redis-secure-0 -c sentinel*' STUB_RC=1 \
277+
STUB_ERR='command terminated with exit code 1' STUB_OUT='NOAUTH Authentication required.' run_redis_step "$rt_dir")
278+
case "$rt_out" in *"
279+
rc=0") printf '%s\n' "$rt_out" >&2; echo "the step passed with a failed sentinel exec" >&2; exit 1 ;; esac
280+
case "$rt_out" in *"15s cap"*) printf '%s\n' "$rt_out" >&2; echo "a refused sentinel command was reported as the 15s cap firing" >&2; exit 1 ;; esac
281+
case "$rt_out" in *"command terminated with exit code 1"*"rfs-redis-secure-0 failed (exit 1): NOAUTH Authentication required."*) ;; *) printf '%s\n' "$rt_out" >&2; echo "the output does not carry what the exec wrote to both streams and its exit status" >&2; exit 1 ;; esac
282+
rm -rf "$rt_dir"
283+
}
284+
285+
@test "a failed Secret read is not reported as a wrong source name" {
286+
# base64 -d exits 0 on empty input, so without pipefail the pipeline takes
287+
# its status and the emptiness check below names the wrong cause.
288+
rt_dir=$(mktemp -d)
289+
write_redis_stubs "$rt_dir"
290+
rt_out=$(STUB_FAIL='*get secret redis-secure.ca-tls -o jsonpath*' STUB_RC=1 \
291+
STUB_ERR='Error from server (ServiceUnavailable): the server is currently unable to handle the request' run_redis_step "$rt_dir")
292+
case "$rt_out" in *"
293+
rc=0") printf '%s\n' "$rt_out" >&2; echo "the step passed with a failed Secret read" >&2; exit 1 ;; esac
294+
case "$rt_out" in *"source name or key is wrong"*) printf '%s\n' "$rt_out" >&2; echo "an API error was reported as a wrong source name" >&2; exit 1 ;; esac
295+
case "$rt_out" in *"ServiceUnavailable"*) ;; *) printf '%s\n' "$rt_out" >&2; echo "the API error is missing from the output" >&2; exit 1 ;; esac
296+
rm -rf "$rt_dir"
297+
}
298+
299+
@test "the private-key check fails when the projection it reads carries a key" {
300+
# The projection is read once above this check. A check that reads it
301+
# again passes when that second read fails, because an empty read holds
302+
# no key material: here the first read returns a key and the second one
303+
# is refused.
304+
rt_dir=$(mktemp -d)
305+
write_redis_stubs "$rt_dir"
306+
rt_out=$(STUB_PROJ_KEY=1 STUB_FAIL='*get secret redis-secure.tenant-ca -o jsonpath*' STUB_FAIL_FROM=2 STUB_RC=1 \
307+
STUB_ERR='Error from server (ServiceUnavailable): the server is currently unable to handle the request' run_redis_step "$rt_dir")
308+
case "$rt_out" in *"
309+
rc=0") printf '%s\n' "$rt_out" >&2; echo "the step passed with key material in the projection" >&2; exit 1 ;; esac
310+
case "$rt_out" in *"projection carries PEM private key material"*) ;; *) printf '%s\n' "$rt_out" >&2; echo "the step did not fail on the key material" >&2; exit 1 ;; esac
311+
rm -rf "$rt_dir"
312+
}
313+
314+
@test "a read that finds the step budget spent does not run and says so" {
315+
rt_dir=$(mktemp -d)
316+
write_redis_stubs "$rt_dir"
317+
rt_out=$(RS_BUDGET=0 run_redis_step "$rt_dir")
318+
case "$rt_out" in *"
319+
rc=0") printf '%s\n' "$rt_out" >&2; echo "the step passed with its budget spent before the first read" >&2; exit 1 ;; esac
320+
case "$rt_out" in *"step budget is spent, so this read did not run: -n tenant-test get secret redis-secure.tenant-ca"*) ;; *) printf '%s\n' "$rt_out" >&2; echo "the first read did not report the spent budget" >&2; exit 1 ;; esac
321+
rm -rf "$rt_dir"
322+
}
323+
324+
@test "a read is clamped to what the step budget has left and names itself" {
325+
# A 2s budget against a read that would hang for 20s: the clamp has to
326+
# end the read at the budget, not at the 30s per-read cap.
327+
rt_dir=$(mktemp -d)
328+
write_redis_stubs "$rt_dir"
329+
rt_start=$(date +%s)
330+
rt_out=$(RS_BUDGET=2 STUB_SLEEP_ON='*get secret redis-secure.tenant-ca -o go-template*' STUB_SLEEP=20 run_redis_step "$rt_dir")
331+
rt_took=$(( $(date +%s) - rt_start ))
332+
case "$rt_out" in *"
333+
rc=0") printf '%s\n' "$rt_out" >&2; echo "the step passed with a hung read" >&2; exit 1 ;; esac
334+
case "$rt_out" in *"ran into its "[12]"s bound: -n tenant-test get secret redis-secure.tenant-ca"*) ;; *) printf '%s\n' "$rt_out" >&2; echo "the hung read did not name itself at the budget's bound" >&2; exit 1 ;; esac
335+
[ "$rt_took" -lt 15 ] || { echo "the step took ${rt_took}s against a 2s budget; the read was not clamped" >&2; exit 1; }
336+
rm -rf "$rt_dir"
337+
}

0 commit comments

Comments
 (0)