Skip to content

Commit 4c7ef0d

Browse files
committed
test(e2e): keep redis-tls failures legible under the op ceiling
The CA-projection step caps each kubectl at 30s, but more capped reads run than its 5m op can cover, so a degraded apiserver could spend the ceiling and leave nothing but "signal: killed". The reads now share a step budget held under the op, and a read that hits its bound or finds the budget spent says which read it was. Two exec checks reported the 15s redis-cli cap for any failure, sending a reader after a slow redis when the exec was refused or the container was missing. They now report the exit status and what the exec printed. The step also runs under bash with pipefail: under dash a failed Secret read took base64's zero status and surfaced as a wrong source name. The private-key check reads the projection already fetched, since a second read that failed made it pass. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <[email protected]>
1 parent f629b8f commit 4c7ef0d

2 files changed

Lines changed: 258 additions & 22 deletions

File tree

‎hack/chainsaw-op-budgets.bats‎

Lines changed: 207 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,12 @@
88
# The gateway step is held by arithmetic: its polls are `timeout N` prefixes
99
# and its cleanup deletes carry `--timeout=Ns`, so their sum is readable from
1010
# the text. Only those two spellings are summed: a wait written any other
11-
# way, a `kubectl wait` or a sleep loop, is invisible to the sum.
11+
# way, a `kubectl wait` or a sleep loop, is invisible to the sum. The
12+
# redis-tls step cannot be summed that way, because some of its
13+
# reads sit in loops and one loop polls, so it carries a step budget that
14+
# every kubectl call is clamped to, and the guard holds that budget under the
15+
# op. Its failure messages are checked by running the step's script against a
16+
# stubbed kubectl.
1217
#
1318
# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its
1419
# own line; a `}` at column zero gets `return 0` inserted ahead of it, so the
@@ -19,10 +24,15 @@
1924

2025
GATEWAY_SUITE="hack/e2e-chainsaw/gateway/chainsaw-test.yaml"
2126
GATEWAY_STEP="parent-child-and-route-drive-listener-set"
27+
REDIS_SUITE="hack/e2e-chainsaw/redis/chainsaw-test.yaml"
28+
REDIS_STEP="verify-tenant-ca-projection"
2229

2330
# Seconds the gateway op keeps above its summed waits for what carries no cap
2431
# of its own: three applies and one label read.
2532
GATEWAY_UNCAPPED=60
33+
# Seconds the redis op keeps above its step budget: the kill grace on the last
34+
# clamped read, one 5s poll sleep, and the openssl and shell work between reads.
35+
REDIS_AFTER_BUDGET=30
2636

2737
# Print the lines of the first `- script:` op inside step $2 of file $1, with
2838
# the op's own indentation, from `- script:` up to the next op or step.
@@ -51,6 +61,15 @@ op_timeout_seconds() {
5161
'
5262
}
5363

64+
# Print the op's shell, or `sh`, the Chainsaw default, when it names none.
65+
op_shell() {
66+
sh_name=$(printf '%s\n' "$1" | awk '
67+
/^[[:space:]]*shell:[[:space:]]*[^[:space:]]+[[:space:]]*$/ { sub(/^[[:space:]]*shell:[[:space:]]*/, ""); sub(/[[:space:]]*$/, ""); print; exit }
68+
/^[[:space:]]*content:/ { exit }
69+
')
70+
printf '%s\n' "${sh_name:-sh}"
71+
}
72+
5473
# Print the op's script, dedented, without the content key.
5574
op_content() {
5675
printf '%s\n' "$1" | awk '
@@ -64,6 +83,81 @@ op_content() {
6483
'
6584
}
6685

86+
# Write a kubectl stub into directory $1 that answers the redis-tls step's
87+
# reads as a healthy cluster would. STUB_FAIL, when set, is a pattern matched
88+
# against the arguments: a matching call prints STUB_ERR to stderr and exits
89+
# STUB_RC instead, after printing STUB_OUT to stdout when that is set, from the
90+
# STUB_FAIL_FROM-th matching call on (default the first). STUB_PROJ_KEY=1 puts
91+
# PEM private key material into the projection. STUB_SLEEP_ON is a pattern
92+
# whose matching calls sleep STUB_SLEEP seconds first; the stub execs sleep,
93+
# so a timeout that kills it leaves no child holding the output pipe.
94+
write_redis_stubs() {
95+
cat > "$1/kubectl" <<'STUB'
96+
#!/bin/sh
97+
args="$*"
98+
if [ -n "${STUB_SLEEP_ON:-}" ]; then
99+
case "$args" in
100+
$STUB_SLEEP_ON) exec sleep "${STUB_SLEEP:-60}" ;;
101+
esac
102+
fi
103+
if [ -n "${STUB_FAIL:-}" ]; then
104+
case "$args" in
105+
$STUB_FAIL)
106+
echo x >> "$(dirname "$0")/fail-matches"
107+
if [ "$(wc -l < "$(dirname "$0")/fail-matches")" -ge "${STUB_FAIL_FROM:-1}" ]; then
108+
[ -z "${STUB_OUT:-}" ] || echo "$STUB_OUT"
109+
echo "${STUB_ERR:-error: stubbed failure}" >&2; exit "${STUB_RC:-1}"
110+
fi ;;
111+
esac
112+
fi
113+
cert_b64=$(printf 'CERT' | base64)
114+
proj_b64=$cert_b64
115+
if [ "${STUB_PROJ_KEY:-}" = 1 ]; then
116+
proj_b64=$(printf 'CERT\n-----BEGIN PRIVATE KEY-----\nKEY\n' | base64 | tr -d '\n')
117+
fi
118+
case "$args" in
119+
*"get secret redis-secure.tenant-ca -o go-template"*) printf 'ca.crt\n' ;;
120+
*"get secret redis-secure.ca-cert -o go-template"*) printf 'ca.crt\n' ;;
121+
*"get secret redis-secure.ca-tls -o go-template"*) printf 'ca.crt\ntls.crt\ntls.key\n' ;;
122+
*"get secret redis-secure.ca-tls -o jsonpath"*) printf '%s' "$cert_b64" ;;
123+
*"get secret redis-secure.tenant-ca -o jsonpath"*) printf '%s' "$proj_b64" ;;
124+
*"get tenantsecret redis-secure.tenant-ca -o jsonpath"*) printf '%s' "$(printf -- '-----BEGIN CERTIFICATE-----' | base64)" ;;
125+
*"get tenantsecret "*) echo "Error from server (NotFound): tenantsecrets \"x\" not found" >&2; exit 1 ;;
126+
*"exec rfr-redis-secure-0 -c redis -- cat /tls/ca.crt"*) printf 'CERT\n' ;;
127+
*"exec rfr-redis-secure-0 -c redis -- timeout 15 redis-cli --tls"*) printf 'PONG\n' ;;
128+
*"exec rfr-redis-secure-0 -c redis -- timeout 15 redis-cli -h"*) printf 'I/O error\n'; exit 1 ;;
129+
*"exec rfr-redis-secure-0 -c redis -- sh -c"*) printf 'role:master\n' ;;
130+
*"exec rfr-redis-secure-1 -c redis -- sh -c"*) printf 'role:slave\nmaster_link_status:up\n' ;;
131+
*"get pod -l app.kubernetes.io/component=sentinel"*) printf 'rfs-redis-secure-0' ;;
132+
*"exec rfs-redis-secure-0 -c sentinel"*) printf '10.0.0.1\n6379\n' ;;
133+
*) echo "kubectl stub: unexpected call: $args" >&2; exit 99 ;;
134+
esac
135+
STUB
136+
cat > "$1/openssl" <<'STUB'
137+
#!/bin/sh
138+
cat >/dev/null
139+
echo "sha256 Fingerprint=AA"
140+
STUB
141+
chmod +x "$1/kubectl" "$1/openssl"
142+
}
143+
144+
# Run the redis-tls step's script, as its op's shell would, against the stubs
145+
# in $1. Prints the script's combined output and then `rc=<status>`. RS_BUDGET,
146+
# when set, replaces the step's budget so the clamp can be driven in seconds.
147+
run_redis_step() {
148+
rs_op=$(op_lines "$REDIS_SUITE" "$REDIS_STEP")
149+
rs_shell=$(op_shell "$rs_op")
150+
op_content "$rs_op" > "$1/step.sh"
151+
if [ -n "${RS_BUDGET:-}" ]; then
152+
sed "s/^step_budget=[0-9]*\$/step_budget=${RS_BUDGET}/" "$1/step.sh" > "$1/step.budget"
153+
if cmp -s "$1/step.sh" "$1/step.budget"; then echo "no step_budget line to override"; echo "rc=97"; return; fi
154+
mv "$1/step.budget" "$1/step.sh"
155+
fi
156+
rs_rc=0
157+
rs_out=$(cd "$1" && PATH="$1:$PATH" NAMESPACE=tenant-test STUB_FAIL="${STUB_FAIL:-}" STUB_FAIL_FROM="${STUB_FAIL_FROM:-1}" STUB_RC="${STUB_RC:-1}" STUB_ERR="${STUB_ERR:-}" STUB_OUT="${STUB_OUT:-}" STUB_SLEEP_ON="${STUB_SLEEP_ON:-}" STUB_SLEEP="${STUB_SLEEP:-}" STUB_PROJ_KEY="${STUB_PROJ_KEY:-}" "$rs_shell" "$1/step.sh" 2>&1) || rs_rc=$?
158+
printf '%s\nrc=%s\n' "$rs_out" "$rs_rc"
159+
}
160+
67161
@test "the gateway listener-set op outlasts the waits inside it" {
68162
gw_op=$(op_lines "$GATEWAY_SUITE" "$GATEWAY_STEP")
69163
[ -n "$gw_op" ] || { echo "no script op found in step $GATEWAY_STEP of $GATEWAY_SUITE" >&2; exit 1; }
@@ -106,3 +200,115 @@ op_content() {
106200
')
107201
[ "$gw_order" = ok ] || { echo "the rparent delete in $GATEWAY_STEP does not sit directly under a successful rchild delete (got '${gw_order}')" >&2; exit 1; }
108202
}
203+
204+
@test "the redis-tls projection op outlasts the step budget its reads are clamped to" {
205+
rt_op=$(op_lines "$REDIS_SUITE" "$REDIS_STEP")
206+
[ -n "$rt_op" ] || { echo "no script op found in step $REDIS_STEP of $REDIS_SUITE" >&2; exit 1; }
207+
rt_timeout=$(op_timeout_seconds "$rt_op")
208+
case "$rt_timeout" in ''|*[!0-9]*) echo "the $REDIS_STEP op states no timeout in m or s (got '${rt_timeout}')" >&2; exit 1 ;; esac
209+
rt_body=$(op_content "$rt_op")
210+
rt_budget=$(printf '%s\n' "$rt_body" | sed -n 's/^step_budget=\([0-9][0-9]*\)$/\1/p')
211+
[ -n "$rt_budget" ] || {
212+
echo "$REDIS_STEP sets no step_budget, so nothing bounds the sum of its reads below the ${rt_timeout}s op" >&2
213+
echo "its kubectl call sites alone come to $(printf '%s\n' "$rt_body" | grep -cE '(^|[^[:alnum:]_])k -n ')" >&2
214+
exit 1
215+
}
216+
[ $(( rt_timeout - rt_budget )) -ge "$REDIS_AFTER_BUDGET" ] || {
217+
echo "the $REDIS_STEP op allows ${rt_timeout}s and the step budget is ${rt_budget}s" >&2
218+
echo "it must keep ${REDIS_AFTER_BUDGET}s above the budget for the last read's kill grace and the work between reads" >&2
219+
exit 1
220+
}
221+
# The budget bounds only the reads that go through k(). A bare kubectl
222+
# outside it would add an uncapped read the arithmetic above never sees.
223+
rt_bare=$(printf '%s\n' "$rt_body" | grep -vE '^[[:space:]]*#' | grep -nE '(^|[^[:alnum:]_-])kubectl([[:space:]]|$)' | grep -v '"\$k_left" kubectl' || true)
224+
[ -z "$rt_bare" ] || { echo "kubectl calls in $REDIS_STEP that bypass the clamped k():" >&2; printf '%s\n' "$rt_bare" >&2; exit 1; }
225+
}
226+
227+
@test "the redis-tls projection script passes against a healthy stub" {
228+
# The positive control for the two failure tests below: without it, a
229+
# stub that broke the script somewhere else would make them pass.
230+
rt_dir=$(mktemp -d)
231+
write_redis_stubs "$rt_dir"
232+
rt_out=$(run_redis_step "$rt_dir")
233+
case "$rt_out" in *"
234+
rc=0") ;; *) printf '%s\n' "$rt_out" >&2; echo "the step failed against a healthy stub" >&2; exit 1 ;; esac
235+
rm -rf "$rt_dir"
236+
}
237+
238+
@test "a failed replication exec reports its own error, not the 15s cap" {
239+
rt_dir=$(mktemp -d)
240+
write_redis_stubs "$rt_dir"
241+
rt_out=$(STUB_FAIL='*exec rfr-redis-secure-1 -c redis -- sh -c*' STUB_RC=1 \
242+
STUB_ERR='Error from server (Forbidden): pods "rfr-redis-secure-1" is forbidden' run_redis_step "$rt_dir")
243+
case "$rt_out" in *"
244+
rc=0") printf '%s\n' "$rt_out" >&2; echo "the step passed with a forbidden exec" >&2; exit 1 ;; esac
245+
case "$rt_out" in *"15s cap"*) printf '%s\n' "$rt_out" >&2; echo "a forbidden exec was reported as the 15s cap firing" >&2; exit 1 ;; esac
246+
case "$rt_out" in *"rfr-redis-secure-1"*"exit 1"*"Forbidden"*) ;; *) printf '%s\n' "$rt_out" >&2; echo "the message does not carry the exit status and what the exec printed" >&2; exit 1 ;; esac
247+
rm -rf "$rt_dir"
248+
}
249+
250+
@test "a failed sentinel exec reports its own error, not the 15s cap" {
251+
rt_dir=$(mktemp -d)
252+
write_redis_stubs "$rt_dir"
253+
rt_out=$(STUB_FAIL='*exec rfs-redis-secure-0 -c sentinel*' STUB_RC=1 \
254+
STUB_ERR='command terminated with exit code 1' STUB_OUT='NOAUTH Authentication required.' run_redis_step "$rt_dir")
255+
case "$rt_out" in *"
256+
rc=0") printf '%s\n' "$rt_out" >&2; echo "the step passed with a failed sentinel exec" >&2; exit 1 ;; esac
257+
case "$rt_out" in *"15s cap"*) printf '%s\n' "$rt_out" >&2; echo "a refused sentinel command was reported as the 15s cap firing" >&2; exit 1 ;; esac
258+
case "$rt_out" in *"command terminated with exit code 1"*"rfs-redis-secure-0 failed (exit 1): NOAUTH Authentication required."*) ;; *) printf '%s\n' "$rt_out" >&2; echo "the output does not carry what the exec wrote to both streams and its exit status" >&2; exit 1 ;; esac
259+
rm -rf "$rt_dir"
260+
}
261+
262+
@test "a failed Secret read is not reported as a wrong source name" {
263+
# base64 -d exits 0 on empty input, so without pipefail the pipeline takes
264+
# its status and the emptiness check below names the wrong cause.
265+
rt_dir=$(mktemp -d)
266+
write_redis_stubs "$rt_dir"
267+
rt_out=$(STUB_FAIL='*get secret redis-secure.ca-tls -o jsonpath*' STUB_RC=1 \
268+
STUB_ERR='Error from server (ServiceUnavailable): the server is currently unable to handle the request' run_redis_step "$rt_dir")
269+
case "$rt_out" in *"
270+
rc=0") printf '%s\n' "$rt_out" >&2; echo "the step passed with a failed Secret read" >&2; exit 1 ;; esac
271+
case "$rt_out" in *"source name or key is wrong"*) printf '%s\n' "$rt_out" >&2; echo "an API error was reported as a wrong source name" >&2; exit 1 ;; esac
272+
case "$rt_out" in *"ServiceUnavailable"*) ;; *) printf '%s\n' "$rt_out" >&2; echo "the API error is missing from the output" >&2; exit 1 ;; esac
273+
rm -rf "$rt_dir"
274+
}
275+
276+
@test "the private-key check fails when the projection it reads carries a key" {
277+
# The projection is read once above this check. A check that reads it
278+
# again passes when that second read fails, because an empty read holds
279+
# no key material: here the first read returns a key and the second one
280+
# is refused.
281+
rt_dir=$(mktemp -d)
282+
write_redis_stubs "$rt_dir"
283+
rt_out=$(STUB_PROJ_KEY=1 STUB_FAIL='*get secret redis-secure.tenant-ca -o jsonpath*' STUB_FAIL_FROM=2 STUB_RC=1 \
284+
STUB_ERR='Error from server (ServiceUnavailable): the server is currently unable to handle the request' run_redis_step "$rt_dir")
285+
case "$rt_out" in *"
286+
rc=0") printf '%s\n' "$rt_out" >&2; echo "the step passed with key material in the projection" >&2; exit 1 ;; esac
287+
case "$rt_out" in *"projection carries PEM private key material"*) ;; *) printf '%s\n' "$rt_out" >&2; echo "the step did not fail on the key material" >&2; exit 1 ;; esac
288+
rm -rf "$rt_dir"
289+
}
290+
291+
@test "a read that finds the step budget spent does not run and says so" {
292+
rt_dir=$(mktemp -d)
293+
write_redis_stubs "$rt_dir"
294+
rt_out=$(RS_BUDGET=0 run_redis_step "$rt_dir")
295+
case "$rt_out" in *"
296+
rc=0") printf '%s\n' "$rt_out" >&2; echo "the step passed with its budget spent before the first read" >&2; exit 1 ;; esac
297+
case "$rt_out" in *"step budget is spent, so this read did not run: -n tenant-test get secret redis-secure.tenant-ca"*) ;; *) printf '%s\n' "$rt_out" >&2; echo "the first read did not report the spent budget" >&2; exit 1 ;; esac
298+
rm -rf "$rt_dir"
299+
}
300+
301+
@test "a read is clamped to what the step budget has left and names itself" {
302+
# A 2s budget against a read that would hang for 20s: the clamp has to
303+
# end the read at the budget, not at the 30s per-read cap.
304+
rt_dir=$(mktemp -d)
305+
write_redis_stubs "$rt_dir"
306+
rt_start=$(date +%s)
307+
rt_out=$(RS_BUDGET=2 STUB_SLEEP_ON='*get secret redis-secure.tenant-ca -o go-template*' STUB_SLEEP=20 run_redis_step "$rt_dir")
308+
rt_took=$(( $(date +%s) - rt_start ))
309+
case "$rt_out" in *"
310+
rc=0") printf '%s\n' "$rt_out" >&2; echo "the step passed with a hung read" >&2; exit 1 ;; esac
311+
case "$rt_out" in *"ran into its "[12]"s bound: -n tenant-test get secret redis-secure.tenant-ca"*) ;; *) printf '%s\n' "$rt_out" >&2; echo "the hung read did not name itself at the budget's bound" >&2; exit 1 ;; esac
312+
[ "$rt_took" -lt 15 ] || { echo "the step took ${rt_took}s against a 2s budget; the read was not clamped" >&2; exit 1; }
313+
rm -rf "$rt_dir"
314+
}

0 commit comments

Comments
 (0)