Skip to content

Commit 5149683

Browse files
fix(e2e): capture tenant cluster snapshot from an in-subshell trap
crust-gather can only reach a tenant Kamaji cluster through the test's localhost port-forward (it connects directly to the kubeconfig server URL — there is no apiserver-proxy mode — and the in-cluster URL is unreachable from the runner). cozytest.sh's generic EXIT trap runs in the parent shell AFTER the failing test subshell has exited and its port-forward is reaped, so a collect there hits a dead socket and crust-gather bails before writing anything (every tenant snapshot so far came back empty). Capture the tenant from an EXIT trap registered INSIDE run_kubernetes_test instead: it fires during THAT subshell's exit, while the supervised port-forward is still alive. Verified end-to-end against the real hack/cozytest.sh under dash (CI's /bin/sh) in an ubuntu:22.04 container — the in-subshell trap fires on failure with the PF alive, the tenant snapshot lands in the same snapshots/<test>/ dir as the host snapshot, the generic cozy_cleanup still runs, and there is no pipeline deadlock. Drop the now-dead tenant loop from cozytest.sh's generic trap (it could only ever see a reaped PF); it keeps the host snapshot. Signed-off-by: Myasnikov Daniil <[email protected]>
1 parent 391c27f commit 5149683

2 files changed

Lines changed: 35 additions & 7 deletions

File tree

‎hack/cozytest.sh‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -79,14 +79,19 @@ TMP_SH=$(mktemp) || { echo "Failed to create temp file" >&2; exit 1; }
7979

8080
# Per-file lifecycle hook. cozytest.sh runs each .bats as a single invocation
8181
# and exit()s on the first failing @test, so this EXIT trap is the one place to:
82-
# 1. on failure, snapshot the cluster(s) with crust-gather BEFORE any cleanup,
83-
# so each failed test keeps its own inspectable state (host + every nested
84-
# tenant cluster via tenantkubeconfig-*) instead of one end-of-suite dump;
82+
# 1. on failure, snapshot the HOST cluster with crust-gather BEFORE any cleanup,
83+
# so each failed test keeps its own inspectable state instead of one
84+
# end-of-suite dump;
8585
# 2. ALWAYS run the file's cozy_cleanup() if it defines one, so a test never
8686
# leaks resources into the shared tenant-test namespace (left-behind PVCs
8787
# otherwise exhaust the tenant quota and cascade-fail every later app).
8888
# cozy_cleanup is a plain shell function a .bats file may define — there are no
8989
# bats setup/teardown directives here, this runner only knows @test + bash.
90+
# NOTE: nested tenant clusters are NOT captured here. This trap runs in the
91+
# parent shell after the failing test subshell has exited and reaped its
92+
# port-forward, and crust-gather can only reach a tenant via that localhost
93+
# forward — so a test that creates tenant clusters (run-kubernetes.sh) captures
94+
# them from its OWN in-subshell EXIT trap, while the forward is still alive.
9095
COZY_REPORT_DIR="${COZY_REPORT_DIR:-_out/cozyreport}"
9196
_cozy_on_exit() {
9297
_rc=$?
@@ -95,10 +100,6 @@ _cozy_on_exit() {
95100
mkdir -p "$_snap" 2>/dev/null || true
96101
echo "» capturing crust-gather snapshot of failed $(basename "$TEST_FILE") -> $_snap"
97102
crust-gather collect --exclude-kind Secret -f "$_snap/host" >/dev/null 2>&1 || true
98-
for _kc in tenantkubeconfig-*; do
99-
[ -f "$_kc" ] || continue
100-
crust-gather collect -k "$_kc" --exclude-kind Secret -f "$_snap/$_kc" >/dev/null 2>&1 || true
101-
done
102103
fi
103104
if command -v cozy_cleanup >/dev/null 2>&1; then
104105
echo "» cozy_cleanup $(basename "$TEST_FILE" .bats)"

‎hack/e2e-apps/run-kubernetes.sh‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,26 @@ cozy_cleanup() {
2121
kubectl -n tenant-test wait kuberneteses.apps.cozystack.io --all --for=delete --timeout=5m 2>/dev/null || true
2222
}
2323

24+
# Snapshot the tenant cluster (its cilium/CSI/coredns internals) on a failed run.
25+
# Registered as an EXIT trap INSIDE run_kubernetes_test so it fires during THIS
26+
# test subshell's exit, while the supervised port-forward is still alive. The
27+
# generic cozytest.sh trap cannot do this: it runs in the parent shell after the
28+
# test subshell has exited and reaped its port-forward, and crust-gather can only
29+
# reach the tenant via that localhost forward (it connects directly to the
30+
# kubeconfig server URL — no host-proxy mode — and the in-cluster URL is
31+
# unreachable from the runner). CURRENT_TENANT_KC is a global so the handler can
32+
# read it regardless of function scope at EXIT-trap time.
33+
_tenant_snapshot_on_fail() {
34+
_rc=$?
35+
[ "$_rc" -eq 0 ] && return 0
36+
command -v crust-gather >/dev/null 2>&1 || return 0
37+
[ -n "${CURRENT_TENANT_KC:-}" ] && [ -f "${CURRENT_TENANT_KC}" ] || return 0
38+
_snap="${COZY_REPORT_DIR:-_out/cozyreport}/snapshots/$(basename "${TEST_FILE:-tenant}" .bats)"
39+
mkdir -p "$_snap" 2>/dev/null || true
40+
echo "» capturing tenant crust-gather snapshot (${CURRENT_TENANT_KC}) before teardown"
41+
crust-gather collect -k "${CURRENT_TENANT_KC}" --exclude-kind Secret -f "$_snap/${CURRENT_TENANT_KC}" >/dev/null 2>&1 || true
42+
}
43+
2444
run_kubernetes_test() {
2545
local version_expr="$1"
2646
local test_name="$2"
@@ -169,6 +189,11 @@ EOF
169189
echo "${PF_SUPERVISOR_PID}" >> tenant-pf-supervisors.pids || true
170190
# Wait for port-forward to be ready before using it
171191
timeout 15 sh -ec 'until curl -sk https://localhost:'"${port}"' >/dev/null 2>&1; do sleep 1; done'
192+
# The kubeconfig + port-forward are live now. Arm the tenant snapshot: any
193+
# failure from here on captures the tenant cluster while the PF is still up
194+
# (see _tenant_snapshot_on_fail). Cleared on the success path below.
195+
CURRENT_TENANT_KC="tenantkubeconfig-${test_name}"
196+
trap '_tenant_snapshot_on_fail' EXIT
172197
# Verify the Kubernetes version matches what we expect (retry for up to 20 seconds)
173198
timeout 20 sh -ec 'until kubectl --kubeconfig tenantkubeconfig-'"${test_name}"' version 2>/dev/null | grep -Fq "Server Version: ${k8s_version}"; do sleep 1; done'
174199

@@ -558,6 +583,8 @@ EOF
558583
exit 1
559584
fi
560585

586+
# Success: disarm the tenant-snapshot trap so it doesn't fire on the clean exit.
587+
trap - EXIT
561588
# Clean up: stop the port-forward supervisor first so it can't respawn, then
562589
# kill the current child port-forward.
563590
kill "${PF_SUPERVISOR_PID}" 2>/dev/null || true

0 commit comments

Comments
 (0)