Skip to content

Commit 60e44b2

Browse files
fix(e2e): make tenant Talos diagnostics reliable
Fix admission, capture setup, timeout headroom, and portability. Assisted-By: GPT-5 <[email protected]> Signed-off-by: Myasnikov Daniil <[email protected]>
1 parent c8ec60c commit 60e44b2

5 files changed

Lines changed: 80 additions & 20 deletions

File tree

‎hack/e2e-chainsaw/_lib/run-kubernetes.sh‎

Lines changed: 29 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,7 @@ metadata:
270270
spec:
271271
duration: 1h
272272
renewBefore: 10m
273+
commonName: ${certificate}
273274
secretName: ${certificate}
274275
secretTemplate:
275276
labels:
@@ -369,9 +370,9 @@ spec:
369370
type: RuntimeDefault
370371
containers:
371372
- name: diagnostics
372-
# talosctl is dynamically linked against glibc; use the same pinned Ubuntu
373-
# base as the E2E sandbox rather than an Alpine/musl helper image.
374-
image: docker.io/library/ubuntu:24.04@sha256:4fbb8e6a8395de5a7550b33509421a2bafbc0aab6c06ba2cef9ebffbc7092d90
373+
# The E2E sandbox ships the statically linked upstream talosctl release, so
374+
# reuse the digest-pinned Alpine helper already pulled by the test setup.
375+
image: docker.io/alpine/k8s:1.36.2@sha256:44ef4942e171939b9c665a4a84beb80e2dcdb9a24330d4651cfdfd2e9deecc47
375376
imagePullPolicy: IfNotPresent
376377
command: ["sh", "-c", "sleep 900"]
377378
securityContext:
@@ -465,18 +466,17 @@ cozy_capture_tenant_talos_node() {
465466
}
466467

467468
# Collect the guest-side evidence requested by issue #3513. This runs only after
468-
# the 18-minute Ready deadline has already failed. The setup budgets 30s for
469-
# stale-resource deletion, 30s for Certificate issuance, 60s for Pod readiness,
470-
# bounded copy operations, and at most 40s of Talos calls per VMI. It runs before
471-
# slower cache diagnostics so the requested guest evidence lands first and stays
472-
# inside the surrounding Chainsaw timeout even when later collectors exhaust a
473-
# budget.
469+
# the 18-minute Ready deadline has already failed. VMIs without a reported IP
470+
# are recorded before any Certificate or Pod is created; when at least one IP is
471+
# available, setup and each Talos command remain bounded. It runs before slower
472+
# cache diagnostics so the requested guest evidence lands first.
474473
cozy_capture_tenant_talos() (
475474
local test_name="$1"
476475
local report_dir="${COZY_REPORT_DIR:-/workspace/_out/cozyreport}/snapshots/${COZY_SNAPSHOT_NAME:-kubernetes}/tenant-talos"
477476
local selector="cluster.x-k8s.io/cluster-name=kubernetes-${test_name},cluster.x-k8s.io/role=worker"
478477
local pod_name="kubernetes-${test_name}-talos-diagnostics"
479478
local workdir vmi_name node_ip
479+
local has_node_ip=false
480480

481481
umask 077
482482
workdir=$(mktemp -d) || return 1
@@ -498,6 +498,25 @@ cozy_capture_tenant_talos() (
498498
return 1
499499
fi
500500

501+
# Record missing addresses before spending time on Certificate issuance or a
502+
# helper Pod. If no worker booted far enough for KubeVirt to report an IP,
503+
# guest-side Talos is unreachable and setup cannot produce more evidence.
504+
while IFS='|' read -r vmi_name node_ip; do
505+
if [ -z "${node_ip}" ]; then
506+
mkdir -p "${report_dir}/${vmi_name}"
507+
printf '%s\n' 'default VMI interface has no reported IP address' \
508+
>"${report_dir}/${vmi_name}/capture-error.log"
509+
continue
510+
fi
511+
has_node_ip=true
512+
done <"${workdir}/vmis.rows"
513+
if [ "${has_node_ip}" != true ]; then
514+
echo "no tenant worker VMI has a reported IP for Talos diagnostics" >&2
515+
printf '%s\n' 'no tenant worker VMI has a reported IP for Talos diagnostics' \
516+
>"${report_dir}/setup-error.log"
517+
return 1
518+
fi
519+
501520
if ! cozy_prepare_tenant_talosconfig "${test_name}" "${workdir}"; then
502521
echo "failed to create short-lived tenant Talos reader config" >&2
503522
printf '%s\n' 'failed to create short-lived tenant Talos reader config' \
@@ -513,12 +532,7 @@ cozy_capture_tenant_talos() (
513532
fi
514533

515534
while IFS='|' read -r vmi_name node_ip; do
516-
if [ -z "${node_ip}" ]; then
517-
mkdir -p "${report_dir}/${vmi_name}"
518-
printf '%s\n' 'default VMI interface has no reported IP address' \
519-
>"${report_dir}/${vmi_name}/capture-error.log"
520-
continue
521-
fi
535+
[ -n "${node_ip}" ] || continue
522536
echo "--- capturing tenant Talos dmesg/kubelet: ${vmi_name} (${node_ip}) ---"
523537
cozy_capture_tenant_talos_node "${pod_name}" "${node_ip}" \
524538
"${report_dir}/${vmi_name}"

‎hack/e2e-chainsaw/kubernetes-latest/chainsaw-test.yaml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,10 @@ spec:
1919
try:
2020
- script:
2121
# ~25m Kamaji bringup + worker join + LB/NFS/ouroboros assertions.
22-
timeout: 40m
22+
# The failure path can spend ~5m10s in bounded guest Talos capture and
23+
# another 6m30s in its in-process tenant snapshot. Keep the outer op
24+
# above those inner budgets with useful scheduling/API-call slack.
25+
timeout: 50m
2326
# Co-locate the tenant crust-gather snapshot (captured by
2427
# run-kubernetes.sh on failure) under the same snapshots/<test>/ dir as
2528
# the global .chainsaw.yaml catch's host snapshot.

‎hack/e2e-chainsaw/kubernetes-previous/chainsaw-test.yaml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,10 @@ spec:
1313
- name: tenant-kubernetes-previous-version
1414
try:
1515
- script:
16-
timeout: 40m
16+
# The failure path can spend ~5m10s in bounded guest Talos capture and
17+
# another 6m30s in its in-process tenant snapshot. Keep the outer op
18+
# above those inner budgets with useful scheduling/API-call slack.
19+
timeout: 50m
1720
# Co-locate the tenant crust-gather snapshot (captured by
1821
# run-kubernetes.sh on failure) under the same snapshots/<test>/ dir as
1922
# the global .chainsaw.yaml catch's host snapshot.

‎hack/e2e-install-cozystack.bats‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,11 @@
106106
(..|select(has("containers"))|.containers[]|.image),
107107
(..|select(has("initContainers"))|.initContainers[]|.image)
108108
' "$kubeovn_yaml" "$linstor_yaml" "$certmanager_yaml" > "$images_list"
109+
# The failure-only Talos diagnostics Pod can land on any node. The sandbox's
110+
# upstream talosctl binary is static, so cache its pinned Alpine base on every
111+
# node while the cluster is healthy instead of cold-pulling on a failure path.
112+
printf '%s\n' 'docker.io/alpine/k8s:1.36.2@sha256:44ef4942e171939b9c665a4a84beb80e2dcdb9a24330d4651cfdfd2e9deecc47' \
113+
>> "$images_list"
109114
hack/e2e-prepull-images.sh < "$images_list"
110115
rm -f "$kubeovn_yaml" "$linstor_yaml" "$certmanager_yaml" "$images_list"
111116
}

‎hack/run-kubernetes-talos-diagnostics_test.bats‎

Lines changed: 38 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@ assert_file_lacks_pattern() {
115115

116116
assert_file_contains 'name: kubernetes-test-latest-version-e2e-talos-reader' "$kubectl_manifest"
117117
assert_file_contains 'duration: 1h' "$kubectl_manifest"
118+
assert_file_contains 'commonName: kubernetes-test-latest-version-e2e-talos-reader' "$kubectl_manifest"
118119
assert_file_contains '- os:reader' "$kubectl_manifest"
119120
assert_file_contains 'name: kubernetes-test-latest-version-talos-ca' "$kubectl_manifest"
120121
assert_file_contains 'cozystack-e2e.io/tenant-talos-diagnostics: "test-latest-version"' "$kubectl_manifest"
@@ -138,7 +139,10 @@ assert_file_lacks_pattern() {
138139
assert_file_contains '-n tenant-test get secret kubernetes-test-latest-version-e2e-talos-reader -o go-template={{index .data "tls.key" | base64decode}}' "$kubectl_calls"
139140
assert_file_contains "--talosconfig $tmp/talosconfig config add kubernetes-test-latest-version --ca $tmp/ca.crt --crt $tmp/client.crt --key $tmp/client.key" "$talosctl_calls"
140141
assert_file_contains "--talosconfig $tmp/talosconfig config context kubernetes-test-latest-version" "$talosctl_calls"
141-
[ "$(stat -c '%a' "$tmp/talosconfig")" = 600 ]
142+
case "$(LC_ALL=C ls -ld "$tmp/talosconfig")" in
143+
-rw-------*) ;;
144+
*) echo "talosconfig permissions are not 0600" >&2; return 1 ;;
145+
esac
142146
}
143147

144148
@test "diagnostics Pod has no API token or Secret volume" {
@@ -155,10 +159,40 @@ assert_file_lacks_pattern() {
155159
assert_file_contains 'readOnlyRootFilesystem: true' "$kubectl_manifest"
156160
assert_file_contains 'drop:' "$kubectl_manifest"
157161
assert_file_contains '- ALL' "$kubectl_manifest"
158-
assert_file_contains 'image: docker.io/library/ubuntu:24.04@sha256:4fbb8e6a8395de5a7550b33509421a2bafbc0aab6c06ba2cef9ebffbc7092d90' "$kubectl_manifest"
162+
assert_file_contains 'image: docker.io/alpine/k8s:1.36.2@sha256:44ef4942e171939b9c665a4a84beb80e2dcdb9a24330d4651cfdfd2e9deecc47' "$kubectl_manifest"
163+
assert_file_contains "'docker.io/alpine/k8s:1.36.2@sha256:44ef4942e171939b9c665a4a84beb80e2dcdb9a24330d4651cfdfd2e9deecc47'" hack/e2e-install-cozystack.bats
159164
assert_file_lacks_pattern 'secret:|secretName:' "$kubectl_manifest"
160165
}
161166

167+
@test "Kubernetes Chainsaw operations leave room for failure diagnostics" {
168+
[ "$(yq 'select(.metadata.name == "kubernetes-latest").spec.steps[0].try[0].script.timeout' hack/e2e-chainsaw/kubernetes-latest/chainsaw-test.yaml)" = 50m ]
169+
[ "$(yq 'select(.metadata.name == "kubernetes-previous").spec.steps[0].try[0].script.timeout' hack/e2e-chainsaw/kubernetes-previous/chainsaw-test.yaml)" = 50m ]
170+
}
171+
172+
@test "orchestrator skips credentials and helper Pod when every VMI lacks an IP" {
173+
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
174+
tmp=$(mktemp -d)
175+
trap 'rm -rf "$tmp"' EXIT
176+
kubectl_calls="$tmp/kubectl.calls"
177+
kubectl_manifest="$tmp/manifest.yaml"
178+
kubectl_vmi_json="$tmp/vmis.json"
179+
talosctl_calls="$tmp/talosctl.calls"
180+
COZY_REPORT_DIR="$tmp/report"
181+
COZY_SNAPSHOT_NAME=diagnostic-no-ip
182+
printf '%s\n' '{"items":[{"metadata":{"name":"worker-a"},"status":{"interfaces":[{"name":"default"}]}},{"metadata":{"name":"worker-b"},"status":{}}]}' >"$kubectl_vmi_json"
183+
184+
capture_rc=0
185+
cozy_capture_tenant_talos test-latest-version || capture_rc=$?
186+
187+
[ "$capture_rc" -eq 1 ]
188+
assert_file_lacks_pattern '(^|[[:space:]])apply([[:space:]]|$)' "$kubectl_calls"
189+
assert_file_lacks_pattern 'delete (certificate|secret|pod)|wait (certificate|pod)|exec' "$kubectl_calls"
190+
[ ! -s "$talosctl_calls" ]
191+
assert_file_contains 'default VMI interface has no reported IP address' "$COZY_REPORT_DIR/snapshots/$COZY_SNAPSHOT_NAME/tenant-talos/worker-a/capture-error.log"
192+
assert_file_contains 'default VMI interface has no reported IP address' "$COZY_REPORT_DIR/snapshots/$COZY_SNAPSHOT_NAME/tenant-talos/worker-b/capture-error.log"
193+
assert_file_contains 'no tenant worker VMI has a reported IP for Talos diagnostics' "$COZY_REPORT_DIR/snapshots/$COZY_SNAPSHOT_NAME/tenant-talos/setup-error.log"
194+
}
195+
162196
@test "node capture uses the VMI IP for endpoint and node with bounded commands" {
163197
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
164198
tmp=$(mktemp -d)
@@ -209,14 +243,15 @@ assert_file_lacks_pattern() {
209243
timeout_fail_node_ip=10.244.1.93
210244
COZY_REPORT_DIR="$tmp/report"
211245
COZY_SNAPSHOT_NAME=diagnostic-smoke
212-
printf '%s\n' '{"items":[{"metadata":{"name":"worker-a"},"status":{"interfaces":[{"name":"default","ipAddress":"10.244.1.93"}]}},{"metadata":{"name":"worker-b"},"status":{"interfaces":[{"name":"default","ipAddress":"10.244.1.94"}]}}]}' >"$kubectl_vmi_json"
246+
printf '%s\n' '{"items":[{"metadata":{"name":"worker-no-ip"},"status":{"interfaces":[{"name":"default"}]}},{"metadata":{"name":"worker-a"},"status":{"interfaces":[{"name":"default","ipAddress":"10.244.1.93"}]}},{"metadata":{"name":"worker-b"},"status":{"interfaces":[{"name":"default","ipAddress":"10.244.1.94"}]}}]}' >"$kubectl_vmi_json"
213247
printf '%s\n' '#!/bin/sh' >"$tmp/talosctl"
214248
chmod 755 "$tmp/talosctl"
215249
cd "$tmp"
216250

217251
cozy_capture_tenant_talos test-latest-version
218252

219253
[ "$(awk '/ exec kubernetes-test-latest-version-talos-diagnostics / { count++ } END { print count + 0 }' "$kubectl_calls")" -eq 4 ]
254+
assert_file_contains 'default VMI interface has no reported IP address' "$COZY_REPORT_DIR/snapshots/$COZY_SNAPSHOT_NAME/tenant-talos/worker-no-ip/capture-error.log"
220255
assert_file_contains '[capture exit code: 124]' "$COZY_REPORT_DIR/snapshots/$COZY_SNAPSHOT_NAME/tenant-talos/worker-a/dmesg.log"
221256
assert_file_contains '[capture exit code: 124]' "$COZY_REPORT_DIR/snapshots/$COZY_SNAPSHOT_NAME/tenant-talos/worker-a/kubelet.log"
222257
assert_file_contains '[capture exit code: 0]' "$COZY_REPORT_DIR/snapshots/$COZY_SNAPSHOT_NAME/tenant-talos/worker-b/dmesg.log"

0 commit comments

Comments
 (0)