Skip to content

Commit 80152b1

Browse files
fix(e2e): capture Talos logs for tenant worker join failures
Tenant Kubernetes worker join failures currently stop at host-side VMI and CSR state because the worker-only topology does not materialize a tenant talosconfig. Mint a short-lived os:reader client certificate and run talosctl from a hardened same-namespace Pod after the Ready deadline. Preserve bounded dmesg and kubelet captures for every worker without masking the original test failure. Assisted-By: GPT-5 <[email protected]> Signed-off-by: Myasnikov Daniil <[email protected]>
1 parent ee26a50 commit 80152b1

2 files changed

Lines changed: 524 additions & 11 deletions

File tree

‎hack/e2e-chainsaw/_lib/run-kubernetes.sh‎

Lines changed: 303 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,16 @@ cozy_cleanup() {
179179
# so they don't leak MetalLB IPs from the shared host pool. Labeled by the
180180
# test so a single selector reaps them all.
181181
kubectl -n tenant-test delete service -l cozystack-e2e.io/tenant-api-lb --ignore-not-found --wait=false 2>/dev/null || true
182+
# A failed node-join capture creates a short-lived reader Certificate and a
183+
# hardened helper Pod. They are labelled separately from the tenant API LB:
184+
# the Secret contains a Talos client key and must be reaped even if the
185+
# diagnostic collector itself returned early.
186+
if ! kubectl -n tenant-test delete certificates.cert-manager.io -l cozystack-e2e.io/tenant-talos-diagnostics --ignore-not-found --wait=true --timeout=30s 2>/dev/null; then
187+
echo "» WARNING: failed to delete tenant Talos diagnostic Certificates" >&2
188+
fi
189+
if ! kubectl -n tenant-test delete pod,secret -l cozystack-e2e.io/tenant-talos-diagnostics --ignore-not-found --wait=false 2>/dev/null; then
190+
echo "» WARNING: failed to delete tenant Talos diagnostic Pod/Secret" >&2
191+
fi
182192
kubectl -n tenant-test delete kuberneteses.apps.cozystack.io --all --ignore-not-found --wait=false 2>/dev/null || true
183193
kubectl -n tenant-test wait kuberneteses.apps.cozystack.io --all --for=delete --timeout=5m 2>/dev/null || true
184194
# The CR delete above finalizes once the Kubernetes CR is gone, which only
@@ -232,6 +242,289 @@ _tenant_snapshot_on_fail() {
232242
esac
233243
}
234244

245+
# Render name|IP rows for worker VMIs from a `kubectl get ... -o json` document.
246+
# Kept pure so an absent default interface is represented by an empty IP instead
247+
# of silently dropping the VMI from the diagnostic report.
248+
cozy_tenant_worker_vmi_rows() {
249+
jq -r '.items[] | [.metadata.name, ([.status.interfaces[]? | select(.name == "default") | .ipAddress][0] // "")] | join("|")'
250+
}
251+
252+
# Ask cert-manager for a short-lived, read-only Talos client certificate. The
253+
# tenant chart deliberately uses TalosConfigTemplate generateType=none, so CABPT
254+
# creates no client talosconfig for this worker-only Kamaji topology. Reusing the
255+
# existing Talos CA Issuer avoids materialising an os:admin credential or copying
256+
# the CA private key out of Kubernetes.
257+
cozy_apply_tenant_talos_reader_certificate() {
258+
local test_name="$1"
259+
local release="kubernetes-${test_name}"
260+
local certificate="${release}-e2e-talos-reader"
261+
262+
kubectl apply -f - <<EOF
263+
apiVersion: cert-manager.io/v1
264+
kind: Certificate
265+
metadata:
266+
name: ${certificate}
267+
namespace: tenant-test
268+
labels:
269+
cozystack-e2e.io/tenant-talos-diagnostics: "${test_name}"
270+
spec:
271+
duration: 1h
272+
renewBefore: 10m
273+
secretName: ${certificate}
274+
secretTemplate:
275+
labels:
276+
cozystack-e2e.io/tenant-talos-diagnostics: "${test_name}"
277+
subject:
278+
organizations:
279+
- os:reader
280+
privateKey:
281+
algorithm: Ed25519
282+
rotationPolicy: Always
283+
usages:
284+
- digital signature
285+
- client auth
286+
issuerRef:
287+
name: ${release}-talos-ca
288+
kind: Issuer
289+
group: cert-manager.io
290+
EOF
291+
}
292+
293+
# Build a local talosconfig from the reader Certificate. All private material
294+
# stays below workdir, which the caller creates with mode 0700 and removes from
295+
# a contained subshell EXIT trap; none of it is copied into cozyreport.
296+
cozy_prepare_tenant_talosconfig() (
297+
local test_name="$1"
298+
local workdir="$2"
299+
local release="kubernetes-${test_name}"
300+
local certificate="${release}-e2e-talos-reader"
301+
302+
umask 077
303+
# An interrupted prior run may have left a reader Secret signed by the old
304+
# tenant CA. Remove the Certificate first so cert-manager cannot recreate that
305+
# Secret between deletion and the new request, then wait for both objects to
306+
# disappear before waiting for issuance below.
307+
kubectl -n tenant-test delete certificate "${certificate}" \
308+
--ignore-not-found --wait=true --timeout=10s >/dev/null 2>&1 || return 1
309+
kubectl -n tenant-test delete secret "${certificate}" \
310+
--ignore-not-found --wait=true --timeout=10s >/dev/null 2>&1 || return 1
311+
cozy_apply_tenant_talos_reader_certificate "${test_name}" || return 1
312+
313+
# cert-manager issuance is asynchronous. Wait on its actual Ready condition;
314+
# on expiry, preserve Certificate diagnostics without ever printing the
315+
# generated Secret.
316+
if ! kubectl -n tenant-test wait certificate "${certificate}" \
317+
--for=condition=Ready --timeout=30s; then
318+
echo "tenant Talos reader Certificate was not issued within 30s" >&2
319+
kubectl -n tenant-test describe certificate "${certificate}" >&2 || true
320+
kubectl -n tenant-test get certificaterequests.cert-manager.io \
321+
-l cert-manager.io/certificate-name="${certificate}" -o wide >&2 || true
322+
return 1
323+
fi
324+
325+
kubectl -n tenant-test get secret "${release}-talos-ca" \
326+
-o go-template='{{index .data "tls.crt" | base64decode}}' >"${workdir}/ca.crt" || return 1
327+
kubectl -n tenant-test get secret "${certificate}" \
328+
-o go-template='{{index .data "tls.crt" | base64decode}}' >"${workdir}/client.crt" || return 1
329+
kubectl -n tenant-test get secret "${certificate}" \
330+
-o go-template='{{index .data "tls.key" | base64decode}}' >"${workdir}/client.key" || return 1
331+
332+
talosctl --talosconfig "${workdir}/talosconfig" config add "${release}" \
333+
--ca "${workdir}/ca.crt" --crt "${workdir}/client.crt" \
334+
--key "${workdir}/client.key" || return 1
335+
talosctl --talosconfig "${workdir}/talosconfig" config context "${release}" || return 1
336+
chmod 600 "${workdir}/talosconfig"
337+
)
338+
339+
# Apply the helper Pod separately from its readiness/copy steps so its security
340+
# invariants have focused unit coverage. The Pod runs on the management cluster
341+
# in the same namespace as bridge-networked worker VMIs, which lets talosctl use
342+
# each real VMI IP as both endpoint and node and keeps server-TLS verification
343+
# valid. A MetalLB or localhost port-forward address would not be in the Talos
344+
# serving certificate SANs.
345+
cozy_apply_tenant_talos_diagnostics_pod() {
346+
local test_name="$1"
347+
local pod_name="kubernetes-${test_name}-talos-diagnostics"
348+
349+
kubectl apply -f - <<EOF
350+
apiVersion: v1
351+
kind: Pod
352+
metadata:
353+
name: ${pod_name}
354+
namespace: tenant-test
355+
labels:
356+
cozystack-e2e.io/tenant-talos-diagnostics: "${test_name}"
357+
spec:
358+
activeDeadlineSeconds: 900
359+
automountServiceAccountToken: false
360+
enableServiceLinks: false
361+
restartPolicy: Never
362+
terminationGracePeriodSeconds: 1
363+
securityContext:
364+
runAsNonRoot: true
365+
runAsUser: 1000
366+
runAsGroup: 1000
367+
fsGroup: 1000
368+
seccompProfile:
369+
type: RuntimeDefault
370+
containers:
371+
- name: diagnostics
372+
# talosctl is dynamically linked against glibc; use the same pinned Ubuntu
373+
# base as the E2E sandbox rather than an Alpine/musl helper image.
374+
image: docker.io/library/ubuntu:24.04@sha256:4fbb8e6a8395de5a7550b33509421a2bafbc0aab6c06ba2cef9ebffbc7092d90
375+
imagePullPolicy: IfNotPresent
376+
command: ["sh", "-c", "sleep 900"]
377+
securityContext:
378+
allowPrivilegeEscalation: false
379+
capabilities:
380+
drop:
381+
- ALL
382+
readOnlyRootFilesystem: true
383+
resources:
384+
requests:
385+
cpu: 10m
386+
memory: 16Mi
387+
limits:
388+
cpu: 200m
389+
memory: 256Mi
390+
volumeMounts:
391+
- name: tmp
392+
mountPath: /tmp
393+
volumes:
394+
- name: tmp
395+
emptyDir: {}
396+
EOF
397+
}
398+
399+
# Start the helper Pod and stream talosctl plus the reader talosconfig into its
400+
# emptyDir. Streaming avoids a Secret volume and keeps the Pod independent of
401+
# kubectl-cp/tar behavior. The final cleanup selector remains the authoritative
402+
# backstop if any preparation step fails.
403+
cozy_prepare_tenant_talos_diagnostics_pod() {
404+
local test_name="$1"
405+
local talosconfig="$2"
406+
local pod_name="kubernetes-${test_name}-talos-diagnostics"
407+
local talosctl_bin
408+
409+
talosctl_bin=$(command -v talosctl) || return 1
410+
kubectl -n tenant-test delete pod "${pod_name}" --ignore-not-found \
411+
--wait=true --timeout=10s >/dev/null 2>&1 || return 1
412+
cozy_apply_tenant_talos_diagnostics_pod "${test_name}" || return 1
413+
414+
if ! kubectl -n tenant-test wait pod "${pod_name}" \
415+
--for=condition=Ready --timeout=60s; then
416+
kubectl -n tenant-test describe pod "${pod_name}" >&2 || true
417+
kubectl -n tenant-test get events \
418+
--field-selector involvedObject.name="${pod_name}" \
419+
--sort-by=.lastTimestamp >&2 || true
420+
return 1
421+
fi
422+
423+
if ! timeout -k 5 60 kubectl -n tenant-test exec -i "${pod_name}" \
424+
-c diagnostics -- sh -ec 'cat > /tmp/talosctl && chmod 500 /tmp/talosctl' \
425+
<"${talosctl_bin}"; then
426+
echo "failed to stream talosctl into tenant diagnostics Pod" >&2
427+
return 1
428+
fi
429+
if ! timeout -k 5 20 kubectl -n tenant-test exec -i "${pod_name}" \
430+
-c diagnostics -- sh -ec 'umask 077; cat > /tmp/talosconfig; chmod 600 /tmp/talosconfig' \
431+
<"${talosconfig}"; then
432+
echo "failed to stream tenant talosconfig into diagnostics Pod" >&2
433+
return 1
434+
fi
435+
}
436+
437+
# Capture one bounded Talos command through the helper Pod and retain the exit
438+
# status alongside partial output. A timeout or unreachable pre-apid worker is
439+
# itself useful evidence and must not erase captures from the other worker.
440+
cozy_capture_tenant_talos_command() {
441+
local pod_name="$1"
442+
local node_ip="$2"
443+
local output="$3"
444+
shift 3
445+
local rc=0
446+
447+
timeout -k 5 20 kubectl -n tenant-test exec "${pod_name}" -c diagnostics -- \
448+
/tmp/talosctl --talosconfig /tmp/talosconfig \
449+
-e "${node_ip}" -n "${node_ip}" "$@" >"${output}" 2>&1 || rc=$?
450+
printf '\n[capture exit code: %s]\n' "${rc}" >>"${output}"
451+
return "${rc}"
452+
}
453+
454+
cozy_capture_tenant_talos_node() {
455+
local pod_name="$1"
456+
local node_ip="$2"
457+
local output_dir="$3"
458+
459+
mkdir -p "${output_dir}"
460+
printf 'endpoint: %s\nnode: %s\n' "${node_ip}" "${node_ip}" >"${output_dir}/target.txt"
461+
cozy_capture_tenant_talos_command "${pod_name}" "${node_ip}" \
462+
"${output_dir}/dmesg.log" dmesg || true
463+
cozy_capture_tenant_talos_command "${pod_name}" "${node_ip}" \
464+
"${output_dir}/kubelet.log" logs kubelet --tail=500 || true
465+
}
466+
467+
# Collect the guest-side evidence requested by issue #3513. This runs only after
468+
# the 18-minute Ready deadline has already failed. The setup budgets 30s for
469+
# stale-resource deletion, 30s for Certificate issuance, 60s for Pod readiness,
470+
# bounded copy operations, and at most 40s of Talos calls per VMI. It runs before
471+
# slower cache diagnostics so the requested guest evidence lands first and stays
472+
# inside the surrounding Chainsaw timeout even when later collectors exhaust a
473+
# budget.
474+
cozy_capture_tenant_talos() (
475+
local test_name="$1"
476+
local report_dir="${COZY_REPORT_DIR:-/workspace/_out/cozyreport}/snapshots/${COZY_SNAPSHOT_NAME:-kubernetes}/tenant-talos"
477+
local selector="cluster.x-k8s.io/cluster-name=kubernetes-${test_name},cluster.x-k8s.io/role=worker"
478+
local pod_name="kubernetes-${test_name}-talos-diagnostics"
479+
local workdir vmi_name node_ip
480+
481+
umask 077
482+
workdir=$(mktemp -d) || return 1
483+
chmod 700 "${workdir}"
484+
trap 'rm -rf "${workdir}"' EXIT
485+
mkdir -p "${report_dir}"
486+
487+
if ! kubectl -n tenant-test get virtualmachineinstances.kubevirt.io \
488+
-l "${selector}" -o json >"${report_dir}/vmis.json" \
489+
2>"${report_dir}/vmis-error.log"; then
490+
echo "failed to list tenant worker VMIs for Talos diagnostics" >&2
491+
return 1
492+
fi
493+
cozy_tenant_worker_vmi_rows <"${report_dir}/vmis.json" >"${workdir}/vmis.rows"
494+
if [ ! -s "${workdir}/vmis.rows" ]; then
495+
echo "no tenant worker VMIs found for Talos diagnostics" >&2
496+
printf 'no tenant worker VMIs matched selector %s\n' "${selector}" \
497+
>"${report_dir}/setup-error.log"
498+
return 1
499+
fi
500+
501+
if ! cozy_prepare_tenant_talosconfig "${test_name}" "${workdir}"; then
502+
echo "failed to create short-lived tenant Talos reader config" >&2
503+
printf '%s\n' 'failed to create short-lived tenant Talos reader config' \
504+
>"${report_dir}/setup-error.log"
505+
return 1
506+
fi
507+
if ! cozy_prepare_tenant_talos_diagnostics_pod \
508+
"${test_name}" "${workdir}/talosconfig"; then
509+
echo "failed to prepare tenant Talos diagnostics Pod" >&2
510+
printf '%s\n' 'failed to prepare tenant Talos diagnostics Pod' \
511+
>"${report_dir}/setup-error.log"
512+
return 1
513+
fi
514+
515+
while IFS='|' read -r vmi_name node_ip; do
516+
if [ -z "${node_ip}" ]; then
517+
mkdir -p "${report_dir}/${vmi_name}"
518+
printf '%s\n' 'default VMI interface has no reported IP address' \
519+
>"${report_dir}/${vmi_name}/capture-error.log"
520+
continue
521+
fi
522+
echo "--- capturing tenant Talos dmesg/kubelet: ${vmi_name} (${node_ip}) ---"
523+
cozy_capture_tenant_talos_node "${pod_name}" "${node_ip}" \
524+
"${report_dir}/${vmi_name}"
525+
done <"${workdir}/vmis.rows"
526+
)
527+
235528
run_kubernetes_test() {
236529
local version_expr="$1"
237530
local test_name="$2"
@@ -491,6 +784,16 @@ EOF
491784
kubectl -n tenant-test get pods -l kubevirt.io=virt-launcher -o wide || true
492785
kubectl -n tenant-test describe pods -l kubevirt.io=virt-launcher || true
493786

787+
# (b) In-guest Talos kernel and kubelet logs. The tenant chart intentionally
788+
# has no admin talosconfig, so mint a one-hour os:reader client from its
789+
# existing cert-manager Issuer and run talosctl from a hardened Pod that can
790+
# reach the bridge-networked VMI IPs without weakening TLS verification.
791+
# A worker that has not reached apid yet will produce a bounded connection
792+
# error while a later-stage worker remains capturable; both outcomes are
793+
# retained in cozyreport. Diagnostic setup failures never mask exit 1 below.
794+
echo "=== (b) in-guest Talos dmesg + kubelet logs ==="
795+
cozy_capture_tenant_talos "${test_name}" || true
796+
494797
# (a2) Worker DataVolume IMPORT stage. A VM stuck "Provisioning" whose
495798
# DataVolume is ImportInProgress at N/A progress with the importer pod
496799
# looping on an HTTP error is a distinct sub-mode of 2a that the VM/VMI
@@ -523,17 +826,6 @@ EOF
523826
kubectl -n tenant-test logs -l kamaji.clastix.io/name="kubernetes-${test_name}" \
524827
-c talos-csr-signer --tail=200 --prefix || true
525828

526-
# (b) In-guest Talos/kubelet state from the worker VMs is intentionally NOT
527-
# captured here. talosctl needs a client talosconfig for the TENANT cluster,
528-
# and the runner has none: the tenant workers are provisioned with their own
529-
# Talos PKI whose CA differs from the sandbox's /workspace/talosconfig (which
530-
# cozyreport.sh uses to reach the MANAGEMENT nodes only), and the chart
531-
# materialises no tenant client talosconfig Secret. Pointing talosctl at the
532-
# worker IPs with the management talosconfig would just fail mTLS and capture
533-
# nothing, so it is skipped rather than shipped as a misleading no-op. (a) +
534-
# (c) carry the 2a-vs-2b split; adding real in-guest capture later requires
535-
# wiring a tenant talosconfig into the runner first.
536-
echo "=== (b) in-guest Talos/kubelet capture skipped: no tenant talosconfig on the runner (a/c cover the 2a-vs-2b split) ==="
537829
exit 1
538830
fi
539831
kubectl --kubeconfig "tenantkubeconfig-${test_name}" get nodes -o wide

0 commit comments

Comments
 (0)