@@ -179,6 +179,16 @@ cozy_cleanup() {
179179 # so they don't leak MetalLB IPs from the shared host pool. Labeled by the
180180 # test so a single selector reaps them all.
181181 kubectl -n tenant-test delete service -l cozystack-e2e.io/tenant-api-lb --ignore-not-found --wait=false 2> /dev/null || true
182+ # A failed node-join capture creates a short-lived reader Certificate and a
183+ # hardened helper Pod. They are labelled separately from the tenant API LB:
184+ # the Secret contains a Talos client key and must be reaped even if the
185+ # diagnostic collector itself returned early.
186+ if ! kubectl -n tenant-test delete certificates.cert-manager.io -l cozystack-e2e.io/tenant-talos-diagnostics --ignore-not-found --wait=true --timeout=30s 2> /dev/null; then
187+ echo " » WARNING: failed to delete tenant Talos diagnostic Certificates" >&2
188+ fi
189+ if ! kubectl -n tenant-test delete pod,secret -l cozystack-e2e.io/tenant-talos-diagnostics --ignore-not-found --wait=false 2> /dev/null; then
190+ echo " » WARNING: failed to delete tenant Talos diagnostic Pod/Secret" >&2
191+ fi
182192 kubectl -n tenant-test delete kuberneteses.apps.cozystack.io --all --ignore-not-found --wait=false 2> /dev/null || true
183193 kubectl -n tenant-test wait kuberneteses.apps.cozystack.io --all --for=delete --timeout=5m 2> /dev/null || true
184194 # The CR delete above finalizes once the Kubernetes CR is gone, which only
@@ -232,6 +242,289 @@ _tenant_snapshot_on_fail() {
232242 esac
233243}
234244
245+ # Render name|IP rows for worker VMIs from a `kubectl get ... -o json` document.
246+ # Kept pure so an absent default interface is represented by an empty IP instead
247+ # of silently dropping the VMI from the diagnostic report.
248+ cozy_tenant_worker_vmi_rows () {
249+ jq -r ' .items[] | [.metadata.name, ([.status.interfaces[]? | select(.name == "default") | .ipAddress][0] // "")] | join("|")'
250+ }
251+
252+ # Ask cert-manager for a short-lived, read-only Talos client certificate. The
253+ # tenant chart deliberately uses TalosConfigTemplate generateType=none, so CABPT
254+ # creates no client talosconfig for this worker-only Kamaji topology. Reusing the
255+ # existing Talos CA Issuer avoids materialising an os:admin credential or copying
256+ # the CA private key out of Kubernetes.
257+ cozy_apply_tenant_talos_reader_certificate () {
258+ local test_name=" $1 "
259+ local release=" kubernetes-${test_name} "
260+ local certificate=" ${release} -e2e-talos-reader"
261+
262+ kubectl apply -f - << EOF
263+ apiVersion: cert-manager.io/v1
264+ kind: Certificate
265+ metadata:
266+ name: ${certificate}
267+ namespace: tenant-test
268+ labels:
269+ cozystack-e2e.io/tenant-talos-diagnostics: "${test_name} "
270+ spec:
271+ duration: 1h
272+ renewBefore: 10m
273+ secretName: ${certificate}
274+ secretTemplate:
275+ labels:
276+ cozystack-e2e.io/tenant-talos-diagnostics: "${test_name} "
277+ subject:
278+ organizations:
279+ - os:reader
280+ privateKey:
281+ algorithm: Ed25519
282+ rotationPolicy: Always
283+ usages:
284+ - digital signature
285+ - client auth
286+ issuerRef:
287+ name: ${release} -talos-ca
288+ kind: Issuer
289+ group: cert-manager.io
290+ EOF
291+ }
292+
293+ # Build a local talosconfig from the reader Certificate. All private material
294+ # stays below workdir, which the caller creates with mode 0700 and removes from
295+ # a contained subshell EXIT trap; none of it is copied into cozyreport.
296+ cozy_prepare_tenant_talosconfig () (
297+ local test_name=" $1 "
298+ local workdir=" $2 "
299+ local release=" kubernetes-${test_name} "
300+ local certificate=" ${release} -e2e-talos-reader"
301+
302+ umask 077
303+ # An interrupted prior run may have left a reader Secret signed by the old
304+ # tenant CA. Remove the Certificate first so cert-manager cannot recreate that
305+ # Secret between deletion and the new request, then wait for both objects to
306+ # disappear before waiting for issuance below.
307+ kubectl -n tenant-test delete certificate " ${certificate} " \
308+ --ignore-not-found --wait=true --timeout=10s > /dev/null 2>&1 || return 1
309+ kubectl -n tenant-test delete secret " ${certificate} " \
310+ --ignore-not-found --wait=true --timeout=10s > /dev/null 2>&1 || return 1
311+ cozy_apply_tenant_talos_reader_certificate " ${test_name} " || return 1
312+
313+ # cert-manager issuance is asynchronous. Wait on its actual Ready condition;
314+ # on expiry, preserve Certificate diagnostics without ever printing the
315+ # generated Secret.
316+ if ! kubectl -n tenant-test wait certificate " ${certificate} " \
317+ --for=condition=Ready --timeout=30s; then
318+ echo " tenant Talos reader Certificate was not issued within 30s" >&2
319+ kubectl -n tenant-test describe certificate " ${certificate} " >&2 || true
320+ kubectl -n tenant-test get certificaterequests.cert-manager.io \
321+ -l cert-manager.io/certificate-name=" ${certificate} " -o wide >&2 || true
322+ return 1
323+ fi
324+
325+ kubectl -n tenant-test get secret " ${release} -talos-ca" \
326+ -o go-template=' {{index .data "tls.crt" | base64decode}}' > " ${workdir} /ca.crt" || return 1
327+ kubectl -n tenant-test get secret " ${certificate} " \
328+ -o go-template=' {{index .data "tls.crt" | base64decode}}' > " ${workdir} /client.crt" || return 1
329+ kubectl -n tenant-test get secret " ${certificate} " \
330+ -o go-template=' {{index .data "tls.key" | base64decode}}' > " ${workdir} /client.key" || return 1
331+
332+ talosctl --talosconfig " ${workdir} /talosconfig" config add " ${release} " \
333+ --ca " ${workdir} /ca.crt" --crt " ${workdir} /client.crt" \
334+ --key " ${workdir} /client.key" || return 1
335+ talosctl --talosconfig " ${workdir} /talosconfig" config context " ${release} " || return 1
336+ chmod 600 " ${workdir} /talosconfig"
337+ )
338+
339+ # Apply the helper Pod separately from its readiness/copy steps so its security
340+ # invariants have focused unit coverage. The Pod runs on the management cluster
341+ # in the same namespace as bridge-networked worker VMIs, which lets talosctl use
342+ # each real VMI IP as both endpoint and node and keeps server-TLS verification
343+ # valid. A MetalLB or localhost port-forward address would not be in the Talos
344+ # serving certificate SANs.
345+ cozy_apply_tenant_talos_diagnostics_pod () {
346+ local test_name=" $1 "
347+ local pod_name=" kubernetes-${test_name} -talos-diagnostics"
348+
349+ kubectl apply -f - << EOF
350+ apiVersion: v1
351+ kind: Pod
352+ metadata:
353+ name: ${pod_name}
354+ namespace: tenant-test
355+ labels:
356+ cozystack-e2e.io/tenant-talos-diagnostics: "${test_name} "
357+ spec:
358+ activeDeadlineSeconds: 900
359+ automountServiceAccountToken: false
360+ enableServiceLinks: false
361+ restartPolicy: Never
362+ terminationGracePeriodSeconds: 1
363+ securityContext:
364+ runAsNonRoot: true
365+ runAsUser: 1000
366+ runAsGroup: 1000
367+ fsGroup: 1000
368+ seccompProfile:
369+ type: RuntimeDefault
370+ containers:
371+ - name: diagnostics
372+ # talosctl is dynamically linked against glibc; use the same pinned Ubuntu
373+ # base as the E2E sandbox rather than an Alpine/musl helper image.
374+ image: docker.io/library/ubuntu:24.04@sha256:4fbb8e6a8395de5a7550b33509421a2bafbc0aab6c06ba2cef9ebffbc7092d90
375+ imagePullPolicy: IfNotPresent
376+ command: ["sh", "-c", "sleep 900"]
377+ securityContext:
378+ allowPrivilegeEscalation: false
379+ capabilities:
380+ drop:
381+ - ALL
382+ readOnlyRootFilesystem: true
383+ resources:
384+ requests:
385+ cpu: 10m
386+ memory: 16Mi
387+ limits:
388+ cpu: 200m
389+ memory: 256Mi
390+ volumeMounts:
391+ - name: tmp
392+ mountPath: /tmp
393+ volumes:
394+ - name: tmp
395+ emptyDir: {}
396+ EOF
397+ }
398+
399+ # Start the helper Pod and stream talosctl plus the reader talosconfig into its
400+ # emptyDir. Streaming avoids a Secret volume and keeps the Pod independent of
401+ # kubectl-cp/tar behavior. The final cleanup selector remains the authoritative
402+ # backstop if any preparation step fails.
403+ cozy_prepare_tenant_talos_diagnostics_pod () {
404+ local test_name=" $1 "
405+ local talosconfig=" $2 "
406+ local pod_name=" kubernetes-${test_name} -talos-diagnostics"
407+ local talosctl_bin
408+
409+ talosctl_bin=$( command -v talosctl) || return 1
410+ kubectl -n tenant-test delete pod " ${pod_name} " --ignore-not-found \
411+ --wait=true --timeout=10s > /dev/null 2>&1 || return 1
412+ cozy_apply_tenant_talos_diagnostics_pod " ${test_name} " || return 1
413+
414+ if ! kubectl -n tenant-test wait pod " ${pod_name} " \
415+ --for=condition=Ready --timeout=60s; then
416+ kubectl -n tenant-test describe pod " ${pod_name} " >&2 || true
417+ kubectl -n tenant-test get events \
418+ --field-selector involvedObject.name=" ${pod_name} " \
419+ --sort-by=.lastTimestamp >&2 || true
420+ return 1
421+ fi
422+
423+ if ! timeout -k 5 60 kubectl -n tenant-test exec -i " ${pod_name} " \
424+ -c diagnostics -- sh -ec ' cat > /tmp/talosctl && chmod 500 /tmp/talosctl' \
425+ < " ${talosctl_bin} " ; then
426+ echo " failed to stream talosctl into tenant diagnostics Pod" >&2
427+ return 1
428+ fi
429+ if ! timeout -k 5 20 kubectl -n tenant-test exec -i " ${pod_name} " \
430+ -c diagnostics -- sh -ec ' umask 077; cat > /tmp/talosconfig; chmod 600 /tmp/talosconfig' \
431+ < " ${talosconfig} " ; then
432+ echo " failed to stream tenant talosconfig into diagnostics Pod" >&2
433+ return 1
434+ fi
435+ }
436+
437+ # Capture one bounded Talos command through the helper Pod and retain the exit
438+ # status alongside partial output. A timeout or unreachable pre-apid worker is
439+ # itself useful evidence and must not erase captures from the other worker.
440+ cozy_capture_tenant_talos_command () {
441+ local pod_name=" $1 "
442+ local node_ip=" $2 "
443+ local output=" $3 "
444+ shift 3
445+ local rc=0
446+
447+ timeout -k 5 20 kubectl -n tenant-test exec " ${pod_name} " -c diagnostics -- \
448+ /tmp/talosctl --talosconfig /tmp/talosconfig \
449+ -e " ${node_ip} " -n " ${node_ip} " " $@ " > " ${output} " 2>&1 || rc=$?
450+ printf ' \n[capture exit code: %s]\n' " ${rc} " >> " ${output} "
451+ return " ${rc} "
452+ }
453+
454+ cozy_capture_tenant_talos_node () {
455+ local pod_name=" $1 "
456+ local node_ip=" $2 "
457+ local output_dir=" $3 "
458+
459+ mkdir -p " ${output_dir} "
460+ printf ' endpoint: %s\nnode: %s\n' " ${node_ip} " " ${node_ip} " > " ${output_dir} /target.txt"
461+ cozy_capture_tenant_talos_command " ${pod_name} " " ${node_ip} " \
462+ " ${output_dir} /dmesg.log" dmesg || true
463+ cozy_capture_tenant_talos_command " ${pod_name} " " ${node_ip} " \
464+ " ${output_dir} /kubelet.log" logs kubelet --tail=500 || true
465+ }
466+
467+ # Collect the guest-side evidence requested by issue #3513. This runs only after
468+ # the 18-minute Ready deadline has already failed. The setup budgets 30s for
469+ # stale-resource deletion, 30s for Certificate issuance, 60s for Pod readiness,
470+ # bounded copy operations, and at most 40s of Talos calls per VMI. It runs before
471+ # slower cache diagnostics so the requested guest evidence lands first and stays
472+ # inside the surrounding Chainsaw timeout even when later collectors exhaust a
473+ # budget.
474+ cozy_capture_tenant_talos () (
475+ local test_name=" $1 "
476+ local report_dir=" ${COZY_REPORT_DIR:-/ workspace/ _out/ cozyreport} /snapshots/${COZY_SNAPSHOT_NAME:- kubernetes} /tenant-talos"
477+ local selector=" cluster.x-k8s.io/cluster-name=kubernetes-${test_name} ,cluster.x-k8s.io/role=worker"
478+ local pod_name=" kubernetes-${test_name} -talos-diagnostics"
479+ local workdir vmi_name node_ip
480+
481+ umask 077
482+ workdir=$( mktemp -d) || return 1
483+ chmod 700 " ${workdir} "
484+ trap ' rm -rf "${workdir}"' EXIT
485+ mkdir -p " ${report_dir} "
486+
487+ if ! kubectl -n tenant-test get virtualmachineinstances.kubevirt.io \
488+ -l " ${selector} " -o json > " ${report_dir} /vmis.json" \
489+ 2> " ${report_dir} /vmis-error.log" ; then
490+ echo " failed to list tenant worker VMIs for Talos diagnostics" >&2
491+ return 1
492+ fi
493+ cozy_tenant_worker_vmi_rows < " ${report_dir} /vmis.json" > " ${workdir} /vmis.rows"
494+ if [ ! -s " ${workdir} /vmis.rows" ]; then
495+ echo " no tenant worker VMIs found for Talos diagnostics" >&2
496+ printf ' no tenant worker VMIs matched selector %s\n' " ${selector} " \
497+ > " ${report_dir} /setup-error.log"
498+ return 1
499+ fi
500+
501+ if ! cozy_prepare_tenant_talosconfig " ${test_name} " " ${workdir} " ; then
502+ echo " failed to create short-lived tenant Talos reader config" >&2
503+ printf ' %s\n' ' failed to create short-lived tenant Talos reader config' \
504+ > " ${report_dir} /setup-error.log"
505+ return 1
506+ fi
507+ if ! cozy_prepare_tenant_talos_diagnostics_pod \
508+ " ${test_name} " " ${workdir} /talosconfig" ; then
509+ echo " failed to prepare tenant Talos diagnostics Pod" >&2
510+ printf ' %s\n' ' failed to prepare tenant Talos diagnostics Pod' \
511+ > " ${report_dir} /setup-error.log"
512+ return 1
513+ fi
514+
515+ while IFS=' |' read -r vmi_name node_ip; do
516+ if [ -z " ${node_ip} " ]; then
517+ mkdir -p " ${report_dir} /${vmi_name} "
518+ printf ' %s\n' ' default VMI interface has no reported IP address' \
519+ > " ${report_dir} /${vmi_name} /capture-error.log"
520+ continue
521+ fi
522+ echo " --- capturing tenant Talos dmesg/kubelet: ${vmi_name} (${node_ip} ) ---"
523+ cozy_capture_tenant_talos_node " ${pod_name} " " ${node_ip} " \
524+ " ${report_dir} /${vmi_name} "
525+ done < " ${workdir} /vmis.rows"
526+ )
527+
235528run_kubernetes_test () {
236529 local version_expr=" $1 "
237530 local test_name=" $2 "
491784 kubectl -n tenant-test get pods -l kubevirt.io=virt-launcher -o wide || true
492785 kubectl -n tenant-test describe pods -l kubevirt.io=virt-launcher || true
493786
787+ # (b) In-guest Talos kernel and kubelet logs. The tenant chart intentionally
788+ # has no admin talosconfig, so mint a one-hour os:reader client from its
789+ # existing cert-manager Issuer and run talosctl from a hardened Pod that can
790+ # reach the bridge-networked VMI IPs without weakening TLS verification.
791+ # A worker that has not reached apid yet will produce a bounded connection
792+ # error while a later-stage worker remains capturable; both outcomes are
793+ # retained in cozyreport. Diagnostic setup failures never mask exit 1 below.
794+ echo " === (b) in-guest Talos dmesg + kubelet logs ==="
795+ cozy_capture_tenant_talos " ${test_name} " || true
796+
494797 # (a2) Worker DataVolume IMPORT stage. A VM stuck "Provisioning" whose
495798 # DataVolume is ImportInProgress at N/A progress with the importer pod
496799 # looping on an HTTP error is a distinct sub-mode of 2a that the VM/VMI
523826 kubectl -n tenant-test logs -l kamaji.clastix.io/name=" kubernetes-${test_name} " \
524827 -c talos-csr-signer --tail=200 --prefix || true
525828
526- # (b) In-guest Talos/kubelet state from the worker VMs is intentionally NOT
527- # captured here. talosctl needs a client talosconfig for the TENANT cluster,
528- # and the runner has none: the tenant workers are provisioned with their own
529- # Talos PKI whose CA differs from the sandbox's /workspace/talosconfig (which
530- # cozyreport.sh uses to reach the MANAGEMENT nodes only), and the chart
531- # materialises no tenant client talosconfig Secret. Pointing talosctl at the
532- # worker IPs with the management talosconfig would just fail mTLS and capture
533- # nothing, so it is skipped rather than shipped as a misleading no-op. (a) +
534- # (c) carry the 2a-vs-2b split; adding real in-guest capture later requires
535- # wiring a tenant talosconfig into the runner first.
536- echo " === (b) in-guest Talos/kubelet capture skipped: no tenant talosconfig on the runner (a/c cover the 2a-vs-2b split) ==="
537829 exit 1
538830 fi
539831 kubectl --kubeconfig " tenantkubeconfig-${test_name} " get nodes -o wide
0 commit comments