Commit a97bc4a

cozy_assert_guest_console_attached() {

local init_names read_rc=0

# The Pod name is in the output for a reason that is not cosmetic: without

# it, a Pod whose initContainers list is absent emits a bare newline, command

# substitution strips it, and "Pods matched, none carries the container" is

# byte-identical to "no Pod matched". Those two must not collapse -- the

# first is the finding this whole check exists for, and on these workers it

# is also the only shape it can take, since guest-console-log is the only

# init container a DataVolume-booted virt-launcher Pod ever has.

init_names=$(timeout -k 5 30 kubectl -n tenant-test get pods \

--request-timeout=30s) || read_rc=$?

if [ "${read_rc}" -ne 0 ]; then

echo "could not read tenant virt-launcher Pods to verify the guest console container (exit ${read_rc})" >&2

echo "this says nothing about logSerialConsole either way; continuing" >&2

return 1

fi

if [ -z "${init_names}" ]; then

echo "no tenant virt-launcher Pod matched; nothing to verify about the guest console container" >&2

echo "this says nothing about logSerialConsole either way; continuing" >&2

return 1

fi

case "${init_names}" in

*guest-console-log*) return 0 ;;

esac

echo "tenant worker virt-launcher Pods carry no guest-console-log container" >&2

echo "the node group asked for logSerialConsole, so either the cluster-wide" >&2

echo "disableSerialConsoleLog won the override or the field never reached the VM" >&2

# Bounded like the read above, and for the same reason: these run on a path

# that ends in exit 1, and a hang here would take the step's whole deadline

# with it -- losing the tenant snapshot that the exit is supposed to reach.

timeout -k 5 30 kubectl -n tenant-test get pods -l kubevirt.io=virt-launcher \

-o jsonpath='{range .items[*]}{.metadata.name}{": "}{.spec.initContainers[*].name}{"\n"}{end}' \

--request-timeout=30s || true

timeout -k 5 30 kubectl -n tenant-test get virtualmachineinstances.kubevirt.io \

-o jsonpath='{range .items[*]}{.metadata.name}{": logSerialConsole="}{.spec.domain.devices.logSerialConsole}{"\n"}{end}' \

--request-timeout=30s || true

return 2

}

cozy_capture_tenant_serial_console() {

local report_dir="${COZY_REPORT_DIR:-/workspace/_out/cozyreport}/snapshots/${COZY_SNAPSHOT_NAME:-kubernetes}/tenant-serial-console"

local pods pod rc

local list_rc=0

local seen=0

local silent=0

local pod_err

local max_pods=6

local err_log="${report_dir}/setup-error.log"

local warn_log="${report_dir}/READ-WARNINGS.txt"

mkdir -p "${report_dir}"

# stderr goes to its own file rather than being folded into the captured

# stdout: a warning kubectl prints on an otherwise successful call would

# otherwise be read back as a Pod name. Which file it goes to is decided by

# the exit status, not by the fact that something was written: an apiserver

# Warning header or a deprecation notice arrives on stderr with a zero exit,

# so a warning beside a healthy read belongs in READ-WARNINGS.txt as

# elsewhere in this tree, while the stderr of a call that actually failed IS

# the diagnosis and belongs in setup-error.log. The list is wrapped in

# timeout for the same reason every read below it is: --request-timeout

# bounds the HTTP request, not a client retrying against a wedged

# apiserver, and a hang here would cost the whole

# section rather than one Pod.

pods=$(timeout -k 5 30 kubectl -n tenant-test get pods -l kubevirt.io=virt-launcher \

--request-timeout=30s 2>"${warn_log}") || list_rc=$?

if [ "${list_rc}" -ne 0 ]; then

echo "failed to list tenant virt-launcher Pods for serial console capture" >&2

mv "${warn_log}" "${err_log}" 2>/dev/null || true

printf '%s\n' \

"failed to list tenant virt-launcher Pods for serial console capture (exit ${list_rc})" \

>>"${err_log}"

return 1

fi

[ -s "${warn_log}" ] || rm -f "${warn_log}"

if [ -z "${pods}" ]; then

echo "no virt-launcher Pod found for serial console capture" >&2

printf '%s\n' 'no virt-launcher Pod found in namespace tenant-test' \

>"${err_log}"

return 1

fi

# Cap the walk. Each read costs up to 35s (30s plus the kill grace) and the

# pool can reach maxReplicas, so an uncapped loop is an unbounded term inside

# a failure path that already spends ~12m on other collectors under one

# chainsaw op timeout — and the bundle this exists to produce is what gets

# truncated when that op is killed. A cap that fired is recorded with both

# counts rather than leaving the shortfall implicit, since a short listing

# otherwise reads as a small pool rather than a truncated walk.

for pod in ${pods}; do

seen=$((seen + 1))

if [ "${seen}" -gt "${max_pods}" ]; then

echo "--- guest serial console capture stopped at ${max_pods} Pods ---"

printf 'capture stopped after %s Pods; %s matched in total\n' \

"${max_pods}" "$(printf '%s\n' ${pods} | wc -l | tr -d ' ')" \

>"${report_dir}/COLLECTION-TRUNCATED.txt"

break

fi

echo "--- capturing guest serial console: ${pod} ---"

rc=0

# stderr goes beside the capture, not into it. Folded in, kubectl's own

# "container guest-console-log is not valid for pod ..." would make the

# file non-empty and the console would look like it had said something.

# That error is the headline case here -- it is what a container wedged in

# Init looks like -- so it has to stay separable from console bytes. Which

# name it lands under is decided by the exit status, the same way the list

# read above decides it: kubectl writes warnings on stderr with a zero

# status too, and a warning beside a healthy capture is not an error.

pod_err="${report_dir}/${pod}.read-error.log"

timeout -k 5 30 kubectl -n tenant-test logs "${pod}" \

-c guest-console-log --limit-bytes=1048576 \

>"${report_dir}/${pod}.log" 2>"${pod_err}" || rc=$?

if [ ! -s "${pod_err}" ]; then

rm -f "${pod_err}"

pod_err=

elif [ "${rc}" -eq 0 ]; then

mv "${pod_err}" "${report_dir}/${pod}.READ-WARNINGS.txt" 2>/dev/null || true

pod_err=

fi

# Tested before the exit-code line is appended, or nothing is ever empty.

# Three outcomes, three labels, because only one of them is a statement

# about what the guest did.

if [ ! -s "${report_dir}/${pod}.log" ] && [ "${rc}" -ne 0 ]; then

silent=$((silent + 1))

# timeout kills the child without a word, so a read can fail having

# written nothing to either stream. Naming a file that was never created

# sends the reader looking for evidence that does not exist.

if [ -n "${pod_err}" ]; then

printf '%s\n' \

'the read returned no console at all; see read-error.log and POD-STATE.txt' \

>>"${report_dir}/${pod}.log"

else

printf '%s\n' \

'the read failed silently and returned no console at all; see POD-STATE.txt' \

>>"${report_dir}/${pod}.log"

fi

elif [ ! -s "${report_dir}/${pod}.log" ]; then

silent=$((silent + 1))

printf '%s\n' \

'the read succeeded and returned nothing; see POD-STATE.txt for whether the container started' \

>>"${report_dir}/${pod}.log"

elif [ "${rc}" -ne 0 ]; then

silent=$((silent + 1))

printf '%s\n' \

'console output is truncated; see POD-STATE.txt for the Pod state' \

>>"${report_dir}/${pod}.log"

fi

printf '\n[capture exit code: %s]\n' "${rc}" >>"${report_dir}/${pod}.log"

done

# An empty console log has two causes that are indistinguishable in the log

# itself: the guest printed nothing, or the container never started and there

# was no stream to read. The second is what kubevirt/kubevirt#15989 produces,

# and it is the reading that would otherwise be filed as a quiet guest --

# silence reported as a result is the failure this collector exists to

# prevent, so it is not allowed to produce one. The Pod's own state and its

# events are what separate the two, and they are read once for the whole

# selector rather than per Pod: several silent workers are one finding, and

# paying per Pod would put back the unbounded term the cap removed.

if [ "${silent}" -gt 0 ]; then

{

printf '=== describe pods -l kubevirt.io=virt-launcher ===\n'

timeout -k 5 30 kubectl -n tenant-test describe pods \

-l kubevirt.io=virt-launcher --request-timeout=30s 2>&1 || true

printf '\n=== Pod events in tenant-test (not filtered to virt-launcher) ===\n'

timeout -k 5 30 kubectl -n tenant-test get events \

--field-selector involvedObject.kind=Pod \

--sort-by=.lastTimestamp --request-timeout=30s 2>&1 || true

} >"${report_dir}/POD-STATE.txt"

fi

}

# (b1) Guest serial console, read from the management cluster. First of the

# in-guest captures because it is the only one that survives a worker which

# never reached apid — the dominant shape of this failure, where no Node

# registers and no certificate request is ever made. (b) below needs that

# same apid to answer, so it cannot describe this class at all.

# Two shapes of empty result are worth telling apart from a silent guest.

# A virt-launcher Pod still Pending has no containers at all yet, which is

# the ordinary shape of a worker that never booted. A Pod wedged in Init on

# guest-console-log is the opposite reading: the console container is what

# stopped it booting, and then neither a console nor an apid exists to

# explain anything else. The container being absent from a Running Pod is

# the rarest of the three, since the per-VM field outranks the cluster-wide

# disable by design and the green path asserts it attached.

echo "=== (b1) tenant worker guest serial console (management cluster, ns tenant-test) ==="

cozy_capture_tenant_serial_console || true

# Last, after everything the suite exists to prove. This checks a debugging

# aid, not the product: a worker boots and serves either way, so letting it

# run first would let a KubeVirt-side change to a diagnostic preempt the

# assertions that actually cover the tenant cluster. It still runs on the

# passing path rather than beside the collector, because the collector only

# runs when node-join fails and would learn the setting was inert in the one

# run where the console can no longer be recovered. Exit 2 alone fails the

# suite; see cozy_assert_guest_console_attached for why its three outcomes

# are not interchangeable.

echo "» verifying the tenant worker guest console container attached"

attach_rc=0

cozy_assert_guest_console_attached || attach_rc=$?

if [ "${attach_rc}" -eq 2 ]; then

exit 1

fi