@@ -553,6 +553,261 @@ cozy_capture_tenant_talos_node() {
553553 " ${output_dir} /kubelet.log" logs kubelet --tail=500 || true
554554}
555555
556+ # Check that KubeVirt actually attached the guest console container, on the
557+ # path where the workers did boot. The capture below runs only when node-join
558+ # fails, so without this the suite would first learn that the node group's
559+ # logSerialConsole never took effect in the run that needed the console, when
560+ # it can no longer be recovered. The platform disables console logging
561+ # cluster-wide (see packages/system/kubevirt), so what this asks is whether
562+ # the per-VM override beat that on the KubeVirt version actually shipping.
563+ #
564+ # The read's own status is kept separate from the answer it returns, and the
565+ # two get different exit codes, because they are different claims. Piping
566+ # kubectl into grep would collapse them: a read that failed produces no output,
567+ # grep finds nothing, and an API blip would be reported as "the override did
568+ # not take effect" -- a cause that was never established, sending the next
569+ # reader at KubeVirt config instead of at the apiserver. A selector that
570+ # matched no Pod at all is the same trap one step further on, and is kept
571+ # apart from a Pod that matched and carries nothing -- the second is the
572+ # finding, the first is not a statement about any Pod.
573+ #
574+ # What it establishes is that SOME virt-launcher Pod in the namespace carries
575+ # the container, not that every one does. That is the direction worth erring
576+ # in for a check whose only job is to catch an inert setting: it can miss, but
577+ # it cannot fail a run over a Pod that is not this test's. The match runs over
578+ # the whole name=containers line, so a Pod named after the container would
579+ # satisfy it -- these names come from the Machine name, so that cannot happen
580+ # here, but a reader changing the format should know the glob spans both.
581+ #
582+ # 0 a Pod carries the container
583+ # 1 the read did not answer, or matched no Pod; not evidence either way
584+ # 2 Pods matched and none carries the container
585+ cozy_assert_guest_console_attached () {
586+ local init_names read_rc=0
587+ local attached total
588+
589+ # The Pod name is in the output for a reason that is not cosmetic: without
590+ # it, a Pod whose initContainers list is absent emits a bare newline, command
591+ # substitution strips it, and "Pods matched, none carries the container" is
592+ # byte-identical to "no Pod matched". Those two must not collapse -- the
593+ # first is the finding this whole check exists for, and on these workers it
594+ # is also the only shape it can take, since guest-console-log is the only
595+ # init container a DataVolume-booted virt-launcher Pod ever has.
596+ init_names=$( timeout -k 5 30 kubectl -n tenant-test get pods \
597+ -l kubevirt.io=virt-launcher \
598+ -o jsonpath=' {range .items[*]}{.metadata.name}{"="}{.spec.initContainers[*].name}{" "}{.spec.containers[*].name}{"\n"}{end}' \
599+ --request-timeout=30s) || read_rc=$?
600+ if [ " ${read_rc} " -ne 0 ]; then
601+ echo " could not read tenant virt-launcher Pods to verify the guest console container (exit ${read_rc} )" >&2
602+ echo " this says nothing about logSerialConsole either way; continuing" >&2
603+ return 1
604+ fi
605+
606+ if [ -z " ${init_names} " ]; then
607+ echo " no tenant virt-launcher Pod matched; nothing to verify about the guest console container" >&2
608+ echo " this says nothing about logSerialConsole either way; continuing" >&2
609+ return 1
610+ fi
611+
612+ case " ${init_names} " in
613+ * guest-console-log* )
614+ # The positive outcome goes on the record too. This run doubles as the
615+ # revalidation of a platform-wide workaround, and a check that returns
616+ # 0 in silence leaves "the override worked" to be inferred from the
617+ # absence of a complaint -- the inference this collector refuses to
618+ # make everywhere else. Counted rather than asserted for every Pod: the
619+ # check is deliberately "some Pod carries it", so the ratio is the
620+ # honest way to say what was seen.
621+ attached=$( printf ' %s\n' " ${init_names} " | grep -c ' guest-console-log' || true)
622+ total=$( printf ' %s\n' " ${init_names} " | grep -c ' =' || true)
623+ echo " » guest-console-log attached on ${attached} of ${total} virt-launcher Pods"
624+ return 0
625+ ;;
626+ esac
627+
628+ echo " tenant worker virt-launcher Pods carry no guest-console-log container" >&2
629+ echo " the node group asked for logSerialConsole, so either the cluster-wide" >&2
630+ echo " disableSerialConsoleLog won the override or the field never reached the VM" >&2
631+ # Bounded like the read above, and for the same reason: these run on a path
632+ # that ends in exit 1, and a hang here would take the step's whole deadline
633+ # with it -- losing the tenant snapshot that the exit is supposed to reach.
634+ timeout -k 5 30 kubectl -n tenant-test get pods -l kubevirt.io=virt-launcher \
635+ -o jsonpath=' {range .items[*]}{.metadata.name}{": "}{.spec.initContainers[*].name}{"\n"}{end}' \
636+ --request-timeout=30s || true
637+ timeout -k 5 30 kubectl -n tenant-test get virtualmachineinstances.kubevirt.io \
638+ -o jsonpath=' {range .items[*]}{.metadata.name}{": logSerialConsole="}{.spec.domain.devices.logSerialConsole}{"\n"}{end}' \
639+ --request-timeout=30s || true
640+ return 2
641+ }
642+
643+ # Capture each tenant worker's guest serial console from the management
644+ # cluster. When a node group sets logSerialConsole, KubeVirt streams the
645+ # console into a guest-console-log container beside virt-launcher, and reading
646+ # it needs nothing from inside the guest: no talosctl, no client certificate,
647+ # no helper Pod, no reachable apid. That is the whole reason it exists. The
648+ # talosctl capture below can only describe a worker whose apid already answers,
649+ # so a worker that stalls earlier -- the case where no Node registers and no
650+ # certificate request is ever made -- is invisible to it by construction, and
651+ # the console is the only remaining surface.
652+ #
653+ # Reads start at the beginning of the stream rather than tailing it, because a
654+ # guest that repaints its console would push the boot output out of any tail
655+ # window while --limit-bytes truncates from the far end instead. That covers
656+ # only what kubectl returns: whatever the kubelet has already rotated out of
657+ # the container log (containerLogMaxSize, 10Mi x 5 by default) is gone before
658+ # the capture runs, and no read option recovers it.
659+ #
660+ # An absent guest-console-log container is the finding, not a gap: it says
661+ # console logging did not take effect on this run, which is a different fact
662+ # from a guest that printed nothing. kubectl's refusal is kept beside the
663+ # capture rather than inside it so the two stay distinguishable, and the
664+ # capture file says which of them happened.
665+ #
666+ # The Pod list is namespace-wide rather than scoped to one release. The two
667+ # virt-launcher captures immediately above the call site already select that
668+ # way, and suites run one at a time. Below the cap a Pod that does not belong
669+ # to this test costs one extra file named after itself and nothing else; at
670+ # the cap it can displace one of this suite's own, since kubectl returns the
671+ # list name-sorted. COLLECTION-TRUNCATED.txt is what keeps that visible.
672+ cozy_capture_tenant_serial_console () {
673+ local report_dir=" ${COZY_REPORT_DIR:-/ workspace/ _out/ cozyreport} /snapshots/${COZY_SNAPSHOT_NAME:- kubernetes} /tenant-serial-console"
674+ local pods pod rc
675+ local list_rc=0
676+ local seen=0
677+ local silent=0
678+ local pod_err
679+ local max_pods=6
680+ local err_log=" ${report_dir} /setup-error.log"
681+ local warn_log=" ${report_dir} /READ-WARNINGS.txt"
682+
683+ mkdir -p " ${report_dir} "
684+ # stderr goes to its own file rather than being folded into the captured
685+ # stdout: a warning kubectl prints on an otherwise successful call would
686+ # otherwise be read back as a Pod name. Which file it goes to is decided by
687+ # the exit status, not by the fact that something was written: an apiserver
688+ # Warning header or a deprecation notice arrives on stderr with a zero exit,
689+ # so a warning beside a healthy read belongs in READ-WARNINGS.txt as
690+ # elsewhere in this tree, while the stderr of a call that actually failed IS
691+ # the diagnosis and belongs in setup-error.log. The list is wrapped in
692+ # timeout for the same reason every read below it is: --request-timeout
693+ # bounds the HTTP request, not a client retrying against a wedged
694+ # apiserver, and a hang here would cost the whole
695+ # section rather than one Pod.
696+ pods=$( timeout -k 5 30 kubectl -n tenant-test get pods -l kubevirt.io=virt-launcher \
697+ -o jsonpath=' {range .items[*]}{.metadata.name}{"\n"}{end}' \
698+ --request-timeout=30s 2> " ${warn_log} " ) || list_rc=$?
699+ if [ " ${list_rc} " -ne 0 ]; then
700+ echo " failed to list tenant virt-launcher Pods for serial console capture" >&2
701+ mv " ${warn_log} " " ${err_log} " 2> /dev/null || true
702+ printf ' %s\n' \
703+ " failed to list tenant virt-launcher Pods for serial console capture (exit ${list_rc} )" \
704+ >> " ${err_log} "
705+ return 1
706+ fi
707+ [ -s " ${warn_log} " ] || rm -f " ${warn_log} "
708+ if [ -z " ${pods} " ]; then
709+ echo " no virt-launcher Pod found for serial console capture" >&2
710+ printf ' %s\n' ' no virt-launcher Pod found in namespace tenant-test' \
711+ > " ${err_log} "
712+ return 1
713+ fi
714+
715+ # Cap the walk. Each read costs up to 35s (30s plus the kill grace) and the
716+ # pool can reach maxReplicas, so an uncapped loop is an unbounded term inside
717+ # a failure path that already spends ~12m on other collectors under one
718+ # chainsaw op timeout — and the bundle this exists to produce is what gets
719+ # truncated when that op is killed. A cap that fired is recorded with both
720+ # counts rather than leaving the shortfall implicit, since a short listing
721+ # otherwise reads as a small pool rather than a truncated walk.
722+ for pod in ${pods} ; do
723+ seen=$(( seen + 1 ))
724+ if [ " ${seen} " -gt " ${max_pods} " ]; then
725+ echo " --- guest serial console capture stopped at ${max_pods} Pods ---"
726+ printf ' capture stopped after %s Pods; %s matched in total\n' \
727+ " ${max_pods} " " $( printf ' %s\n' ${pods} | wc -l | tr -d ' ' ) " \
728+ > " ${report_dir} /COLLECTION-TRUNCATED.txt"
729+ break
730+ fi
731+ echo " --- capturing guest serial console: ${pod} ---"
732+ rc=0
733+ # stderr goes beside the capture, not into it. Folded in, kubectl's own
734+ # "container guest-console-log is not valid for pod ..." would make the
735+ # file non-empty and the console would look like it had said something.
736+ # That error is the headline case here -- it is what a container wedged in
737+ # Init looks like -- so it has to stay separable from console bytes. Which
738+ # name it lands under is decided by the exit status, the same way the list
739+ # read above decides it: kubectl writes warnings on stderr with a zero
740+ # status too, and a warning beside a healthy capture is not an error.
741+ pod_err=" ${report_dir} /${pod} .read-error.log"
742+ timeout -k 5 30 kubectl -n tenant-test logs " ${pod} " \
743+ -c guest-console-log --limit-bytes=1048576 \
744+ > " ${report_dir} /${pod} .log" 2> " ${pod_err} " || rc=$?
745+ if [ ! -s " ${pod_err} " ]; then
746+ rm -f " ${pod_err} "
747+ pod_err=
748+ elif [ " ${rc} " -eq 0 ]; then
749+ mv " ${pod_err} " " ${report_dir} /${pod} .READ-WARNINGS.txt" 2> /dev/null || true
750+ pod_err=
751+ fi
752+ # Tested before the exit-code line is appended, or nothing is ever empty.
753+ # Three outcomes, three labels, because only one of them is a statement
754+ # about what the guest did.
755+ if [ ! -s " ${report_dir} /${pod} .log" ] && [ " ${rc} " -ne 0 ]; then
756+ silent=$(( silent + 1 ))
757+ # timeout kills the child without a word, so a read can fail having
758+ # written nothing to either stream. Naming a file that was never created
759+ # sends the reader looking for evidence that does not exist.
760+ if [ -n " ${pod_err} " ]; then
761+ printf ' %s\n' \
762+ ' the read returned no console at all; see read-error.log and POD-STATE.txt' \
763+ >> " ${report_dir} /${pod} .log"
764+ else
765+ printf ' %s\n' \
766+ ' the read failed silently and returned no console at all; see POD-STATE.txt' \
767+ >> " ${report_dir} /${pod} .log"
768+ fi
769+ elif [ ! -s " ${report_dir} /${pod} .log" ]; then
770+ silent=$(( silent + 1 ))
771+ printf ' %s\n' \
772+ ' the read succeeded and returned nothing; see POD-STATE.txt for whether the container started' \
773+ >> " ${report_dir} /${pod} .log"
774+ elif [ " ${rc} " -ne 0 ]; then
775+ silent=$(( silent + 1 ))
776+ if [ -n " ${pod_err} " ]; then
777+ printf ' %s\n' \
778+ ' console output is truncated; see read-error.log and POD-STATE.txt' \
779+ >> " ${report_dir} /${pod} .log"
780+ else
781+ printf ' %s\n' \
782+ ' console output is truncated; see POD-STATE.txt for the Pod state' \
783+ >> " ${report_dir} /${pod} .log"
784+ fi
785+ fi
786+ printf ' \n[capture exit code: %s]\n' " ${rc} " >> " ${report_dir} /${pod} .log"
787+ done
788+
789+ # An empty console log has two causes that are indistinguishable in the log
790+ # itself: the guest printed nothing, or the container never started and there
791+ # was no stream to read. The second is what kubevirt/kubevirt#15989 produces,
792+ # and it is the reading that would otherwise be filed as a quiet guest --
793+ # silence reported as a result is the failure this collector exists to
794+ # prevent, so it is not allowed to produce one. The Pod's own state and its
795+ # events are what separate the two, and they are read once for the whole
796+ # selector rather than per Pod: several silent workers are one finding, and
797+ # paying per Pod would put back the unbounded term the cap removed.
798+ if [ " ${silent} " -gt 0 ]; then
799+ {
800+ printf ' === describe pods -l kubevirt.io=virt-launcher ===\n'
801+ timeout -k 5 30 kubectl -n tenant-test describe pods \
802+ -l kubevirt.io=virt-launcher --request-timeout=30s 2>&1 || true
803+ printf ' \n=== Pod events in tenant-test (not filtered to virt-launcher) ===\n'
804+ timeout -k 5 30 kubectl -n tenant-test get events \
805+ --field-selector involvedObject.kind=Pod \
806+ --sort-by=.lastTimestamp --request-timeout=30s 2>&1 || true
807+ } > " ${report_dir} /POD-STATE.txt"
808+ fi
809+ }
810+
556811# Collect the guest-side evidence requested by issue #3513. This runs only after
557812# the 18-minute Ready deadline has already failed. VMIs without a reported IP
558813# are recorded before any Certificate or Pod is created; when at least one IP is
@@ -728,6 +983,12 @@ ${ouroboros_addon}
728983 diskSize: 20Gi
729984 gpus: []
730985 instanceType: u1.medium
986+ # The failure this suite keeps hitting is a worker that stalls in the
987+ # guest before Talos apid answers, which leaves nothing to read on the
988+ # management side and nothing for talosctl to connect to. Turning this on
989+ # attaches KubeVirt's guest-console-log container, the only artifact that
990+ # covers that window.
991+ logSerialConsole: true
731992 maxReplicas: 10
732993 minReplicas: 2
733994 resources: {}
@@ -886,6 +1147,22 @@ EOF
8861147 kubectl -n tenant-test get pods -l kubevirt.io=virt-launcher -o wide || true
8871148 kubectl -n tenant-test describe pods -l kubevirt.io=virt-launcher || true
8881149
1150+ # (b1) Guest serial console, read from the management cluster. First of the
1151+ # in-guest captures because it is the only one that survives a worker which
1152+ # never reached apid — the dominant shape of this failure, where no Node
1153+ # registers and no certificate request is ever made. (b) below needs that
1154+ # same apid to answer, so it cannot describe this class at all.
1155+ # Two shapes of empty result are worth telling apart from a silent guest.
1156+ # A virt-launcher Pod still Pending has no containers at all yet, which is
1157+ # the ordinary shape of a worker that never booted. A Pod wedged in Init on
1158+ # guest-console-log is the opposite reading: the console container is what
1159+ # stopped it booting, and then neither a console nor an apid exists to
1160+ # explain anything else. The container being absent from a Running Pod is
1161+ # the rarest of the three, since the per-VM field outranks the cluster-wide
1162+ # disable by design and the green path asserts it attached.
1163+ echo " === (b1) tenant worker guest serial console (management cluster, ns tenant-test) ==="
1164+ cozy_capture_tenant_serial_console || true
1165+
8891166 # (b) In-guest Talos kernel and kubelet logs. The tenant chart intentionally
8901167 # has no admin talosconfig, so mint a one-hour os:reader client from its
8911168 # existing cert-manager Issuer and run talosctl from a hardened Pod that can
@@ -1484,6 +1761,22 @@ EOF
14841761 exit 1
14851762 fi
14861763
1764+ # Last, after everything the suite exists to prove. This checks a debugging
1765+ # aid, not the product: a worker boots and serves either way, so letting it
1766+ # run first would let a KubeVirt-side change to a diagnostic preempt the
1767+ # assertions that actually cover the tenant cluster. It still runs on the
1768+ # passing path rather than beside the collector, because the collector only
1769+ # runs when node-join fails and would learn the setting was inert in the one
1770+ # run where the console can no longer be recovered. Exit 2 alone fails the
1771+ # suite; see cozy_assert_guest_console_attached for why its three outcomes
1772+ # are not interchangeable.
1773+ echo " » verifying the tenant worker guest console container attached"
1774+ attach_rc=0
1775+ cozy_assert_guest_console_attached || attach_rc=$?
1776+ if [ " ${attach_rc} " -eq 2 ]; then
1777+ exit 1
1778+ fi
1779+
14871780 # Success: disarm the tenant-snapshot trap so it doesn't fire on the clean exit.
14881781 trap - EXIT
14891782 # Clean up: delete the test-scoped tenant API LoadBalancer (frees its MetalLB
0 commit comments