11# shellcheck shell=bash
22# Sourced by the chainsaw kubernetes-latest/previous Tests after cd to repo root.
33. hack/e2e-chainsaw/_lib/remediation-guard.sh
4- . hack/e2e-chainsaw/_lib/talos-image-cache.sh
54. hack/e2e-chainsaw/_lib/ghcr-mirror.sh
65
7- # talos_spec_block: emit the tenant Kubernetes CR `spec.talos` block combining the
8- # Talos OS image cache (imageFactoryURL) and the ghcr.io worker image-pull mirror
9- # (registryMirrors), each included only when its in-sandbox mirror is up. Prints
10- # nothing when both fall back to the public defaults , so the chart defaults apply.
11- # Indented for insertion directly under `spec:` in the heredoc below.
6+ # talos_spec_block: emit the tenant Kubernetes CR `spec.talos` block carrying the
7+ # ghcr.io worker image-pull mirror (registryMirrors), included only when the
8+ # in-sandbox mirror is up. Prints nothing when it falls back to the public
9+ # default , so the chart defaults apply. Indented for insertion directly under
10+ # `spec:` in the heredoc below.
1211#
13- # No trailing newline, unlike the single-key helper this replaced: the caller reads
14- # it through `$(...)`, which strips one anyway, and `${talos_block}` sits on a line
15- # of its own in the heredoc, which supplies the break before the next `spec` key.
16- # An empty result therefore leaves a blank line, which is valid YAML.
12+ # This block no longer carries an imageFactoryURL. Workers boot by CDI-cloning the
13+ # golden Talos image in cozy-public (nodeGroups.md0.image.builtin below), so no
14+ # worker imports the OS image over HTTP and the per-worker Talos image-cache mirror
15+ # this once selected has been removed; only the kubelet image pull still leaves the
16+ # sandbox, which is what the ghcr.io mirror covers.
17+ #
18+ # No trailing newline: the caller reads it through `$(...)`, which strips one
19+ # anyway, and `${talos_block}` sits on a line of its own in the heredoc, which
20+ # supplies the break before the next `spec` key. An empty result therefore leaves a
21+ # blank line, which is valid YAML.
1722talos_spec_block () {
18- local url ghcr mirrors
19- url=$( resolve_talos_image_factory_url)
23+ local ghcr mirrors
2024 ghcr=$( resolve_ghcr_mirror_endpoint)
2125 mirrors=$( ghcr_registry_mirrors_block " $ghcr " )
22- [ -n " $url " ] || [ -n " $ mirrors" ] || return 0
26+ [ -n " $mirrors " ] || return 0
2327 printf ' talos:\n'
24- [ -n " $url " ] && printf ' imageFactoryURL: %s\n' " $url "
25- [ -n " $mirrors " ] && printf ' %s' " $mirrors "
28+ printf ' %s' " $mirrors "
2629 return 0
2730}
2831
@@ -3277,12 +3280,13 @@ COZY_DIAG_RATE_INTERVAL_DEFAULT=12
32773280# that moved it, and it is two costs rather than one.
32783281#
32793282# A shorter budget declines more, and what it declines first is what is gated
3280- # last -- ghcr_mirror_diagnose and talos_image_cache_diagnose. Neither loses its
3281- # subject entirely. The mirror's state is partly recoverable from the request
3282- # counts and response durations the mirror capture records against its own log,
3283- # and from the kube-system snapshot the host report takes; the image cache's
3284- # re-probe has no substitute and is the collector this budget gives up first,
3285- # which was already true at 480.
3283+ # last -- ghcr_mirror_diagnose. It does not lose its subject entirely: the
3284+ # mirror's state is partly recoverable from the request counts and response
3285+ # durations the mirror capture records against its own log, and from the
3286+ # kube-system snapshot the host report takes. It used to share the tail with a
3287+ # talos-image-cache re-probe, the one collector here whose subject had no
3288+ # substitute; that cache is gone with the per-worker HTTP import, and the golden
3289+ # it was replaced by is read at (a2), inside the budget rather than at its mercy.
32863290#
32873291# The second cost lands on the guest captures rather than on the tail. Two
32883292# collectors read the node's metric stream ahead of the console -- (d2) and (d3)
@@ -3669,18 +3673,22 @@ cozy_report_node_join_failure() {
36693673 cozy_diag_read ' virt-launcher Pod detail' \
36703674 kubectl -n tenant-test describe pods -l kubevirt.io=virt-launcher " ${request_timeout} "
36713675
3672- # (a2) Worker DataVolume IMPORT stage. A VM stuck "Provisioning" whose
3673- # DataVolume is ImportInProgress at N/A progress with the importer pod
3674- # looping on an HTTP error is a distinct sub-mode of 2a that the VM/VMI
3675- # state alone does not show: the OS image never finishes importing, so the
3676- # VM never boots. This is what took out PR #2826's CI — the CDI importer
3677- # could not reach the talos-image-cache ClusterIP (`dial tcp <svc>:80: i/o
3678- # timeout`) even though the cache pod was healthy. Show the DataVolume/PVC
3679- # phases and the importer pod logs here; the cache ClusterIP re-probe that tells
3680- # "cache path went dead mid-run" apart from "upstream factory slow/flaky" belongs
3681- # to this section too, but it creates a Pod and waits on curl, so it sits with
3682- # the other minute-scale collectors further down rather than here.
3683- echo " === (a2) tenant worker DataVolume import stage (management cluster, ns tenant-test) ==="
3676+ # (a2) Worker DataVolume boot-disk stage. A VM stuck "Provisioning" whose
3677+ # disk-system DataVolume never reaches Succeeded is a distinct sub-mode of 2a
3678+ # that the VM/VMI state alone does not show: the OS disk never populates, so the
3679+ # VM never boots.
3680+ #
3681+ # These suites set nodeGroups.md0.image.builtin, so a worker disk is a CDI CLONE
3682+ # of the golden Talos image in cozy-public (packages/system/kubernetes-worker-image)
3683+ # rather than a per-worker HTTP import. The usual culprit is therefore a clone
3684+ # blocked on a golden that has not finished importing, which is why the golden's
3685+ # own DataVolume/PVC is dumped alongside the worker ones below.
3686+ #
3687+ # The importer-Pod walk further down stays, but read its result with that in mind:
3688+ # on the clone path there is no importer Pod in tenant-test and "matched none" is
3689+ # the healthy state, not a finding. An importer-* Pod appearing at all means the
3690+ # group fell back to the HTTP import, and its log then carries the factory error.
3691+ echo " === (a2) tenant worker DataVolume boot-disk stage (management cluster) ==="
36843692 # The two greps keep these dumps readable. kubectl's stderr is no longer folded
36853693 # into them: `2>&1 | grep` sent a refusal into the filter, which dropped it for
36863694 # not matching, so a read that never happened looked the same as one that found
@@ -3691,6 +3699,12 @@ cozy_report_node_join_failure() {
36913699 cozy_diag_read ' worker DataVolume detail' \
36923700 kubectl -n tenant-test describe datavolume " ${request_timeout} " \
36933701 | grep -Ei ' Name:|Phase:|Progress:|Restart|Reason:|Message:|Running Condition|Bound Condition' || true
3702+ # The clone SOURCE. A worker clone sits pending indefinitely while its golden is
3703+ # still importing, so a worker DataVolume that never leaves Pending means nothing
3704+ # until this read says whether the golden reached Succeeded.
3705+ cozy_diag_read ' golden Talos worker image (clone source, ns cozy-public)' \
3706+ kubectl -n cozy-public get datavolume,pvc -o wide " ${request_timeout} " \
3707+ | grep -E ' NAME|talos-worker' || true
36943708
36953709 # The listing is read on its own rather than inline in the `for`, so a listing
36963710 # that never answered stays distinguishable from a namespace with no importer
@@ -3989,35 +4003,27 @@ cozy_report_node_join_failure() {
39894003 cozy_capture_tenant_talos " ${test_name} " || true
39904004 fi
39914005
3992- # The OS-image cache and the ghcr.io mirror fail independently and produce the
3993- # same symptom from outside the guest, so both get dumped: this one answers
3994- # whether the worker's kubelet-image pull reached the mirror or fell back to
3995- # public ghcr.io, which the node-join failure alone cannot distinguish. Gated
3996- # like its neighbours, and after the guest captures. Bounded read by read
3997- # like the collector at (d2), but five of them at COZY_DIAG_READ_TIMEOUT plus
3998- # grace, so it can spend a quarter of the phase budget -- and time is the
3999- # only thing the gate rations, so a quarter spent here is a quarter the
4000- # guest captures do not get. Cost is not what settles the order, though, or
4001- # (d2) would sit here too: what settles it is whether the answer survives
4002- # being declined. The console evidence this would starve is irreplaceable
4003- # and (d2)'s question has no other answer in the tree, while the mirror's
4004- # state is partly recoverable from the reads above -- so those two go first
4005- # and this one waits, whichever of them is cheaper. Cheaper than the
4006- # talos-image-cache re-probe below, which creates a Pod and waits on curl
4007- # retries, so it goes ahead of it.
4006+ # Last: this answers whether the worker's kubelet-image pull reached the mirror
4007+ # or fell back to public ghcr.io, which the node-join failure alone cannot
4008+ # distinguish. Gated like its neighbours, and after the guest captures. Bounded
4009+ # read by read like the collector at (d2), but five of them at
4010+ # COZY_DIAG_READ_TIMEOUT plus grace, so it can spend a quarter of the phase
4011+ # budget -- and time is the only thing the gate rations, so a quarter spent here
4012+ # is a quarter the guest captures do not get. Cost is not what settles the order,
4013+ # though, or (d2) would sit here too: what settles it is whether the answer
4014+ # survives being declined. The console evidence this would starve is
4015+ # irreplaceable and (d2)'s question has no other answer in the tree, while the
4016+ # mirror's state is partly recoverable from the reads above -- so those two go
4017+ # first and this one waits.
4018+ #
4019+ # It used to have a neighbour here, a re-probe of the talos-image-cache
4020+ # ClusterIP. The worker OS disk is cloned from the golden in cozy-public now
4021+ # rather than imported per worker over HTTP, so there is no cache to re-probe;
4022+ # the golden's own state is read at (a2) instead, off the same one read.
40084023 if cozy_diag_phase_has_time ' ghcr-mirror state, access log and warm-up Job' ; then
40094024 echo " --- ghcr-mirror state, access log and warm-up Job ---"
40104025 ghcr_mirror_diagnose || true
40114026 fi
4012-
4013- # Last, and gated on the phase as well as bounded per read, because it is the
4014- # collector that most needs both: its reachability re-probe creates a Pod, waits
4015- # on curl retries, and makes seven unbounded management-cluster calls of its own,
4016- # so it has no ceiling here at all.
4017- if cozy_diag_phase_has_time ' re-probe talos-image-cache ClusterIP + cacher debug bundle' ; then
4018- echo " --- re-probe talos-image-cache ClusterIP + cacher debug bundle ---"
4019- talos_image_cache_diagnose || true
4020- fi
40214027}
40224028
40234029run_kubernetes_test () {
@@ -4063,22 +4069,21 @@ YAML
40634069)
40644070 fi
40654071
4066- # Point worker DataVolume imports at the in-sandbox Talos OS image cache and
4067- # worker image pulls at the in-sandbox ghcr.io mirror when each is up (falls back
4068- # to the public defaults otherwise). Emitted right under spec: as
4069- # `talos: { imageFactoryURL: ..., registryMirrors: {...} }`, or nothing when both
4070- # defaults apply.
4072+ # Point worker image pulls at the in-sandbox ghcr.io mirror when it is up (falls
4073+ # back to the public default otherwise). Emitted right under spec: as
4074+ # `talos: { registryMirrors: {...} }`, or nothing when the default applies. The
4075+ # worker OS disk is not sourced here at all — it is cloned from the golden below.
40714076 local talos_block
40724077 talos_block=$( talos_spec_block)
40734078
4079+
40744080 kubectl apply -f - << EOF
40754081apiVersion: apps.cozystack.io/v1alpha1
40764082kind: Kubernetes
40774083metadata:
40784084 name: "${test_name} "
40794085 namespace: tenant-test
40804086spec:
4081- ${talos_block}
40824087 addons:
40834088 certManager:
40844089 enabled: false
0 commit comments