11# shellcheck shell=bash
22# Sourced by the chainsaw kubernetes-latest/previous Tests after cd to repo root.
33. hack/e2e-chainsaw/_lib/remediation-guard.sh
4- . hack/e2e-chainsaw/_lib/talos-image-cache.sh
54. hack/e2e-chainsaw/_lib/ghcr-mirror.sh
65
7- # talos_spec_block: emit the tenant Kubernetes CR `spec.talos` block combining the
8- # Talos OS image cache (imageFactoryURL) and the ghcr.io worker image-pull mirror
9- # (registryMirrors), each included only when its in-sandbox mirror is up. Prints
10- # nothing when both fall back to the public defaults , so the chart defaults apply.
11- # Indented for insertion directly under `spec:` in the heredoc below.
6+ # talos_spec_block: emit the tenant Kubernetes CR `spec.talos` block carrying the
7+ # ghcr.io worker image-pull mirror (registryMirrors), included only when the
8+ # in-sandbox mirror is up. Prints nothing when it falls back to the public
9+ # default , so the chart defaults apply. Indented for insertion directly under
10+ # `spec:` in the heredoc below.
1211#
13- # No trailing newline, unlike the single-key helper this replaced: the caller reads
14- # it through `$(...)`, which strips one anyway, and `${talos_block}` sits on a line
15- # of its own in the heredoc, which supplies the break before the next `spec` key.
16- # An empty result therefore leaves a blank line, which is valid YAML.
12+ # This block no longer carries an imageFactoryURL. Workers boot by CDI-cloning the
13+ # golden Talos image in cozy-public (nodeGroups.md0.image.builtin below), so no
14+ # worker imports the OS image over HTTP and the per-worker Talos image-cache mirror
15+ # this once selected has been removed; only the kubelet image pull still leaves the
16+ # sandbox, which is what the ghcr.io mirror covers.
17+ #
18+ # No trailing newline: the caller reads it through `$(...)`, which strips one
19+ # anyway, and `${talos_block}` sits on a line of its own in the heredoc, which
20+ # supplies the break before the next `spec` key. An empty result therefore leaves a
21+ # blank line, which is valid YAML.
1722talos_spec_block () {
18- local url ghcr mirrors
19- url=$( resolve_talos_image_factory_url)
23+ local ghcr mirrors
2024 ghcr=$( resolve_ghcr_mirror_endpoint)
2125 mirrors=$( ghcr_registry_mirrors_block " $ghcr " )
22- [ -n " $url " ] || [ -n " $ mirrors" ] || return 0
26+ [ -n " $mirrors " ] || return 0
2327 printf ' talos:\n'
24- [ -n " $url " ] && printf ' imageFactoryURL: %s\n' " $url "
25- [ -n " $mirrors " ] && printf ' %s' " $mirrors "
28+ printf ' %s' " $mirrors "
2629 return 0
2730}
2831
@@ -2881,12 +2884,13 @@ COZY_DIAG_RATE_INTERVAL_DEFAULT=12
28812884# that moved it, and it is two costs rather than one.
28822885#
28832886# A shorter budget declines more, and what it declines first is what is gated
2884- # last -- ghcr_mirror_diagnose and talos_image_cache_diagnose. Neither loses its
2885- # subject entirely. The mirror's state is partly recoverable from the request
2886- # counts and response durations the mirror capture records against its own log,
2887- # and from the kube-system snapshot the host report takes; the image cache's
2888- # re-probe has no substitute and is the collector this budget gives up first,
2889- # which was already true at 480.
2887+ # last -- ghcr_mirror_diagnose. It does not lose its subject entirely: the
2888+ # mirror's state is partly recoverable from the request counts and response
2889+ # durations the mirror capture records against its own log, and from the
2890+ # kube-system snapshot the host report takes. It used to share the tail with a
2891+ # talos-image-cache re-probe, the one collector here whose subject had no
2892+ # substitute; that cache is gone with the per-worker HTTP import, and the golden
2893+ # it was replaced by is read at (a2), inside the budget rather than at its mercy.
28902894#
28912895# The second cost lands on the guest captures rather than on the tail. Two
28922896# collectors read the node's metric stream ahead of the console -- (d2) and (d3)
@@ -3260,18 +3264,22 @@ cozy_report_node_join_failure() {
32603264 cozy_diag_read ' virt-launcher Pod detail' \
32613265 kubectl -n tenant-test describe pods -l kubevirt.io=virt-launcher " ${request_timeout} "
32623266
3263- # (a2) Worker DataVolume IMPORT stage. A VM stuck "Provisioning" whose
3264- # DataVolume is ImportInProgress at N/A progress with the importer pod
3265- # looping on an HTTP error is a distinct sub-mode of 2a that the VM/VMI
3266- # state alone does not show: the OS image never finishes importing, so the
3267- # VM never boots. This is what took out PR #2826's CI — the CDI importer
3268- # could not reach the talos-image-cache ClusterIP (`dial tcp <svc>:80: i/o
3269- # timeout`) even though the cache pod was healthy. Show the DataVolume/PVC
3270- # phases and the importer pod logs here; the cache ClusterIP re-probe that tells
3271- # "cache path went dead mid-run" apart from "upstream factory slow/flaky" belongs
3272- # to this section too, but it creates a Pod and waits on curl, so it sits with
3273- # the other minute-scale collectors further down rather than here.
3274- echo " === (a2) tenant worker DataVolume import stage (management cluster, ns tenant-test) ==="
3267+ # (a2) Worker DataVolume boot-disk stage. A VM stuck "Provisioning" whose
3268+ # disk-system DataVolume never reaches Succeeded is a distinct sub-mode of 2a
3269+ # that the VM/VMI state alone does not show: the OS disk never populates, so the
3270+ # VM never boots.
3271+ #
3272+ # These suites set nodeGroups.md0.image.builtin, so a worker disk is a CDI CLONE
3273+ # of the golden Talos image in cozy-public (packages/system/kubernetes-worker-image)
3274+ # rather than a per-worker HTTP import. The usual culprit is therefore a clone
3275+ # blocked on a golden that has not finished importing, which is why the golden's
3276+ # own DataVolume/PVC is dumped alongside the worker ones below.
3277+ #
3278+ # The importer-Pod walk further down stays, but read its result with that in mind:
3279+ # on the clone path there is no importer Pod in tenant-test and "matched none" is
3280+ # the healthy state, not a finding. An importer-* Pod appearing at all means the
3281+ # group fell back to the HTTP import, and its log then carries the factory error.
3282+ echo " === (a2) tenant worker DataVolume boot-disk stage (management cluster) ==="
32753283 # The two greps keep these dumps readable. kubectl's stderr is no longer folded
32763284 # into them: `2>&1 | grep` sent a refusal into the filter, which dropped it for
32773285 # not matching, so a read that never happened looked the same as one that found
@@ -3282,6 +3290,12 @@ cozy_report_node_join_failure() {
32823290 cozy_diag_read ' worker DataVolume detail' \
32833291 kubectl -n tenant-test describe datavolume " ${request_timeout} " \
32843292 | grep -Ei ' Name:|Phase:|Progress:|Restart|Reason:|Message:|Running Condition|Bound Condition' || true
3293+ # The clone SOURCE. A worker clone sits pending indefinitely while its golden is
3294+ # still importing, so a worker DataVolume that never leaves Pending means nothing
3295+ # until this read says whether the golden reached Succeeded.
3296+ cozy_diag_read ' golden Talos worker image (clone source, ns cozy-public)' \
3297+ kubectl -n cozy-public get datavolume,pvc -o wide " ${request_timeout} " \
3298+ | grep -E ' NAME|talos-worker' || true
32853299
32863300 # The listing is read on its own rather than inline in the `for`, so a listing
32873301 # that never answered stays distinguishable from a namespace with no importer
@@ -3580,35 +3594,27 @@ cozy_report_node_join_failure() {
35803594 cozy_capture_tenant_talos " ${test_name} " || true
35813595 fi
35823596
3583- # The OS-image cache and the ghcr.io mirror fail independently and produce the
3584- # same symptom from outside the guest, so both get dumped: this one answers
3585- # whether the worker's kubelet-image pull reached the mirror or fell back to
3586- # public ghcr.io, which the node-join failure alone cannot distinguish. Gated
3587- # like its neighbours, and after the guest captures. Bounded read by read
3588- # like the collector at (d2), but five of them at COZY_DIAG_READ_TIMEOUT plus
3589- # grace, so it can spend a quarter of the phase budget -- and time is the
3590- # only thing the gate rations, so a quarter spent here is a quarter the
3591- # guest captures do not get. Cost is not what settles the order, though, or
3592- # (d2) would sit here too: what settles it is whether the answer survives
3593- # being declined. The console evidence this would starve is irreplaceable
3594- # and (d2)'s question has no other answer in the tree, while the mirror's
3595- # state is partly recoverable from the reads above -- so those two go first
3596- # and this one waits, whichever of them is cheaper. Cheaper than the
3597- # talos-image-cache re-probe below, which creates a Pod and waits on curl
3598- # retries, so it goes ahead of it.
3597+ # Last: this answers whether the worker's kubelet-image pull reached the mirror
3598+ # or fell back to public ghcr.io, which the node-join failure alone cannot
3599+ # distinguish. Gated like its neighbours, and after the guest captures. Bounded
3600+ # read by read like the collector at (d2), but five of them at
3601+ # COZY_DIAG_READ_TIMEOUT plus grace, so it can spend a quarter of the phase
3602+ # budget -- and time is the only thing the gate rations, so a quarter spent here
3603+ # is a quarter the guest captures do not get. Cost is not what settles the order,
3604+ # though, or (d2) would sit here too: what settles it is whether the answer
3605+ # survives being declined. The console evidence this would starve is
3606+ # irreplaceable and (d2)'s question has no other answer in the tree, while the
3607+ # mirror's state is partly recoverable from the reads above -- so those two go
3608+ # first and this one waits.
3609+ #
3610+ # It used to have a neighbour here, a re-probe of the talos-image-cache
3611+ # ClusterIP. The worker OS disk is cloned from the golden in cozy-public now
3612+ # rather than imported per worker over HTTP, so there is no cache to re-probe;
3613+ # the golden's own state is read at (a2) instead, off the same one read.
35993614 if cozy_diag_phase_has_time ' ghcr-mirror state, access log and warm-up Job' ; then
36003615 echo " --- ghcr-mirror state, access log and warm-up Job ---"
36013616 ghcr_mirror_diagnose || true
36023617 fi
3603-
3604- # Last, and gated on the phase as well as bounded per read, because it is the
3605- # collector that most needs both: its reachability re-probe creates a Pod, waits
3606- # on curl retries, and makes seven unbounded management-cluster calls of its own,
3607- # so it has no ceiling here at all.
3608- if cozy_diag_phase_has_time ' re-probe talos-image-cache ClusterIP + cacher debug bundle' ; then
3609- echo " --- re-probe talos-image-cache ClusterIP + cacher debug bundle ---"
3610- talos_image_cache_diagnose || true
3611- fi
36123618}
36133619
36143620run_kubernetes_test () {
@@ -3654,22 +3660,21 @@ YAML
36543660)
36553661 fi
36563662
3657- # Point worker DataVolume imports at the in-sandbox Talos OS image cache and
3658- # worker image pulls at the in-sandbox ghcr.io mirror when each is up (falls back
3659- # to the public defaults otherwise). Emitted right under spec: as
3660- # `talos: { imageFactoryURL: ..., registryMirrors: {...} }`, or nothing when both
3661- # defaults apply.
3663+ # Point worker image pulls at the in-sandbox ghcr.io mirror when it is up (falls
3664+ # back to the public default otherwise). Emitted right under spec: as
3665+ # `talos: { registryMirrors: {...} }`, or nothing when the default applies. The
3666+ # worker OS disk is not sourced here at all — it is cloned from the golden below.
36623667 local talos_block
36633668 talos_block=$( talos_spec_block)
36643669
3670+
36653671 kubectl apply -f - << EOF
36663672apiVersion: apps.cozystack.io/v1alpha1
36673673kind: Kubernetes
36683674metadata:
36693675 name: "${test_name} "
36703676 namespace: tenant-test
36713677spec:
3672- ${talos_block}
36733678 addons:
36743679 certManager:
36753680 enabled: false
0 commit comments