22# e2e CRs.
33#
44# Prefer the in-sandbox caching mirror (hack/e2e-talos-image-cache.yaml, deployed
5- # by hack/e2e-install-cozystack.bats) when it is Available, otherwise fall back to
6- # the chart default (the public factory.talos.dev) by emitting nothing. The
7- # mirror rides out the public factory's flaky/range-less egress that otherwise
8- # stalls worker DataVolume imports past the 12-minute node-join deadline — see
9- # cozystack/cozystack#3231. Falling back to the default means the mirror can only
10- # help, never make CI worse.
5+ # by hack/e2e-install-cozystack.bats) when a worker's CDI importer can actually
6+ # reach it, otherwise fall back to the chart default (the public factory.talos.dev)
7+ # by emitting nothing. The mirror rides out the public factory's flaky/range-less
8+ # egress that otherwise stalls worker DataVolume imports past the 12-minute
9+ # node-join deadline — see cozystack/cozystack#3231. Falling back to the default
10+ # means the mirror can only help, never make CI worse.
11+ #
12+ # Reachability is not assumed. Tenant namespaces run under a default-deny Cilium
13+ # egress installed by the tenant chart: a worker's importer may reach the `world`
14+ # entity (hence the public factory) and the tenant's own tree, but not an
15+ # arbitrary kube-system Service. Without an explicit allow, the importer resolves
16+ # talos-image-cache.kube-system.svc yet its TCP connect to the ClusterIP is
17+ # silently dropped and the disk import never starts. This helper installs that
18+ # allow (the CiliumClusterwideNetworkPolicy shipped in the mirror manifest) and
19+ # then verifies the path end to end from a Pod that faces the exact same egress
20+ # rules as a real importer, so the mirror is used only when it genuinely works.
21+ #
22+ # Root cause + fix are from cozystack/cozystack#3254 (@lexfrei), ported here to
23+ # the Chainsaw layout (hack/e2e-chainsaw/_lib/).
1124
1225TALOS_IMAGE_CACHE_SVC_URL=" ${TALOS_IMAGE_CACHE_SVC_URL:- http:// talos-image-cache.kube-system.svc} "
1326_TALOS_IMAGE_FACTORY_DECISION_FILE=" ${_TALOS_IMAGE_FACTORY_DECISION_FILE:-/ tmp/ e2e-talos-image-factory-url} "
27+ TALOS_IMAGE_CACHE_MANIFEST=" ${TALOS_IMAGE_CACHE_MANIFEST:- hack/ e2e-talos-image-cache.yaml} "
28+ # Tenant namespace the kubernetes-* CRs are created in (run-kubernetes.sh and the
29+ # oidc suites all target tenant-test); the reachability probe runs here so it
30+ # inherits that namespace's default-deny egress, exactly as a worker importer does.
31+ TALOS_IMAGE_CACHE_PROBE_NS=" ${TALOS_IMAGE_CACHE_PROBE_NS:- tenant-test} "
32+ TALOS_IMAGE_CACHE_PROBE_POD=" ${TALOS_IMAGE_CACHE_PROBE_POD:- talos-image-cache-probe} "
33+
34+ # _apply_talos_image_cache_egress_policy: install the CiliumClusterwideNetworkPolicy
35+ # that lets tenant CDI importer Pods egress to the mirror. It lives in the mirror
36+ # manifest but is applied here, not in hack/e2e-install-cozystack.bats, because that
37+ # manifest is applied before Cozystack (and thus before Cilium's CRDs) exists.
38+ # Idempotent; best-effort — a failure just leaves the probe below to fall back.
39+ _apply_talos_image_cache_egress_policy () {
40+ command -v yq > /dev/null 2>&1 || return 1
41+ yq ' select(.kind == "CiliumClusterwideNetworkPolicy")' " $TALOS_IMAGE_CACHE_MANIFEST " 2> /dev/null \
42+ | kubectl apply -f - > /dev/null 2>&1
43+ }
44+
45+ # _talos_image_cache_probe_overrides: emit the kubectl --overrides JSON for the
46+ # probe Pod. $1 is the container image, $2 the shell command it runs. The single
47+ # container replaces the one `kubectl run` generates (merge patch semantics), and
48+ # the security context satisfies the tenant namespace's restricted PSA.
49+ #
50+ # Pure string builder, kept separate so it can be unit-tested: an edit that
51+ # unbalances a quote here would emit invalid JSON, every probe would fail, and CI
52+ # would silently fall back to the public factory instead of erroring.
53+ _talos_image_cache_probe_overrides () {
54+ printf ' {"spec":{"securityContext":{"runAsNonRoot":true,"runAsUser":1000,"seccompProfile":{"type":"RuntimeDefault"}},"containers":[{"name":"c","image":"%s","command":["sh","-ec","%s"],"securityContext":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]}}}]}}' \
55+ " $1 " " $2 "
56+ }
57+
58+ # _talos_image_cache_probe_succeeded: succeed iff the probe output ($1) carries the
59+ # strict 206 marker.
60+ #
61+ # Gate strictly on 206. The old localhost check kept a reachable-but-range-less
62+ # mirror and only warned; here any non-206 — unreachable (code=000) or a plain 200 —
63+ # falls back to the public factory instead. serve.py always answers a valid range
64+ # with 206, so the 200 case is unreachable in practice; the strict gate keeps the
65+ # "use the mirror only when it is fully working" rule simple.
66+ #
67+ # Pure predicate, kept separate so it can be unit-tested: this single comparison is
68+ # what decides whether tenant workers are pointed at the mirror at all.
69+ _talos_image_cache_probe_succeeded () {
70+ printf ' %s' " $1 " | grep -q ' code=206'
71+ }
1472
15- # _talos_image_cache_serves_ranges: succeed iff the mirror answers a byte-range
16- # request with 206 Partial Content. Runs curl inside the serve pod against
17- # localhost, range-probing the seeded image file it finds under /data, so the
18- # check works without the sandbox being able to reach the ClusterIP.
19- _talos_image_cache_serves_ranges () {
20- local pod code
73+ # _talos_image_cache_reachable_from_tenant: succeed iff a Pod that faces the exact
74+ # same tenant egress restrictions and network path as a worker's CDI importer can
75+ # reach the mirror Service ClusterIP with a byte-range (206) response.
76+ #
77+ # This is the load-bearing gate: a green result guarantees a real importer will
78+ # reach the mirror. It runs a throwaway Pod in the tenant namespace, labelled
79+ # cdi.kubevirt.io=importer (so the tenant egress default-deny plus the allow
80+ # policy above apply to it just as they do to a real importer), and has it curl
81+ # the ClusterIP Service for the seeded image with a Range header — exercising DNS,
82+ # the egress policy, ClusterIP translation, and range support together, not merely
83+ # that the server answers on localhost.
84+ #
85+ # cdi.kubevirt.io=importer is CDI's own importer-pod label. The probe and the
86+ # egress policy's endpointSelector must both use it; if a future CDI renames it,
87+ # they would still agree with each other but no longer with reality, and the probe
88+ # would pass while real importers stayed blocked. It is correct as of the CDI
89+ # version this platform ships (packages/system/kubevirt-cdi-operator).
90+ _talos_image_cache_reachable_from_tenant () {
91+ local pod rel img curl_cmd overrides out
2192 pod=$( kubectl -n kube-system get pod \
2293 -l app.kubernetes.io/name=talos-image-cache \
2394 --field-selector=status.phase=Running \
2495 -o jsonpath=' {.items[0].metadata.name}' 2> /dev/null) || return 1
2596 [ -n " $pod " ] || return 1
26- code=$( kubectl -n kube-system exec " $pod " -c serve -- sh -ec '
27- f=$(find /data -name "*.raw.xz" 2>/dev/null | head -n1)
28- [ -n "$f" ] || exit 1
29- curl -s -o /dev/null -w "%{http_code}" -r 0-0 "http://127.0.0.1:8080/${f#/data/}"
30- ' 2> /dev/null) || return 1
31- [ " $code " = " 206" ]
97+ # Discover the seeded file (its presence also confirms the seed finished) and
98+ # probe that exact path, so the probe requests what a worker actually requests.
99+ rel=$( kubectl -n kube-system exec " $pod " -c serve -- sh -ec \
100+ ' f=$(find /data -name "*.raw.xz" 2>/dev/null | head -n1); [ -n "$f" ] || exit 1; printf "%s" "${f#/data/}"' \
101+ 2> /dev/null) || return 1
102+ [ -n " $rel " ] || return 1
103+ # Reuse the mirror's own digest-pinned image for the probe Pod. It is the same
104+ # image many CI hooks share, so by this late stage of the suite it is usually
105+ # already cached on whatever node the probe lands on and no pull is needed; if
106+ # the probe does land on a node without it, the pull is bounded by the Pod's
107+ # --pod-running-timeout below and a timeout just falls back to the public factory.
108+ img=$( kubectl -n kube-system get deploy talos-image-cache \
109+ -o jsonpath=' {.spec.template.spec.containers[0].image}' 2> /dev/null) || return 1
110+ [ -n " $img " ] || return 1
111+
112+ # --retry rides out the seconds Cilium needs to program the freshly-applied allow
113+ # policy; a persistently unreachable ClusterIP still fails fast per attempt
114+ # (connect-timeout) and returns a non-206, falling back to the public factory.
115+ # The total budget is deliberately generous: this decision is cached once for the
116+ # whole suite, so a single slow policy-programming window must not permanently
117+ # disable the mirror. --max-time still bounds a hung attempt (a genuinely
118+ # unreachable ClusterIP resolves well within it and falls back).
119+ curl_cmd=" curl -s -o /dev/null -w code=%{http_code} --retry 8 --retry-all-errors --retry-delay 5 --connect-timeout 5 --max-time 45 -r 0-0 ${TALOS_IMAGE_CACHE_SVC_URL} /${rel} "
120+ overrides=$( _talos_image_cache_probe_overrides " $img " " $curl_cmd " )
121+
122+ kubectl -n " $TALOS_IMAGE_CACHE_PROBE_NS " delete pod " $TALOS_IMAGE_CACHE_PROBE_POD " \
123+ --ignore-not-found > /dev/null 2>&1
124+ # No --rm: with --rm the Pod is gone the instant it exits, and --attach can lose
125+ # the tail of a fast-completing Pod's stream, leaving $out empty — a false negative
126+ # that would fall back to the flaky public factory this mirror exists to avoid.
127+ # Keep the Pod so its logs can be re-read if the attach stream was lost, then
128+ # delete it explicitly below.
129+ out=$( kubectl -n " $TALOS_IMAGE_CACHE_PROBE_NS " run " $TALOS_IMAGE_CACHE_PROBE_POD " \
130+ --restart=Never --attach --pod-running-timeout=90s \
131+ --image=" $img " --labels=' cdi.kubevirt.io=importer' \
132+ --overrides=" $overrides " 2>&1 ) || true
133+ # --attach returns only after the container exits, so its logs are ready now;
134+ # fall back to them when the attach stream carried no result marker.
135+ if ! printf ' %s' " $out " | grep -q ' code=' ; then
136+ out=$( kubectl -n " $TALOS_IMAGE_CACHE_PROBE_NS " logs " $TALOS_IMAGE_CACHE_PROBE_POD " 2> /dev/null || true)
137+ fi
138+ # Fire-and-forget: the result is already captured in $out, so nothing depends on
139+ # the Pod actually being gone. The pre-run delete above must NOT do this — there,
140+ # `kubectl run` would race a still-terminating Pod of the same name.
141+ kubectl -n " $TALOS_IMAGE_CACHE_PROBE_NS " delete pod " $TALOS_IMAGE_CACHE_PROBE_POD " \
142+ --ignore-not-found --wait=false > /dev/null 2>&1
143+ _talos_image_cache_probe_succeeded " $out "
32144}
33145
34146# resolve_talos_image_factory_url: print the imageFactoryURL to use, or an empty
@@ -44,21 +156,21 @@ resolve_talos_image_factory_url() {
44156 local url=" "
45157 if kubectl -n kube-system get deploy talos-image-cache > /dev/null 2>&1 ; then
46158 # Available only after the seed initContainer has fully fetched the image, so
47- # this is the signal that the mirror is warm and safe to point tenants at .
159+ # this is the signal that the mirror is warm.
48160 if kubectl -n kube-system rollout status deploy/talos-image-cache --timeout=12m > /dev/null 2>&1 ; then
49- url= " $TALOS_IMAGE_CACHE_SVC_URL "
50- # Available only proves the seed finished and the port is open. Verify the
51- # server actually answers a byte-range request with 206 (not a range-less
52- # 200): range support is what lets CDI skip the slow copy-to-scratch path,
53- # and a future swap to a range-incapable server would otherwise regress it
54- # silently. Probe from inside the serve pod so this does not depend on the
55- # sandbox reaching a ClusterIP. A non-206 still imports correctly (just
56- # without the fast path), so warn loudly rather than discard the mirror and
57- # reintroduce the public factory's egress flakiness.
58- if _talos_image_cache_serves_ranges ; then
59- echo " talos-image-cache mirror ready (byte-range verified) — tenant workers import from ${url} " >&2
161+ # Available proves only that the seed finished and the server answers on
162+ # localhost — NOT that a worker's importer, penned in by the tenant egress
163+ # default-deny, can reach the ClusterIP. Install the allow policy (Cilium is
164+ # up by now, unlike at seed-deploy time) and verify the whole importer path
165+ # end to end. Point tenants at the mirror only when that check passes;
166+ # otherwise fall back to the public factory so the mirror can only help,
167+ # never point workers at an unreachable ClusterIP and make CI worse.
168+ _apply_talos_image_cache_egress_policy || true
169+ if _talos_image_cache_reachable_from_tenant ; then
170+ url= " $TALOS_IMAGE_CACHE_SVC_URL "
171+ echo " talos-image-cache mirror reachable from tenant namespace (byte-range 206 verified) — tenant workers import from ${url} " >&2
60172 else
61- echo " WARNING: talos-image-cache is Available but did not answer a range request with 206 — tenant workers still import from ${url} , but CDI will fall back to a full copy " >&2
173+ echo " WARNING: talos-image-cache is Available but a tenant-scoped probe could not reach its ClusterIP with a 206 — tenant workers fall back to public factory.talos.dev " >&2
62174 fi
63175 else
64176 echo " WARNING: talos-image-cache mirror not Available in time — tenant workers fall back to public factory.talos.dev" >&2
@@ -70,6 +182,34 @@ resolve_talos_image_factory_url() {
70182 printf ' %s' " $url "
71183}
72184
185+ # talos_image_cache_diagnose: on-demand diagnostic for the node-join failure path
186+ # in run-kubernetes.sh (Chainsaw-branch addition, not in #3254). Re-probes the
187+ # mirror from the tenant namespace — the same path a real importer faces — and
188+ # dumps the cacher + egress-policy state, so an operator can tell "the egress
189+ # allow / ClusterIP path is broken" apart from "the upstream factory was slow".
190+ # All output to stderr; never fails the caller.
191+ talos_image_cache_diagnose () {
192+ if ! kubectl -n kube-system get deploy talos-image-cache > /dev/null 2>&1 ; then
193+ echo " talos-image-cache not deployed — tenant workers used the public factory; nothing to diagnose" >&2
194+ return 0
195+ fi
196+ {
197+ if _talos_image_cache_reachable_from_tenant; then
198+ echo " talos-image-cache reachable from ${TALOS_IMAGE_CACHE_PROBE_NS} at diagnose time (byte-range 206)"
199+ else
200+ echo " talos-image-cache NOT reachable from ${TALOS_IMAGE_CACHE_PROBE_NS} at diagnose time (tenant egress allow missing/unprogrammed, or cache down)"
201+ fi
202+ echo " --- cache deploy/pod/svc/endpointslice ---"
203+ kubectl -n kube-system get deploy,pod,svc,endpointslice \
204+ -l app.kubernetes.io/name=talos-image-cache -o wide 2>&1 || true
205+ echo " --- importer egress allow policy ---"
206+ kubectl get ciliumclusterwidenetworkpolicy e2e-talos-image-cache-importer-egress -o wide 2>&1 || true
207+ echo " --- serve container log (tail) ---"
208+ kubectl -n kube-system logs -l app.kubernetes.io/name=talos-image-cache \
209+ -c serve --tail=50 --prefix 2>&1 || true
210+ } >&2
211+ }
212+
73213# talos_image_factory_spec_block: emit a two-line YAML block
74214# talos:
75215# imageFactoryURL: <url>
0 commit comments