@@ -4,12 +4,36 @@ run_kubernetes_test() {
44 local version_expr=" $1 "
55 local test_name=" $2 "
66 local port=" $3 "
7+ # Optional: when "true", enable the ouroboros addon on the Kubernetes CR
8+ # and run the hairpin-NAT reconciliation assertions after the cluster is
9+ # Ready. Folded in here so we don't pay a second ~25m Kamaji bringup just
10+ # to flip one addon flag — kubernetes-latest passes "true", kubernetes-
11+ # previous leaves it empty.
12+ local enable_ouroboros=" ${4:- } "
713 local k8s_version=$( yq " $version_expr " packages/apps/kubernetes/files/versions.yaml)
814
915 # Clean up stale resources from a previous failed retry
1016 kubectl -n tenant-test delete kuberneteses.apps.cozystack.io " ${test_name} " --ignore-not-found --wait=false 2> /dev/null || true
1117 kubectl -n tenant-test wait kuberneteses.apps.cozystack.io " ${test_name} " --for=delete --timeout=2m 2> /dev/null || true
1218
19+ # Compose the optional ouroboros addon block. Indentation matches the
20+ # surrounding addons map (4 spaces).
21+ local ouroboros_addon=" "
22+ if [ " ${enable_ouroboros} " = " true" ]; then
23+ ouroboros_addon=$( cat << 'YAML '
24+ ouroboros:
25+ enabled: true
26+ # logLevel=debug surfaces controller informer events for failure
27+ # diagnosis; scoped to the e2e fixture only, production tenants stay
28+ # on the upstream chart default (info).
29+ valuesOverride:
30+ ouroboros:
31+ controller:
32+ logLevel: debug
33+ YAML
34+ )
35+ fi
36+
1337 kubectl apply -f - << EOF
1438apiVersion: apps.cozystack.io/v1alpha1
1539kind: Kubernetes
3862 monitoringAgents:
3963 enabled: false
4064 valuesOverride: {}
65+ ${ouroboros_addon}
4166 verticalPodAutoscaler:
4267 valuesOverride: {}
4368 controlPlane:
7196 # Wait for the tenant-test namespace to be active
7297 kubectl wait namespace tenant-test --timeout=20s --for=jsonpath=' {.status.phase}' =Active
7398
74- # Wait for the Kamaji control plane to be created (retry for up to 10 seconds)
75- timeout 10 sh -ec ' until kubectl get kamajicontrolplane -n tenant-test kubernetes-' " ${test_name} " ' ; do sleep 1; done'
99+ # Wait for the Kamaji control plane to be created. Under Flux v2.8
100+ # kstatus-based health checks helm-controller can take 20-30s to dispatch
101+ # the new Kubernetes HR before it renders the KamajiControlPlane CR; the
102+ # old 10s budget was tight on v2.7 and consistently fails on v2.8.
103+ timeout 2m sh -ec ' until kubectl get kamajicontrolplane -n tenant-test kubernetes-' " ${test_name} " ' ; do sleep 1; done'
76104
77105 # Wait for the tenant control plane to be fully created (timeout after 4 minutes)
78106 kubectl wait --for=condition=TenantControlPlaneCreated kamajicontrolplane -n tenant-test kubernetes-${test_name} --timeout=4m
@@ -323,10 +351,125 @@ EOF
323351 kubectl wait machinedeployment kubernetes-${test_name} -md0 -n tenant-test --timeout=10m --for=jsonpath=' {.status.v1beta2.readyReplicas}' =2
324352
325353 for component in cilium coredns csi vsnap-crd; do
326- kubectl wait hr kubernetes-${test_name} -${component} -n tenant-test --timeout=1m --for=condition=ready
354+ kubectl wait hr kubernetes-${test_name} -${component} -n tenant-test --timeout=5m --for=condition=ready
327355 done
328356 kubectl wait hr kubernetes-${test_name} -ingress-nginx -n tenant-test --timeout=5m --for=condition=ready
329357
358+ # Optional ouroboros addon assertions. Folded in from the standalone
359+ # ouroboros.bats so the test reuses this cluster instead of spinning up a
360+ # second ~25m Kamaji bringup. The assertions cover: HR Ready, controller
361+ # pod Running, Ingress->coredns-custom rewrite line injection, and the
362+ # end-to-end DNS resolution proof from inside the tenant cluster.
363+ if [ " ${enable_ouroboros} " = " true" ]; then
364+ kubectl wait hr " kubernetes-${test_name} -ouroboros" -n tenant-test \
365+ --timeout=10m --for=condition=ready
366+
367+ # cozystack coredns wrapper renders an empty coredns-custom ConfigMap in
368+ # kube-system; the ouroboros controller writes the rewrite snippet into
369+ # its ouroboros.override key.
370+ kubectl --kubeconfig " tenantkubeconfig-${test_name} " -n kube-system \
371+ get configmap coredns-custom
372+
373+ # Upstream chart ships no readiness probe — wait covers pod Running only;
374+ # the rewrite-snippet check below is the real reconciliation assertion.
375+ kubectl --kubeconfig " tenantkubeconfig-${test_name} " -n cozy-ouroboros \
376+ wait pod --selector=app.kubernetes.io/component=controller \
377+ --timeout=5m --for=condition=ready
378+
379+ local hairpin_host=hairpin-cozystack-e2e.example.invalid
380+ kubectl --kubeconfig " tenantkubeconfig-${test_name} " -n default apply -f - << EOF
381+ apiVersion: networking.k8s.io/v1
382+ kind: Ingress
383+ metadata:
384+ name: hairpin-probe
385+ spec:
386+ ingressClassName: nginx
387+ tls:
388+ - hosts:
389+ - ${hairpin_host}
390+ secretName: hairpin-probe-tls
391+ rules:
392+ - host: ${hairpin_host}
393+ http:
394+ paths:
395+ - path: /
396+ pathType: Prefix
397+ backend:
398+ service:
399+ name: hairpin-probe
400+ port:
401+ number: 80
402+ EOF
403+
404+ # Poll the import ConfigMap for the rewrite line. Dump-the-whole-map
405+ # form avoids the silent-empty kubectl jsonpath bracket-notation trap
406+ # on ConfigMap keys with dots (e.g. ouroboros.override).
407+ local deadline=$(( $(date +% s) + 300 ))
408+ local snippet=
409+ while [ " $( date +%s) " -lt " ${deadline} " ]; do
410+ snippet=$( kubectl --kubeconfig " tenantkubeconfig-${test_name} " -n kube-system \
411+ get configmap coredns coredns-custom \
412+ -o ' jsonpath={range .items[*]}{.metadata.name}{"\n"}{.data}{"\n---\n"}{end}' \
413+ 2> /dev/null || true)
414+ if echo " ${snippet} " | grep -q " rewrite name ${hairpin_host} " ; then break ; fi
415+ sleep 5
416+ done
417+ if ! echo " ${snippet} " | grep -q " rewrite name ${hairpin_host} " ; then
418+ echo " ouroboros rewrite snippet for ${hairpin_host} not written to coredns-custom within 5m" >&2
419+ kubectl --kubeconfig " tenantkubeconfig-${test_name} " -n cozy-ouroboros \
420+ logs --selector=app.kubernetes.io/component=controller --tail=200 --all-containers || true
421+ exit 1
422+ fi
423+
424+ # End-to-end proof: resolve the hairpin host from inside the tenant.
425+ # CoreDNS reload-period default is 30s, so the in-pod loop is needed.
426+ local proxy_ip
427+ proxy_ip=$( kubectl --kubeconfig " tenantkubeconfig-${test_name} " -n cozy-ouroboros \
428+ get service ouroboros-proxy -o jsonpath=' {.spec.clusterIP}' 2> /dev/null || true)
429+ if [ -z " ${proxy_ip} " ]; then
430+ echo " ouroboros-proxy Service has no ClusterIP" >&2
431+ exit 1
432+ fi
433+ kubectl --kubeconfig " tenantkubeconfig-${test_name} " -n default \
434+ delete pod dnscheck --ignore-not-found 2> /dev/null || true
435+ kubectl --kubeconfig " tenantkubeconfig-${test_name} " -n default \
436+ run dnscheck --image=nicolaka/netshoot:v0.13 --restart=Never \
437+ --command -- sh -c "
438+ deadline=\$ (( \$ (date +%s) + 120 ))
439+ while [ \"\$ (date +%s)\" -lt \"\$ {deadline}\" ]; do
440+ addr=\$ (dig +short +tries=2 +time=5 ${hairpin_host} | head -n 1)
441+ echo \" resolved: \$ {addr:-<empty>}\"
442+ if [ \"\$ {addr}\" = \" ${proxy_ip} \" ]; then
443+ exit 0
444+ fi
445+ sleep 5
446+ done
447+ echo \" timed out waiting for ${hairpin_host} to resolve to ${proxy_ip} \"
448+ exit 1
449+ "
450+ local dns_deadline=$(( $(date +% s) + 180 ))
451+ local phase=
452+ while [ " $( date +%s) " -lt " ${dns_deadline} " ]; do
453+ phase=$( kubectl --kubeconfig " tenantkubeconfig-${test_name} " -n default \
454+ get pod dnscheck -o jsonpath=' {.status.phase}' 2> /dev/null || true)
455+ case " ${phase} " in
456+ Succeeded|Failed) break ;;
457+ esac
458+ sleep 3
459+ done
460+ kubectl --kubeconfig " tenantkubeconfig-${test_name} " -n default \
461+ logs dnscheck 2>&1 | sed ' s/^/ dnscheck: /' || true
462+ if [ " ${phase:- } " != " Succeeded" ]; then
463+ echo " dnscheck pod did not reach Succeeded phase (last seen: ${phase:- <empty>} )" >&2
464+ exit 1
465+ fi
466+
467+ kubectl --kubeconfig " tenantkubeconfig-${test_name} " -n default \
468+ delete pod dnscheck --ignore-not-found 2> /dev/null || true
469+ kubectl --kubeconfig " tenantkubeconfig-${test_name} " -n default \
470+ delete ingress hairpin-probe --ignore-not-found 2> /dev/null || true
471+ fi
472+
330473 # Guard: parent HelmRelease must not have entered an install/upgrade remediation cycle.
331474 # A non-zero installFailures/upgradeFailures indicates the helm-wait budget expired while
332475 # admin-kubeconfig was still being provisioned, which would trigger uninstall remediation
0 commit comments