Skip to content

Commit e56e66a

Browse files
test(kubernetes-nodes): fail when the pool and cluster CIDRs disagree
The tenant pod and service CIDRs are literals in two charts: the kubernetes chart writes them into the CAPI Cluster, the pool chart into every worker's Talos machine config. The parent's comment says they must match and keeps them "in one place", but since the pools moved out the second copy has had nothing tying it to the first, and a mismatch renders cleanly on both sides while workers route over a range the control plane does not use. The test renders both charts and compares the lists, reading the worker side from the TalosConfigTemplate the reconcile Job's heredoc actually emits rather than grepping the Job text. Assisted-by: LLM Signed-off-by: Myasnikov Daniil <[email protected]>
1 parent 2572161 commit e56e66a

1 file changed

Lines changed: 104 additions & 0 deletions

File tree

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
#!/usr/bin/env bats
2+
# -----------------------------------------------------------------------------
3+
# The tenant cluster's pod and service CIDRs are written by two charts:
4+
#
5+
# * packages/apps/kubernetes renders them into the CAPI Cluster's
6+
# spec.clusterNetwork, which the control plane is built from;
7+
# * packages/apps/kubernetes-nodes renders them into every worker's Talos
8+
# machine config (cluster.network.podSubnets / serviceSubnets), inside the
9+
# TalosConfigTemplate the talos-reconcile Job applies.
10+
#
11+
# Each chart carries its own literal, and neither render fails when the two
12+
# disagree: the workers boot, join, and route pod and Service traffic over
13+
# ranges the control plane does not use. This guard renders both charts and
14+
# compares the lists, so a range moved on one side only fails here.
15+
#
16+
# The worker side is read the way the Job produces it: the heredoc is cut out
17+
# of the Job's command, run through sh, and the TalosConfigTemplate it prints is
18+
# parsed, then the machine config inside it. A text grep of the Job would also
19+
# match a comment.
20+
#
21+
# Requires: helm, yq (mikefarah v4+). Runs under both `bats` and the in-repo
22+
# cozytest.sh runner, so TMP is provisioned inline per test and cleaned without
23+
# an EXIT trap.
24+
# -----------------------------------------------------------------------------
25+
26+
load test_helper
27+
28+
REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
29+
30+
# $1: output directory. Writes pods and services, one CIDR per line.
31+
cluster_cidrs() {
32+
helm template kubernetes-myk8s "$REPO_ROOT/packages/apps/kubernetes" \
33+
--namespace tenant-test \
34+
-f "$REPO_ROOT/packages/apps/kubernetes/tests/values-ci.yaml" \
35+
> "$1/cluster.yaml" 2> "$1/cluster.err" \
36+
|| { cat "$1/cluster.err" >&2; return 1; }
37+
yq 'select(.kind == "Cluster" and .apiVersion == "cluster.x-k8s.io/v1beta1") | .spec.clusterNetwork.pods.cidrBlocks[]' \
38+
"$1/cluster.yaml" > "$1/cluster.pods"
39+
yq 'select(.kind == "Cluster" and .apiVersion == "cluster.x-k8s.io/v1beta1") | .spec.clusterNetwork.services.cidrBlocks[]' \
40+
"$1/cluster.yaml" > "$1/cluster.services"
41+
}
42+
43+
# $1: output directory. Writes pool.pods and pool.services, one CIDR per line.
44+
pool_cidrs() {
45+
helm template kubernetes-nodes-myk8s-md0 "$REPO_ROOT/packages/apps/kubernetes-nodes" \
46+
--namespace tenant-test \
47+
--set cluster=myk8s \
48+
--set _cluster.cluster-domain=cozy.local \
49+
--set instanceType="" \
50+
--set resources.cpu=2 --set resources.memory=4Gi \
51+
--show-only templates/talos-reconcile-job.yaml \
52+
> "$1/pool.yaml" 2> "$1/pool.err" \
53+
|| { cat "$1/pool.err" >&2; return 1; }
54+
yq 'select(.kind == "Job") | .spec.template.spec.containers[0].command[2]' "$1/pool.yaml" > "$1/cmd.sh"
55+
awk '
56+
/(^|="?\$\()cat <<EOF/ { print "cat <<EOF"; inblock=1; next }
57+
inblock && /^EOF$/ { print "EOF"; inblock=0; next }
58+
inblock { print }
59+
' "$1/cmd.sh" > "$1/heredoc.sh"
60+
sh "$1/heredoc.sh" > "$1/tct.yaml" 2> "$1/heredoc.err" \
61+
|| { cat "$1/heredoc.err" >&2; return 1; }
62+
yq 'select(.kind == "TalosConfigTemplate") | .spec.template.spec.data' "$1/tct.yaml" > "$1/machineconfig.yaml"
63+
yq '.cluster.network.podSubnets[]' "$1/machineconfig.yaml" > "$1/pool.pods"
64+
yq '.cluster.network.serviceSubnets[]' "$1/machineconfig.yaml" > "$1/pool.services"
65+
}
66+
67+
# $1: directory, $2: pods or services, $3: what to call it in a message.
68+
assert_agree() {
69+
cluster=$(cat "$1/cluster.$2")
70+
pool=$(cat "$1/pool.$2")
71+
# An empty read means a field moved or a render came out short; comparing
72+
# "" with "" would pass on nothing.
73+
if [ -z "$cluster" ] || [ -z "$pool" ]; then
74+
echo "FAIL: could not read the $3 CIDRs from both renders."
75+
echo " kubernetes Cluster.spec.clusterNetwork: '${cluster:-<missing>}'"
76+
echo " kubernetes-nodes machine config cluster.network: '${pool:-<missing>}'"
77+
return 1
78+
fi
79+
if [ "$cluster" != "$pool" ]; then
80+
echo "FAIL: the tenant $3 CIDRs differ between the two charts."
81+
echo " packages/apps/kubernetes (CAPI Cluster): $cluster"
82+
echo " packages/apps/kubernetes-nodes (worker machine config): $pool"
83+
echo "Workers would route $3 traffic over a range the control plane does not"
84+
echo "use. Change both charts together. Moving the worker side also renames"
85+
echo "the content-hashed TalosConfigTemplate and rolls every worker."
86+
return 1
87+
fi
88+
}
89+
90+
@test "kubernetes and kubernetes-nodes render the same pod CIDRs" {
91+
TMP=$(mktemp -d)
92+
cluster_cidrs "$TMP"
93+
pool_cidrs "$TMP"
94+
assert_agree "$TMP" pods pod
95+
rm -rf "$TMP"
96+
}
97+
98+
@test "kubernetes and kubernetes-nodes render the same service CIDRs" {
99+
TMP=$(mktemp -d)
100+
cluster_cidrs "$TMP"
101+
pool_cidrs "$TMP"
102+
assert_agree "$TMP" services service
103+
rm -rf "$TMP"
104+
}

0 commit comments

Comments
 (0)