Skip to content

Commit 4bc6621

Browse files
committed
test(kubernetes,kubernetes-nodes): assert both sides of every substrate branch
Every `if substrate` in both charts is asserted on both sides, so an inverted condition cannot pass: - the reconcile Job reads SVC_IP from the LoadBalancer status on proxmox and from the ClusterIP on kubevirt; - the machineconfig carries the pool's nameservers on proxmox and none on kubevirt; - the proxmox gate on each kubevirt-only template in the parent chart, the two RBAC bindings included, so a proxmox cluster renders no binding to a Role, ClusterRole or ServiceAccount it does not render; - the sizing conversion at 1500m and 8G, where ceil and floor disagree (2 cores and 7630 MiB against 1 and 7629); - the old-template retention loop keeps a ProxmoxMachineTemplate a MachineSet still references; - the proxmox worker monitor carries no Pod threshold while the kubevirt one does; - a live cluster and a live pool refuse a substrate switch while a first install still renders; - both charts refuse a dnsServers entry that is not an address, the parent chart included, where the value reaches ProxmoxCluster.spec.dnsServers. The negations carry `any: true`: without it a negated containsDocument passes whether or not the document is in the render. hack/talos-reconcile-heredoc_test.bats gains the dnsServers cases: the render refuses a command substitution and names the entry; an entry with a colon in it is refused too, which pins the IPv6 branch of the validation and not only the IPv4 one; and a real address survives the Job's shell unchanged. The heredoc is extracted by shape — `cat <<EOF` at the start of a line or right after `=$(` / `="$(` — so the same case keeps working whichever way the Job writes it. Signed-off-by: Marian Koreniuk <[email protected]> Assisted-by: LLM
1 parent 517baf9 commit 4bc6621

7 files changed

Lines changed: 760 additions & 0 deletions

‎hack/talos-reconcile-heredoc_test.bats‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,3 +141,113 @@ VALS
141141
printf '%s' "$out" | grep -qF 'http://m$(id)`x`y' || { echo "kubernetes-nodes hostile endpoint not preserved literally" >&2; rm -rf "$work"; exit 1; }
142142
rm -rf "$work"
143143
}
144+
145+
# The proxmox substrate adds the only tenant-controlled value in this heredoc
146+
# that is not shell-escaped: proxmox.dnsServers. `quote` there is YAML quoting
147+
# and leaves $(...) and backticks for the Job's shell to run — rendered with a
148+
# substitution, the shell executed it and its output ended up in the worker's
149+
# resolver list. The fix is render-time validation rather than escaping, because
150+
# Talos takes addresses here and an address cannot carry a metacharacter, so
151+
# these two tests hold both halves: the render refuses what is not an address,
152+
# and what is an address survives the shell untouched.
153+
@test "kubernetes-nodes refuses a proxmox dnsServers entry that is not an address" {
154+
work=$(mktemp -d)
155+
cat > "$work/vals.yaml" <<'VALS'
156+
cluster: myk8s
157+
_cluster:
158+
cluster-domain: cozy.local
159+
version: "v1.35"
160+
minReplicas: 0
161+
maxReplicas: 3
162+
instanceType: ""
163+
diskSize: 20Gi
164+
storageClass: replicated
165+
roles: [ingress-nginx]
166+
resources: {cpu: "2", memory: 4Gi}
167+
substrate: proxmox
168+
proxmox:
169+
templateTags: [talos]
170+
network: {bridge: vmbr0}
171+
dnsServers: ["10.0.0.1", "$(id -u)"]
172+
VALS
173+
if helm template kubernetes-nodes-myk8s-md0 packages/apps/kubernetes-nodes -n tenant-test -f "$work/vals.yaml" \
174+
--show-only templates/talos-reconcile-job.yaml > "$work/out.yaml" 2>"$work/err"; then
175+
echo "render accepted a command substitution in proxmox.dnsServers" >&2
176+
rm -rf "$work"; exit 1
177+
fi
178+
grep -qF 'proxmox.dnsServers[1]' "$work/err" || { echo "refusal does not name the offending entry" >&2; cat "$work/err" >&2; rm -rf "$work"; exit 1; }
179+
rm -rf "$work"
180+
}
181+
182+
# The validator has two branches and a value without a colon never reaches the
183+
# second one. This case does: `::1$(id -u)` looks enough like an IPv6 address to
184+
# take the IPv6 path, so it is what pins that path's character class. Widen the
185+
# class and this entry renders, with the substitution back in the heredoc, while
186+
# the case above still passes on its IPv4 branch.
187+
@test "kubernetes-nodes refuses an IPv6-shaped dnsServers entry carrying a substitution" {
188+
work=$(mktemp -d)
189+
cat > "$work/vals.yaml" <<'VALS'
190+
cluster: myk8s
191+
_cluster:
192+
cluster-domain: cozy.local
193+
version: "v1.35"
194+
minReplicas: 0
195+
maxReplicas: 3
196+
instanceType: ""
197+
diskSize: 20Gi
198+
storageClass: replicated
199+
roles: [ingress-nginx]
200+
resources: {cpu: "2", memory: 4Gi}
201+
substrate: proxmox
202+
proxmox:
203+
templateTags: [talos]
204+
network: {bridge: vmbr0}
205+
dnsServers: ["10.0.0.1", "::1$(id -u)"]
206+
VALS
207+
if helm template kubernetes-nodes-myk8s-md0 packages/apps/kubernetes-nodes -n tenant-test -f "$work/vals.yaml" \
208+
--show-only templates/talos-reconcile-job.yaml > "$work/out.yaml" 2>"$work/err"; then
209+
echo "render accepted a substitution inside an IPv6-shaped entry" >&2
210+
rm -rf "$work"; exit 1
211+
fi
212+
grep -qF 'proxmox.dnsServers[1]' "$work/err" || { echo "refusal does not name the IPv6-shaped entry" >&2; cat "$work/err" >&2; rm -rf "$work"; exit 1; }
213+
rm -rf "$work"
214+
}
215+
216+
@test "kubernetes-nodes proxmox heredoc emits the nameservers it was given" {
217+
work=$(mktemp -d)
218+
cat > "$work/vals.yaml" <<'VALS'
219+
cluster: myk8s
220+
_cluster:
221+
cluster-domain: cozy.local
222+
version: "v1.35"
223+
minReplicas: 0
224+
maxReplicas: 3
225+
instanceType: ""
226+
diskSize: 20Gi
227+
storageClass: replicated
228+
roles: [ingress-nginx]
229+
resources: {cpu: "2", memory: 4Gi}
230+
substrate: proxmox
231+
proxmox:
232+
templateTags: [talos]
233+
network: {bridge: vmbr0}
234+
dnsServers: ["10.0.0.1", "2001:4860:4860::8888"]
235+
VALS
236+
helm template kubernetes-nodes-myk8s-md0 packages/apps/kubernetes-nodes -n tenant-test -f "$work/vals.yaml" \
237+
--show-only templates/talos-reconcile-job.yaml \
238+
| yq 'select(.kind == "Job") | .spec.template.spec.containers[0].command[2]' \
239+
> "$work/cmd.sh"
240+
[ -s "$work/cmd.sh" ] || { echo "proxmox render produced no Job command" >&2; rm -rf "$work"; exit 1; }
241+
awk '
242+
# Matched on shape: `cat <<EOF` at the start of a line, or right after
243+
# `=$(` / `="$(` when the heredoc is captured into a variable.
244+
/(^|=\"?\$\()cat <<EOF/ { print "cat <<EOF"; inblock=1; next }
245+
inblock && /^EOF$/ { print "EOF"; inblock=0; next }
246+
inblock { print }
247+
' "$work/cmd.sh" > "$work/heredoc.sh"
248+
grep -q '^cat <<EOF$' "$work/heredoc.sh" || { echo "could not extract the proxmox heredoc" >&2; rm -rf "$work"; exit 1; }
249+
out=$(sh "$work/heredoc.sh" 2>"$work/err") || { echo "proxmox heredoc shell exited non-zero" >&2; cat "$work/err" >&2; rm -rf "$work"; exit 1; }
250+
printf '%s' "$out" | grep -qF -e '- "10.0.0.1"' || { echo "IPv4 nameserver missing from the emitted config" >&2; rm -rf "$work"; exit 1; }
251+
printf '%s' "$out" | grep -qF -e '- "2001:4860:4860::8888"' || { echo "IPv6 nameserver missing from the emitted config" >&2; rm -rf "$work"; exit 1; }
252+
rm -rf "$work"
253+
}
Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
suite: substrate branches in the pool render
2+
3+
# Every `if substrate` in this chart, asserted on BOTH sides: the SVC_IP source
4+
# (the LoadBalancer address on proxmox, the ClusterIP on kubevirt — a KubeVirt
5+
# pool waiting for a LoadBalancer address on a ClusterIP Service would never
6+
# write its TalosConfigTemplate) and the nameservers (the pool's own on proxmox,
7+
# none on kubevirt, where the workers reach the management CoreDNS).
8+
#
9+
# The sizing case uses 1500m and 8G, where ceil and floor disagree; at 4 CPUs
10+
# and 8Gi they agree, and the rounding values.yaml promises would go unpinned.
11+
templates:
12+
- templates/nodegroup.yaml
13+
- templates/talos-reconcile-job.yaml
14+
release:
15+
name: kubernetes-nodes-myk8s-md0
16+
namespace: tenant-test
17+
set: &base
18+
cluster: myk8s
19+
_cluster:
20+
cluster-domain: cozy.local
21+
version: "v1.35"
22+
minReplicas: 2
23+
maxReplicas: 3
24+
instanceType: ""
25+
diskSize: 20Gi
26+
storageClass: replicated
27+
roles:
28+
- ingress-nginx
29+
resources:
30+
cpu: "2"
31+
memory: 4Gi
32+
tests:
33+
- it: kubevirt takes the apiserver Service ClusterIP
34+
template: templates/talos-reconcile-job.yaml
35+
documentSelector:
36+
path: kind
37+
value: Job
38+
asserts:
39+
- matchRegex:
40+
path: spec.template.spec.containers[0].command[2]
41+
pattern: 'SVC_IP=\$\(kubectl -n "\$NS" get svc "\$RELEASE" -o jsonpath=''\{\.spec\.clusterIP\}'''
42+
- notMatchRegex:
43+
path: spec.template.spec.containers[0].command[2]
44+
pattern: 'loadBalancer\.ingress'
45+
46+
- it: proxmox takes the Service LoadBalancer address, which is what an off-cluster VM can reach
47+
set: &proxmox
48+
substrate: proxmox
49+
proxmox:
50+
templateTags: [talos]
51+
network:
52+
bridge: vmbr0
53+
dnsServers: ["10.0.0.1"]
54+
template: templates/talos-reconcile-job.yaml
55+
documentSelector:
56+
path: kind
57+
value: Job
58+
asserts:
59+
- matchRegex:
60+
path: spec.template.spec.containers[0].command[2]
61+
pattern: 'SVC_IP=\$\(kubectl -n "\$NS" get svc "\$RELEASE" -o jsonpath=''\{\.status\.loadBalancer\.ingress\[0\]\.ip\}'''
62+
# kube-dns is still read with {.spec.clusterIP}; what must not appear on
63+
# proxmox is the apiserver Service being read that way.
64+
- notMatchRegex:
65+
path: spec.template.spec.containers[0].command[2]
66+
pattern: 'SVC_IP=\$\(kubectl -n "\$NS" get svc "\$RELEASE" -o jsonpath=''\{\.spec\.clusterIP\}'''
67+
68+
- it: kubevirt workers resolve through the management cluster CoreDNS
69+
template: templates/talos-reconcile-job.yaml
70+
documentSelector:
71+
path: kind
72+
value: Job
73+
asserts:
74+
- matchRegex:
75+
path: spec.template.spec.containers[0].command[2]
76+
pattern: 'nameservers:\s*\n\s*- \$\{COREDNS_IP\}'
77+
78+
- it: proxmox workers carry their own resolvers, because that ClusterIP is unroutable for them
79+
set: *proxmox
80+
template: templates/talos-reconcile-job.yaml
81+
documentSelector:
82+
path: kind
83+
value: Job
84+
asserts:
85+
- matchRegex:
86+
path: spec.template.spec.containers[0].command[2]
87+
pattern: 'nameservers:\s*\n\s*- "10\.0\.0\.1"'
88+
- notMatchRegex:
89+
path: spec.template.spec.containers[0].command[2]
90+
pattern: 'nameservers:\s*\n\s*- \$\{COREDNS_IP\}'
91+
92+
- it: rounds fractional CPU up, because a VM cannot have 1.5 cores
93+
set:
94+
substrate: proxmox
95+
proxmox:
96+
templateTags: [talos]
97+
network:
98+
bridge: vmbr0
99+
dnsServers: ["10.0.0.1"]
100+
resources:
101+
cpu: "1500m"
102+
memory: "8G"
103+
template: templates/nodegroup.yaml
104+
documentSelector:
105+
path: kind
106+
value: ProxmoxMachineTemplate
107+
asserts:
108+
# floor would give 1 core and 7629 MiB; both numbers are chosen to differ.
109+
- equal:
110+
path: spec.template.spec.numCores
111+
value: 2
112+
- equal:
113+
path: spec.template.spec.memoryMiB
114+
value: 7630
115+
116+
# osImage picks where a KubeVirt worker's disk comes from. A Proxmox worker's
117+
# disk is a clone of the VM template, so the field cannot be honoured — but
118+
# half of it looks as if it were: osImage.factory.version reaches the
119+
# machineconfig's installer image and changes the Job, while the boot image
120+
# stays whatever the template carries. Refused rather than ignored for that
121+
# reason.
122+
- it: and still takes it on kubevirt
123+
set:
124+
osImage:
125+
factory:
126+
version: v1.12.6
127+
template: templates/nodegroup.yaml
128+
documentSelector:
129+
path: kind
130+
value: MachineDeployment
131+
asserts:
132+
- exists:
133+
path: metadata.name
134+
135+
- it: the worker monitor keeps its Pod threshold on kubevirt
136+
template: templates/nodegroup.yaml
137+
documentSelector:
138+
path: kind
139+
value: WorkloadMonitor
140+
asserts:
141+
- equal:
142+
path: spec.minReplicas
143+
value: 2
144+
145+
- it: and drops it on proxmox, where there are no Pods to count
146+
set: *proxmox
147+
template: templates/nodegroup.yaml
148+
documentSelector:
149+
path: kind
150+
value: WorkloadMonitor
151+
asserts:
152+
# The controller lists Pods matching the selector and reports
153+
# Operational=false below minReplicas. A proxmox worker has no Pod here,
154+
# so any threshold pins the pool to false while its nodes are healthy.
155+
- notExists:
156+
path: spec.minReplicas
157+
- exists:
158+
path: spec.selector
Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
suite: old machine templates survive a rerender on proxmox
2+
3+
# A MachineSet draining its last workers still references the template those
4+
# workers were created from. If the render stops emitting it, Helm deletes the
5+
# infrastructureRef out from under them. The retention loop exists for that, and
6+
# it looks templates up BY KIND — so on a proxmox pool it has to look for
7+
# ProxmoxMachineTemplate. Pinned here because pinning the lookup back to the
8+
# KubeVirt kind left all 99 tests green: no proxmox case reached that loop.
9+
templates:
10+
- templates/nodegroup.yaml
11+
release:
12+
name: kubernetes-nodes-myk8s-md0
13+
namespace: tenant-test
14+
set:
15+
cluster: myk8s
16+
_cluster:
17+
cluster-domain: cozy.local
18+
version: "v1.35"
19+
minReplicas: 0
20+
maxReplicas: 3
21+
instanceType: ""
22+
diskSize: 20Gi
23+
storageClass: replicated
24+
roles:
25+
- ingress-nginx
26+
resources:
27+
cpu: "2"
28+
memory: 4Gi
29+
substrate: proxmox
30+
proxmox:
31+
templateTags: [talos]
32+
network:
33+
bridge: vmbr0
34+
dnsServers: ["10.0.0.1"]
35+
tests:
36+
- it: keeps a ProxmoxMachineTemplate a MachineSet still references
37+
kubernetesProvider:
38+
scheme:
39+
"cluster.x-k8s.io/v1beta1/MachineDeployment":
40+
gvr: {group: "cluster.x-k8s.io", version: "v1beta1", resource: "machinedeployments"}
41+
namespaced: true
42+
"cluster.x-k8s.io/v1beta1/Cluster":
43+
gvr: {group: "cluster.x-k8s.io", version: "v1beta1", resource: "clusters"}
44+
namespaced: true
45+
"cluster.x-k8s.io/v1beta1/MachineSet":
46+
gvr: {group: "cluster.x-k8s.io", version: "v1beta1", resource: "machinesets"}
47+
namespaced: true
48+
"infrastructure.cluster.x-k8s.io/v1alpha1/ProxmoxMachineTemplate":
49+
gvr: {group: "infrastructure.cluster.x-k8s.io", version: "v1alpha1", resource: "proxmoxmachinetemplates"}
50+
namespaced: true
51+
"infrastructure.cluster.x-k8s.io/v1alpha1/KubevirtMachineTemplate":
52+
gvr: {group: "infrastructure.cluster.x-k8s.io", version: "v1alpha1", resource: "kubevirtmachinetemplates"}
53+
namespaced: true
54+
objects:
55+
- kind: Cluster
56+
apiVersion: cluster.x-k8s.io/v1beta1
57+
metadata: {name: kubernetes-myk8s, namespace: tenant-test}
58+
spec:
59+
infrastructureRef: {kind: ProxmoxCluster, name: kubernetes-myk8s}
60+
- kind: ProxmoxMachineTemplate
61+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
62+
metadata:
63+
name: kubernetes-myk8s-md0-0ld0ld
64+
namespace: tenant-test
65+
annotations:
66+
meta.helm.sh/release-name: kubernetes-nodes-myk8s-md0
67+
ownerReferences:
68+
- kind: Cluster
69+
apiVersion: cluster.x-k8s.io/v1beta1
70+
name: kubernetes-myk8s
71+
uid: "00000000-0000-0000-0000-000000000000"
72+
spec:
73+
template:
74+
spec:
75+
sourceNode: prox-home
76+
- kind: MachineSet
77+
apiVersion: cluster.x-k8s.io/v1beta1
78+
metadata: {name: kubernetes-myk8s-md0-draining, namespace: tenant-test}
79+
spec:
80+
template:
81+
spec:
82+
infrastructureRef:
83+
kind: ProxmoxMachineTemplate
84+
name: kubernetes-myk8s-md0-0ld0ld
85+
- kind: KubevirtMachineTemplate
86+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
87+
metadata: {name: scheme-registration-only, namespace: scheme-registration-only}
88+
spec: {}
89+
asserts:
90+
- containsDocument:
91+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
92+
kind: ProxmoxMachineTemplate
93+
name: kubernetes-myk8s-md0-0ld0ld
94+
any: true

0 commit comments

Comments
 (0)