|
| 1 | +{{- /* |
| 2 | + Talos worker machine config for one node group, rendered as the `spec:` |
| 3 | + block of a TalosConfigTemplate. |
| 4 | +
|
| 5 | + Single source of truth for that spec, shared by the two consumers that have |
| 6 | + to agree on it byte-for-byte: |
| 7 | +
|
| 8 | + * the talos-reconcile Job, which embeds it in the heredoc it |
| 9 | + `kubectl apply`s once the runtime inputs (apiserver Service ClusterIP, |
| 10 | + Talos CA, tenant Kubernetes CA, management CoreDNS ClusterIP) exist; |
| 11 | + * the MachineDeployment, which points |
| 12 | + spec.template.spec.bootstrap.configRef at the TalosConfigTemplate named |
| 13 | + after a hash of this spec. |
| 14 | +
|
| 15 | + Why the hash. TalosConfigTemplate.spec is immutable — the |
| 16 | + vtalosconfigtemplate.cluster.x-k8s.io webhook rejects any mutation with |
| 17 | + "TalosConfigTemplate.Spec is immutable" — so the object can never be updated |
| 18 | + in place. Applying it under a fixed name means that once it exists, no change |
| 19 | + to the rendered machine config can ever reach the cluster: the apply is |
| 20 | + denied server-side and the stale spec stays (cozystack/cozystack#3515, |
| 21 | + observed with machine.network.searchDomains after a clusterDomain |
| 22 | + correction; the same holds for certSANs, machine.registries, nameservers and |
| 23 | + any future field). Naming the template after a content hash — the idiom |
| 24 | + KubevirtMachineTemplate already uses in this chart — turns a content change |
| 25 | + into a NEW template instead of a rejected mutation, and CAPI propagates it |
| 26 | + the way it is meant to: a new MachineSet rolls the workers onto the new |
| 27 | + bootstrap config. |
| 28 | +
|
| 29 | + The `${...}` placeholders are shell variables the Job expands at apply time, |
| 30 | + and are deliberately NOT part of the hash: the CA certificates, the Service |
| 31 | + ClusterIP, the CoreDNS ClusterIP and the Talos/bootstrap tokens are runtime |
| 32 | + facts of an already-provisioned cluster rather than chart inputs. Hashing |
| 33 | + them would roll every worker on a certificate rotation or a Service |
| 34 | + re-address. |
| 35 | +
|
| 36 | + Input: a dict with `root` (the chart root context) and `group` (the node |
| 37 | + group's values — instanceType / resources / gpus / kubelet / osImage). Everything else |
| 38 | + is derived here rather than passed in, so the two call sites cannot drift |
| 39 | + apart and compute different hashes for the same node group. |
| 40 | +*/}} |
| 41 | +{{- define "kubernetes.talosConfigTemplateSpec" -}} |
| 42 | +{{- $root := .root }} |
| 43 | +{{- $group := .group }} |
| 44 | +{{- $kubeletVersion := include "kubernetes.versionMap" $root | trim }} |
| 45 | +{{- /* Resolve this pool's worker image (schematicID, version) from .Values.osImage |
| 46 | + — the same selection templates/nodegroup.yaml uses for the boot disk — so the |
| 47 | + in-guest Talos installer image matches the OS this pool actually booted. |
| 48 | + Defaults to the pool's talos.*; a pool with osImage.builtin / osImage.factory |
| 49 | + pins its own flavor and version so an in-guest upgrade does not flip it back to |
| 50 | + the pool default. */}} |
| 51 | +{{- /* Shared with nodegroup.yaml's boot-disk source rather than mirrored here: |
| 52 | + a disk booted from one Talos flavor and an installer pinned to another is |
| 53 | + the silently-broken pairing this chart's support matrix was factored out |
| 54 | + to prevent, and two hand-kept copies of the resolution are how the two |
| 55 | + drift apart. The helper also carries the format checks, so this file no |
| 56 | + longer depends on the other one aborting the shared render first. */}} |
| 57 | +{{- $resolvedImage := dict }} |
| 58 | +{{- include "kubernetes-nodes.resolveOsImage" (dict |
| 59 | + "osImage" $group.osImage |
| 60 | + "talos" $root.Values.talos |
| 61 | + "out" $resolvedImage |
| 62 | + "groupName" (include "kubernetes-nodes.groupName" $root)) }} |
| 63 | +{{- $talosSchematicID := $resolvedImage.schematicID }} |
| 64 | +{{- $talosVersion := $resolvedImage.version }} |
| 65 | +{{- $podCIDR := "10.243.0.0/16" }} |
| 66 | +{{- $serviceCIDR := "10.95.0.0/16" }} |
| 67 | +{{- $dnsDomain := include "kubernetes.tenantClusterDomain" $root }} |
| 68 | +{{- /* Management cluster DNS domain. Tenant Kubernetes workers run as |
| 69 | + KubeVirt VMs inside the management cluster's pod network and need |
| 70 | + to resolve management-cluster service short names ("<svc>.<ns>.svc") |
| 71 | + the same way an Ubuntu+kubeadm worker did via cloud-init DHCP. The |
| 72 | + kubevirt-csi NFS mount path is the load-bearing case: kubelet on |
| 73 | + the worker host calls `mount.nfs linstor-csi-nfs.cozy-linstor.svc` |
| 74 | + (no domain suffix), and without searchDomains in /etc/resolv.conf |
| 75 | + the host resolver hands that name to management CoreDNS verbatim |
| 76 | + and gets NXDOMAIN. |
| 77 | +
|
| 78 | + Distinct from tenantClusterDomain ("cluster.local"), which is the |
| 79 | + *tenant* kubelet --cluster-dns zone. This value is the |
| 80 | + *management* cluster-domain, sourced from .Values._cluster |
| 81 | + (populated by cozystack-controller from the platform config). */}} |
| 82 | +{{- $mgmtClusterDomain := (index ($root.Values._cluster | default dict) "cluster-domain") | default "cozy.local" }} |
| 83 | +{{- /* Kubelet reservations, mirroring the computation the MachineDeployment |
| 84 | + side validates: auto-computed system/kube reserved memory = 5% of |
| 85 | + effective memory clamped to [256Mi, 1Gi]; auto-computed system/kube |
| 86 | + reserved CPU = 5% of effective CPU clamped to [50m, 500m]. |
| 87 | + instanceType lookup is optional — operators that override |
| 88 | + resources.cpu / resources.memory on the nodeGroup get those, otherwise |
| 89 | + the chart falls back to the same static-defaults pair the |
| 90 | + MachineDeployment validator uses on a no-instanceType setup. */}} |
| 91 | +{{- $instanceType := dict }} |
| 92 | +{{- if $group.instanceType }} |
| 93 | +{{- $instanceType = lookup "instancetype.kubevirt.io/v1beta1" "VirtualMachineClusterInstancetype" "" $group.instanceType }} |
| 94 | +{{- $instanceType = $instanceType | default dict }} |
| 95 | +{{- end }} |
| 96 | +{{- $effectiveMemory := "" }} |
| 97 | +{{- if and $group.resources $group.resources.memory }} |
| 98 | +{{- $effectiveMemory = $group.resources.memory | toString }} |
| 99 | +{{- else if and $instanceType $instanceType.spec $instanceType.spec.memory $instanceType.spec.memory.guest }} |
| 100 | +{{- $effectiveMemory = $instanceType.spec.memory.guest | toString }} |
| 101 | +{{- end }} |
| 102 | +{{- $effectiveCpu := "" }} |
| 103 | +{{- if and $group.resources $group.resources.cpu }} |
| 104 | +{{- $effectiveCpu = $group.resources.cpu | toString }} |
| 105 | +{{- else if and $instanceType $instanceType.spec $instanceType.spec.cpu $instanceType.spec.cpu.guest }} |
| 106 | +{{- $effectiveCpu = $instanceType.spec.cpu.guest | toString }} |
| 107 | +{{- end }} |
| 108 | +{{- $autoReservedMi := 256 }} |
| 109 | +{{- if $effectiveMemory }} |
| 110 | +{{- $effectiveMemMi := divf (include "cozy-lib.resources.toFloat" $effectiveMemory | float64) 1048576.0 | int }} |
| 111 | +{{- $fivePercentMi := mulf ($effectiveMemMi | float64) 0.05 | int }} |
| 112 | +{{- $autoReservedMi = min (max $fivePercentMi 256) 1024 }} |
| 113 | +{{- end }} |
| 114 | +{{- $autoReservedMillicores := 50 }} |
| 115 | +{{- if $effectiveCpu }} |
| 116 | +{{- $effectiveCpuMillicores := include "kubernetes.cpuToMillicores" $effectiveCpu | int }} |
| 117 | +{{- $fivePercentMillis := mulf ($effectiveCpuMillicores | float64) 0.05 | int }} |
| 118 | +{{- $autoReservedMillicores = min (max $fivePercentMillis 50) 500 }} |
| 119 | +{{- end }} |
| 120 | +{{- $kubeletOverride := $group.kubelet | default dict }} |
| 121 | +{{- $systemReservedMemory := $kubeletOverride.systemReservedMemory | default (printf "%dMi" $autoReservedMi) }} |
| 122 | +{{- $kubeReservedMemory := $kubeletOverride.kubeReservedMemory | default (printf "%dMi" $autoReservedMi) }} |
| 123 | +{{- $systemReservedCpu := $kubeletOverride.systemReservedCpu | default (printf "%dm" $autoReservedMillicores) }} |
| 124 | +{{- $kubeReservedCpu := $kubeletOverride.kubeReservedCpu | default (printf "%dm" $autoReservedMillicores) }} |
| 125 | +{{- $evictionHardMemory := $kubeletOverride.evictionHardMemory | default "7%" }} |
| 126 | +{{- $evictionSoftMemory := $kubeletOverride.evictionSoftMemory | default "10%" -}} |
| 127 | +{{- /* The spec proper. Everything above only computes inputs; nothing above |
| 128 | + this line may emit output, or the leading newline lands in the caller's |
| 129 | + heredoc and in the hash. */ -}} |
| 130 | +spec: |
| 131 | + template: |
| 132 | + spec: |
| 133 | + generateType: none |
| 134 | + talosVersion: "{{ $talosVersion | replace "\\" "\\\\" | replace "$" "\\$" | replace "`" "\\`" }}" |
| 135 | + {{- /* INVARIANT (cozystack/cozystack#3513): the data block below is emitted from the unquoted `cat <<EOF | kubectl apply` heredoc in the talos-reconcile Job, so it is subject to shell parameter expansion and command substitution at Job runtime. Every tenant-controlled free-form value interpolated into it must therefore be either shell-escaped for backslash, dollar and backtick (as talosVersion here and registryMirrors below are) or render-time pattern-validated to exclude those bytes (as the kubelet reservation fields are in nodegroup.yaml). A new such field added here without one of the two silently reopens command injection into a host-cluster Pod. Note: an asterisk-slash sequence inside this Helm comment closes it early and breaks the render, so the kubelet field names are spelled out here rather than written as globs. This is a Helm comment: it is stripped at render, so its own text never reaches the heredoc. */}} |
| 136 | + data: | |
| 137 | + version: v1alpha1 |
| 138 | + persist: true |
| 139 | + machine: |
| 140 | + type: worker |
| 141 | + token: ${TALOS_TOKEN} |
| 142 | + network: |
| 143 | + # Management CoreDNS as the worker's host |
| 144 | + # resolver, plus management cluster search |
| 145 | + # domains so partial names like |
| 146 | + # "linstor-csi-nfs.cozy-linstor.svc" resolve |
| 147 | + # the same way they do on Ubuntu+kubeadm |
| 148 | + # workers in main (which got these via |
| 149 | + # cloud-init/DHCP). Without searchDomains the |
| 150 | + # kubevirt-csi NFS mount fails NXDOMAIN |
| 151 | + # because mount.nfs hands the partial name |
| 152 | + # to the resolver verbatim and management |
| 153 | + # CoreDNS only knows full FQDNs. |
| 154 | + # |
| 155 | + # Tenant in-cluster pods are not affected: |
| 156 | + # dnsPolicy: ClusterFirst routes them through |
| 157 | + # kubelet --cluster-dns (tenant CoreDNS). |
| 158 | + # This block only configures host-side |
| 159 | + # resolution (kubelet mount syscalls, image |
| 160 | + # pulls, pods opted out of cluster DNS). |
| 161 | + nameservers: |
| 162 | + {{- /* A kubevirt worker resolves through the management |
| 163 | + cluster's CoreDNS over the pod network. A proxmox |
| 164 | + worker is off-cluster and cannot reach that |
| 165 | + ClusterIP any more than it can reach the |
| 166 | + apiserver's, so it carries its own resolvers. */}} |
| 167 | + {{- if eq ($root.Values.substrate | default "kubevirt") "proxmox" }} |
| 168 | + {{- range $root.Values.proxmox.dnsServers }} |
| 169 | + - {{ . | quote }} |
| 170 | + {{- end }} |
| 171 | + {{- else }} |
| 172 | + - ${COREDNS_IP} |
| 173 | + {{- end }} |
| 174 | + searchDomains: |
| 175 | + - svc.{{ $mgmtClusterDomain }} |
| 176 | + - {{ $mgmtClusterDomain }} |
| 177 | + {{- if ne $mgmtClusterDomain "cluster.local" }} |
| 178 | + - svc.cluster.local |
| 179 | + - cluster.local |
| 180 | + {{- end }} |
| 181 | + extraHostEntries: |
| 182 | + - ip: ${SVC_IP} |
| 183 | + aliases: |
| 184 | + - ${RELEASE}.${NS}.svc |
| 185 | + - ${RELEASE}.${NS}.svc.{{ $dnsDomain }} |
| 186 | + ca: |
| 187 | + crt: ${TALOS_CA_B64} |
| 188 | + {{- if $group.gpus }} |
| 189 | + # GPU node-groups: label every node `gpu=on` so |
| 190 | + # HAMi's hami-device-plugin DaemonSet (nodeSelector |
| 191 | + # gpu=on) schedules and advertises nvidia.com/gpu. |
| 192 | + # Without the label the plugin stays at DESIRED=0 |
| 193 | + # and no GPUs are exposed to the tenant. Mirrors |
| 194 | + # main's kubeadm config (kubeletExtraArgs node- |
| 195 | + # labels: "gpu=on") so HAMi behaviour is identical |
| 196 | + # before and after the Talos worker rollover. |
| 197 | + nodeLabels: |
| 198 | + gpu: "on" |
| 199 | + {{- end }} |
| 200 | + kubelet: |
| 201 | + image: ghcr.io/siderolabs/kubelet:{{ $kubeletVersion }} |
| 202 | + extraConfig: |
| 203 | + systemReserved: |
| 204 | + cpu: "{{ $systemReservedCpu }}" |
| 205 | + memory: "{{ $systemReservedMemory }}" |
| 206 | + kubeReserved: |
| 207 | + cpu: "{{ $kubeReservedCpu }}" |
| 208 | + memory: "{{ $kubeReservedMemory }}" |
| 209 | + evictionHard: |
| 210 | + memory.available: "{{ $evictionHardMemory }}" |
| 211 | + nodefs.available: "10%" |
| 212 | + imagefs.available: "15%" |
| 213 | + nodefs.inodesFree: "5%" |
| 214 | + evictionSoft: |
| 215 | + memory.available: "{{ $evictionSoftMemory }}" |
| 216 | + nodefs.available: "15%" |
| 217 | + imagefs.available: "20%" |
| 218 | + evictionSoftGracePeriod: |
| 219 | + memory.available: "1m30s" |
| 220 | + nodefs.available: "1m30s" |
| 221 | + imagefs.available: "1m30s" |
| 222 | + evictionMinimumReclaim: |
| 223 | + memory.available: "256Mi" |
| 224 | + install: |
| 225 | + disk: /dev/vda |
| 226 | + image: {{ $root.Values.talos.installerRepository | trimSuffix "/" | replace "\\" "\\\\" | replace "$" "\\$" | replace "`" "\\`" }}/{{ $talosSchematicID | replace "\\" "\\\\" | replace "$" "\\$" | replace "`" "\\`" }}:{{ $talosVersion | replace "\\" "\\\\" | replace "$" "\\$" | replace "`" "\\`" }} |
| 227 | + wipe: false |
| 228 | + features: |
| 229 | + rbac: true |
| 230 | + kubePrism: |
| 231 | + enabled: false |
| 232 | + {{- with $root.Values.talos.registryMirrors }} |
| 233 | + registries: |
| 234 | + mirrors: |
| 235 | + {{- /* registryMirrors is free-form tenant-facing input rendered into the unquoted reconcile-Job heredoc, so the value is escaped (backslash, dollar, backtick) to render as a literal and never be shell-expanded or command-substituted at Job runtime. This is a Helm comment: it is stripped at render, so its own text never reaches the heredoc. See cozystack/cozystack#3513. */}} |
| 236 | + {{- toYaml . | replace "\\" "\\\\" | replace "$" "\\$" | replace "`" "\\`" | nindent 14 }} |
| 237 | + {{- end }} |
| 238 | + cluster: |
| 239 | + id: ${CLUSTER_ID} |
| 240 | + secret: ${CLUSTER_SECRET} |
| 241 | + controlPlane: |
| 242 | + endpoint: https://${RELEASE}.${NS}.svc:6443 |
| 243 | + clusterName: ${RELEASE} |
| 244 | + network: |
| 245 | + dnsDomain: {{ $dnsDomain }} |
| 246 | + podSubnets: |
| 247 | + - {{ $podCIDR }} |
| 248 | + serviceSubnets: |
| 249 | + - {{ $serviceCIDR }} |
| 250 | + token: ${BOOTSTRAP_TOKEN} |
| 251 | + ca: |
| 252 | + crt: ${K8S_CA_B64} |
| 253 | + discovery: |
| 254 | + enabled: true |
| 255 | + registries: |
| 256 | + kubernetes: |
| 257 | + disabled: true |
| 258 | + service: |
| 259 | + disabled: true |
| 260 | +{{- end }} |
| 261 | + |
| 262 | +{{- /* |
| 263 | + Content hash of the TalosConfigTemplate spec, truncated the same way the |
| 264 | + KubevirtMachineTemplate hash in this chart is. Takes the same dict as |
| 265 | + kubernetes.talosConfigTemplateSpec, so the Job that creates the template and |
| 266 | + the MachineDeployment that references it always agree on the name. |
| 267 | +*/}} |
| 268 | +{{- define "kubernetes.talosConfigTemplateHash" -}} |
| 269 | +{{- include "kubernetes.talosConfigTemplateSpec" . | sha256sum | trunc 6 -}} |
| 270 | +{{- end }} |
0 commit comments