|
| 1 | +{{- /* |
| 2 | + Talos worker machine config for one node group, rendered as the `spec:` |
| 3 | + block of a TalosConfigTemplate. |
| 4 | +
|
| 5 | + Single source of truth for that spec, shared by the two consumers that have |
| 6 | + to agree on it byte-for-byte: |
| 7 | +
|
| 8 | + * the talos-reconcile Job, which embeds it in the heredoc it |
| 9 | + `kubectl apply`s once the runtime inputs (apiserver Service ClusterIP, |
| 10 | + Talos CA, tenant Kubernetes CA, management CoreDNS ClusterIP) exist; |
| 11 | + * the MachineDeployment, which points |
| 12 | + spec.template.spec.bootstrap.configRef at the TalosConfigTemplate named |
| 13 | + after a hash of this spec. |
| 14 | +
|
| 15 | + Why the hash. TalosConfigTemplate.spec is immutable — the |
| 16 | + vtalosconfigtemplate.cluster.x-k8s.io webhook rejects any mutation with |
| 17 | + "TalosConfigTemplate.Spec is immutable" — so the object can never be updated |
| 18 | + in place. Applying it under a fixed name means that once it exists, no change |
| 19 | + to the rendered machine config can ever reach the cluster: the apply is |
| 20 | + denied server-side and the stale spec stays (cozystack/cozystack#3515, |
| 21 | + observed with machine.network.searchDomains after a clusterDomain |
| 22 | + correction; the same holds for certSANs, machine.registries, nameservers and |
| 23 | + any future field). Naming the template after a content hash — the idiom |
| 24 | + KubevirtMachineTemplate already uses in this chart — turns a content change |
| 25 | + into a NEW template instead of a rejected mutation, and CAPI propagates it |
| 26 | + the way it is meant to: a new MachineSet rolls the workers onto the new |
| 27 | + bootstrap config. |
| 28 | +
|
| 29 | + The `${...}` placeholders are shell variables the Job expands at apply time, |
| 30 | + and are deliberately NOT part of the hash: the CA certificates, the Service |
| 31 | + ClusterIP, the CoreDNS ClusterIP and the Talos/bootstrap tokens are runtime |
| 32 | + facts of an already-provisioned cluster rather than chart inputs. Hashing |
| 33 | + them would roll every worker on a certificate rotation or a Service |
| 34 | + re-address. |
| 35 | +
|
| 36 | + Input: a dict with `root` (the chart root context) and `group` (the node |
| 37 | + group's values — instanceType / resources / gpus / kubelet). Everything else |
| 38 | + is derived here rather than passed in, so the two call sites cannot drift |
| 39 | + apart and compute different hashes for the same node group. |
| 40 | +*/}} |
| 41 | +{{- define "kubernetes.talosConfigTemplateSpec" -}} |
| 42 | +{{- $root := .root }} |
| 43 | +{{- $group := .group }} |
| 44 | +{{- $kubeletVersion := include "kubernetes.versionMap" $root | trim }} |
| 45 | +{{- $talosVersion := $root.Values.talos.version }} |
| 46 | +{{- $podCIDR := "10.243.0.0/16" }} |
| 47 | +{{- $serviceCIDR := "10.95.0.0/16" }} |
| 48 | +{{- $dnsDomain := include "kubernetes.tenantClusterDomain" $root }} |
| 49 | +{{- /* Management cluster DNS domain. Tenant Kubernetes workers run as |
| 50 | + KubeVirt VMs inside the management cluster's pod network and need |
| 51 | + to resolve management-cluster service short names ("<svc>.<ns>.svc") |
| 52 | + the same way an Ubuntu+kubeadm worker did via cloud-init DHCP. The |
| 53 | + kubevirt-csi NFS mount path is the load-bearing case: kubelet on |
| 54 | + the worker host calls `mount.nfs linstor-csi-nfs.cozy-linstor.svc` |
| 55 | + (no domain suffix), and without searchDomains in /etc/resolv.conf |
| 56 | + the host resolver hands that name to management CoreDNS verbatim |
| 57 | + and gets NXDOMAIN. |
| 58 | +
|
| 59 | + Distinct from tenantClusterDomain ("cluster.local"), which is the |
| 60 | + *tenant* kubelet --cluster-dns zone. This value is the |
| 61 | + *management* cluster-domain, sourced from .Values._cluster |
| 62 | + (populated by cozystack-controller from the platform config). */}} |
| 63 | +{{- $mgmtClusterDomain := (index ($root.Values._cluster | default dict) "cluster-domain") | default "cozy.local" }} |
| 64 | +{{- /* Kubelet reservations, mirroring the computation the MachineDeployment |
| 65 | + side validates: auto-computed system/kube reserved memory = 5% of |
| 66 | + effective memory clamped to [256Mi, 1Gi]; auto-computed system/kube |
| 67 | + reserved CPU = 5% of effective CPU clamped to [50m, 500m]. |
| 68 | + instanceType lookup is optional — operators that override |
| 69 | + resources.cpu / resources.memory on the nodeGroup get those, otherwise |
| 70 | + the chart falls back to the same static-defaults pair the |
| 71 | + MachineDeployment validator uses on a no-instanceType setup. */}} |
| 72 | +{{- $instanceType := dict }} |
| 73 | +{{- if $group.instanceType }} |
| 74 | +{{- $instanceType = lookup "instancetype.kubevirt.io/v1beta1" "VirtualMachineClusterInstancetype" "" $group.instanceType }} |
| 75 | +{{- $instanceType = $instanceType | default dict }} |
| 76 | +{{- end }} |
| 77 | +{{- $effectiveMemory := "" }} |
| 78 | +{{- if and $group.resources $group.resources.memory }} |
| 79 | +{{- $effectiveMemory = $group.resources.memory | toString }} |
| 80 | +{{- else if and $instanceType $instanceType.spec $instanceType.spec.memory $instanceType.spec.memory.guest }} |
| 81 | +{{- $effectiveMemory = $instanceType.spec.memory.guest | toString }} |
| 82 | +{{- end }} |
| 83 | +{{- $effectiveCpu := "" }} |
| 84 | +{{- if and $group.resources $group.resources.cpu }} |
| 85 | +{{- $effectiveCpu = $group.resources.cpu | toString }} |
| 86 | +{{- else if and $instanceType $instanceType.spec $instanceType.spec.cpu $instanceType.spec.cpu.guest }} |
| 87 | +{{- $effectiveCpu = $instanceType.spec.cpu.guest | toString }} |
| 88 | +{{- end }} |
| 89 | +{{- $autoReservedMi := 256 }} |
| 90 | +{{- if $effectiveMemory }} |
| 91 | +{{- $effectiveMemMi := divf (include "cozy-lib.resources.toFloat" $effectiveMemory | float64) 1048576.0 | int }} |
| 92 | +{{- $fivePercentMi := mulf ($effectiveMemMi | float64) 0.05 | int }} |
| 93 | +{{- $autoReservedMi = min (max $fivePercentMi 256) 1024 }} |
| 94 | +{{- end }} |
| 95 | +{{- $autoReservedMillicores := 50 }} |
| 96 | +{{- if $effectiveCpu }} |
| 97 | +{{- $effectiveCpuMillicores := include "kubernetes.cpuToMillicores" $effectiveCpu | int }} |
| 98 | +{{- $fivePercentMillis := mulf ($effectiveCpuMillicores | float64) 0.05 | int }} |
| 99 | +{{- $autoReservedMillicores = min (max $fivePercentMillis 50) 500 }} |
| 100 | +{{- end }} |
| 101 | +{{- $kubeletOverride := $group.kubelet | default dict }} |
| 102 | +{{- $systemReservedMemory := $kubeletOverride.systemReservedMemory | default (printf "%dMi" $autoReservedMi) }} |
| 103 | +{{- $kubeReservedMemory := $kubeletOverride.kubeReservedMemory | default (printf "%dMi" $autoReservedMi) }} |
| 104 | +{{- $systemReservedCpu := $kubeletOverride.systemReservedCpu | default (printf "%dm" $autoReservedMillicores) }} |
| 105 | +{{- $kubeReservedCpu := $kubeletOverride.kubeReservedCpu | default (printf "%dm" $autoReservedMillicores) }} |
| 106 | +{{- $evictionHardMemory := $kubeletOverride.evictionHardMemory | default "7%" }} |
| 107 | +{{- $evictionSoftMemory := $kubeletOverride.evictionSoftMemory | default "10%" -}} |
| 108 | +{{- /* The spec proper. Everything above only computes inputs; nothing above |
| 109 | + this line may emit output, or the leading newline lands in the caller's |
| 110 | + heredoc and in the hash. */ -}} |
| 111 | +spec: |
| 112 | + template: |
| 113 | + spec: |
| 114 | + generateType: none |
| 115 | + talosVersion: "{{ $talosVersion }}" |
| 116 | + data: | |
| 117 | + version: v1alpha1 |
| 118 | + persist: true |
| 119 | + machine: |
| 120 | + type: worker |
| 121 | + token: ${TALOS_TOKEN} |
| 122 | + network: |
| 123 | + # Management CoreDNS as the worker's host |
| 124 | + # resolver, plus management cluster search |
| 125 | + # domains so partial names like |
| 126 | + # "linstor-csi-nfs.cozy-linstor.svc" resolve |
| 127 | + # the same way they do on Ubuntu+kubeadm |
| 128 | + # workers in main (which got these via |
| 129 | + # cloud-init/DHCP). Without searchDomains the |
| 130 | + # kubevirt-csi NFS mount fails NXDOMAIN |
| 131 | + # because mount.nfs hands the partial name |
| 132 | + # to the resolver verbatim and management |
| 133 | + # CoreDNS only knows full FQDNs. |
| 134 | + # |
| 135 | + # Tenant in-cluster pods are not affected: |
| 136 | + # dnsPolicy: ClusterFirst routes them through |
| 137 | + # kubelet --cluster-dns (tenant CoreDNS). |
| 138 | + # This block only configures host-side |
| 139 | + # resolution (kubelet mount syscalls, image |
| 140 | + # pulls, pods opted out of cluster DNS). |
| 141 | + nameservers: |
| 142 | + - ${COREDNS_IP} |
| 143 | + searchDomains: |
| 144 | + - svc.{{ $mgmtClusterDomain }} |
| 145 | + - {{ $mgmtClusterDomain }} |
| 146 | + {{- if ne $mgmtClusterDomain "cluster.local" }} |
| 147 | + - svc.cluster.local |
| 148 | + - cluster.local |
| 149 | + {{- end }} |
| 150 | + extraHostEntries: |
| 151 | + - ip: ${SVC_IP} |
| 152 | + aliases: |
| 153 | + - ${RELEASE}.${NS}.svc |
| 154 | + - ${RELEASE}.${NS}.svc.{{ $dnsDomain }} |
| 155 | + ca: |
| 156 | + crt: ${TALOS_CA_B64} |
| 157 | + {{- if $group.gpus }} |
| 158 | + # GPU node-groups: label every node `gpu=on` so |
| 159 | + # HAMi's hami-device-plugin DaemonSet (nodeSelector |
| 160 | + # gpu=on) schedules and advertises nvidia.com/gpu. |
| 161 | + # Without the label the plugin stays at DESIRED=0 |
| 162 | + # and no GPUs are exposed to the tenant. Mirrors |
| 163 | + # main's kubeadm config (kubeletExtraArgs node- |
| 164 | + # labels: "gpu=on") so HAMi behaviour is identical |
| 165 | + # before and after the Talos worker rollover. |
| 166 | + nodeLabels: |
| 167 | + gpu: "on" |
| 168 | + {{- end }} |
| 169 | + kubelet: |
| 170 | + image: ghcr.io/siderolabs/kubelet:{{ $kubeletVersion }} |
| 171 | + extraConfig: |
| 172 | + systemReserved: |
| 173 | + cpu: "{{ $systemReservedCpu }}" |
| 174 | + memory: "{{ $systemReservedMemory }}" |
| 175 | + kubeReserved: |
| 176 | + cpu: "{{ $kubeReservedCpu }}" |
| 177 | + memory: "{{ $kubeReservedMemory }}" |
| 178 | + evictionHard: |
| 179 | + memory.available: "{{ $evictionHardMemory }}" |
| 180 | + nodefs.available: "10%" |
| 181 | + imagefs.available: "15%" |
| 182 | + nodefs.inodesFree: "5%" |
| 183 | + evictionSoft: |
| 184 | + memory.available: "{{ $evictionSoftMemory }}" |
| 185 | + nodefs.available: "15%" |
| 186 | + imagefs.available: "20%" |
| 187 | + evictionSoftGracePeriod: |
| 188 | + memory.available: "1m30s" |
| 189 | + nodefs.available: "1m30s" |
| 190 | + imagefs.available: "1m30s" |
| 191 | + evictionMinimumReclaim: |
| 192 | + memory.available: "256Mi" |
| 193 | + install: |
| 194 | + disk: /dev/vda |
| 195 | + image: {{ $root.Values.talos.installerRepository | trimSuffix "/" }}/{{ $root.Values.talos.schematicID }}:{{ $talosVersion }} |
| 196 | + wipe: false |
| 197 | + features: |
| 198 | + rbac: true |
| 199 | + kubePrism: |
| 200 | + enabled: false |
| 201 | + cluster: |
| 202 | + id: ${CLUSTER_ID} |
| 203 | + secret: ${CLUSTER_SECRET} |
| 204 | + controlPlane: |
| 205 | + endpoint: https://${RELEASE}.${NS}.svc:6443 |
| 206 | + clusterName: ${RELEASE} |
| 207 | + network: |
| 208 | + dnsDomain: {{ $dnsDomain }} |
| 209 | + podSubnets: |
| 210 | + - {{ $podCIDR }} |
| 211 | + serviceSubnets: |
| 212 | + - {{ $serviceCIDR }} |
| 213 | + token: ${BOOTSTRAP_TOKEN} |
| 214 | + ca: |
| 215 | + crt: ${K8S_CA_B64} |
| 216 | + discovery: |
| 217 | + enabled: true |
| 218 | + registries: |
| 219 | + kubernetes: |
| 220 | + disabled: true |
| 221 | + service: |
| 222 | + disabled: true |
| 223 | +{{- end }} |
| 224 | + |
| 225 | +{{- /* |
| 226 | + Content hash of the TalosConfigTemplate spec, truncated the same way the |
| 227 | + KubevirtMachineTemplate hash in this chart is. Takes the same dict as |
| 228 | + kubernetes.talosConfigTemplateSpec, so the Job that creates the template and |
| 229 | + the MachineDeployment that references it always agree on the name. |
| 230 | +*/}} |
| 231 | +{{- define "kubernetes.talosConfigTemplateHash" -}} |
| 232 | +{{- include "kubernetes.talosConfigTemplateSpec" . | sha256sum | trunc 6 -}} |
| 233 | +{{- end }} |
0 commit comments