Skip to content

Commit 702a622

Browse files
fix(kubernetes): make MachineHealthCheck maxUnhealthy configurable
The MachineHealthCheck was hardcoded to maxUnhealthy: 0. In Cluster API semantics this blocks all remediation: as soon as a single machine is unhealthy the threshold is exceeded, so crashed worker nodes are never auto-replaced. Expose maxUnhealthy per node group (integer or percentage) with a default of "100%" so auto-remediation works out of the box while staying tunable for sensitive pools. Signed-off-by: mattia-eleuteri <[email protected]>
1 parent e16d987 commit 702a622

5 files changed

Lines changed: 165 additions & 155 deletions

File tree

‎packages/apps/kubernetes/README.md‎

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -91,21 +91,22 @@ See the reference for components utilized in this service:
9191
9292
### Application-specific Parameters
9393
94-
| Name | Description | Type | Value |
95-
| ----------------------------------- | ---------------------------------------------------------------------------------------------- | ------------------- | ----------- |
96-
| `nodeGroups` | Worker nodes configuration map. | `map[string]object` | `{...}` |
97-
| `nodeGroups[name].minReplicas` | Minimum number of replicas. | `int` | `0` |
98-
| `nodeGroups[name].maxReplicas` | Maximum number of replicas. | `int` | `10` |
99-
| `nodeGroups[name].instanceType` | Virtual machine instance type. | `string` | `u1.medium` |
100-
| `nodeGroups[name].ephemeralStorage` | Ephemeral storage size. | `quantity` | `20Gi` |
101-
| `nodeGroups[name].roles` | List of node roles. | `[]string` | `[]` |
102-
| `nodeGroups[name].resources` | CPU and memory resources for each worker node. | `object` | `{}` |
103-
| `nodeGroups[name].resources.cpu` | CPU available. | `quantity` | `""` |
104-
| `nodeGroups[name].resources.memory` | Memory (RAM) available. | `quantity` | `""` |
105-
| `nodeGroups[name].gpus` | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). | `[]object` | `[]` |
106-
| `nodeGroups[name].gpus[i].name` | Name of GPU, such as "nvidia.com/AD102GL_L40S". | `string` | `""` |
107-
| `version` | Kubernetes major.minor version to deploy | `string` | `v1.35` |
108-
| `host` | External hostname for Kubernetes cluster. Defaults to `<cluster-name>.<tenant-host>` if empty. | `string` | `""` |
94+
| Name | Description | Type | Value |
95+
| ----------------------------------- | ----------------------------------------------------------------------------------------------------------- | ------------------- | ----------- |
96+
| `nodeGroups` | Worker nodes configuration map. | `map[string]object` | `{...}` |
97+
| `nodeGroups[name].minReplicas` | Minimum number of replicas. | `int` | `0` |
98+
| `nodeGroups[name].maxReplicas` | Maximum number of replicas. | `int` | `10` |
99+
| `nodeGroups[name].maxUnhealthy` | Max unhealthy nodes (integer or percentage) tolerated before the MachineHealthCheck stops auto-remediation. | `string` | `100%` |
100+
| `nodeGroups[name].instanceType` | Virtual machine instance type. | `string` | `u1.medium` |
101+
| `nodeGroups[name].ephemeralStorage` | Ephemeral storage size. | `quantity` | `20Gi` |
102+
| `nodeGroups[name].roles` | List of node roles. | `[]string` | `[]` |
103+
| `nodeGroups[name].resources` | CPU and memory resources for each worker node. | `object` | `{}` |
104+
| `nodeGroups[name].resources.cpu` | CPU available. | `quantity` | `""` |
105+
| `nodeGroups[name].resources.memory` | Memory (RAM) available. | `quantity` | `""` |
106+
| `nodeGroups[name].gpus` | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). | `[]object` | `[]` |
107+
| `nodeGroups[name].gpus[i].name` | Name of GPU, such as "nvidia.com/AD102GL_L40S". | `string` | `""` |
108+
| `version` | Kubernetes major.minor version to deploy | `string` | `v1.35` |
109+
| `host` | External hostname for Kubernetes cluster. Defaults to `<cluster-name>.<tenant-host>` if empty. | `string` | `""` |
109110
110111
111112
### Cluster Addons

‎packages/apps/kubernetes/templates/cluster.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -340,7 +340,7 @@ metadata:
340340
namespace: {{ $.Release.Namespace }}
341341
spec:
342342
clusterName: {{ $.Release.Name }}
343-
maxUnhealthy: 0
343+
maxUnhealthy: {{ $group.maxUnhealthy | default "100%" }}
344344
nodeStartupTimeout: 10m
345345
selector:
346346
matchLabels:

‎packages/apps/kubernetes/values.schema.json‎

Lines changed: 144 additions & 137 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,149 @@
22
"title": "Chart Values",
33
"type": "object",
44
"properties": {
5+
"storageClass": {
6+
"description": "StorageClass used to store the data.",
7+
"type": "string",
8+
"default": "replicated"
9+
},
10+
"nodeGroups": {
11+
"description": "Worker nodes configuration map.",
12+
"type": "object",
13+
"default": {
14+
"md0": {
15+
"ephemeralStorage": "20Gi",
16+
"gpus": [],
17+
"instanceType": "u1.medium",
18+
"maxReplicas": 10,
19+
"maxUnhealthy": "100%",
20+
"minReplicas": 0,
21+
"resources": {},
22+
"roles": [
23+
"ingress-nginx"
24+
]
25+
}
26+
},
27+
"additionalProperties": {
28+
"type": "object",
29+
"required": [
30+
"ephemeralStorage",
31+
"instanceType",
32+
"maxReplicas",
33+
"maxUnhealthy",
34+
"minReplicas",
35+
"resources"
36+
],
37+
"properties": {
38+
"ephemeralStorage": {
39+
"description": "Ephemeral storage size.",
40+
"default": "20Gi",
41+
"pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
42+
"anyOf": [
43+
{
44+
"type": "integer"
45+
},
46+
{
47+
"type": "string"
48+
}
49+
],
50+
"x-kubernetes-int-or-string": true
51+
},
52+
"gpus": {
53+
"description": "List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).",
54+
"type": "array",
55+
"items": {
56+
"type": "object",
57+
"required": [
58+
"name"
59+
],
60+
"properties": {
61+
"name": {
62+
"description": "Name of GPU, such as \"nvidia.com/AD102GL_L40S\".",
63+
"type": "string"
64+
}
65+
}
66+
}
67+
},
68+
"instanceType": {
69+
"description": "Virtual machine instance type.",
70+
"type": "string",
71+
"default": "u1.medium"
72+
},
73+
"maxReplicas": {
74+
"description": "Maximum number of replicas.",
75+
"type": "integer",
76+
"default": 10
77+
},
78+
"maxUnhealthy": {
79+
"description": "Max unhealthy nodes (integer or percentage) tolerated before the MachineHealthCheck stops auto-remediation.",
80+
"type": "string",
81+
"default": "100%"
82+
},
83+
"minReplicas": {
84+
"description": "Minimum number of replicas.",
85+
"type": "integer",
86+
"default": 0
87+
},
88+
"resources": {
89+
"description": "CPU and memory resources for each worker node.",
90+
"type": "object",
91+
"properties": {
92+
"cpu": {
93+
"description": "CPU available.",
94+
"pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
95+
"anyOf": [
96+
{
97+
"type": "integer"
98+
},
99+
{
100+
"type": "string"
101+
}
102+
],
103+
"x-kubernetes-int-or-string": true
104+
},
105+
"memory": {
106+
"description": "Memory (RAM) available.",
107+
"pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
108+
"anyOf": [
109+
{
110+
"type": "integer"
111+
},
112+
{
113+
"type": "string"
114+
}
115+
],
116+
"x-kubernetes-int-or-string": true
117+
}
118+
}
119+
},
120+
"roles": {
121+
"description": "List of node roles.",
122+
"type": "array",
123+
"items": {
124+
"type": "string"
125+
}
126+
}
127+
}
128+
}
129+
},
130+
"version": {
131+
"description": "Kubernetes major.minor version to deploy",
132+
"type": "string",
133+
"default": "v1.35",
134+
"enum": [
135+
"v1.35",
136+
"v1.34",
137+
"v1.33",
138+
"v1.32",
139+
"v1.31",
140+
"v1.30"
141+
]
142+
},
143+
"host": {
144+
"description": "External hostname for Kubernetes cluster. Defaults to `\u003ccluster-name\u003e.\u003ctenant-host\u003e` if empty.",
145+
"type": "string",
146+
"default": ""
147+
},
5148
"addons": {
6149
"description": "Cluster addons configuration.",
7150
"type": "object",
@@ -494,142 +637,6 @@
494637
}
495638
}
496639
}
497-
},
498-
"host": {
499-
"description": "External hostname for Kubernetes cluster. Defaults to `\u003ccluster-name\u003e.\u003ctenant-host\u003e` if empty.",
500-
"type": "string",
501-
"default": ""
502-
},
503-
"nodeGroups": {
504-
"description": "Worker nodes configuration map.",
505-
"type": "object",
506-
"default": {
507-
"md0": {
508-
"ephemeralStorage": "20Gi",
509-
"gpus": [],
510-
"instanceType": "u1.medium",
511-
"maxReplicas": 10,
512-
"minReplicas": 0,
513-
"resources": {},
514-
"roles": [
515-
"ingress-nginx"
516-
]
517-
}
518-
},
519-
"additionalProperties": {
520-
"type": "object",
521-
"required": [
522-
"ephemeralStorage",
523-
"instanceType",
524-
"maxReplicas",
525-
"minReplicas",
526-
"resources"
527-
],
528-
"properties": {
529-
"ephemeralStorage": {
530-
"description": "Ephemeral storage size.",
531-
"default": "20Gi",
532-
"pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
533-
"anyOf": [
534-
{
535-
"type": "integer"
536-
},
537-
{
538-
"type": "string"
539-
}
540-
],
541-
"x-kubernetes-int-or-string": true
542-
},
543-
"gpus": {
544-
"description": "List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).",
545-
"type": "array",
546-
"items": {
547-
"type": "object",
548-
"required": [
549-
"name"
550-
],
551-
"properties": {
552-
"name": {
553-
"description": "Name of GPU, such as \"nvidia.com/AD102GL_L40S\".",
554-
"type": "string"
555-
}
556-
}
557-
}
558-
},
559-
"instanceType": {
560-
"description": "Virtual machine instance type.",
561-
"type": "string",
562-
"default": "u1.medium"
563-
},
564-
"maxReplicas": {
565-
"description": "Maximum number of replicas.",
566-
"type": "integer",
567-
"default": 10
568-
},
569-
"minReplicas": {
570-
"description": "Minimum number of replicas.",
571-
"type": "integer",
572-
"default": 0
573-
},
574-
"resources": {
575-
"description": "CPU and memory resources for each worker node.",
576-
"type": "object",
577-
"properties": {
578-
"cpu": {
579-
"description": "CPU available.",
580-
"pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
581-
"anyOf": [
582-
{
583-
"type": "integer"
584-
},
585-
{
586-
"type": "string"
587-
}
588-
],
589-
"x-kubernetes-int-or-string": true
590-
},
591-
"memory": {
592-
"description": "Memory (RAM) available.",
593-
"pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
594-
"anyOf": [
595-
{
596-
"type": "integer"
597-
},
598-
{
599-
"type": "string"
600-
}
601-
],
602-
"x-kubernetes-int-or-string": true
603-
}
604-
}
605-
},
606-
"roles": {
607-
"description": "List of node roles.",
608-
"type": "array",
609-
"items": {
610-
"type": "string"
611-
}
612-
}
613-
}
614-
}
615-
},
616-
"storageClass": {
617-
"description": "StorageClass used to store the data.",
618-
"type": "string",
619-
"default": "replicated"
620-
},
621-
"version": {
622-
"description": "Kubernetes major.minor version to deploy",
623-
"type": "string",
624-
"default": "v1.35",
625-
"enum": [
626-
"v1.35",
627-
"v1.34",
628-
"v1.33",
629-
"v1.32",
630-
"v1.31",
631-
"v1.30"
632-
]
633640
}
634641
}
635-
}
642+
}

‎packages/apps/kubernetes/values.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ storageClass: replicated
2828
## @typedef {struct} NodeGroup - Worker node group configuration.
2929
## @field {int} minReplicas=0 - Minimum number of replicas.
3030
## @field {int} maxReplicas=10 - Maximum number of replicas.
31+
## @field {string} maxUnhealthy="100%" - Max unhealthy nodes (integer or percentage) tolerated before the MachineHealthCheck stops auto-remediation.
3132
## @field {string} instanceType="u1.medium" - Virtual machine instance type.
3233
## @field {quantity} ephemeralStorage="20Gi" - Ephemeral storage size.
3334
## @field {[]string} roles - List of node roles.
@@ -39,6 +40,7 @@ nodeGroups:
3940
md0:
4041
minReplicas: 0
4142
maxReplicas: 10
43+
maxUnhealthy: "100%"
4244
instanceType: "u1.medium"
4345
ephemeralStorage: 20Gi
4446
roles:

0 commit comments

Comments
 (0)