Skip to content

Commit f2a8c3d

Browse files
authored
[kubernetes] User-selectable cluster version (#1191)
## What this PR does This patch adds a new version field to the kubernetes chart, letting end-users specify the version of kubernetes they want to deploy. ### Release note ```release-note [kubernetes] Let users specify desired version of tenant k8s cluster. ``` <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Added a configurable Kubernetes version parameter, allowing selection of specific minor versions for cluster deployments. * Introduced a version mapping system to ensure clusters use precise Kubernetes patch versions. * **Bug Fixes** * Ensured only supported Kubernetes versions can be selected, reducing configuration errors. * **Documentation** * Updated documentation to describe the new version parameter and its usage. * **Tests** * Enhanced end-to-end tests to cover deployments with both the latest and previous Kubernetes versions. * **Chores** * Consolidated version references for multiple packages to streamline version management. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
2 parents d0fba98 + 5b6ebbc commit f2a8c3d

10 files changed

Lines changed: 88 additions & 18 deletions

File tree

‎hack/e2e-apps/kubernetes.bats‎

Lines changed: 50 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,16 @@
11
#!/usr/bin/env bats
22

3-
@test "Create a tenant Kubernetes control plane" {
3+
run_kubernetes_test() {
4+
local version_expr="$1"
5+
local test_name="$2"
6+
local port="$3"
7+
local k8s_version=$(yq "$version_expr" packages/apps/kubernetes/files/versions.yaml)
8+
49
kubectl apply -f - <<EOF
510
apiVersion: apps.cozystack.io/v1alpha1
611
kind: Kubernetes
712
metadata:
8-
name: test
13+
name: "${test_name}"
914
namespace: tenant-test
1015
spec:
1116
addons:
@@ -60,13 +65,49 @@ spec:
6065
roles:
6166
- ingress-nginx
6267
storageClass: replicated
68+
version: "${k8s_version}"
6369
EOF
70+
# Wait for the tenant-test namespace to be active
6471
kubectl wait namespace tenant-test --timeout=20s --for=jsonpath='{.status.phase}'=Active
65-
timeout 10 sh -ec 'until kubectl get kamajicontrolplane -n tenant-test kubernetes-test; do sleep 1; done'
66-
kubectl wait --for=condition=TenantControlPlaneCreated kamajicontrolplane -n tenant-test kubernetes-test --timeout=4m
67-
kubectl wait tcp -n tenant-test kubernetes-test --timeout=2m --for=jsonpath='{.status.kubernetesResources.version.status}'=Ready
68-
kubectl wait deploy --timeout=4m --for=condition=available -n tenant-test kubernetes-test kubernetes-test-cluster-autoscaler kubernetes-test-kccm kubernetes-test-kcsi-controller
69-
kubectl wait machinedeployment kubernetes-test-md0 -n tenant-test --timeout=1m --for=jsonpath='{.status.replicas}'=2
70-
kubectl wait machinedeployment kubernetes-test-md0 -n tenant-test --timeout=10m --for=jsonpath='{.status.v1beta2.readyReplicas}'=2
71-
kubectl -n tenant-test delete kuberneteses.apps.cozystack.io test
72+
73+
# Wait for the Kamaji control plane to be created (retry for up to 10 seconds)
74+
timeout 10 sh -ec 'until kubectl get kamajicontrolplane -n tenant-test kubernetes-'"${test_name}"'; do sleep 1; done'
75+
76+
# Wait for the tenant control plane to be fully created (timeout after 4 minutes)
77+
kubectl wait --for=condition=TenantControlPlaneCreated kamajicontrolplane -n tenant-test kubernetes-${test_name} --timeout=4m
78+
79+
# Wait for Kubernetes resources to be ready (timeout after 2 minutes)
80+
kubectl wait tcp -n tenant-test kubernetes-${test_name} --timeout=2m --for=jsonpath='{.status.kubernetesResources.version.status}'=Ready
81+
82+
# Wait for all required deployments to be available (timeout after 4 minutes)
83+
kubectl wait deploy --timeout=4m --for=condition=available -n tenant-test kubernetes-${test_name} kubernetes-${test_name}-cluster-autoscaler kubernetes-${test_name}-kccm kubernetes-${test_name}-kcsi-controller
84+
85+
# Wait for the machine deployment to scale to 2 replicas (timeout after 1 minute)
86+
kubectl wait machinedeployment kubernetes-${test_name}-md0 -n tenant-test --timeout=1m --for=jsonpath='{.status.replicas}'=2
87+
88+
# Get the admin kubeconfig and save it to a file
89+
kubectl get secret kubernetes-${test_name}-admin-kubeconfig -ojsonpath='{.data.super-admin\.conf}' -n tenant-test | base64 -d > tenantkubeconfig
90+
91+
# Update the kubeconfig to use localhost for the API server
92+
yq -i ".clusters[0].cluster.server = \"https://localhost:${port}\"" tenantkubeconfig
93+
94+
# Set up port forwarding to the Kubernetes API server for a 40 second timeout
95+
bash -c 'timeout 40s kubectl port-forward service/kubernetes-'"${test_name}"' -n tenant-test '"${port}"':6443 > /dev/null 2>&1 &'
96+
97+
# Verify the Kubernetes version matches what we expect (retry for up to 20 seconds)
98+
timeout 20 sh -ec 'until kubectl --kubeconfig tenantkubeconfig version 2>/dev/null | grep -Fq "Server Version: ${k8s_version}"; do sleep 5; done'
99+
100+
# Wait for all machine deployment replicas to be ready (timeout after 10 minutes)
101+
kubectl wait machinedeployment kubernetes-${test_name}-md0 -n tenant-test --timeout=10m --for=jsonpath='{.status.v1beta2.readyReplicas}'=2
102+
103+
# Clean up by deleting the Kubernetes resource
104+
kubectl -n tenant-test delete kuberneteses.apps.cozystack.io $test_name
105+
106+
}
107+
108+
@test "Create a tenant Kubernetes control plane with latest version" {
109+
run_kubernetes_test 'keys | sort_by(.) | .[-1]' 'test-latest-version' '59991'
110+
}
111+
@test "Create a tenant Kubernetes control plane with previous version" {
112+
run_kubernetes_test 'keys | sort_by(.) | .[-2]' 'test-previous-version' '59992'
72113
}

‎packages/apps/kubernetes/Chart.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,10 @@ type: application
1616
# This is the chart version. This version number should be incremented each time you make changes
1717
# to the chart and its templates, including the app version.
1818
# Versions are expected to follow Semantic Versioning (https://semver.org/)
19-
version: 0.25.3
19+
version: 0.26.0
2020

2121
# This is the version number of the application being deployed. This version number should be
2222
# incremented each time you make changes to the application. Versions are not expected to
2323
# follow Semantic Versioning. They should reflect the version the application is using.
2424
# It is recommended to use it with quotes.
25-
appVersion: 1.32.4
25+
appVersion: 1.32.6

‎packages/apps/kubernetes/Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ include ../../../scripts/package.mk
77

88
generate:
99
readme-generator -v values.yaml -s values.schema.json -r README.md
10+
yq -o=json -i '.properties.version.enum = (load("files/versions.yaml") | keys)' values.schema.json
1011
yq -o json -i '.properties.addons.properties.ingressNginx.properties.exposeMethod.enum = ["Proxied","LoadBalancer"]' values.schema.json
1112
yq -o json -i '.properties.controlPlane.properties.apiServer.properties.resourcesPreset.enum = $(PRESET_ENUM)' values.schema.json
1213
yq -o json -i '.properties.controlPlane.properties.controllerManager.properties.resourcesPreset.enum = $(PRESET_ENUM)' values.schema.json

‎packages/apps/kubernetes/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ See the reference for components utilized in this service:
8686
| `host` | Hostname used to access the Kubernetes cluster externally. Defaults to `<cluster-name>.<tenant-host>` when empty. | `""` |
8787
| `controlPlane.replicas` | Number of replicas for Kubernetes control-plane components. | `2` |
8888
| `storageClass` | StorageClass used to store user data. | `replicated` |
89+
| `version` | Kubernetes version given as vMAJOR.MINOR | `v1.32` |
8990
| `nodeGroups` | nodeGroups configuration | `{}` |
9091
9192
### Cluster Addons
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
"v1.28": "v1.28.15"
2+
"v1.29": "v1.29.15"
3+
"v1.30": "v1.30.14"
4+
"v1.31": "v1.31.10"
5+
"v1.32": "v1.32.6"
6+
"v1.33": "v1.33.0"
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{{- define "kubernetes.versionMap" }}
2+
{{- $versionMap := .Files.Get "files/versions.yaml" | fromYaml }}
3+
{{- if not (hasKey $versionMap .Values.version) }}
4+
{{- printf `Kubernetes version %s is not supported, allowed versions are %s` $.Values.version (keys $versionMap) | fail }}
5+
{{- end }}
6+
{{- index $versionMap .Values.version }}
7+
{{- end }}

‎packages/apps/kubernetes/templates/cluster.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ spec:
151151
labels:
152152
policy.cozystack.io/allow-to-etcd: "true"
153153
replicas: 2
154-
version: {{ $.Chart.AppVersion }}
154+
version: {{ include "kubernetes.versionMap" $ }}
155155
---
156156
apiVersion: cozystack.io/v1alpha1
157157
kind: WorkloadMonitor
@@ -290,7 +290,7 @@ spec:
290290
kind: KubevirtMachineTemplate
291291
name: {{ $.Release.Name }}-{{ $groupName }}-{{ $kubevirtmachinetemplateHash }}
292292
namespace: {{ $.Release.Namespace }}
293-
version: v{{ $.Chart.AppVersion }}
293+
version: {{ include "kubernetes.versionMap" $}}
294294
---
295295
apiVersion: cluster.x-k8s.io/v1beta1
296296
kind: MachineHealthCheck

‎packages/apps/kubernetes/values.schema.json‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,19 @@
123123
"description": "StorageClass used to store user data.",
124124
"default": "replicated"
125125
},
126+
"version": {
127+
"type": "string",
128+
"description": "Kubernetes version given as vMAJOR.MINOR",
129+
"default": "v1.32",
130+
"enum": [
131+
"v1.28",
132+
"v1.29",
133+
"v1.30",
134+
"v1.31",
135+
"v1.32",
136+
"v1.33"
137+
]
138+
},
126139
"addons": {
127140
"type": "object",
128141
"properties": {

‎packages/apps/kubernetes/values.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,11 @@
33
## @param host Hostname used to access the Kubernetes cluster externally. Defaults to `<cluster-name>.<tenant-host>` when empty.
44
## @param controlPlane.replicas Number of replicas for Kubernetes control-plane components.
55
## @param storageClass StorageClass used to store user data.
6+
## @param version Kubernetes version given as vMAJOR.MINOR
67
##
78
host: ""
89
storageClass: replicated
9-
10+
version: "v1.32"
1011
## @param nodeGroups [object] nodeGroups configuration
1112
##
1213
nodeGroups:

‎packages/apps/versions_map‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ kubernetes 0.24.0 62cb694d
6161
kubernetes 0.25.0 70f82667
6262
kubernetes 0.25.1 acd4663a
6363
kubernetes 0.25.2 08cb7c0f
64-
kubernetes 0.25.3 HEAD
64+
kubernetes 0.26.0 HEAD
6565
mysql 0.1.0 263e47be
6666
mysql 0.2.0 c24a103f
6767
mysql 0.3.0 53f2365e
@@ -150,7 +150,7 @@ tcp-balancer 0.4.2 4369b031
150150
tcp-balancer 0.5.0 08cb7c0f
151151
tcp-balancer 0.5.1 HEAD
152152
tenant 1.10.0 4369b031
153-
tenant 1.11.0 70f82667
153+
tenant 1.11.0 08cb7c0f
154154
tenant 1.11.1 HEAD
155155
virtual-machine 0.1.4 f2015d65
156156
virtual-machine 0.1.5 263e47be
@@ -169,7 +169,7 @@ virtual-machine 0.9.1 93bdf411
169169
virtual-machine 0.10.0 6130f43d
170170
virtual-machine 0.10.2 632224a3
171171
virtual-machine 0.11.0 4369b031
172-
virtual-machine 0.12.0 70f82667
172+
virtual-machine 0.12.0 acd4663a
173173
virtual-machine 0.12.1 HEAD
174174
vm-disk 0.1.0 d971f2ff
175175
vm-disk 0.1.1 6130f43d
@@ -187,7 +187,7 @@ vm-instance 0.6.0 721c12a7
187187
vm-instance 0.7.0 6130f43d
188188
vm-instance 0.7.2 632224a3
189189
vm-instance 0.8.0 4369b031
190-
vm-instance 0.9.0 70f82667
190+
vm-instance 0.9.0 acd4663a
191191
vm-instance 0.10.0 HEAD
192192
vpn 0.1.0 263e47be
193193
vpn 0.2.0 53f2365e

0 commit comments

Comments
 (0)