Skip to content

Commit 53485bc

Browse files
committed
[kubernetes] add check for deployed Kubernetes server version using kubectl
Signed-off-by: IvanHunters <[email protected]>
1 parent 7ac0d9b commit 53485bc

3 files changed

Lines changed: 207 additions & 73 deletions

File tree

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
#!/usr/bin/env bats
2+
3+
@test "Create a tenant Kubernetes control plane" {
4+
LATEST_K8S_VERSION=$(yq 'keys | .[-1]' ../packages/apps/kubernetes/files/versions.yaml)
5+
TEMPORAL_TENANT_PORT=59991
6+
7+
kubectl apply -f - <<EOF
8+
apiVersion: apps.cozystack.io/v1alpha1
9+
kind: Kubernetes
10+
metadata:
11+
name: test-latest
12+
namespace: tenant-test
13+
spec:
14+
addons:
15+
certManager:
16+
enabled: false
17+
valuesOverride: {}
18+
cilium:
19+
valuesOverride: {}
20+
fluxcd:
21+
enabled: false
22+
valuesOverride: {}
23+
gatewayAPI:
24+
enabled: false
25+
gpuOperator:
26+
enabled: false
27+
valuesOverride: {}
28+
ingressNginx:
29+
enabled: true
30+
hosts: []
31+
valuesOverride: {}
32+
monitoringAgents:
33+
enabled: false
34+
valuesOverride: {}
35+
verticalPodAutoscaler:
36+
valuesOverride: {}
37+
controlPlane:
38+
apiServer:
39+
resources: {}
40+
resourcesPreset: small
41+
controllerManager:
42+
resources: {}
43+
resourcesPreset: micro
44+
konnectivity:
45+
server:
46+
resources: {}
47+
resourcesPreset: micro
48+
replicas: 2
49+
scheduler:
50+
resources: {}
51+
resourcesPreset: micro
52+
host: ""
53+
nodeGroups:
54+
md0:
55+
ephemeralStorage: 20Gi
56+
gpus: []
57+
instanceType: u1.medium
58+
maxReplicas: 10
59+
minReplicas: 0
60+
resources:
61+
cpu: ""
62+
memory: ""
63+
roles:
64+
- ingress-nginx
65+
storageClass: replicated
66+
version: "$LATEST_K8S_VERSION"
67+
EOF
68+
# Wait for the tenant-test namespace to be active
69+
kubectl wait namespace tenant-test --timeout=20s --for=jsonpath='{.status.phase}'=Active
70+
71+
# Wait for the Kamaji control plane to be created (retry for up to 10 seconds)
72+
timeout 10 sh -ec 'until kubectl get kamajicontrolplane -n tenant-test kubernetes-test-latest; do sleep 1; done'
73+
74+
# Wait for the tenant control plane to be fully created (timeout after 4 minutes)
75+
kubectl wait --for=condition=TenantControlPlaneCreated kamajicontrolplane -n tenant-test kubernetes-test-latest --timeout=4m
76+
77+
# Wait for Kubernetes resources to be ready (timeout after 2 minutes)
78+
kubectl wait tcp -n tenant-test kubernetes-test-latest --timeout=2m --for=jsonpath='{.status.kubernetesResources.version.status}'=Ready
79+
80+
# Wait for all required deployments to be available (timeout after 4 minutes)
81+
kubectl wait deploy --timeout=4m --for=condition=available -n tenant-test kubernetes-test-latest kubernetes-test-latest-cluster-autoscaler kubernetes-test-latest-kccm kubernetes-test-latest-kcsi-controller
82+
83+
# Wait for the machine deployment to scale to 2 replicas (timeout after 1 minute)
84+
kubectl wait machinedeployment kubernetes-test-latest-md0 -n tenant-test --timeout=1m --for=jsonpath='{.status.replicas}'=2
85+
86+
# Get the admin kubeconfig and save it to a file
87+
kubectl get secret kubernetes-test-latest-admin-kubeconfig -ojsonpath='{.data.super-admin\.conf}' -n tenant-test | base64 -d > tenantkubeconfig
88+
89+
# Update the kubeconfig to use localhost for the API server
90+
yq -i ".clusters[0].cluster.server = \"https://localhost:${TEMPORAL_TENANT_PORT}\"" tenantkubeconfig
91+
92+
# Set up port forwarding to the Kubernetes API server for a 40 second timeout
93+
bash -c 'timeout 40s kubectl port-forward service/kubernetes-test-latest -n tenant-test '"${TEMPORAL_TENANT_PORT}"':6443 > /dev/null 2>&1 &'
94+
95+
# Verify the Kubernetes version matches what we expect (retry for up to 20 seconds)
96+
timeout 20 sh -ec 'until kubectl --kubeconfig tenantkubeconfig version 2>/dev/null | grep "Server Version: $LATEST_K8S_VERSION"; do sleep 5; done'
97+
98+
# Wait for all machine deployment replicas to be ready (timeout after 10 minutes)
99+
kubectl wait machinedeployment kubernetes-test-latest-md0 -n tenant-test --timeout=10m --for=jsonpath='{.status.v1beta2.readyReplicas}'=2
100+
101+
# Clean up by deleting the Kubernetes resource
102+
kubectl -n tenant-test delete kuberneteses.apps.cozystack.io test-latest
103+
}
Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
#!/usr/bin/env bats
2+
3+
@test "Create a previous version tenant Kubernetes control plane" {
4+
# Get latest version of k8s from gcr
5+
PREVIOUS_K8S_VERSION=$(yq 'keys | .[-2]' ../packages/apps/kubernetes/files/versions.yaml)
6+
TEMPORAL_TENANT_PORT=59992
7+
8+
kubectl apply -f - <<EOF
9+
apiVersion: apps.cozystack.io/v1alpha1
10+
kind: Kubernetes
11+
metadata:
12+
name: test-previous
13+
namespace: tenant-test
14+
spec:
15+
addons:
16+
certManager:
17+
enabled: false
18+
valuesOverride: {}
19+
cilium:
20+
valuesOverride: {}
21+
fluxcd:
22+
enabled: false
23+
valuesOverride: {}
24+
gatewayAPI:
25+
enabled: false
26+
gpuOperator:
27+
enabled: false
28+
valuesOverride: {}
29+
ingressNginx:
30+
enabled: true
31+
hosts: []
32+
valuesOverride: {}
33+
monitoringAgents:
34+
enabled: false
35+
valuesOverride: {}
36+
verticalPodAutoscaler:
37+
valuesOverride: {}
38+
controlPlane:
39+
apiServer:
40+
resources: {}
41+
resourcesPreset: small
42+
controllerManager:
43+
resources: {}
44+
resourcesPreset: micro
45+
konnectivity:
46+
server:
47+
resources: {}
48+
resourcesPreset: micro
49+
replicas: 2
50+
scheduler:
51+
resources: {}
52+
resourcesPreset: micro
53+
host: ""
54+
nodeGroups:
55+
md0:
56+
ephemeralStorage: 20Gi
57+
gpus: []
58+
instanceType: u1.medium
59+
maxReplicas: 10
60+
minReplicas: 0
61+
resources:
62+
cpu: ""
63+
memory: ""
64+
roles:
65+
- ingress-nginx
66+
storageClass: replicated
67+
version: "$PREVIOUS_K8S_VERSION"
68+
EOF
69+
# Wait for the tenant-test namespace to be active
70+
kubectl wait namespace tenant-test --timeout=20s --for=jsonpath='{.status.phase}'=Active
71+
72+
# Wait for the Kamaji control plane to be created (retry for up to 10 seconds)
73+
timeout 10 sh -ec 'until kubectl get kamajicontrolplane -n tenant-test kubernetes-test-previous; do sleep 1; done'
74+
75+
# Wait for the tenant control plane to be fully created (timeout after 4 minutes)
76+
kubectl wait --for=condition=TenantControlPlaneCreated kamajicontrolplane -n tenant-test kubernetes-test-previous --timeout=4m
77+
78+
# Wait for Kubernetes resources to be ready (timeout after 2 minutes)
79+
kubectl wait tcp -n tenant-test kubernetes-test-previous --timeout=2m --for=jsonpath='{.status.kubernetesResources.version.status}'=Ready
80+
81+
# Wait for all required deployments to be available (timeout after 4 minutes)
82+
kubectl wait deploy --timeout=4m --for=condition=available -n tenant-test kubernetes-test-previous kubernetes-test-previous-cluster-autoscaler kubernetes-test-previous-kccm kubernetes-test-previous-kcsi-controller
83+
84+
# Wait for the machine deployment to scale to 2 replicas (timeout after 1 minute)
85+
kubectl wait machinedeployment kubernetes-test-previous-md0 -n tenant-test --timeout=1m --for=jsonpath='{.status.replicas}'=2
86+
87+
# Get the admin kubeconfig and save it to a file
88+
kubectl get secret kubernetes-test-previous-admin-kubeconfig -ojsonpath='{.data.super-admin\.conf}' -n tenant-test | base64 -d > tenantkubeconfig
89+
90+
# Update the kubeconfig to use localhost for the API server
91+
yq -i ".clusters[0].cluster.server = \"https://localhost:${TEMPORAL_TENANT_PORT}\"" tenantkubeconfig
92+
93+
# Set up port forwarding to the Kubernetes API server for a 40 second timeout
94+
bash -c 'timeout 40s kubectl port-forward service/kubernetes-test-previous -n tenant-test '"${TEMPORAL_TENANT_PORT}"':6443 > /dev/null 2>&1 &'
95+
96+
# Verify the Kubernetes version matches what we expect (retry for up to 20 seconds)
97+
timeout 20 sh -ec 'until kubectl --kubeconfig tenantkubeconfig version 2>/dev/null | grep "Server Version: $PREVIOUS_K8S_VERSION"; do sleep 5; done'
98+
99+
# Wait for all machine deployment replicas to be ready (timeout after 10 minutes)
100+
kubectl wait machinedeployment kubernetes-test-previous-md0 -n tenant-test --timeout=10m --for=jsonpath='{.status.v1beta2.readyReplicas}'=2
101+
102+
# Clean up by deleting the Kubernetes resource
103+
kubectl -n tenant-test delete kuberneteses.apps.cozystack.io test-previous
104+
}

‎hack/e2e-apps/kubernetes.bats‎

Lines changed: 0 additions & 73 deletions
This file was deleted.

0 commit comments

Comments
 (0)