Skip to content

Commit d02f6bf

Browse files
committed
Introduce new test framework
Signed-off-by: Andrei Kvapil <[email protected]>
1 parent 609e7ed commit d02f6bf

4 files changed

Lines changed: 500 additions & 371 deletions

File tree

‎hack/e2e.bats‎

Lines changed: 385 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,385 @@
1+
#!/usr/bin/env bats
2+
# -----------------------------------------------------------------------------
3+
# Cozystack end‑to‑end provisioning test (Bats)
4+
# -----------------------------------------------------------------------------
5+
6+
export TALOSCONFIG=$PWD/talosconfig
7+
export KUBECONFIG=$PWD/kubeconfig
8+
9+
@test "Environment variable COZYSTACK_INSTALLER_YAML is defined" {
10+
if [ -z "${COZYSTACK_INSTALLER_YAML:-}" ]; then
11+
echo 'COZYSTACK_INSTALLER_YAML environment variable is not set!' >&2
12+
echo >&2
13+
echo 'Please export it with the following command:' >&2
14+
echo ' export COZYSTACK_INSTALLER_YAML=$(helm template -n cozy-system installer packages/core/installer)' >&2
15+
fi
16+
}
17+
18+
@test "IPv4 forwarding is enabled" {
19+
if [ "$(cat /proc/sys/net/ipv4/ip_forward)" != 1 ]; then
20+
echo "IPv4 forwarding is disabled!" >&2
21+
echo >&2
22+
echo "Enable it with:" >&2
23+
echo " echo 1 > /proc/sys/net/ipv4/ip_forward" >&2
24+
fi
25+
}
26+
27+
@test "Clean previous VMs" {
28+
kill $(cat srv1/qemu.pid srv2/qemu.pid srv3/qemu.pid 2>/dev/null) 2>/dev/null || true
29+
rm -rf srv1 srv2 srv3
30+
}
31+
32+
@test "Prepare networking and masquerading" {
33+
ip link del cozy-br0 2>/dev/null || true
34+
ip link add cozy-br0 type bridge
35+
ip link set cozy-br0 up
36+
ip address add 192.168.123.1/24 dev cozy-br0
37+
38+
# Masquerading rule – idempotent (delete first, then add)
39+
iptables -t nat -D POSTROUTING -s 192.168.123.0/24 ! -d 192.168.123.0/24 -j MASQUERADE 2>/dev/null || true
40+
iptables -t nat -A POSTROUTING -s 192.168.123.0/24 ! -d 192.168.123.0/24 -j MASQUERADE
41+
}
42+
43+
@test "Prepare cloud‑init drive for VMs" {
44+
mkdir -p srv1 srv2 srv3
45+
46+
# Generate cloud‑init ISOs
47+
for i in 1 2 3; do
48+
echo "hostname: srv${i}" > "srv${i}/meta-data"
49+
50+
cat > "srv${i}/user-data" <<'EOF'
51+
#cloud-config
52+
EOF
53+
54+
cat > "srv${i}/network-config" <<EOF
55+
version: 2
56+
ethernets:
57+
eth0:
58+
dhcp4: false
59+
addresses:
60+
- "192.168.123.1${i}/26"
61+
gateway4: "192.168.123.1"
62+
nameservers:
63+
search: [cluster.local]
64+
addresses: [8.8.8.8]
65+
EOF
66+
67+
( cd "srv${i}" && genisoimage \
68+
-output seed.img \
69+
-volid cidata -rational-rock -joliet \
70+
user-data meta-data network-config )
71+
done
72+
}
73+
74+
@test "Download Talos NoCloud image" {
75+
if [ ! -f nocloud-amd64.raw ]; then
76+
wget https://github.com/cozystack/cozystack/releases/latest/download/nocloud-amd64.raw.xz \
77+
-O nocloud-amd64.raw.xz --show-progress --output-file /dev/stdout --progress=dot:giga 2>/dev/null
78+
rm -f nocloud-amd64.raw
79+
xz --decompress nocloud-amd64.raw.xz
80+
fi
81+
}
82+
83+
@test "Prepare VM disks" {
84+
for i in 1 2 3; do
85+
cp nocloud-amd64.raw srv${i}/system.img
86+
qemu-img resize srv${i}/system.img 20G
87+
qemu-img create srv${i}/data.img 100G
88+
done
89+
}
90+
91+
@test "Create tap devices" {
92+
for i in 1 2 3; do
93+
ip link del cozy-srv${i} 2>/dev/null || true
94+
ip tuntap add dev cozy-srv${i} mode tap
95+
ip link set cozy-srv${i} up
96+
ip link set cozy-srv${i} master cozy-br0
97+
done
98+
}
99+
100+
@test "Boot QEMU VMs" {
101+
for i in 1 2 3; do
102+
qemu-system-x86_64 -machine type=pc,accel=kvm -cpu host -smp 8 -m 16384 \
103+
-device virtio-net,netdev=net0,mac=52:54:00:12:34:5${i} \
104+
-netdev tap,id=net0,ifname=cozy-srv${i},script=no,downscript=no \
105+
-drive file=srv${i}/system.img,if=virtio,format=raw \
106+
-drive file=srv${i}/seed.img,if=virtio,format=raw \
107+
-drive file=srv${i}/data.img,if=virtio,format=raw \
108+
-display none -daemonize -pidfile srv${i}/qemu.pid
109+
done
110+
111+
# Give qemu a few seconds to start up networking
112+
sleep 5
113+
}
114+
115+
@test "Wait until Talos API port 50000 is reachable on all machines" {
116+
timeout 60 sh -ec 'until nc -nz 192.168.123.11 50000 && nc -nz 192.168.123.12 50000 && nc -nz 192.168.123.13 50000; do sleep 1; done'
117+
}
118+
119+
@test "Generate Talos cluster configuration" {
120+
# Cluster‑wide patches
121+
cat > patch.yaml <<'EOF'
122+
machine:
123+
kubelet:
124+
nodeIP:
125+
validSubnets:
126+
- 192.168.123.0/24
127+
extraConfig:
128+
maxPods: 512
129+
kernel:
130+
modules:
131+
- name: openvswitch
132+
- name: drbd
133+
parameters:
134+
- usermode_helper=disabled
135+
- name: zfs
136+
- name: spl
137+
registries:
138+
mirrors:
139+
docker.io:
140+
endpoints:
141+
- https://mirror.gcr.io
142+
files:
143+
- content: |
144+
[plugins]
145+
[plugins."io.containerd.cri.v1.runtime"]
146+
device_ownership_from_security_context = true
147+
path: /etc/cri/conf.d/20-customization.part
148+
op: create
149+
150+
cluster:
151+
apiServer:
152+
extraArgs:
153+
oidc-issuer-url: "https://keycloak.example.org/realms/cozy"
154+
oidc-client-id: "kubernetes"
155+
oidc-username-claim: "preferred_username"
156+
oidc-groups-claim: "groups"
157+
network:
158+
cni:
159+
name: none
160+
dnsDomain: cozy.local
161+
podSubnets:
162+
- 10.244.0.0/16
163+
serviceSubnets:
164+
- 10.96.0.0/16
165+
EOF
166+
167+
# Control‑plane‑only patches
168+
cat > patch-controlplane.yaml <<'EOF'
169+
machine:
170+
nodeLabels:
171+
node.kubernetes.io/exclude-from-external-load-balancers:
172+
$patch: delete
173+
network:
174+
interfaces:
175+
- interface: eth0
176+
vip:
177+
ip: 192.168.123.10
178+
cluster:
179+
allowSchedulingOnControlPlanes: true
180+
controllerManager:
181+
extraArgs:
182+
bind-address: 0.0.0.0
183+
scheduler:
184+
extraArgs:
185+
bind-address: 0.0.0.0
186+
apiServer:
187+
certSANs:
188+
- 127.0.0.1
189+
proxy:
190+
disabled: true
191+
discovery:
192+
enabled: false
193+
etcd:
194+
advertisedSubnets:
195+
- 192.168.123.0/24
196+
EOF
197+
198+
# Generate secrets once
199+
if [ ! -f secrets.yaml ]; then
200+
talosctl gen secrets
201+
fi
202+
203+
rm -f controlplane.yaml worker.yaml talosconfig kubeconfig
204+
talosctl gen config --with-secrets secrets.yaml cozystack https://192.168.123.10:6443 \
205+
[email protected] --config-patch-control-plane @patch-controlplane.yaml
206+
}
207+
208+
@test "Apply Talos configuration to the node" {
209+
# Apply the configuration to all three nodes
210+
for node in 11 12 13; do
211+
talosctl apply -f controlplane.yaml -n 192.168.123.${node} -e 192.168.123.${node} -i
212+
done
213+
214+
# Wait for Talos services to come up again
215+
timeout 60 sh -ec 'until nc -nz 192.168.123.11 50000 && nc -nz 192.168.123.12 50000 && nc -nz 192.168.123.13 50000; do sleep 1; done'
216+
}
217+
218+
@test "Bootstrap Talos cluster" {
219+
# Bootstrap etcd on the first node
220+
timeout 10 sh -ec 'until talosctl bootstrap -n 192.168.123.11 -e 192.168.123.11; do sleep 1; done'
221+
222+
# Wait until etcd is healthy
223+
timeout 180 sh -ec 'until talosctl etcd members -n 192.168.123.11,192.168.123.12,192.168.123.13 -e 192.168.123.10 >/dev/null 2>&1; do sleep 1; done'
224+
timeout 60 sh -ec 'while talosctl etcd members -n 192.168.123.11,192.168.123.12,192.168.123.13 -e 192.168.123.10 2>&1 | grep -q "rpc error"; do sleep 1; done'
225+
226+
# Retrieve kubeconfig
227+
rm -f kubeconfig
228+
talosctl kubeconfig kubeconfig -e 192.168.123.10 -n 192.168.123.10
229+
230+
# Wait until all three nodes register in Kubernetes
231+
timeout 60 sh -ec 'until [ $(kubectl get node --no-headers | wc -l) -eq 3 ]; do sleep 1; done'
232+
}
233+
234+
@test "Install Cozystack" {
235+
# Create namespace & configmap required by installer
236+
kubectl create namespace cozy-system --dry-run=client -o yaml | kubectl apply -f -
237+
kubectl create configmap cozystack -n cozy-system \
238+
--from-literal=bundle-name=paas-full \
239+
--from-literal=ipv4-pod-cidr=10.244.0.0/16 \
240+
--from-literal=ipv4-pod-gateway=10.244.0.1 \
241+
--from-literal=ipv4-svc-cidr=10.96.0.0/16 \
242+
--from-literal=ipv4-join-cidr=100.64.0.0/16 \
243+
--from-literal=root-host=example.org \
244+
--from-literal=api-server-endpoint=https://192.168.123.10:6443 \
245+
--dry-run=client -o yaml | kubectl apply -f -
246+
247+
# Apply installer manifests from env variable
248+
echo "$COZYSTACK_INSTALLER_YAML" | kubectl apply -f -
249+
250+
# Wait for the installer deployment to become available
251+
kubectl wait deployment/cozystack -n cozy-system --timeout=1m --for=condition=Available
252+
253+
# Wait until HelmReleases appear & reconcile them
254+
timeout 60 sh -ec 'until kubectl get hr -A | grep -q cozys; do sleep 1; done'
255+
sleep 5
256+
kubectl get hr -A | awk 'NR>1 {print "kubectl wait --timeout=15m --for=condition=ready -n "$1" hr/"$2" &"} END {print "wait"}' | sh -ex
257+
258+
# Fail the test if any HelmRelease is not Ready
259+
if kubectl get hr -A | grep -v " True " | grep -v NAME; then
260+
kubectl get hr -A
261+
fail "Some HelmReleases failed to reconcile"
262+
fi
263+
}
264+
265+
@test "Wait for Cluster‑API provider deployments" {
266+
# Wait for Cluster‑API provider deployments
267+
timeout 60 sh -ec 'until kubectl get deploy -n cozy-cluster-api capi-controller-manager capi-kamaji-controller-manager capi-kubeadm-bootstrap-controller-manager capi-operator-cluster-api-operator capk-controller-manager >/dev/null 2>&1; do sleep 1; done'
268+
kubectl wait deployment/capi-controller-manager deployment/capi-kamaji-controller-manager deployment/capi-kubeadm-bootstrap-controller-manager deployment/capi-operator-cluster-api-operator deployment/capk-controller-manager -n cozy-cluster-api --timeout=1m --for=condition=available
269+
}
270+
271+
@test "Wait for LINSTOR and configure storage" {
272+
# Linstor controller and nodes
273+
kubectl wait deployment/linstor-controller -n cozy-linstor --timeout=5m --for=condition=available
274+
timeout 60 sh -ec 'until [ $(kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor node list | grep -c Online) -eq 3 ]; do sleep 1; done'
275+
276+
for node in srv1 srv2 srv3; do
277+
kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor ps cdp zfs ${node} /dev/vdc --pool-name data --storage-pool data
278+
done
279+
280+
# Storage classes
281+
kubectl apply -f - <<'EOF'
282+
---
283+
apiVersion: storage.k8s.io/v1
284+
kind: StorageClass
285+
metadata:
286+
name: local
287+
annotations:
288+
storageclass.kubernetes.io/is-default-class: "true"
289+
provisioner: linstor.csi.linbit.com
290+
parameters:
291+
linstor.csi.linbit.com/storagePool: "data"
292+
linstor.csi.linbit.com/layerList: "storage"
293+
linstor.csi.linbit.com/allowRemoteVolumeAccess: "false"
294+
volumeBindingMode: WaitForFirstConsumer
295+
allowVolumeExpansion: true
296+
---
297+
apiVersion: storage.k8s.io/v1
298+
kind: StorageClass
299+
metadata:
300+
name: replicated
301+
provisioner: linstor.csi.linbit.com
302+
parameters:
303+
linstor.csi.linbit.com/storagePool: "data"
304+
linstor.csi.linbit.com/autoPlace: "3"
305+
linstor.csi.linbit.com/layerList: "drbd storage"
306+
linstor.csi.linbit.com/allowRemoteVolumeAccess: "true"
307+
property.linstor.csi.linbit.com/DrbdOptions/auto-quorum: suspend-io
308+
property.linstor.csi.linbit.com/DrbdOptions/Resource/on-no-data-accessible: suspend-io
309+
property.linstor.csi.linbit.com/DrbdOptions/Resource/on-suspended-primary-outdated: force-secondary
310+
property.linstor.csi.linbit.com/DrbdOptions/Net/rr-conflict: retry-connect
311+
volumeBindingMode: WaitForFirstConsumer
312+
allowVolumeExpansion: true
313+
EOF
314+
}
315+
316+
@test "Wait for MetalLB and configure address pool" {
317+
# MetalLB address pool
318+
kubectl apply -f - <<'EOF'
319+
---
320+
apiVersion: metallb.io/v1beta1
321+
kind: L2Advertisement
322+
metadata:
323+
name: cozystack
324+
namespace: cozy-metallb
325+
spec:
326+
ipAddressPools: [cozystack]
327+
---
328+
apiVersion: metallb.io/v1beta1
329+
kind: IPAddressPool
330+
metadata:
331+
name: cozystack
332+
namespace: cozy-metallb
333+
spec:
334+
addresses: [192.168.123.200-192.168.123.250]
335+
autoAssign: true
336+
avoidBuggyIPs: false
337+
EOF
338+
}
339+
340+
@test "Check Cozystack API service" {
341+
kubectl wait --for=condition=Available apiservices/v1alpha1.apps.cozystack.io --timeout=2m
342+
}
343+
344+
@test "Configure Tenant and wait for applications" {
345+
# Patch root tenant and wait for its releases
346+
kubectl patch tenants/root -n tenant-root --type merge -p '{"spec":{"host":"example.org","ingress":true,"monitoring":true,"etcd":true,"isolated":true}}'
347+
348+
timeout 60 sh -ec 'until kubectl get hr -n tenant-root etcd ingress monitoring tenant-root >/dev/null 2>&1; do sleep 1; done'
349+
kubectl wait hr/etcd hr/ingress hr/tenant-root -n tenant-root --timeout=2m --for=condition=ready
350+
351+
if ! kubectl wait hr/monitoring -n tenant-root --timeout=2m --for=condition=ready; then
352+
flux reconcile hr monitoring -n tenant-root --force
353+
kubectl wait hr/monitoring -n tenant-root --timeout=2m --for=condition=ready
354+
fi
355+
356+
# Expose Cozystack services through ingress
357+
kubectl patch configmap/cozystack -n cozy-system --type merge -p '{"data":{"expose-services":"api,dashboard,cdi-uploadproxy,vm-exportproxy,keycloak"}}'
358+
359+
# NGINX ingress controller
360+
timeout 60 sh -ec 'until kubectl get deploy root-ingress-controller -n tenant-root >/dev/null 2>&1; do sleep 1; done'
361+
kubectl wait deploy/root-ingress-controller -n tenant-root --timeout=5m --for=condition=available
362+
363+
# etcd statefulset
364+
kubectl wait sts/etcd -n tenant-root --for=jsonpath='{.status.readyReplicas}'=3 --timeout=5m
365+
366+
# VictoriaMetrics components
367+
kubectl wait vmalert/vmalert-shortterm vmalertmanager/alertmanager -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=5m
368+
kubectl wait vlogs/generic -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=5m
369+
kubectl wait vmcluster/shortterm vmcluster/longterm -n tenant-root --for=jsonpath='{.status.clusterStatus}'=operational --timeout=5m
370+
371+
# Grafana
372+
kubectl wait clusters.postgresql.cnpg.io/grafana-db -n tenant-root --for=condition=ready --timeout=5m
373+
kubectl wait deploy/grafana-deployment -n tenant-root --for=condition=available --timeout=5m
374+
375+
# Verify Grafana via ingress
376+
ingress_ip=$(kubectl get svc root-ingress-controller -n tenant-root -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
377+
curl -sS -k "https://${ingress_ip}" -H 'Host: grafana.example.org' | grep -q Found
378+
}
379+
380+
@test "Keycloak OIDC stack is healthy" {
381+
kubectl patch configmap/cozystack -n cozy-system --type merge -p '{"data":{"oidc-enabled":"true"}}'
382+
383+
timeout 120 sh -ec 'until kubectl get hr -n cozy-keycloak keycloak keycloak-configure keycloak-operator >/dev/null 2>&1; do sleep 1; done'
384+
kubectl wait hr/keycloak hr/keycloak-configure hr/keycloak-operator -n cozy-keycloak --timeout=10m --for=condition=ready
385+
}

0 commit comments

Comments
 (0)