|
| 1 | +#!/usr/bin/env bats |
| 2 | +# ----------------------------------------------------------------------------- |
| 3 | +# Cozystack end‑to‑end provisioning test (Bats) |
| 4 | +# ----------------------------------------------------------------------------- |
| 5 | + |
| 6 | +export TALOSCONFIG=$PWD/talosconfig |
| 7 | +export KUBECONFIG=$PWD/kubeconfig |
| 8 | + |
| 9 | +@test "Environment variable COZYSTACK_INSTALLER_YAML is defined" { |
| 10 | + if [ -z "${COZYSTACK_INSTALLER_YAML:-}" ]; then |
| 11 | + echo 'COZYSTACK_INSTALLER_YAML environment variable is not set!' >&2 |
| 12 | + echo >&2 |
| 13 | + echo 'Please export it with the following command:' >&2 |
| 14 | + echo ' export COZYSTACK_INSTALLER_YAML=$(helm template -n cozy-system installer packages/core/installer)' >&2 |
| 15 | + fi |
| 16 | +} |
| 17 | + |
| 18 | +@test "IPv4 forwarding is enabled" { |
| 19 | + if [ "$(cat /proc/sys/net/ipv4/ip_forward)" != 1 ]; then |
| 20 | + echo "IPv4 forwarding is disabled!" >&2 |
| 21 | + echo >&2 |
| 22 | + echo "Enable it with:" >&2 |
| 23 | + echo " echo 1 > /proc/sys/net/ipv4/ip_forward" >&2 |
| 24 | + fi |
| 25 | +} |
| 26 | + |
| 27 | +@test "Clean previous VMs" { |
| 28 | + kill $(cat srv1/qemu.pid srv2/qemu.pid srv3/qemu.pid 2>/dev/null) 2>/dev/null || true |
| 29 | + rm -rf srv1 srv2 srv3 |
| 30 | +} |
| 31 | + |
| 32 | +@test "Prepare networking and masquerading" { |
| 33 | + ip link del cozy-br0 2>/dev/null || true |
| 34 | + ip link add cozy-br0 type bridge |
| 35 | + ip link set cozy-br0 up |
| 36 | + ip address add 192.168.123.1/24 dev cozy-br0 |
| 37 | + |
| 38 | + # Masquerading rule – idempotent (delete first, then add) |
| 39 | + iptables -t nat -D POSTROUTING -s 192.168.123.0/24 ! -d 192.168.123.0/24 -j MASQUERADE 2>/dev/null || true |
| 40 | + iptables -t nat -A POSTROUTING -s 192.168.123.0/24 ! -d 192.168.123.0/24 -j MASQUERADE |
| 41 | +} |
| 42 | + |
| 43 | +@test "Prepare cloud‑init drive for VMs" { |
| 44 | + mkdir -p srv1 srv2 srv3 |
| 45 | + |
| 46 | + # Generate cloud‑init ISOs |
| 47 | + for i in 1 2 3; do |
| 48 | + echo "hostname: srv${i}" > "srv${i}/meta-data" |
| 49 | + |
| 50 | + cat > "srv${i}/user-data" <<'EOF' |
| 51 | +#cloud-config |
| 52 | +EOF |
| 53 | + |
| 54 | + cat > "srv${i}/network-config" <<EOF |
| 55 | +version: 2 |
| 56 | +ethernets: |
| 57 | + eth0: |
| 58 | + dhcp4: false |
| 59 | + addresses: |
| 60 | + - "192.168.123.1${i}/26" |
| 61 | + gateway4: "192.168.123.1" |
| 62 | + nameservers: |
| 63 | + search: [cluster.local] |
| 64 | + addresses: [8.8.8.8] |
| 65 | +EOF |
| 66 | + |
| 67 | + ( cd "srv${i}" && genisoimage \ |
| 68 | + -output seed.img \ |
| 69 | + -volid cidata -rational-rock -joliet \ |
| 70 | + user-data meta-data network-config ) |
| 71 | + done |
| 72 | +} |
| 73 | + |
| 74 | +@test "Download Talos NoCloud image" { |
| 75 | + if [ ! -f nocloud-amd64.raw ]; then |
| 76 | + wget https://github.com/cozystack/cozystack/releases/latest/download/nocloud-amd64.raw.xz \ |
| 77 | + -O nocloud-amd64.raw.xz --show-progress --output-file /dev/stdout --progress=dot:giga 2>/dev/null |
| 78 | + rm -f nocloud-amd64.raw |
| 79 | + xz --decompress nocloud-amd64.raw.xz |
| 80 | + fi |
| 81 | +} |
| 82 | + |
| 83 | +@test "Prepare VM disks" { |
| 84 | + for i in 1 2 3; do |
| 85 | + cp nocloud-amd64.raw srv${i}/system.img |
| 86 | + qemu-img resize srv${i}/system.img 20G |
| 87 | + qemu-img create srv${i}/data.img 100G |
| 88 | + done |
| 89 | +} |
| 90 | + |
| 91 | +@test "Create tap devices" { |
| 92 | + for i in 1 2 3; do |
| 93 | + ip link del cozy-srv${i} 2>/dev/null || true |
| 94 | + ip tuntap add dev cozy-srv${i} mode tap |
| 95 | + ip link set cozy-srv${i} up |
| 96 | + ip link set cozy-srv${i} master cozy-br0 |
| 97 | + done |
| 98 | +} |
| 99 | + |
| 100 | +@test "Boot QEMU VMs" { |
| 101 | + for i in 1 2 3; do |
| 102 | + qemu-system-x86_64 -machine type=pc,accel=kvm -cpu host -smp 8 -m 16384 \ |
| 103 | + -device virtio-net,netdev=net0,mac=52:54:00:12:34:5${i} \ |
| 104 | + -netdev tap,id=net0,ifname=cozy-srv${i},script=no,downscript=no \ |
| 105 | + -drive file=srv${i}/system.img,if=virtio,format=raw \ |
| 106 | + -drive file=srv${i}/seed.img,if=virtio,format=raw \ |
| 107 | + -drive file=srv${i}/data.img,if=virtio,format=raw \ |
| 108 | + -display none -daemonize -pidfile srv${i}/qemu.pid |
| 109 | + done |
| 110 | + |
| 111 | + # Give qemu a few seconds to start up networking |
| 112 | + sleep 5 |
| 113 | +} |
| 114 | + |
| 115 | +@test "Wait until Talos API port 50000 is reachable on all machines" { |
| 116 | + timeout 60 sh -ec 'until nc -nz 192.168.123.11 50000 && nc -nz 192.168.123.12 50000 && nc -nz 192.168.123.13 50000; do sleep 1; done' |
| 117 | +} |
| 118 | + |
| 119 | +@test "Generate Talos cluster configuration" { |
| 120 | + # Cluster‑wide patches |
| 121 | + cat > patch.yaml <<'EOF' |
| 122 | +machine: |
| 123 | + kubelet: |
| 124 | + nodeIP: |
| 125 | + validSubnets: |
| 126 | + - 192.168.123.0/24 |
| 127 | + extraConfig: |
| 128 | + maxPods: 512 |
| 129 | + kernel: |
| 130 | + modules: |
| 131 | + - name: openvswitch |
| 132 | + - name: drbd |
| 133 | + parameters: |
| 134 | + - usermode_helper=disabled |
| 135 | + - name: zfs |
| 136 | + - name: spl |
| 137 | + registries: |
| 138 | + mirrors: |
| 139 | + docker.io: |
| 140 | + endpoints: |
| 141 | + - https://mirror.gcr.io |
| 142 | + files: |
| 143 | + - content: | |
| 144 | + [plugins] |
| 145 | + [plugins."io.containerd.cri.v1.runtime"] |
| 146 | + device_ownership_from_security_context = true |
| 147 | + path: /etc/cri/conf.d/20-customization.part |
| 148 | + op: create |
| 149 | +
|
| 150 | +cluster: |
| 151 | + apiServer: |
| 152 | + extraArgs: |
| 153 | + oidc-issuer-url: "https://keycloak.example.org/realms/cozy" |
| 154 | + oidc-client-id: "kubernetes" |
| 155 | + oidc-username-claim: "preferred_username" |
| 156 | + oidc-groups-claim: "groups" |
| 157 | + network: |
| 158 | + cni: |
| 159 | + name: none |
| 160 | + dnsDomain: cozy.local |
| 161 | + podSubnets: |
| 162 | + - 10.244.0.0/16 |
| 163 | + serviceSubnets: |
| 164 | + - 10.96.0.0/16 |
| 165 | +EOF |
| 166 | + |
| 167 | + # Control‑plane‑only patches |
| 168 | + cat > patch-controlplane.yaml <<'EOF' |
| 169 | +machine: |
| 170 | + nodeLabels: |
| 171 | + node.kubernetes.io/exclude-from-external-load-balancers: |
| 172 | + $patch: delete |
| 173 | + network: |
| 174 | + interfaces: |
| 175 | + - interface: eth0 |
| 176 | + vip: |
| 177 | + ip: 192.168.123.10 |
| 178 | +cluster: |
| 179 | + allowSchedulingOnControlPlanes: true |
| 180 | + controllerManager: |
| 181 | + extraArgs: |
| 182 | + bind-address: 0.0.0.0 |
| 183 | + scheduler: |
| 184 | + extraArgs: |
| 185 | + bind-address: 0.0.0.0 |
| 186 | + apiServer: |
| 187 | + certSANs: |
| 188 | + - 127.0.0.1 |
| 189 | + proxy: |
| 190 | + disabled: true |
| 191 | + discovery: |
| 192 | + enabled: false |
| 193 | + etcd: |
| 194 | + advertisedSubnets: |
| 195 | + - 192.168.123.0/24 |
| 196 | +EOF |
| 197 | + |
| 198 | + # Generate secrets once |
| 199 | + if [ ! -f secrets.yaml ]; then |
| 200 | + talosctl gen secrets |
| 201 | + fi |
| 202 | + |
| 203 | + rm -f controlplane.yaml worker.yaml talosconfig kubeconfig |
| 204 | + talosctl gen config --with-secrets secrets.yaml cozystack https://192.168.123.10:6443 \ |
| 205 | + [email protected] --config-patch-control-plane @patch-controlplane.yaml |
| 206 | +} |
| 207 | + |
| 208 | +@test "Apply Talos configuration to the node" { |
| 209 | + # Apply the configuration to all three nodes |
| 210 | + for node in 11 12 13; do |
| 211 | + talosctl apply -f controlplane.yaml -n 192.168.123.${node} -e 192.168.123.${node} -i |
| 212 | + done |
| 213 | + |
| 214 | + # Wait for Talos services to come up again |
| 215 | + timeout 60 sh -ec 'until nc -nz 192.168.123.11 50000 && nc -nz 192.168.123.12 50000 && nc -nz 192.168.123.13 50000; do sleep 1; done' |
| 216 | +} |
| 217 | + |
| 218 | +@test "Bootstrap Talos cluster" { |
| 219 | + # Bootstrap etcd on the first node |
| 220 | + timeout 10 sh -ec 'until talosctl bootstrap -n 192.168.123.11 -e 192.168.123.11; do sleep 1; done' |
| 221 | + |
| 222 | + # Wait until etcd is healthy |
| 223 | + timeout 180 sh -ec 'until talosctl etcd members -n 192.168.123.11,192.168.123.12,192.168.123.13 -e 192.168.123.10 >/dev/null 2>&1; do sleep 1; done' |
| 224 | + timeout 60 sh -ec 'while talosctl etcd members -n 192.168.123.11,192.168.123.12,192.168.123.13 -e 192.168.123.10 2>&1 | grep -q "rpc error"; do sleep 1; done' |
| 225 | + |
| 226 | + # Retrieve kubeconfig |
| 227 | + rm -f kubeconfig |
| 228 | + talosctl kubeconfig kubeconfig -e 192.168.123.10 -n 192.168.123.10 |
| 229 | + |
| 230 | + # Wait until all three nodes register in Kubernetes |
| 231 | + timeout 60 sh -ec 'until [ $(kubectl get node --no-headers | wc -l) -eq 3 ]; do sleep 1; done' |
| 232 | +} |
| 233 | + |
| 234 | +@test "Install Cozystack" { |
| 235 | + # Create namespace & configmap required by installer |
| 236 | + kubectl create namespace cozy-system --dry-run=client -o yaml | kubectl apply -f - |
| 237 | + kubectl create configmap cozystack -n cozy-system \ |
| 238 | + --from-literal=bundle-name=paas-full \ |
| 239 | + --from-literal=ipv4-pod-cidr=10.244.0.0/16 \ |
| 240 | + --from-literal=ipv4-pod-gateway=10.244.0.1 \ |
| 241 | + --from-literal=ipv4-svc-cidr=10.96.0.0/16 \ |
| 242 | + --from-literal=ipv4-join-cidr=100.64.0.0/16 \ |
| 243 | + --from-literal=root-host=example.org \ |
| 244 | + --from-literal=api-server-endpoint=https://192.168.123.10:6443 \ |
| 245 | + --dry-run=client -o yaml | kubectl apply -f - |
| 246 | + |
| 247 | + # Apply installer manifests from env variable |
| 248 | + echo "$COZYSTACK_INSTALLER_YAML" | kubectl apply -f - |
| 249 | + |
| 250 | + # Wait for the installer deployment to become available |
| 251 | + kubectl wait deployment/cozystack -n cozy-system --timeout=1m --for=condition=Available |
| 252 | + |
| 253 | + # Wait until HelmReleases appear & reconcile them |
| 254 | + timeout 60 sh -ec 'until kubectl get hr -A | grep -q cozys; do sleep 1; done' |
| 255 | + sleep 5 |
| 256 | + kubectl get hr -A | awk 'NR>1 {print "kubectl wait --timeout=15m --for=condition=ready -n "$1" hr/"$2" &"} END {print "wait"}' | sh -ex |
| 257 | + |
| 258 | + # Fail the test if any HelmRelease is not Ready |
| 259 | + if kubectl get hr -A | grep -v " True " | grep -v NAME; then |
| 260 | + kubectl get hr -A |
| 261 | + fail "Some HelmReleases failed to reconcile" |
| 262 | + fi |
| 263 | +} |
| 264 | + |
| 265 | +@test "Wait for Cluster‑API provider deployments" { |
| 266 | + # Wait for Cluster‑API provider deployments |
| 267 | + timeout 60 sh -ec 'until kubectl get deploy -n cozy-cluster-api capi-controller-manager capi-kamaji-controller-manager capi-kubeadm-bootstrap-controller-manager capi-operator-cluster-api-operator capk-controller-manager >/dev/null 2>&1; do sleep 1; done' |
| 268 | + kubectl wait deployment/capi-controller-manager deployment/capi-kamaji-controller-manager deployment/capi-kubeadm-bootstrap-controller-manager deployment/capi-operator-cluster-api-operator deployment/capk-controller-manager -n cozy-cluster-api --timeout=1m --for=condition=available |
| 269 | +} |
| 270 | + |
| 271 | +@test "Wait for LINSTOR and configure storage" { |
| 272 | + # Linstor controller and nodes |
| 273 | + kubectl wait deployment/linstor-controller -n cozy-linstor --timeout=5m --for=condition=available |
| 274 | + timeout 60 sh -ec 'until [ $(kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor node list | grep -c Online) -eq 3 ]; do sleep 1; done' |
| 275 | + |
| 276 | + for node in srv1 srv2 srv3; do |
| 277 | + kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor ps cdp zfs ${node} /dev/vdc --pool-name data --storage-pool data |
| 278 | + done |
| 279 | + |
| 280 | + # Storage classes |
| 281 | + kubectl apply -f - <<'EOF' |
| 282 | +--- |
| 283 | +apiVersion: storage.k8s.io/v1 |
| 284 | +kind: StorageClass |
| 285 | +metadata: |
| 286 | + name: local |
| 287 | + annotations: |
| 288 | + storageclass.kubernetes.io/is-default-class: "true" |
| 289 | +provisioner: linstor.csi.linbit.com |
| 290 | +parameters: |
| 291 | + linstor.csi.linbit.com/storagePool: "data" |
| 292 | + linstor.csi.linbit.com/layerList: "storage" |
| 293 | + linstor.csi.linbit.com/allowRemoteVolumeAccess: "false" |
| 294 | +volumeBindingMode: WaitForFirstConsumer |
| 295 | +allowVolumeExpansion: true |
| 296 | +--- |
| 297 | +apiVersion: storage.k8s.io/v1 |
| 298 | +kind: StorageClass |
| 299 | +metadata: |
| 300 | + name: replicated |
| 301 | +provisioner: linstor.csi.linbit.com |
| 302 | +parameters: |
| 303 | + linstor.csi.linbit.com/storagePool: "data" |
| 304 | + linstor.csi.linbit.com/autoPlace: "3" |
| 305 | + linstor.csi.linbit.com/layerList: "drbd storage" |
| 306 | + linstor.csi.linbit.com/allowRemoteVolumeAccess: "true" |
| 307 | + property.linstor.csi.linbit.com/DrbdOptions/auto-quorum: suspend-io |
| 308 | + property.linstor.csi.linbit.com/DrbdOptions/Resource/on-no-data-accessible: suspend-io |
| 309 | + property.linstor.csi.linbit.com/DrbdOptions/Resource/on-suspended-primary-outdated: force-secondary |
| 310 | + property.linstor.csi.linbit.com/DrbdOptions/Net/rr-conflict: retry-connect |
| 311 | +volumeBindingMode: WaitForFirstConsumer |
| 312 | +allowVolumeExpansion: true |
| 313 | +EOF |
| 314 | +} |
| 315 | + |
| 316 | +@test "Wait for MetalLB and configure address pool" { |
| 317 | + # MetalLB address pool |
| 318 | + kubectl apply -f - <<'EOF' |
| 319 | +--- |
| 320 | +apiVersion: metallb.io/v1beta1 |
| 321 | +kind: L2Advertisement |
| 322 | +metadata: |
| 323 | + name: cozystack |
| 324 | + namespace: cozy-metallb |
| 325 | +spec: |
| 326 | + ipAddressPools: [cozystack] |
| 327 | +--- |
| 328 | +apiVersion: metallb.io/v1beta1 |
| 329 | +kind: IPAddressPool |
| 330 | +metadata: |
| 331 | + name: cozystack |
| 332 | + namespace: cozy-metallb |
| 333 | +spec: |
| 334 | + addresses: [192.168.123.200-192.168.123.250] |
| 335 | + autoAssign: true |
| 336 | + avoidBuggyIPs: false |
| 337 | +EOF |
| 338 | +} |
| 339 | + |
| 340 | +@test "Check Cozystack API service" { |
| 341 | + kubectl wait --for=condition=Available apiservices/v1alpha1.apps.cozystack.io --timeout=2m |
| 342 | +} |
| 343 | + |
| 344 | +@test "Configure Tenant and wait for applications" { |
| 345 | + # Patch root tenant and wait for its releases |
| 346 | + kubectl patch tenants/root -n tenant-root --type merge -p '{"spec":{"host":"example.org","ingress":true,"monitoring":true,"etcd":true,"isolated":true}}' |
| 347 | + |
| 348 | + timeout 60 sh -ec 'until kubectl get hr -n tenant-root etcd ingress monitoring tenant-root >/dev/null 2>&1; do sleep 1; done' |
| 349 | + kubectl wait hr/etcd hr/ingress hr/tenant-root -n tenant-root --timeout=2m --for=condition=ready |
| 350 | + |
| 351 | + if ! kubectl wait hr/monitoring -n tenant-root --timeout=2m --for=condition=ready; then |
| 352 | + flux reconcile hr monitoring -n tenant-root --force |
| 353 | + kubectl wait hr/monitoring -n tenant-root --timeout=2m --for=condition=ready |
| 354 | + fi |
| 355 | + |
| 356 | + # Expose Cozystack services through ingress |
| 357 | + kubectl patch configmap/cozystack -n cozy-system --type merge -p '{"data":{"expose-services":"api,dashboard,cdi-uploadproxy,vm-exportproxy,keycloak"}}' |
| 358 | + |
| 359 | + # NGINX ingress controller |
| 360 | + timeout 60 sh -ec 'until kubectl get deploy root-ingress-controller -n tenant-root >/dev/null 2>&1; do sleep 1; done' |
| 361 | + kubectl wait deploy/root-ingress-controller -n tenant-root --timeout=5m --for=condition=available |
| 362 | + |
| 363 | + # etcd statefulset |
| 364 | + kubectl wait sts/etcd -n tenant-root --for=jsonpath='{.status.readyReplicas}'=3 --timeout=5m |
| 365 | + |
| 366 | + # VictoriaMetrics components |
| 367 | + kubectl wait vmalert/vmalert-shortterm vmalertmanager/alertmanager -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=5m |
| 368 | + kubectl wait vlogs/generic -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=5m |
| 369 | + kubectl wait vmcluster/shortterm vmcluster/longterm -n tenant-root --for=jsonpath='{.status.clusterStatus}'=operational --timeout=5m |
| 370 | + |
| 371 | + # Grafana |
| 372 | + kubectl wait clusters.postgresql.cnpg.io/grafana-db -n tenant-root --for=condition=ready --timeout=5m |
| 373 | + kubectl wait deploy/grafana-deployment -n tenant-root --for=condition=available --timeout=5m |
| 374 | + |
| 375 | + # Verify Grafana via ingress |
| 376 | + ingress_ip=$(kubectl get svc root-ingress-controller -n tenant-root -o jsonpath='{.status.loadBalancer.ingress[0].ip}') |
| 377 | + curl -sS -k "https://${ingress_ip}" -H 'Host: grafana.example.org' | grep -q Found |
| 378 | +} |
| 379 | + |
| 380 | +@test "Keycloak OIDC stack is healthy" { |
| 381 | + kubectl patch configmap/cozystack -n cozy-system --type merge -p '{"data":{"oidc-enabled":"true"}}' |
| 382 | + |
| 383 | + timeout 120 sh -ec 'until kubectl get hr -n cozy-keycloak keycloak keycloak-configure keycloak-operator >/dev/null 2>&1; do sleep 1; done' |
| 384 | + kubectl wait hr/keycloak hr/keycloak-configure hr/keycloak-operator -n cozy-keycloak --timeout=10m --for=condition=ready |
| 385 | +} |
0 commit comments