|
240 | 240 | case "$storage_class" in |
241 | 241 | replicated) ;; |
242 | 242 | local) |
243 | | - # VictoriaLogs is the only root-tenant install workload that explicitly |
| 243 | + # The Tenant patch below leaves root monitoring off, so VictoriaLogs does |
| 244 | + # not render in this run; the override is here for a run that turns it |
| 245 | + # on. VictoriaLogs is the only root-tenant workload that explicitly |
244 | 246 | # defaults to replicated instead of inheriting the cluster's default |
245 | | - # StorageClass. The container lane has no DRBD and intentionally exposes |
246 | | - # only local. Put the override in the values Secret BEFORE monitoring is |
247 | | - # enabled, so the monitoring HelmRelease's first render creates local |
| 247 | + # StorageClass, and the container lane has no DRBD and intentionally |
| 248 | + # exposes only local. The override goes into the values Secret before the |
| 249 | + # Tenant patch, so the monitoring HelmRelease's first render creates local |
248 | 250 | # claims. Patching that HelmRelease after creation races the operator: a |
249 | 251 | # replicated PVC can be created first, and storageClassName is immutable. |
250 | 252 | # Flux drift detection is disabled, so the owning cozystack-basics release |
|
268 | 270 | ;; |
269 | 271 | esac |
270 | 272 |
|
271 | | - kubectl patch tenants/root -n tenant-root --type merge -p '{"spec":{"host":"example.org","ingress":true,"monitoring":true,"etcd":true,"isolated":true, "seaweedfs": true}}' |
| 273 | + # Root monitoring stays off: no suite inspects the tenant-root instance, and |
| 274 | + # its bring-up (VictoriaMetrics, VictoriaLogs, Grafana and its CNPG database) |
| 275 | + # competes with the rest of the install for node capacity, so a slow start |
| 276 | + # could fail a pull request that never touched monitoring. The monitoring |
| 277 | + # package itself stays installed, because tenant-application declares |
| 278 | + # dependsOn cozystack.monitoring-application. |
| 279 | + kubectl patch tenants/root -n tenant-root --type merge -p '{"spec":{"host":"example.org","ingress":true,"monitoring":false,"etcd":true,"isolated":true, "seaweedfs": true}}' |
272 | 280 |
|
273 | | - timeout 60 sh -ec 'until kubectl get hr -n tenant-root etcd ingress monitoring seaweedfs tenant-root >/dev/null 2>&1; do sleep 1; done' |
| 281 | + timeout 60 sh -ec 'until kubectl get hr -n tenant-root etcd ingress seaweedfs tenant-root >/dev/null 2>&1; do sleep 1; done' |
274 | 282 | # tenant-root parent HR only flips Ready after every child HR is Ready, |
275 | | - # so listing all four top-level children plus the parent gives precise |
| 283 | + # so listing the children this patch enables plus the parent gives precise |
276 | 284 | # failure messages without redundant separate waits. seaweedfs now |
277 | 285 | # installs as a serial chain seaweedfs-db (CNPG bootstrap) -> |
278 | 286 | # seaweedfs-system (master raft quorum) -> seaweedfs wrapper, which |
|
281 | 289 | # parent's Ready can land past the HR's single 15m timeout window; the HR |
282 | 290 | # re-reconciles every 1m until it converges, so this wait is 20m to observe |
283 | 291 | # that eventual Ready rather than expiring first. |
284 | | - kubectl wait hr/etcd hr/ingress hr/monitoring hr/seaweedfs hr/tenant-root \ |
| 292 | + kubectl wait hr/etcd hr/ingress hr/seaweedfs hr/tenant-root \ |
285 | 293 | -n tenant-root --timeout=20m --for=condition=ready |
286 | 294 |
|
287 | 295 |
|
|
301 | 309 | kubectl -n tenant-root wait pod \ |
302 | 310 | -l app.kubernetes.io/name=etcd,app.kubernetes.io/instance=etcd,app.kubernetes.io/managed-by=etcd-operator \ |
303 | 311 | --for=condition=ready --timeout=10m |
304 | | - |
305 | | - # VictoriaMetrics components. vmalert/vmalertmanager, vlclusters/generic and |
306 | | - # vmcluster/shortterm+longterm are all vm-operator-managed resources that flip |
307 | | - # updateStatus=operational only once their workloads are scheduled and Ready. |
308 | | - # During platform bring-up they contend for node resources with the rest of |
309 | | - # the install, so convergence is load-sensitive: on a calm sandbox each reaches |
310 | | - # operational in under a second, but under install-time load (concurrent e2e |
311 | | - # sandboxes on one runner) monitoring bring-up is slow. vmalert already uses a |
312 | | - # 15m budget; vlclusters and vmcluster used 10m, so this block carried a |
313 | | - # 10m/15m split even though all three contend for the same node capacity and a |
314 | | - # slow VictoriaLogs bring-up can fail the install on a PR that never touched |
315 | | - # monitoring. Unify the block on one 15m budget (near-zero cost in the happy |
316 | | - # path, comfortably inside the E2E job budget) and dump live status on timeout |
317 | | - # so a genuine stuck-not-slow regression stays legible instead of surfacing as |
318 | | - # a bare "timed out" line. |
319 | | - timeout 60 sh -ec 'until kubectl get vmalert/vmalert-shortterm -n tenant-root >/dev/null 2>&1; do sleep 2; done' |
320 | | - timeout 60 sh -ec 'until kubectl get vmalertmanager/alertmanager -n tenant-root >/dev/null 2>&1; do sleep 2; done' |
321 | | - kubectl wait vmalert/vmalert-shortterm vmalertmanager/alertmanager -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=15m || { |
322 | | - echo "=== vmalert/vmalert-shortterm, vmalertmanager/alertmanager did not reach updateStatus=operational ===" |
323 | | - kubectl get vmalert/vmalert-shortterm vmalertmanager/alertmanager -n tenant-root -o yaml 2>&1 || true |
324 | | - echo "=== tenant-root pods ===" |
325 | | - kubectl get pods -n tenant-root -o wide 2>&1 || true |
326 | | - false |
327 | | - } |
328 | | - timeout 60 sh -ec 'until kubectl get vlclusters/generic -n tenant-root >/dev/null 2>&1; do sleep 2; done' |
329 | | - kubectl wait vlclusters/generic -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=15m || { |
330 | | - echo "=== vlclusters/generic did not reach updateStatus=operational ===" |
331 | | - kubectl get vlclusters/generic -n tenant-root -o yaml 2>&1 || true |
332 | | - echo "=== tenant-root pods ===" |
333 | | - kubectl get pods -n tenant-root -o wide 2>&1 || true |
334 | | - false |
335 | | - } |
336 | | - timeout 60 sh -ec 'until kubectl get vmcluster/shortterm vmcluster/longterm -n tenant-root >/dev/null 2>&1; do sleep 2; done' |
337 | | - kubectl wait vmcluster/shortterm vmcluster/longterm -n tenant-root --for=jsonpath='{.status.updateStatus}'=operational --timeout=15m || { |
338 | | - echo "=== vmcluster/shortterm,longterm did not reach updateStatus=operational ===" |
339 | | - kubectl get vmcluster/shortterm vmcluster/longterm -n tenant-root -o yaml 2>&1 || true |
340 | | - echo "=== tenant-root pods ===" |
341 | | - kubectl get pods -n tenant-root -o wide 2>&1 || true |
342 | | - false |
343 | | - } |
344 | | - |
345 | | - # Grafana. The grafana-db CNPG cluster and the grafana-deployment Deployment |
346 | | - # complete the tenant-root monitoring bring-up and contend for the same node |
347 | | - # resources as the VictoriaMetrics stack above during install. Under |
348 | | - # install-time load (concurrent e2e sandboxes on one runner) either can be slow |
349 | | - # and fail the install on a PR that never touched monitoring, so both move from |
350 | | - # their 10m budget to the same uniform 15m as the vm-operator waits above and |
351 | | - # dump live status on timeout to keep a genuine stuck-not-slow regression |
352 | | - # legible instead of surfacing as a bare "timed out" line. |
353 | | - timeout 60 sh -ec 'until kubectl get clusters.postgresql.cnpg.io/grafana-db -n tenant-root >/dev/null 2>&1; do sleep 2; done' |
354 | | - kubectl wait clusters.postgresql.cnpg.io/grafana-db -n tenant-root --for=condition=ready --timeout=15m || { |
355 | | - echo "=== clusters.postgresql.cnpg.io/grafana-db did not reach condition=ready ===" |
356 | | - kubectl get clusters.postgresql.cnpg.io/grafana-db -n tenant-root -o yaml 2>&1 || true |
357 | | - echo "=== tenant-root pods ===" |
358 | | - kubectl get pods -n tenant-root -o wide 2>&1 || true |
359 | | - false |
360 | | - } |
361 | | - timeout 60 sh -ec 'until kubectl get deploy/grafana-deployment -n tenant-root >/dev/null 2>&1; do sleep 2; done' |
362 | | - kubectl wait deploy/grafana-deployment -n tenant-root --for=condition=available --timeout=15m || { |
363 | | - echo "=== deploy/grafana-deployment did not reach condition=available ===" |
364 | | - kubectl get deploy/grafana-deployment -n tenant-root -o yaml 2>&1 || true |
365 | | - echo "=== tenant-root pods ===" |
366 | | - kubectl get pods -n tenant-root -o wide 2>&1 || true |
367 | | - false |
368 | | - } |
369 | | - |
370 | | - # Verify Grafana via ingress |
371 | | - ingress_ip=$(kubectl get svc root-ingress-controller -n tenant-root -o jsonpath='{.status.loadBalancer.ingress[0].ip}') |
372 | | - if ! curl -sS -k "https://${ingress_ip}" -H 'Host: grafana.example.org' --max-time 30 | grep -q Found; then |
373 | | - echo "Failed to access Grafana via ingress at ${ingress_ip}" >&2 |
374 | | - exit 1 |
375 | | - fi |
376 | 312 | } |
377 | 313 |
|
378 | 314 | @test "Keycloak OIDC stack is healthy" { |
|
0 commit comments