|
388 | 388 | # endpoints (both Kamaji control-plane pods), so a single apiserver pod restart |
389 | 389 | # is routed around transparently. `kubectl port-forward` instead pins to one |
390 | 390 | # pod and dies when that pod blips: a lone kube-apiserver restart was observed |
391 | | - # leaving localhost refusing connections for the entire 12m node-Ready wait |
| 391 | + # leaving localhost refusing connections for the entire 18m node-Ready wait |
392 | 392 | # while the cluster was in fact healthy (CAPI NodeHealthy=True on both nodes), |
393 | 393 | # failing the test on a dead tunnel. The LB endpoint is also stable until |
394 | 394 | # teardown, so the failure snapshot can still reach the tenant. Test-scoped and |
@@ -441,14 +441,30 @@ EOF |
441 | 441 | # two budgets starve each other under load: a slow KubeVirt VM boot consumes |
442 | 442 | # the join budget, then the tenant cluster's cilium CNI needs several more |
443 | 443 | # minutes to make the freshly-joined nodes Ready — overflowing the fixed 3m |
444 | | - # Ready window even though the CNI converges fine. One 12m deadline that polls |
| 444 | + # Ready window even though the CNI converges fine. One deadline that polls |
445 | 445 | # for ">=2 nodes Ready" is robust to wherever the time goes. |
446 | | - if ! timeout 12m bash -c ' |
| 446 | + # |
| 447 | + # 18m, not the earlier 12m: this single budget has to absorb the *entire* |
| 448 | + # worker bring-up, and the `machinedeployment .status.replicas=2` gate above |
| 449 | + # clears while the KubeVirt VMs are still only Machine objects — the clock |
| 450 | + # here starts ~2m before the guest VMIs even exist. Under host storage |
| 451 | + # pressure that margin evaporates: in run 30260770694 a transient |
| 452 | + # `drbd.linbit.com/lost-quorum` taint delayed the worker DataVolume imports, |
| 453 | + # the worker VMIs were created ~2m into this wait, and the guests then had |
| 454 | + # only ~10m to import Talos, boot, register a kubelet and let cilium turn the |
| 455 | + # nodes Ready. They did not make it: both kubernetes-latest and |
| 456 | + # kubernetes-previous failed here at exactly 12m with zero Nodes registered |
| 457 | + # and the tenant cilium HR still mid-install. A less-loaded fleet run passed |
| 458 | + # the same suites unchanged, so this is load-induced slowness, not a stuck |
| 459 | + # bring-up; 18m restores margin and still sits well inside the 40m step |
| 460 | + # timeout (the downstream LB/NFS/ouroboros checks add ~10-15m on the happy |
| 461 | + # path). |
| 462 | + if ! timeout 18m bash -c ' |
447 | 463 | until [ "$(kubectl --kubeconfig tenantkubeconfig-'"${test_name}"' get nodes --no-headers 2>/dev/null | grep -cw Ready)" -ge 2 ]; do |
448 | 464 | sleep 5 |
449 | 465 | done |
450 | 466 | '; then |
451 | | - # Node-join failed: fewer than 2 tenant nodes became Ready inside the 12m |
| 467 | + # Node-join failed: fewer than 2 tenant nodes became Ready inside the 18m |
452 | 468 | # deadline. Dump scoped diagnostics that split the failure sub-modes, then |
453 | 469 | # fail fast — no point running LB/NFS tests without Ready nodes. |
454 | 470 | # |
|
460 | 476 | # (2b) is the failure mode a follow-up fix has to target, and it cannot be |
461 | 477 | # designed without this artifact. Every capture is guarded with `|| true` |
462 | 478 | # so a capture failure never masks the real `exit 1`. |
463 | | - echo "=== node-join failed: fewer than 2 tenant nodes Ready within 12m — diagnostics follow ===" |
| 479 | + echo "=== node-join failed: fewer than 2 tenant nodes Ready within 18m — diagnostics follow ===" |
464 | 480 | kubectl --kubeconfig "tenantkubeconfig-${test_name}" describe nodes || true |
465 | 481 | kubectl -n tenant-test get hr || true |
466 | 482 |
|
|
0 commit comments