|
3 | 3 | # their respective prerequisites are reachable. Designed to run in the |
4 | 4 | # background during the platform HR reconcile wait, so its wall-clock cost |
5 | 5 | # overlaps with the wait instead of compounding it. |
| 6 | +# |
| 7 | +# Each LINSTOR prerequisite below sits at the end of a multi-hop reconcile |
| 8 | +# chain: cozystack-operator -> platform HR -> linstor HR -> piraeus-operator |
| 9 | +# -> cert-manager issues the controller TLS -> linstor-controller Deployment |
| 10 | +# -> controller pod -> DB migration. On a loaded CI runner that chain has been |
| 11 | +# observed to take 7-9 min end to end, and the operator alone needs ~70s just |
| 12 | +# to emit the linstor HR. The earlier per-step "object exists" budgets |
| 13 | +# (timeout 60 / timeout 300) were anchored to this script's start, so they |
| 14 | +# raced that reconcile latency; when one lost (linstor HR appeared at ~+70s |
| 15 | +# against a 60s budget) `set -e` aborted the whole script and the install |
| 16 | +# failed. Instead, drive every wait off one shared deadline -- the same 15m |
| 17 | +# window the installer's `kubectl wait hr --all` uses -- and tolerate |
| 18 | +# not-yet-created objects without aborting, while still failing hard if a |
| 19 | +# resource never becomes ready inside the budget. |
6 | 20 | set -eu |
7 | 21 |
|
8 | | -echo "[post-install-prep] waiting for linstor HelmRelease object to exist" |
9 | | -timeout 60 sh -ec 'until kubectl get hr/linstor -n cozy-linstor >/dev/null 2>&1; do sleep 2; done' |
| 22 | +DEADLINE=$(( $(date +%s) + 900 )) |
10 | 23 |
|
11 | | -echo "[post-install-prep] waiting for linstor HelmRelease to be Ready" |
12 | | -kubectl wait helmrelease/linstor -n cozy-linstor --for=condition=Ready --timeout=15m |
| 24 | +# wait_for <description> <kubectl-wait-args...> |
| 25 | +# Polls `kubectl wait` until it succeeds or the shared deadline elapses. |
| 26 | +# kubectl wait exits non-zero immediately when the object does not exist yet, |
| 27 | +# so the loop tolerates "not created yet" without the set -e cliff that a bare |
| 28 | +# `kubectl wait` would trigger on a NotFound. The per-attempt timeout shrinks |
| 29 | +# to the budget remaining, so the final attempt can consume the rest of it. |
| 30 | +wait_for() { |
| 31 | + desc=$1 |
| 32 | + shift |
| 33 | + echo "[post-install-prep] waiting for ${desc}" |
| 34 | + while :; do |
| 35 | + remaining=$(( DEADLINE - $(date +%s) )) |
| 36 | + if [ "$remaining" -le 0 ]; then |
| 37 | + echo "[post-install-prep] timed out waiting for ${desc}" >&2 |
| 38 | + return 1 |
| 39 | + fi |
| 40 | + if kubectl wait "$@" --timeout="${remaining}s" 2>/dev/null; then |
| 41 | + return 0 |
| 42 | + fi |
| 43 | + sleep 5 |
| 44 | + done |
| 45 | +} |
13 | 46 |
|
14 | | -echo "[post-install-prep] waiting for linstor-controller deployment to exist" |
15 | | -timeout 300 sh -ec 'until kubectl get deploy/linstor-controller -n cozy-linstor >/dev/null 2>&1; do sleep 2; done' |
16 | | - |
17 | | -echo "[post-install-prep] waiting for linstor-controller to be Available" |
18 | | -kubectl wait deployment/linstor-controller -n cozy-linstor --timeout=5m --for=condition=available |
| 47 | +wait_for "linstor HelmRelease to be Ready" \ |
| 48 | + helmrelease/linstor -n cozy-linstor --for=condition=Ready |
| 49 | +wait_for "linstor-controller Deployment to be Available" \ |
| 50 | + deployment/linstor-controller -n cozy-linstor --for=condition=available |
19 | 51 |
|
20 | 52 | echo "[post-install-prep] waiting for 3 LINSTOR nodes Online" |
21 | | -timeout 60 sh -ec 'until [ $(kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor node list | grep -c Online) -eq 3 ]; do sleep 1; done' |
| 53 | +timeout 300 sh -ec 'until [ $(kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor node list | grep -c Online) -eq 3 ]; do sleep 2; done' |
22 | 54 |
|
23 | 55 | echo "[post-install-prep] creating LINSTOR storage pools (parallel across nodes)" |
24 | 56 | created_pools=$(kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor sp l -s data --pastable | awk '$2 == "data" {printf " " $4} END{printf " "}') |
|
0 commit comments