Skip to content

Commit 5ef02ee

Browse files
myasnikovdaniilmyasnikovdaniil
committed
test(e2e): bump app HR-Ready waits for Flux v2.8 kstatus
Flux helm-controller v1.5.0 (shipped in Flux v2.8) replaced the legacy Helm readiness logic with kstatus-based health checking, which polls Deployment/StatefulSet/Service status and waits for kstatus to assess rollout-ready. This typically adds 10-30s of latency between resource apply and HR condition Ready=True compared to v2.7. Our per-app bats tests had `kubectl wait hr/... --for=condition=ready --timeout=` budgets of 20s, 30s, 60s, 100s — sized for v2.7's faster readiness flip and previously masked by the 3x retry on Run E2E tests (dropped in #2558). Under v2.8 with no retry, every app that runs after the first few hit its HR wait timeout exactly, surfacing as 13 of 16 test failures on PR #2619. Standardise on 5m for HR Ready across all app tests. Tight enough to catch genuine bring-up failures within the per-app job budget, loose enough to absorb kstatus dispatch jitter on a management cluster that is concurrently reconciling multiple tenant-test HRs. Post-HR-Ready checks for app-specific resources (StatefulSet/Service/ PVC/operator CRs) remain — for CR-based apps kstatus only observes that the chart applied, not that the operator finished. Audit of which of those are actually redundant under v2.8 kstatus is queued as a follow-up. Also bumps the 10s "kamajicontrolplane appears" wait in run-kubernetes.sh to 2m, and the kubernetes-addon HR Ready loop from 1m to 5m for the same reason. Assisted-By: Claude <[email protected]> Signed-off-by: Myasnikov Daniil <[email protected]>
1 parent 29e95bd commit 5ef02ee

13 files changed

Lines changed: 21 additions & 21 deletions

‎hack/e2e-apps/bucket.bats‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ spec:
1919
EOF
2020

2121
# Wait for the bucket to be ready
22-
kubectl -n tenant-test wait hr bucket-${name} --timeout=100s --for=condition=ready
22+
kubectl -n tenant-test wait hr bucket-${name} --timeout=5m --for=condition=ready
2323
timeout 60 sh -ec "until kubectl -n tenant-test get bucketclaims.objectstorage.k8s.io bucket-${name} >/dev/null 2>&1; do sleep 2; done"
2424
kubectl -n tenant-test wait bucketclaims.objectstorage.k8s.io bucket-${name} --timeout=300s --for=jsonpath='{.status.bucketReady}'
2525
timeout 60 sh -ec "until kubectl -n tenant-test get bucketaccesses.objectstorage.k8s.io bucket-${name}-admin >/dev/null 2>&1; do sleep 2; done"

‎hack/e2e-apps/clickhouse.bats‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ EOF
3838
# Wait for the operator to materialise the HelmRelease before kubectl wait
3939
# kicks in (kubectl wait errors immediately if the object does not exist yet).
4040
timeout 60 sh -ec "until kubectl -n tenant-test get hr clickhouse-$name >/dev/null 2>&1; do sleep 2; done"
41-
kubectl -n tenant-test wait hr clickhouse-$name --timeout=20s --for=condition=ready
41+
kubectl -n tenant-test wait hr clickhouse-$name --timeout=5m --for=condition=ready
4242
timeout 180 sh -ec "until kubectl -n tenant-test get svc chendpoint-clickhouse-$name -o jsonpath='{.spec.ports[*].port}' | grep -q '8123 9000'; do sleep 10; done"
4343
timeout 60 sh -ec "until kubectl -n tenant-test get statefulset.apps/chi-clickhouse-$name-clickhouse-0-0 >/dev/null 2>&1; do sleep 2; done"
4444
kubectl -n tenant-test wait statefulset.apps/chi-clickhouse-$name-clickhouse-0-0 --timeout=120s --for=jsonpath='{.status.replicas}'=1

‎hack/e2e-apps/etcd.bats‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ dump_diagnostics() {
2626
# resource, so poll for existence first.
2727
wait_etcd_hr_ready() {
2828
timeout 60 sh -ec 'until kubectl -n tenant-test get hr/etcd >/dev/null 2>&1; do sleep 2; done'
29-
kubectl -n tenant-test wait hr/etcd --timeout=60s --for=condition=ready
29+
kubectl -n tenant-test wait hr/etcd --timeout=5m --for=condition=ready
3030
}
3131

3232
@test "Create Etcd" {

‎hack/e2e-apps/external-dns.bats‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@ EOF
2121
# Wait for the operator to materialise the HelmRelease before kubectl wait
2222
# kicks in (kubectl wait errors immediately if the object does not exist yet).
2323
timeout 60 sh -ec "until kubectl -n tenant-test get hr ${name} >/dev/null 2>&1; do sleep 2; done"
24-
kubectl -n tenant-test wait hr ${name} --timeout=120s --for=condition=ready
25-
kubectl -n tenant-test wait hr ${name}-system --timeout=120s --for=condition=ready
24+
kubectl -n tenant-test wait hr ${name} --timeout=5m --for=condition=ready
25+
kubectl -n tenant-test wait hr ${name}-system --timeout=5m --for=condition=ready
2626
timeout 60 sh -ec "until kubectl -n tenant-test get deploy -l app.kubernetes.io/instance=${name}-system -o jsonpath='{.items[0].status.readyReplicas}' | grep -q '1'; do sleep 5; done"
2727

2828
kubectl -n tenant-test delete externaldns.apps.cozystack.io ${name}
@@ -46,8 +46,8 @@ EOF
4646
# Wait for the operator to materialise the HelmRelease before kubectl wait
4747
# kicks in (kubectl wait errors immediately if the object does not exist yet).
4848
timeout 60 sh -ec "until kubectl -n tenant-test get hr ${name} >/dev/null 2>&1; do sleep 2; done"
49-
kubectl -n tenant-test wait hr ${name} --timeout=120s --for=condition=ready
50-
kubectl -n tenant-test wait hr ${name}-system --timeout=120s --for=condition=ready
49+
kubectl -n tenant-test wait hr ${name} --timeout=5m --for=condition=ready
50+
kubectl -n tenant-test wait hr ${name}-system --timeout=5m --for=condition=ready
5151
timeout 60 sh -ec "until kubectl -n tenant-test get deploy -l app.kubernetes.io/instance=${name}-system -o jsonpath='{.items[0].status.readyReplicas}' | grep -q '1'; do sleep 5; done"
5252

5353
kubectl -n tenant-test delete externaldns.apps.cozystack.io ${name}

‎hack/e2e-apps/harbor.bats‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ EOF
4141
# Wait for the operator to materialise the HelmRelease before kubectl wait
4242
# kicks in (kubectl wait errors immediately if the object does not exist yet).
4343
timeout 60 sh -ec "until kubectl -n tenant-test get hr $release >/dev/null 2>&1; do sleep 2; done"
44-
kubectl -n tenant-test wait hr $release --timeout=60s --for=condition=ready
44+
kubectl -n tenant-test wait hr $release --timeout=5m --for=condition=ready
4545

4646
# Wait for COSI to provision bucket
4747
timeout 60 sh -ec "until kubectl -n tenant-test get bucketclaims.objectstorage.k8s.io $release-registry >/dev/null 2>&1; do sleep 2; done"

‎hack/e2e-apps/kafka.bats‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ EOF
4242
# Wait for the operator to materialise the HelmRelease before kubectl wait
4343
# kicks in (kubectl wait errors immediately if the object does not exist yet).
4444
timeout 60 sh -ec "until kubectl -n tenant-test get hr kafka-$name >/dev/null 2>&1; do sleep 2; done"
45-
kubectl -n tenant-test wait hr kafka-$name --timeout=30s --for=condition=ready
45+
kubectl -n tenant-test wait hr kafka-$name --timeout=5m --for=condition=ready
4646
timeout 60 sh -ec "until kubectl -n tenant-test get kafkas $name >/dev/null 2>&1; do sleep 2; done"
4747
kubectl wait kafkas -n tenant-test $name --timeout=300s --for=condition=ready
4848
timeout 60 sh -ec "until kubectl -n tenant-test get pvc data-kafka-$name-zookeeper-0; do sleep 10; done"

‎hack/e2e-apps/mariadb.bats‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ EOF
3838
# Wait for the operator to materialise the HelmRelease before kubectl wait
3939
# kicks in (kubectl wait errors immediately if the object does not exist yet).
4040
timeout 60 sh -ec "until kubectl -n tenant-test get hr mariadb-$name >/dev/null 2>&1; do sleep 2; done"
41-
kubectl -n tenant-test wait hr mariadb-$name --timeout=30s --for=condition=ready
41+
kubectl -n tenant-test wait hr mariadb-$name --timeout=5m --for=condition=ready
4242
timeout 80 sh -ec "until kubectl -n tenant-test get svc mariadb-$name -o jsonpath='{.spec.ports[0].port}' | grep -q '3306'; do sleep 10; done"
4343
timeout 80 sh -ec "until kubectl -n tenant-test get endpoints mariadb-$name -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done"
4444
timeout 60 sh -ec "until kubectl -n tenant-test get statefulset.apps/mariadb-$name >/dev/null 2>&1; do sleep 2; done"

‎hack/e2e-apps/mongodb.bats‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ EOF
3030
# kicks in (kubectl wait errors immediately if the object does not exist yet).
3131
timeout 60 sh -ec "until kubectl -n tenant-test get hr mongodb-$name >/dev/null 2>&1; do sleep 2; done"
3232
# Wait for HelmRelease
33-
kubectl -n tenant-test wait hr mongodb-$name --timeout=60s --for=condition=ready
33+
kubectl -n tenant-test wait hr mongodb-$name --timeout=5m --for=condition=ready
3434
# Wait for MongoDB service (port 27017)
3535
timeout 120 sh -ec "until kubectl -n tenant-test get svc mongodb-$name-rs0 -o jsonpath='{.spec.ports[0].port}' | grep -q '27017'; do sleep 10; done"
3636
# Wait for endpoints

‎hack/e2e-apps/openbao.bats‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@ EOF
2121
# Wait for the operator to materialise the HelmRelease before kubectl wait
2222
# kicks in (kubectl wait errors immediately if the object does not exist yet).
2323
timeout 60 sh -ec "until kubectl -n tenant-test get hr openbao-$name >/dev/null 2>&1; do sleep 2; done"
24-
kubectl -n tenant-test wait hr openbao-$name --timeout=60s --for=condition=ready
25-
kubectl -n tenant-test wait hr openbao-$name-system --timeout=120s --for=condition=ready
24+
kubectl -n tenant-test wait hr openbao-$name --timeout=5m --for=condition=ready
25+
kubectl -n tenant-test wait hr openbao-$name-system --timeout=5m --for=condition=ready
2626

2727
# Wait for container to be started (pod Running does not guarantee container is ready for exec on slow CI)
2828
if ! timeout 120 sh -ec "until kubectl -n tenant-test get pod openbao-$name-0 --output jsonpath='{.status.containerStatuses[0].started}' 2>/dev/null | grep -q true; do sleep 5; done"; then

‎hack/e2e-apps/postgres.bats‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ EOF
4343
# Wait for the operator to materialise the HelmRelease before kubectl wait
4444
# kicks in (kubectl wait errors immediately if the object does not exist yet).
4545
timeout 60 sh -ec "until kubectl -n tenant-test get hr postgres-$name >/dev/null 2>&1; do sleep 2; done"
46-
kubectl -n tenant-test wait hr postgres-$name --timeout=100s --for=condition=ready
46+
kubectl -n tenant-test wait hr postgres-$name --timeout=5m --for=condition=ready
4747
timeout 60 sh -ec "until kubectl -n tenant-test get job.batch postgres-$name-init-job >/dev/null 2>&1; do sleep 2; done"
4848
kubectl -n tenant-test wait job.batch postgres-$name-init-job --timeout=50s --for=condition=Complete
4949
timeout 40 sh -ec "until kubectl -n tenant-test get svc postgres-$name-r -o jsonpath='{.spec.ports[0].port}' | grep -q '5432'; do sleep 10; done"

0 commit comments

Comments
 (0)