Skip to content

Commit 3447b55

Browse files
potiukpotiuk
authored
More stable kubernetes port forwarding (#11538)
Seems that port forwarding during kubernetes tests started to behave erratically - seems that kubectl port forward sometimes might hang indefinitely rather than connect or fail. We change the strategy a bit to try to allocate increasing port numbers in case something like that happens.
1 parent 765d29e commit 3447b55

File tree

5 files changed

+53
-6
lines changed

5 files changed

+53
-6
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -638,10 +638,14 @@ jobs:
638638
- name: "Prepare PROD Image"
639639
run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
640640
- name: "Deploy airflow to cluster"
641+
id: deploy-app
641642
run: ./scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh
642643
env:
643644
# We have the right image pulled already by the previous step
644645
SKIP_BUILDING_PROD_IMAGE: "true"
646+
# due to some instabilities, in CI we try to increase port numbers when trying to establish
647+
# port forwarding
648+
INCREASE_PORT_NUMBER_FOR_KUBERNETES: "true"
645649
- name: "Cache virtualenv for kubernetes testing"
646650
uses: actions/cache@v2
647651
env:

‎kubernetes_tests/test_kubernetes_executor.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@
2727
from requests.adapters import HTTPAdapter
2828
from urllib3.util.retry import Retry
2929

30-
KUBERNETES_HOST_PORT = (os.environ.get('CLUSTER_HOST') or "localhost") + ":8080"
30+
CLUSTER_FORWARDED_PORT = (os.environ.get('CLUSTER_FORWARDED_PORT') or "8080")
31+
KUBERNETES_HOST_PORT = (os.environ.get('CLUSTER_HOST') or "localhost") + ":" + CLUSTER_FORWARDED_PORT
3132

3233
print()
3334
print(f"Cluster host/port used: ${KUBERNETES_HOST_PORT}")

‎scripts/ci/kubernetes/ci_run_kubernetes_tests.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
kind::make_sure_kubernetes_tools_are_installed
2222
kind::get_kind_cluster_name
2323

24+
traps::add_trap kind::stop_kubectl EXIT HUP INT TERM
2425
traps::add_trap kind::dump_kind_logs EXIT HUP INT TERM
2526

2627
interactive="false"

‎scripts/ci/libraries/_initialization.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -767,3 +767,7 @@ function initialization::ga_output() {
767767
echo "::set-output name=${1}::${2}"
768768
echo "${1}=${2}"
769769
}
770+
771+
function initialization::ga_env() {
772+
echo "${1}=${2}" >> "${GITHUB_ENV}"
773+
}

‎scripts/ci/libraries/_kind.sh‎

Lines changed: 42 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -268,11 +268,44 @@ function kind::load_image_to_kind_cluster() {
268268
kind load docker-image --name "${KIND_CLUSTER_NAME}" "${AIRFLOW_PROD_IMAGE_KUBERNETES}"
269269
}
270270

271+
MAX_NUM_TRIES_FOR_PORT_FORWARD=12
272+
readonly MAX_NUM_TRIES_FOR_PORT_FORWARD
273+
274+
SLEEP_TIME_FOR_PORT_FORWARD=10
275+
readonly SLEEP_TIME_FOR_PORT_FORWARD
276+
277+
forwarded_port_number=8080
278+
279+
function kind::start_kubectl_forward() {
280+
echo
281+
echo "Trying to forward port ${forwarded_port_number} to 8080 on server"
282+
echo
283+
kubectl port-forward svc/airflow-webserver "${forwarded_port_number}:8080" --namespace airflow >/dev/null &
284+
}
285+
286+
function kind::stop_kubectl() {
287+
echo
288+
echo "Stops all kubectl instances"
289+
echo
290+
killall kubectl || true
291+
sleep 10
292+
killall -s KILL kubectl || true
293+
294+
}
295+
271296
function kind::forward_port_to_kind_webserver() {
272297
num_tries=0
273298
set +e
274-
while ! curl http://localhost:8080/health -s | grep -q healthy; do
275-
if [[ ${num_tries} == 6 ]]; then
299+
kind::start_kubectl_forward
300+
sleep "${SLEEP_TIME_FOR_PORT_FORWARD}"
301+
while ! curl "http://localhost:${forwarded_port_number}/health" -s | grep -q healthy; do
302+
echo
303+
echo "Trying to establish port forwarding to 'airflow webserver'"
304+
echo
305+
if [[ ${INCREASE_PORT_NUMBER_FOR_KUBERNETES} == "true" ]] ; then
306+
forwarded_port_number=$(( forwarded_port_number + 1 ))
307+
fi
308+
if [[ ${num_tries} == "${MAX_NUM_TRIES_FOR_PORT_FORWARD}" ]]; then
276309
echo >&2
277310
echo >&2 "ERROR! Could not setup a forward port to Airflow's webserver after ${num_tries}! Exiting."
278311
echo >&2
@@ -281,11 +314,15 @@ function kind::forward_port_to_kind_webserver() {
281314
echo
282315
echo "Trying to establish port forwarding to 'airflow webserver'"
283316
echo
284-
kubectl port-forward svc/airflow-webserver 8080:8080 --namespace airflow >/dev/null &
285-
sleep 10
317+
kind::start_kubectl_forward
318+
sleep "${SLEEP_TIME_FOR_PORT_FORWARD}"
286319
num_tries=$(( num_tries + 1))
287320
done
288-
echo "Connection to 'airflow webserver' established"
321+
echo
322+
echo "Connection to 'airflow webserver' established on port ${forwarded_port_number}"
323+
echo
324+
initialization::ga_env CLUSTER_FORWARDED_PORT "${forwarded_port_number}"
325+
export CLUSTER_FORWARDED_PORT="${forwarded_port_number}"
289326
set -e
290327
}
291328

0 commit comments

Comments
 (0)