Skip to content

Commit e62056b

Browse files
gerardopotiuk
authored andcommitted
[AIRFLOW-5223] Use kind for Kubernetes in CI (apache#5837)
This PR reimplements Kubernetes integration testing using kind, a tool for running local Kubernetes clusters using Docker container "nodes". The "nodes" are deployed to a separate docker daemon (dind) started through docker-compose.
1 parent cb0dbe3 commit e62056b

28 files changed

Lines changed: 119 additions & 439 deletions

‎.pre-commit-config.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ repos:
8383
- --fuzzy-match-generates-todo
8484
- id: insert-license
8585
name: Add licence for all JINJA template files
86-
files: ^airflow/www/templates/.*\.html$|^docs/templates/.*\.html$|^airflow/contrib/plugins/metastore_browser/templates/.*\.html$ # yamllint disable-line rule:line-length
86+
files: ^airflow/www/templates/.*\.html$|^docs/templates/.*\.html$|^airflow/contrib/plugins/metastore_browser/templates/.*\.html$ # yamllint disable-line rule:line-length
8787
exclude: ^\.github/.*$|^airflow/_vendor/.*$
8888
args:
8989
- --comment-style

‎.travis.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,11 +44,11 @@ jobs:
4444
python: "3.7"
4545
stage: test
4646
- name: "Tests postgres kubernetes python 3.6 (persistent)"
47-
env: BACKEND=postgres ENV=kubernetes KUBERNETES_VERSION=v1.13.0 KUBERNETES_MODE=persistent_mode
47+
env: BACKEND=postgres ENV=kubernetes KUBERNETES_VERSION=v1.15.0 KUBERNETES_MODE=persistent_mode
4848
python: "3.6"
4949
stage: test
5050
- name: "Tests postgres kubernetes python 3.6 (git)"
51-
env: BACKEND=postgres ENV=kubernetes KUBERNETES_VERSION=v1.13.0 KUBERNETES_MODE=git_mode
51+
env: BACKEND=postgres ENV=kubernetes KUBERNETES_VERSION=v1.15.0 KUBERNETES_MODE=git_mode
5252
python: "3.6"
5353
stage: test
5454
- name: "Static checks (no pylint, no licence check)"

‎Dockerfile‎

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,6 @@ RUN apt-get update \
5656
&& apt-get clean \
5757
&& rm -rf /var/lib/apt/lists/*
5858

59-
6059
# Install basic apt dependencies
6160
RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - \
6261
&& apt-get update \
@@ -135,6 +134,10 @@ ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
135134

136135
ARG APT_DEPS_IMAGE="airflow-apt-deps-ci-slim"
137136
ENV APT_DEPS_IMAGE=${APT_DEPS_IMAGE}
137+
ARG KUBERNETES_VERSION="v1.15.0"
138+
ENV KUBERNETES_VERSION=${KUBERNETES_VERSION}
139+
ARG KIND_VERSION="v0.5.0"
140+
ENV KIND_VERSION=${KIND_VERSION}
138141

139142
RUN echo "${APT_DEPS_IMAGE}"
140143

@@ -148,6 +151,9 @@ RUN if [[ "${APT_DEPS_IMAGE}" == "airflow-apt-deps-ci" ]]; then \
148151
&& apt-get update \
149152
&& apt-get install --no-install-recommends -y \
150153
gnupg \
154+
apt-transport-https \
155+
ca-certificates \
156+
software-properties-common \
151157
krb5-user \
152158
ldap-utils \
153159
less \
@@ -171,6 +177,44 @@ RUN if [[ "${APT_DEPS_IMAGE}" == "airflow-apt-deps-ci" ]]; then \
171177
# TODO: We should think about removing those and moving them into docker-compose dependencies.
172178
COPY scripts/ci/docker_build/ci_build_install_deps.sh /tmp/ci_build_install_deps.sh
173179

180+
# Kubernetes dependencies
181+
RUN \
182+
if [[ "${APT_DEPS_IMAGE}" == "airflow-apt-deps-ci" ]]; then \
183+
curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \
184+
&& add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian stretch stable" \
185+
&& apt-get update \
186+
&& apt-get -y install --no-install-recommends docker-ce \
187+
&& apt-get autoremove -yqq --purge \
188+
&& apt-get clean && rm -rf /var/lib/apt/lists/* \
189+
;\
190+
fi
191+
192+
RUN \
193+
if [[ "${APT_DEPS_IMAGE}" == "airflow-apt-deps-ci" ]]; then \
194+
curl -Lo kubectl \
195+
"https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl" \
196+
&& chmod +x kubectl \
197+
&& mv kubectl /usr/local/bin/kubectl \
198+
;\
199+
fi
200+
201+
RUN \
202+
if [[ "${APT_DEPS_IMAGE}" == "airflow-apt-deps-ci" ]]; then \
203+
curl -Lo kind \
204+
"https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64" \
205+
&& chmod +x kind \
206+
&& mv kind /usr/local/bin/kind \
207+
;\
208+
fi
209+
210+
ENV HADOOP_DISTRO=cdh \
211+
HADOOP_MAJOR=5 \
212+
HADOOP_DISTRO_VERSION=5.11.0 \
213+
HADOOP_VERSION=2.6.0 \
214+
HIVE_VERSION=1.1.0
215+
ENV HADOOP_URL=https://archive.cloudera.com/${HADOOP_DISTRO}${HADOOP_MAJOR}/${HADOOP_DISTRO}/${HADOOP_MAJOR}/
216+
ENV HADOOP_HOME=/tmp/hadoop-cdh HIVE_HOME=/tmp/hive
217+
174218
RUN if [[ "${APT_DEPS_IMAGE}" == "airflow-apt-deps-ci" ]]; then /tmp/ci_build_install_deps.sh; fi
175219

176220
ENV PATH "${PATH}:/tmp/hive/bin"

‎scripts/ci/ci_before_install.sh‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,12 +30,6 @@ script_start
3030
build_image_on_ci
3131

3232
KUBERNETES_VERSION=${KUBERNETES_VERSION:=""}
33-
# Required for K8s v1.10.x. See
34-
# https://github.com/kubernetes/kubernetes/issues/61058#issuecomment-372764783
35-
if [[ "${KUBERNETES_VERSION}" == "" ]]; then
36-
sudo mount --make-shared /
37-
sudo service docker restart
38-
fi
3933

4034
sudo pip install pre-commit
4135

‎scripts/ci/ci_run_airflow_testing.sh‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -74,18 +74,11 @@ elif [[ "${ENV}" == "kubernetes" ]]; then
7474
echo
7575
echo "Running kubernetes tests in ${KUBERNETES_MODE}"
7676
echo
77-
"${MY_DIR}/kubernetes/minikube/stop_minikube.sh"
78-
"${MY_DIR}/kubernetes/setup_kubernetes.sh"
79-
"${MY_DIR}/kubernetes/kube/deploy.sh" -d "${KUBERNETES_MODE}"
80-
MINIKUBE_IP=$(minikube ip)
81-
export MINIKUBE_IP
8277
docker-compose --log-level ERROR \
8378
-f "${MY_DIR}/docker-compose.yml" \
8479
-f "${MY_DIR}/docker-compose-${BACKEND}.yml" \
85-
-f "${MY_DIR}/docker-compose-kubernetes.yml" \
8680
"${DOCKER_COMPOSE_LOCAL[@]}" \
87-
run --no-deps airflow-testing /opt/airflow/scripts/ci/in_container/entrypoint_ci.sh;
88-
"${MY_DIR}/kubernetes/minikube/stop_minikube.sh"
81+
run airflow-testing /opt/airflow/scripts/ci/in_container/entrypoint_ci.sh;
8982
echo
9083
echo "Finished Running kubernetes tests in ${KUBERNETES_MODE}"
9184
echo

‎scripts/ci/docker-compose.yml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ services:
2424
- POSTGRES_USER=postgres
2525
- POSTGRES_PASSWORD=airflow
2626
- POSTGRES_DB=airflow
27+
2728
mysql:
2829
image: mysql:5.6
2930
environment:
@@ -54,11 +55,19 @@ services:
5455
- LDAP_CONFIG_PASSWORD=insecure
5556
volumes:
5657
- ./openldap/ldif:/container/service/slapd/assets/config/bootstrap/ldif/custom:ro
58+
5759
krb5-kdc-server:
5860
image: godatadriven/krb5-kdc-server
5961
hostname: krb5-kdc-server
6062
domainname: example.com
6163

64+
docker:
65+
image: docker:19.03.2-dind
66+
privileged: true
67+
environment:
68+
- DOCKER_TLS_CERTDIR=
69+
- DOCKER_TLS_VERIFY=
70+
6271
airflow-testing:
6372
image: ${AIRFLOW_CONTAINER_DOCKER_IMAGE}
6473
init: true
@@ -74,6 +83,7 @@ services:
7483
- KRB5_CONFIG=/etc/krb5.conf
7584
- KRB5_KTNAME=/etc/airflow.keytab
7685
- CELERY_BROKER_URLS=amqp://guest:guest@rabbitmq:5672,redis://redis:6379/0
86+
- DOCKER_HOST=tcp://docker:2375
7787
- ENV
7888
- BACKEND
7989
- CI
@@ -91,6 +101,9 @@ services:
91101
- RUN_TESTS
92102
- AIRFLOW_CI_VERBOSE
93103
- AIRFLOW_CI_SILENT
104+
- AIRFLOW_CONTAINER_DOCKER_IMAGE
105+
- KUBERNETES_VERSION
106+
- KUBERNETES_MODE
94107
depends_on:
95108
- postgres
96109
- mysql
@@ -100,3 +113,4 @@ services:
100113
- redis
101114
- openldap
102115
- krb5-kdc-server
116+
- docker

‎scripts/ci/in_container/entrypoint_ci.sh‎

Lines changed: 6 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
#
2020
# Bash sanity settings (error on exit, complain for undefined vars, error when pipe fails)
2121
set -euo pipefail
22+
2223
MY_DIR=$(cd "$(dirname "$0")" || exit 1; pwd)
2324

2425
if [[ ${AIRFLOW_CI_VERBOSE:="false"} == "true" ]]; then
@@ -132,11 +133,6 @@ if [[ ! -h /home/travis/build/apache/airflow ]]; then
132133
sudo ln -s "${AIRFLOW_SOURCES}" /home/travis/build/apache/airflow
133134
fi
134135

135-
# Fix file permissions
136-
if [[ -d "${HOME}/.minikube" ]]; then
137-
sudo chown -R "${AIRFLOW_USER}.${AIRFLOW_USER}" "${HOME}/.kube" "${HOME}/.minikube"
138-
fi
139-
140136
# Cleanup the logs, tmp when entering the environment
141137
sudo rm -rf "${AIRFLOW_SOURCES}"/logs/*
142138
sudo rm -rf "${AIRFLOW_SOURCES}"/tmp/*
@@ -255,19 +251,14 @@ if [[ -z "${KUBERNETES_VERSION}" ]]; then
255251
echo
256252
"${MY_DIR}/run_ci_tests.sh" "${ARGS[@]}"
257253
else
258-
export KUBERNETES_VERSION
259-
export MINIKUBE_IP
260-
# This script runs inside a container, the path of the kubernetes certificate
261-
# is /home/travis/.minikube/client.crt but the user in the container is `root`
262-
# TODO: Check this. This should be made travis-independent :D
263-
if [[ ! -d /home/travis ]]; then
264-
sudo mkdir -p /home/travis
265-
fi
266-
sudo ln -s /root/.minikube /home/travis/.minikube
254+
echo "Set up Kubernetes cluster for tests"
255+
"${MY_DIR}/../kubernetes/setup_kubernetes.sh"
256+
"${MY_DIR}/../kubernetes/app/deploy_app.sh" -d "${KUBERNETES_MODE}"
257+
267258
echo
268259
echo "Running CI tests with ${ARGS[*]}"
269260
echo
270-
"${MY_DIR}/run_ci_tests.sh" tests.minikube "${ARGS[@]}"
261+
"${MY_DIR}/run_ci_tests.sh" tests.integration.kubernetes "${ARGS[@]}"
271262
fi
272263

273264
in_container_script_end

‎scripts/ci/kubernetes/README.md‎

Lines changed: 0 additions & 42 deletions
This file was deleted.
Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -150,12 +150,6 @@ ${SED_COMMAND} -i "s|{{CONFIGMAP_DAGS_VOLUME_CLAIM}}|${CONFIGMAP_DAGS_VOLUME_CLA
150150
cat "${BUILD_DIRNAME}/airflow.yaml"
151151
cat "${BUILD_DIRNAME}/configmaps.yaml"
152152

153-
# Fix file permissions
154-
# TODO: Check this - this should be TRAVIS-independent
155-
if [[ "${TRAVIS}" == true ]]; then
156-
sudo chown -R travis.travis "$HOME/.kube" "$HOME/.minikube"
157-
fi
158-
159153
kubectl delete -f "${DIRNAME}/postgres.yaml"
160154
kubectl delete -f "${BUILD_DIRNAME}/airflow.yaml"
161155
kubectl delete -f "${DIRNAME}/secrets.yaml"
@@ -212,13 +206,13 @@ else
212206
fi
213207

214208
# Wait until Airflow webserver is up
215-
MINIKUBE_IP=$(minikube ip)
209+
KUBERNETES_HOST=docker
216210
AIRFLOW_WEBSERVER_IS_READY="0"
217211
CONSECUTIVE_SUCCESS_CALLS=0
218212
for i in {1..30}
219213
do
220214
echo "------- Wait until webserver is up: $i -------"
221-
HTTP_CODE=$(curl -LI "http://${MINIKUBE_IP}:30809/health" -o /dev/null -w '%{http_code}\n' -sS) || true
215+
HTTP_CODE=$(curl -LI "http://${KUBERNETES_HOST}:30809/health" -o /dev/null -w '%{http_code}\n' -sS) || true
222216
if [[ "${HTTP_CODE}" == 200 ]]; then
223217
(( CONSECUTIVE_SUCCESS_CALLS+=1 ))
224218
else

0 commit comments

Comments
 (0)