Skip to content

Commit 23d5ea0

Browse files
astro-sql-decoratorkaxil
astro-sql-decorator
authored and
kaxil
committed
Use production image for k8s tests (#9038)
* Use production image for k8s tests The CI image has become too large to load into KinD, it also only really makes sense to use the production image for integration tests * nit Co-authored-by: Daniel Imberman <[email protected]> (cherry picked from commit e4d811d)
1 parent 5664e36 commit 23d5ea0

File tree

15 files changed

+220
-155
lines changed

15 files changed

+220
-155
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ jobs:
5353
- name: Cache pre-commit env
5454
uses: actions/cache@v1
5555
env:
56-
cache-name: cache-pre-commit
56+
cache-name: cache-pre-commit-epoch2
5757
with:
5858
path: ~/.cache/pre-commit
5959
key: ${{ env.cache-name }}-${{ github.job }}-${{ hashFiles('.pre-commit-config.yaml') }}
@@ -96,6 +96,44 @@ jobs:
9696
- name: "Build PROD image ${{ matrix.python-version }}"
9797
run: ./scripts/ci/ci_build_production_images.sh
9898

99+
tests-kubernetes:
100+
timeout-minutes: 80
101+
name: "kubernetes-test-${{matrix.kube-mode}}-kube"
102+
runs-on: ubuntu-latest
103+
needs: [static-checks]
104+
strategy:
105+
matrix:
106+
kube-mode:
107+
- persistent_mode
108+
# #TODO These tests will be unblocked when k8s tests run on host
109+
# - git_mode
110+
kubernetes-version:
111+
- "v1.15.3"
112+
fail-fast: false
113+
env:
114+
BACKEND: postgres
115+
TEST_TYPE: ${{ matrix.test-type }}
116+
RUN_TESTS: "true"
117+
CI_JOB_TYPE: "Tests"
118+
PYTHON_MAJOR_MINOR_VERSION: "3.6"
119+
SKIP_CI_IMAGE_CHECK: "true"
120+
RUNTIME: "kubernetes"
121+
ENABLE_KIND_CLUSTER: "true"
122+
KUBERNETES_MODE: "${{ matrix.kube-mode }}"
123+
KUBERNETES_VERSION: "${{ matrix.kubernetes-version }}"
124+
steps:
125+
- uses: actions/checkout@master
126+
- uses: actions/setup-python@v1
127+
with:
128+
python-version: '3.x'
129+
- name: "Free space"
130+
run: ./scripts/ci/ci_free_space_on_ci.sh
131+
- name: "Build CI image ${{ matrix.python-version }}"
132+
run: ./scripts/ci/ci_prepare_image_on_ci.sh
133+
- name: "Build PROD image ${{ matrix.python-version }}"
134+
run: ./scripts/ci/ci_build_production_images.sh
135+
- name: "Tests"
136+
run: ./scripts/ci/ci_run_airflow_testing.sh
99137
tests-postgres:
100138
timeout-minutes: 80
101139
name: "${{matrix.test-type}}:Pg${{matrix.postgres-version}},Py${{matrix.python-version}}"

‎.travis.yml‎

Lines changed: 0 additions & 81 deletions
This file was deleted.

‎airflow/config_templates/config.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2120,7 +2120,7 @@
21202120
version_added: 1.10.3
21212121
type: string
21222122
example: ~
2123-
default: ""
2123+
default: "50000"
21242124
- name: fs_group
21252125
description: |
21262126
Specifies a gid to associate with all containers in the worker pods

‎airflow/config_templates/default_airflow.cfg‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1005,7 +1005,7 @@ tolerations =
10051005
kube_client_request_args =
10061006

10071007
# Specifies the uid to run the first process of the worker pods containers as
1008-
run_as_user =
1008+
run_as_user = 50000
10091009

10101010
# Specifies a gid to associate with all containers in the worker pods
10111011
# if using a git_ssh_key_secret_name use an fs_group

airflow/contrib/example_dags/example_kubernetes_executor_config.py renamed to airflow/example_dags/example_kubernetes_executor_config.py

File renamed without changes.

‎scripts/ci/in_container/kubernetes/app/deploy_app.sh‎

Lines changed: 30 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -56,19 +56,19 @@ rm -f "${BUILD_DIRNAME}"/*
5656
if [[ "${KUBERNETES_MODE}" == "persistent_mode" ]]; then
5757
INIT_DAGS_VOLUME_NAME=airflow-dags
5858
POD_AIRFLOW_DAGS_VOLUME_NAME=airflow-dags
59-
CONFIGMAP_DAGS_FOLDER=/root/airflow/dags
59+
CONFIGMAP_DAGS_FOLDER=/opt/airflow/dags
6060
CONFIGMAP_GIT_DAGS_FOLDER_MOUNT_POINT=
6161
CONFIGMAP_DAGS_VOLUME_CLAIM=airflow-dags
6262
else
6363
INIT_DAGS_VOLUME_NAME=airflow-dags-fake
6464
POD_AIRFLOW_DAGS_VOLUME_NAME=airflow-dags-git
65-
CONFIGMAP_DAGS_FOLDER=/root/airflow/dags/repo/airflow/contrib/example_dags
66-
CONFIGMAP_GIT_DAGS_FOLDER_MOUNT_POINT=/root/airflow/dags
65+
CONFIGMAP_DAGS_FOLDER=/opt/airflow/dags/repo/airflow/example_dags
66+
CONFIGMAP_GIT_DAGS_FOLDER_MOUNT_POINT=/opt/airflow/dags
6767
CONFIGMAP_DAGS_VOLUME_CLAIM=
6868
fi
6969

70-
CONFIGMAP_GIT_REPO=${CI_TARGET_REPO:-apache/airflow}
71-
CONFIGMAP_BRANCH=${CI_TARGET_BRANCH:=master}
70+
CONFIGMAP_GIT_REPO=${GITHUB_REPOSITORY:-apache/airflow}
71+
CONFIGMAP_BRANCH=${GITHUB_BASE_REF:=master}
7272

7373
if [[ "${KUBERNETES_MODE}" == "persistent_mode" ]]; then
7474
sed -e "s/{{INIT_GIT_SYNC}}//g" \
@@ -111,9 +111,28 @@ kubectl apply -f "${MY_DIR}/secrets.yaml"
111111
kubectl apply -f "${BUILD_DIRNAME}/configmaps.yaml"
112112
kubectl apply -f "${MY_DIR}/postgres.yaml"
113113
kubectl apply -f "${MY_DIR}/volumes.yaml"
114+
115+
set +x
116+
set +o pipefail
117+
PODS_ARE_READY="0"
118+
for i in {1..150}; do
119+
echo "------- Running kubectl get pods: $i -------"
120+
PODS=$(kubectl get pods | awk 'NR>1 {print $0}')
121+
echo "$PODS"
122+
NUM_POSTGRES_READY=$(echo "${PODS}" | grep postgres | awk '{print $2}' | grep -cE '([1-9])\/(\1)' | xargs)
123+
if [[ "${NUM_POSTGRES_READY}" == "1" ]]; then
124+
PODS_ARE_READY="1"
125+
break
126+
fi
127+
sleep 4
128+
done
129+
130+
sleep 7
131+
114132
kubectl apply -f "${BUILD_DIRNAME}/airflow.yaml"
115133

116134
dump_logs() {
135+
echo "dumping logs"
117136
POD=$(kubectl get pods -o go-template --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}' | grep airflow | head -1)
118137
echo "------- pod description -------"
119138
kubectl describe pod "${POD}"
@@ -134,20 +153,20 @@ set +x
134153
set +o pipefail
135154
# wait for up to 10 minutes for everything to be deployed
136155
PODS_ARE_READY="0"
137-
for i in {1..150}; do
156+
for i in {1..20}; do
138157
echo "------- Running kubectl get pods: $i -------"
139158
PODS=$(kubectl get pods | awk 'NR>1 {print $0}')
140159
echo "$PODS"
141-
NUM_AIRFLOW_READY=$(echo "${PODS}" | grep airflow | awk '{print $2}' | grep -cE '([0-9])\/(\1)' | xargs)
142-
NUM_POSTGRES_READY=$(echo "${PODS}" | grep postgres | awk '{print $2}' | grep -cE '([0-9])\/(\1)' | xargs)
160+
NUM_AIRFLOW_READY=$(echo "${PODS}" | grep airflow | awk '{print $2}' | grep -cE '([2-9])\/(\1)' | xargs)
161+
NUM_POSTGRES_READY=$(echo "${PODS}" | grep postgres | awk '{print $2}' | grep -cE '([1-9])\/(\1)' | xargs)
143162
if [[ "${NUM_AIRFLOW_READY}" == "1" && "${NUM_POSTGRES_READY}" == "1" ]]; then
144163
PODS_ARE_READY="1"
145164
break
146165
fi
147166
sleep 4
148167
done
149168
POD=$(kubectl get pods -o go-template --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}' | grep airflow | head -1)
150-
169+
dump_logs
151170
if [[ "${PODS_ARE_READY}" == "1" ]]; then
152171
echo "PODS are ready."
153172
else
@@ -160,7 +179,7 @@ fi
160179
KUBERNETES_HOST=${CLUSTER_NAME}-worker
161180
AIRFLOW_WEBSERVER_IS_READY="0"
162181
CONSECUTIVE_SUCCESS_CALLS=0
163-
for i in {1..30}; do
182+
for i in {1..20}; do
164183
echo "------- Wait until webserver is up: $i -------"
165184
PODS=$(kubectl get pods | awk 'NR>1 {print $0}')
166185
echo "$PODS"
@@ -182,5 +201,6 @@ if [[ "${AIRFLOW_WEBSERVER_IS_READY}" == "1" ]]; then
182201
echo "Airflow webserver is ready."
183202
else
184203
echo >&2 "Airflow webserver is not ready after waiting for a long time. Exiting..."
204+
dump_logs
185205
exit 1
186206
fi

‎scripts/ci/in_container/kubernetes/app/templates/airflow.template.yaml‎

Lines changed: 13 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -41,18 +41,20 @@ spec:
4141
labels:
4242
name: airflow
4343
spec:
44+
securityContext:
45+
runAsUser: 50000
46+
runAsGroup: 50000
47+
fsGroup: 50000
4448
initContainers:
4549
- name: "init"
4650
image: {{AIRFLOW_KUBERNETES_IMAGE}}
4751
imagePullPolicy: IfNotPresent
4852
volumeMounts:
4953
- name: airflow-configmap
50-
mountPath: /root/airflow/airflow.cfg
54+
mountPath: /opt/airflow/airflow.cfg
5155
subPath: airflow.cfg
5256
- name: {{INIT_DAGS_VOLUME_NAME}}
53-
mountPath: /root/airflow/dags
54-
- name: test-volume
55-
mountPath: /root/test_volume
57+
mountPath: /opt/airflow/dags
5658
env:
5759
- name: SQL_ALCHEMY_CONN
5860
valueFrom:
@@ -85,12 +87,12 @@ spec:
8587
key: sql_alchemy_conn
8688
volumeMounts:
8789
- name: airflow-configmap
88-
mountPath: /root/airflow/airflow.cfg
90+
mountPath: /opt/airflow/airflow.cfg
8991
subPath: airflow.cfg
9092
- name: {{POD_AIRFLOW_DAGS_VOLUME_NAME}}
91-
mountPath: /root/airflow/dags
93+
mountPath: /opt/airflow/dags
9294
- name: airflow-logs
93-
mountPath: /root/airflow/logs
95+
mountPath: /opt/airflow/logs
9496
- name: scheduler
9597
image: {{AIRFLOW_KUBERNETES_IMAGE}}
9698
imagePullPolicy: IfNotPresent
@@ -107,12 +109,12 @@ spec:
107109
key: sql_alchemy_conn
108110
volumeMounts:
109111
- name: airflow-configmap
110-
mountPath: /root/airflow/airflow.cfg
112+
mountPath: /opt/airflow/airflow.cfg
111113
subPath: airflow.cfg
112114
- name: {{POD_AIRFLOW_DAGS_VOLUME_NAME}}
113-
mountPath: /root/airflow/dags
115+
mountPath: /opt/airflow/dags
114116
- name: airflow-logs
115-
mountPath: /root/airflow/logs
117+
mountPath: /opt/airflow/logs
116118
volumes:
117119
- name: airflow-dags
118120
persistentVolumeClaim:
@@ -121,12 +123,8 @@ spec:
121123
emptyDir: {}
122124
- name: airflow-dags-git
123125
emptyDir: {}
124-
- name: test-volume
125-
persistentVolumeClaim:
126-
claimName: test-volume
127126
- name: airflow-logs
128-
persistentVolumeClaim:
129-
claimName: airflow-logs
127+
emptyDir: {}
130128
- name: airflow-configmap
131129
configMap:
132130
name: airflow-configmap

‎scripts/ci/in_container/kubernetes/app/templates/configmaps.template.yaml‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,18 +24,18 @@ data:
2424
airflow.cfg: |
2525
[core]
2626
dags_folder = {{CONFIGMAP_DAGS_FOLDER}}
27-
base_log_folder = /root/airflow/logs
27+
base_log_folder = /opt/airflow/logs
2828
logging_level = INFO
2929
executor = KubernetesExecutor
3030
parallelism = 32
3131
load_examples = False
3232
load_default_connections = True
33-
plugins_folder = /root/airflow/plugins
33+
plugins_folder = /opt/airflow/plugins
3434
sql_alchemy_conn = $SQL_ALCHEMY_CONN
3535
3636
[scheduler]
3737
dag_dir_list_interval = 300
38-
child_process_log_directory = /root/airflow/logs/scheduler
38+
child_process_log_directory = /opt/airflow/logs/scheduler
3939
# Task instances listen for external kill signal (when you clear tasks
4040
# from the CLI or the UI), this defines the frequency at which they should
4141
# listen (in seconds).
@@ -193,12 +193,13 @@ data:
193193
git_dags_folder_mount_point = {{CONFIGMAP_GIT_DAGS_FOLDER_MOUNT_POINT}}
194194
dags_volume_claim = {{CONFIGMAP_DAGS_VOLUME_CLAIM}}
195195
dags_volume_subpath =
196-
logs_volume_claim = airflow-logs
196+
logs_volume_claim =
197197
logs_volume_subpath =
198198
dags_volume_host =
199199
logs_volume_host =
200200
in_cluster = True
201201
namespace = default
202+
run_as_user = 50000
202203
gcp_service_account_keys =
203204
204205
# Example affinity and toleration definitions.

‎scripts/ci/in_container/kubernetes/app/volumes.yaml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ kind: PersistentVolume
4242
apiVersion: v1
4343
metadata:
4444
name: airflow-logs
45+
annotations:
46+
pv.beta.kubernetes.io/gid: "50000"
4547
spec:
4648
accessModes:
4749
- ReadWriteMany
@@ -67,7 +69,7 @@ metadata:
6769
name: test-volume
6870
spec:
6971
accessModes:
70-
- ReadWriteOnce
72+
- ReadWriteMany
7173
capacity:
7274
storage: 2Gi
7375
hostPath:

‎scripts/ci/in_container/kubernetes/docker/airflow-test-env-init.sh‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,6 @@
1818

1919
set -x
2020

21-
cd /opt/airflow/airflow && \
22-
cp -R example_dags/* /root/airflow/dags/ && \
23-
cp -R contrib/example_dags/example_kubernetes_*.py /root/airflow/dags/ && \
24-
cp -a contrib/example_dags/libs /root/airflow/dags/ && \
21+
cd /opt/airflow && \
2522
airflow initdb && \
26-
alembic upgrade heads && \
27-
(airflow create_user -u airflow -l airflow -f jon -e [email protected] -r Admin -p airflow || true) && \
28-
echo "retrieved from mount" > /root/test_volume/test.txt
23+
(airflow create_user -u airflow -l airflow -f jon -e [email protected] -r Admin -p airflow || true)

0 commit comments

Comments
 (0)