Skip to content

Commit 306a666

Browse files
potiukpotiuk
authored
Docker images are now consistently labelled and a bit smaller (#10387)
Extracted from #10368
1 parent d6f6d53 commit 306a666

File tree

5 files changed

+76
-24
lines changed

5 files changed

+76
-24
lines changed

‎.dockerignore‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,20 @@
3434
!docs
3535
!licenses
3636
!metastore_browser
37-
!scripts
37+
38+
# Add those folders to the context so that they are available in the CI container
39+
!scripts/ci/in_container
40+
!scripts/ci/pre_commit
41+
!scripts/prod
42+
!scripts/perf
43+
!scripts/tools
44+
45+
# Add backport packages to the context
46+
!backport_packages
47+
48+
# Add tests and kubernetes_tests to context.
3849
!tests
50+
!kubernetes_tests
3951

4052
!.coveragerc
4153
!.rat-excludes

‎Dockerfile‎

Lines changed: 32 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -54,12 +54,6 @@ ARG PYTHON_MAJOR_MINOR_VERSION="3.6"
5454
FROM ${PYTHON_BASE_IMAGE} as airflow-build-image
5555
SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
5656

57-
LABEL org.apache.airflow.distro="debian"
58-
LABEL org.apache.airflow.distro.version="buster"
59-
LABEL org.apache.airflow.module="airflow"
60-
LABEL org.apache.airflow.component="airflow"
61-
LABEL org.apache.airflow.image="airflow-build-image"
62-
6357
ARG PYTHON_BASE_IMAGE
6458
ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE}
6559

@@ -171,6 +165,9 @@ ARG AIRFLOW_CONSTRAINTS_REFERENCE="constraints-master"
171165
ARG AIRFLOW_CONSTRAINTS_URL="https://raw.githubusercontent.com/apache/airflow/${AIRFLOW_CONSTRAINTS_REFERENCE}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt"
172166
ENV AIRFLOW_CONSTRAINTS_URL=${AIRFLOW_CONSTRAINTS_URL}
173167

168+
ENV PATH=${PATH}:/root/.local/bin
169+
RUN mkdir -p /root/.local/bin
170+
174171
# In case of Production build image segment we want to pre-install master version of airflow
175172
# dependencies from github so that we do not have to always reinstall it from the scratch.
176173
RUN pip install --user \
@@ -202,8 +199,6 @@ ENV AIRFLOW_INSTALL_VERSION=${AIRFLOW_INSTALL_VERSION}
202199

203200
WORKDIR /opt/airflow
204201

205-
ENV PATH=${PATH}:/root/.local/bin
206-
207202
RUN pip install --user "${AIRFLOW_INSTALL_SOURCES}[${AIRFLOW_EXTRAS}]${AIRFLOW_INSTALL_VERSION}" \
208203
--constraint "${AIRFLOW_CONSTRAINTS_URL}" && \
209204
if [ -n "${ADDITIONAL_PYTHON_DEPS}" ]; then pip install --user ${ADDITIONAL_PYTHON_DEPS} \
@@ -228,6 +223,20 @@ RUN AIRFLOW_SITE_PACKAGE="/root/.local/lib/python${PYTHON_MAJOR_MINOR_VERSION}/s
228223
RUN find /root/.local -executable -print0 | xargs --null chmod g+x && \
229224
find /root/.local -print0 | xargs --null chmod g+rw
230225

226+
LABEL org.apache.airflow.distro="debian"
227+
LABEL org.apache.airflow.distro.version="buster"
228+
LABEL org.apache.airflow.module="airflow"
229+
LABEL org.apache.airflow.component="airflow"
230+
LABEL org.apache.airflow.image="airflow-build-image"
231+
232+
ARG BUILD_ID
233+
ENV BUILD_ID=${BUILD_ID}
234+
ARG COMMIT_SHA
235+
ENV COMMIT_SHA=${COMMIT_SHA}
236+
237+
LABEL org.apache.airflow.buildImage.buildId=${BUILD_ID}
238+
LABEL org.apache.airflow.buildImage.commitSha=${COMMIT_SHA}
239+
231240
##############################################################################################
232241
# This is the actual Airflow image - much smaller than the build one. We copy
233242
# installed Airflow and all it's dependencies from the build image to make it smaller.
@@ -372,5 +381,20 @@ EXPOSE 8080
372381

373382
USER ${AIRFLOW_UID}
374383

384+
ARG BUILD_ID
385+
ENV BUILD_ID=${BUILD_ID}
386+
ARG COMMIT_SHA
387+
ENV COMMIT_SHA=${COMMIT_SHA}
388+
389+
LABEL org.apache.airflow.distro="debian"
390+
LABEL org.apache.airflow.distro.version="buster"
391+
LABEL org.apache.airflow.module="airflow"
392+
LABEL org.apache.airflow.component="airflow"
393+
LABEL org.apache.airflow.image="airflow"
394+
LABEL org.apache.airflow.uid="${AIRFLOW_UID}"
395+
LABEL org.apache.airflow.gid="${AIRFLOW_GID}"
396+
LABEL org.apache.airflow.mainImage.buildId=${BUILD_ID}
397+
LABEL org.apache.airflow.mainImage.commitSha=${COMMIT_SHA}
398+
375399
ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"]
376400
CMD ["--help"]

‎Dockerfile.ci‎

Lines changed: 20 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -281,20 +281,11 @@ RUN yarn --cwd airflow/www run prod
281281
COPY scripts/ci/in_container/entrypoint_ci.sh /entrypoint
282282
RUN chmod a+x /entrypoint
283283

284-
# Copy selected subdirectories only
285-
COPY .github/ ${AIRFLOW_SOURCES}/.github/
286-
COPY metastore_browser/ ${AIRFLOW_SOURCES}/metastore_browser/
287-
COPY dags/ ${AIRFLOW_SOURCES}/dags/
288-
COPY common/ ${AIRFLOW_SOURCES}/common/
289-
COPY licenses/ ${AIRFLOW_SOURCES}/licenses/
290-
COPY scripts/ ${AIRFLOW_SOURCES}/scripts/
291-
COPY docs/ ${AIRFLOW_SOURCES}/docs/
292-
COPY tests/ ${AIRFLOW_SOURCES}/tests/
293-
COPY airflow/ ${AIRFLOW_SOURCES}/airflow/
294-
COPY .coveragerc .rat-excludes .flake8 pylintrc LICENSE MANIFEST.in NOTICE CHANGELOG.txt \
295-
.github pytest.ini \
296-
setup.cfg setup.py \
297-
${AIRFLOW_SOURCES}/
284+
# We can copy everything here. The Context is filtered by dockerignore. This makes sure we are not
285+
# copying over stuff that is accidentally generated or that we do not need (such as .egginfo)
286+
# if you want to add something that is missing and you expect to see it in the image you can
287+
# add it with ! in .dockerignore next to the airflow, test etc. directories there
288+
COPY . ${AIRFLOW_SOURCES}/
298289

299290
# Install autocomplete for airflow
300291
RUN register-python-argcomplete airflow >> ~/.bashrc
@@ -318,6 +309,21 @@ ENV PATH="${HOME}:${PATH}"
318309
# Needed to stop Gunicorn from crashing when /tmp is now mounted from host
319310
ENV GUNICORN_CMD_ARGS="--worker-tmp-dir /dev/shm/"
320311

312+
ARG BUILD_ID
313+
ENV BUILD_ID=${BUILD_ID}
314+
ARG COMMIT_SHA
315+
ENV COMMIT_SHA=${COMMIT_SHA}
316+
317+
LABEL org.apache.airflow.distro="debian"
318+
LABEL org.apache.airflow.distro.version="buster"
319+
LABEL org.apache.airflow.module="airflow"
320+
LABEL org.apache.airflow.component="airflow"
321+
LABEL org.apache.airflow.image="airflow-ci"
322+
LABEL org.apache.airflow.uid="0"
323+
LABEL org.apache.airflow.gid="0"
324+
LABEL org.apache.airflow.buildId=${BUILD_ID}
325+
LABEL org.apache.airflow.commitSha=${COMMIT_SHA}
326+
321327
EXPOSE 8080
322328

323329
ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"]

‎scripts/ci/libraries/_build_images.sh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -576,6 +576,8 @@ Docker building ${AIRFLOW_CI_IMAGE}.
576576
--build-arg ADDITIONAL_DEV_DEPS="${ADDITIONAL_DEV_DEPS}" \
577577
--build-arg ADDITIONAL_RUNTIME_DEPS="${ADDITIONAL_RUNTIME_DEPS}" \
578578
--build-arg UPGRADE_TO_LATEST_CONSTRAINTS="${UPGRADE_TO_LATEST_CONSTRAINTS}" \
579+
--build-arg BUILD_ID="${CI_BUILD_ID}" \
580+
--build-arg COMMIT_SHA="${COMMIT_SHA}" \
579581
"${DOCKER_CACHE_CI_DIRECTIVE[@]}" \
580582
-t "${AIRFLOW_CI_IMAGE}" \
581583
--target "main" \
@@ -724,6 +726,8 @@ function build_prod_image() {
724726
--build-arg ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS}" \
725727
--build-arg ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS}" \
726728
--build-arg ADDITIONAL_DEV_DEPS="${ADDITIONAL_DEV_DEPS}" \
729+
--build-arg BUILD_ID="${CI_BUILD_ID}" \
730+
--build-arg COMMIT_SHA="${COMMIT_SHA}" \
727731
"${DOCKER_CACHE_PROD_BUILD_DIRECTIVE[@]}" \
728732
-t "${AIRFLOW_PROD_BUILD_IMAGE}" \
729733
--target "airflow-build-image" \
@@ -740,6 +744,8 @@ function build_prod_image() {
740744
--build-arg AIRFLOW_BRANCH="${AIRFLOW_BRANCH_FOR_PYPI_PRELOADING}" \
741745
--build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \
742746
--build-arg EMBEDDED_DAGS="${EMBEDDED_DAGS}" \
747+
--build-arg BUILD_ID="${CI_BUILD_ID}" \
748+
--build-arg COMMIT_SHA="${COMMIT_SHA}" \
743749
"${DOCKER_CACHE_PROD_DIRECTIVE[@]}" \
744750
-t "${AIRFLOW_PROD_IMAGE}" \
745751
--target "main" \

‎scripts/ci/libraries/_initialization.sh‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,10 @@ function initialize_common_environment {
220220
# Namespace where airflow is installed via helm
221221
export HELM_AIRFLOW_NAMESPACE="airflow"
222222

223+
COMMIT_SHA="$(git rev-parse HEAD || echo "Unknown")"
224+
export COMMIT_SHA
225+
226+
export CI_BUILD_ID="0"
223227
}
224228

225229
# Retrieves CI environment variables needed - depending on the CI system we run it in.
@@ -233,7 +237,7 @@ function get_environment_for_builds_on_ci() {
233237
export CI_TARGET_BRANCH="master"
234238
export CI_SOURCE_REPO="apache/airflow"
235239
export CI_SOURCE_BRANCH="master"
236-
export CI_BUILD_ID="default-build-id"
240+
export CI_BUILD_ID="0"
237241
export CI_JOB_ID="default-job-id"
238242
if [[ ${CI:=} != "true" ]]; then
239243
echo

0 commit comments

Comments
 (0)