Skip to content

Commit 3f43846

Browse files
potiukashb
potiuk
authored and
ashb
committed
Update setup.py to get non-conflicting set of dependencies (#12636)
This change upgrades setup.py and setup.cfg to provide non-conflicting `pip check` valid set of constraints for CI image.
1 parent 74e3c56 commit 3f43846

13 files changed

+210
-115
lines changed

‎BREEZE.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1355,6 +1355,7 @@ This is the current syntax for `./breeze <./breeze>`_:
13551355
13561356
If you use this flag, automatically --github-registry is enabled.
13571357
1358+
13581359
Default: latest.
13591360
13601361
-v, --verbose
@@ -1508,6 +1509,7 @@ This is the current syntax for `./breeze <./breeze>`_:
15081509
15091510
If you use this flag, automatically --github-registry is enabled.
15101511
1512+
15111513
Default: latest.
15121514
15131515
-v, --verbose
@@ -2276,6 +2278,7 @@ This is the current syntax for `./breeze <./breeze>`_:
22762278
22772279
If you use this flag, automatically --github-registry is enabled.
22782280
2281+
22792282
Default: latest.
22802283
22812284
****************************************************************************************************

‎CI.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -253,7 +253,7 @@ You can use those variables when you try to reproduce the build locally.
253253
| Image build variables |
254254
+-----------------------------------------+-------------+-------------+------------+-------------------------------------------------+
255255
| ``UPGRADE_TO_LATEST_CONSTRAINTS`` | false | false | false | Determines whether the build should |
256-
| | | | (x) | attempt to eagerly upgrade all |
256+
| | | | (x) | attempt to upgrade all |
257257
| | | | | PIP dependencies to latest ones matching |
258258
| | | | | ``setup.py`` limits. This tries to replicate |
259259
| | | | | the situation of "fresh" user who just installs |

‎CONTRIBUTING.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -321,7 +321,7 @@ Step 4: Prepare PR
321321
the "full tests needed" label is set for your PR. Additional check is set that prevents from
322322
accidental merging of the request until full matrix of tests succeeds for the PR.
323323

324-
* when your change has "upgrade to latest dependencies" label set, constraints will be automatically
324+
* when your change has "upgrade to newer dependencies" label set, constraints will be automatically
325325
upgraded to latest constraints matching your setup.py. This is useful in case you want to force
326326
upgrade to a latest version of dependencies. You can ask committers to set the label for you
327327
when you need it in your PR.

‎breeze‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1078,6 +1078,7 @@ function breeze::parse_arguments() {
10781078
echo
10791079
echo "Force pulling the image, using github registry and skip mounting local sources."
10801080
echo "This is in order to get the exact same version as used in CI environment for SHA/RUN_ID!."
1081+
echo "You can specify --skip-mounting-local-sources to not mount local sources. "
10811082
echo
10821083
export FORCE_PULL_IMAGES="true"
10831084
export USE_GITHUB_REGISTRY="true"
@@ -2385,6 +2386,7 @@ function breeze::flag_pull_push_docker_images() {
23852386
23862387
If you use this flag, automatically --github-registry is enabled.
23872388
2389+
23882390
Default: ${_breeze_default_github_image_id:=}.
23892391
23902392
"

‎scripts/ci/images/ci_prepare_ci_image_on_ci.sh‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,5 +59,4 @@ function build_ci_image_on_ci() {
5959
export CHECK_IMAGE_FOR_REBUILD="false"
6060
}
6161

62-
6362
build_ci_image_on_ci

‎scripts/ci/images/ci_wait_for_all_ci_images.sh‎

Lines changed: 3 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -15,42 +15,12 @@
1515
# KIND, either express or implied. See the License for the
1616
# specific language governing permissions and limitations
1717
# under the License.
18-
export AIRFLOW_SOURCES="${AIRFLOW_SOURCES:=$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../.." && pwd )}"
1918
echo
20-
echo "Airflow sources: ${AIRFLOW_SOURCES}"
19+
echo "Waiting for all CI images to appear: ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}"
2120
echo
2221

23-
if [[ ${USE_GITHUB_REGISTRY} != "true" || ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} != "true" ]]; then
24-
echo
25-
echo "This script should not be called"
26-
echo "It need both USE_GITHUB_REGISTRY and GITHUB_REGISTRY_WAIT_FOR_IMAGE to true!"
27-
echo
28-
echo "USE_GITHUB_REGISTRY = ${USE_GITHUB_REGISTRY}"
29-
echo "GITHUB_REGISTRY_WAIT_FOR_IMAGE =${GITHUB_REGISTRY_WAIT_FOR_IMAGE}"
30-
echo
31-
exit 1
32-
fi
33-
34-
echo
35-
echo "Waiting for all images to appear: ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}"
36-
echo
37-
38-
echo
39-
echo "Check if jq is installed"
40-
echo
41-
command -v jq >/dev/null || (echo "ERROR! You must have 'jq' tool installed!" && exit 1)
42-
43-
echo
44-
echo "The jq version $(jq --version)"
45-
echo
46-
47-
# shellcheck source=scripts/ci/libraries/_all_libs.sh
48-
source "${AIRFLOW_SOURCES}/scripts/ci/libraries/_all_libs.sh"
49-
50-
initialization::initialize_common_environment
51-
5222
for PYTHON_MAJOR_MINOR_VERSION in ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}
5323
do
54-
export AIRFLOW_CI_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci"
55-
push_pull_remove_images::wait_for_github_registry_image "${AIRFLOW_CI_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
24+
export PYTHON_MAJOR_MINOR_VERSION
25+
"$( dirname "${BASH_SOURCE[0]}" )/ci_wait_for_ci_image.sh"
5626
done

‎scripts/ci/images/ci_wait_for_all_prod_images.sh‎

Lines changed: 3 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -15,44 +15,12 @@
1515
# KIND, either express or implied. See the License for the
1616
# specific language governing permissions and limitations
1717
# under the License.
18-
export AIRFLOW_SOURCES="${AIRFLOW_SOURCES:=$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../.." && pwd )}"
1918
echo
20-
echo "Airflow sources: ${AIRFLOW_SOURCES}"
19+
echo "Waiting for all PROD images to appear: ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}"
2120
echo
2221

23-
if [[ ${USE_GITHUB_REGISTRY} != "true" || ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} != "true" ]]; then
24-
echo
25-
echo "This script should not be called"
26-
echo "It need both USE_GITHUB_REGISTRY and GITHUB_REGISTRY_WAIT_FOR_IMAGE to true!"
27-
echo
28-
echo "USE_GITHUB_REGISTRY = ${USE_GITHUB_REGISTRY}"
29-
echo "GITHUB_REGISTRY_WAIT_FOR_IMAGE =${GITHUB_REGISTRY_WAIT_FOR_IMAGE}"
30-
echo
31-
exit 1
32-
fi
33-
34-
echo
35-
echo "Waiting for all images to appear: ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}"
36-
echo
37-
38-
echo
39-
echo "Check if jq is installed"
40-
echo
41-
command -v jq >/dev/null || (echo "ERROR! You must have 'jq' tool installed!" && exit 1)
42-
43-
echo
44-
echo "The jq version $(jq --version)"
45-
echo
46-
47-
# shellcheck source=scripts/ci/libraries/_all_libs.sh
48-
source "${AIRFLOW_SOURCES}/scripts/ci/libraries/_all_libs.sh"
49-
50-
initialization::initialize_common_environment
51-
5222
for PYTHON_MAJOR_MINOR_VERSION in ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}
5323
do
54-
export AIRFLOW_PROD_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}"
55-
export AIRFLOW_PROD_BUILD_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-build"
56-
push_pull_remove_images::wait_for_github_registry_image "${AIRFLOW_PROD_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
57-
push_pull_remove_images::wait_for_github_registry_image "${AIRFLOW_PROD_BUILD_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
24+
export PYTHON_MAJOR_MINOR_VERSION
25+
"$( dirname "${BASH_SOURCE[0]}" )/ci_wait_for_prod_image.sh"
5826
done

‎scripts/ci/images/ci_wait_for_ci_image.sh‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
#!/usr/bin/env bash
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
# shellcheck source=scripts/ci/libraries/_script_init.sh
19+
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
20+
21+
function verify_ci_image_dependencies {
22+
echo
23+
echo "Checking if Airflow dependencies are non-conflicting in CI image."
24+
echo
25+
26+
push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_CI_IMAGE}" \
27+
"${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
28+
29+
# TODO: remove after we have it fully working
30+
docker run --rm --entrypoint /bin/bash "${AIRFLOW_CI_IMAGE}" -c 'pip check' || true
31+
}
32+
33+
push_pull_remove_images::check_if_github_registry_wait_for_image_enabled
34+
35+
push_pull_remove_images::check_if_jq_installed
36+
37+
build_image::login_to_github_registry_if_needed
38+
39+
export AIRFLOW_CI_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci"
40+
41+
echo
42+
echo "Waiting for image to appear: ${AIRFLOW_CI_IMAGE_NAME}"
43+
echo
44+
45+
push_pull_remove_images::wait_for_github_registry_image \
46+
"${AIRFLOW_CI_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
47+
48+
echo
49+
echo "Verifying the ${AIRFLOW_CI_IMAGE_NAME} image after pulling it"
50+
echo
51+
52+
verify_ci_image_dependencies

‎scripts/ci/images/ci_wait_for_prod_image.sh‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
#!/usr/bin/env bash
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
# shellcheck source=scripts/ci/libraries/_script_init.sh
19+
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
20+
21+
function verify_prod_image_dependencies {
22+
echo
23+
echo "Checking if Airflow dependencies are non-conflicting in PROD image."
24+
echo
25+
26+
push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_PROD_IMAGE}" \
27+
"${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
28+
29+
# TODO: remove the | true after we fixed pip check for prod image
30+
docker run --rm --entrypoint /bin/bash "${AIRFLOW_PROD_IMAGE}" -c 'pip check' || true
31+
}
32+
33+
push_pull_remove_images::check_if_github_registry_wait_for_image_enabled
34+
35+
push_pull_remove_images::check_if_jq_installed
36+
37+
build_image::login_to_github_registry_if_needed
38+
39+
export AIRFLOW_PROD_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}"
40+
41+
echo
42+
echo "Waiting for image to appear: ${AIRFLOW_PROD_IMAGE_NAME}"
43+
echo
44+
45+
push_pull_remove_images::wait_for_github_registry_image \
46+
"${AIRFLOW_PROD_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
47+
48+
echo
49+
echo "Verifying the ${AIRFLOW_PROD_IMAGE_NAME} image after pulling it"
50+
echo
51+
52+
verify_prod_image_dependencies

‎scripts/ci/libraries/_build_images.sh‎

Lines changed: 21 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -346,25 +346,35 @@ function build_images::get_docker_image_names() {
346346

347347
# File that is touched when the CI image is built for the first time locally
348348
export BUILT_CI_IMAGE_FLAG_FILE="${BUILD_CACHE_DIR}/${BRANCH_NAME}/.built_${PYTHON_MAJOR_MINOR_VERSION}"
349+
350+
# GitHub Registry names must be lowercase :(
351+
github_repository_lowercase="$(echo "${GITHUB_REPOSITORY}" |tr '[:upper:]' '[:lower:]')"
352+
export GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/${AIRFLOW_PROD_BASE_TAG}"
353+
export GITHUB_REGISTRY_AIRFLOW_PROD_BUILD_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/${AIRFLOW_PROD_BASE_TAG}-build"
354+
export GITHUB_REGISTRY_PYTHON_BASE_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster"
355+
356+
export GITHUB_REGISTRY_AIRFLOW_CI_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/${AIRFLOW_CI_BASE_TAG}"
357+
export GITHUB_REGISTRY_PYTHON_BASE_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster"
349358
}
350359

351-
# Prepares all variables needed by the CI build. Depending on the configuration used (python version
352-
# DockerHub user etc. the variables are set so that other functions can use those variables.
353-
function build_images::prepare_ci_build() {
354-
export AIRFLOW_CI_LOCAL_MANIFEST_IMAGE="local/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}-manifest"
355-
export AIRFLOW_CI_REMOTE_MANIFEST_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}-manifest"
360+
# If GitHub Registry is used, login to the registry using GITHUB_USERNAME and GITHUB_TOKEN
361+
function build_image::login_to_github_registry_if_needed() {
356362
if [[ ${USE_GITHUB_REGISTRY} == "true" ]]; then
357363
if [[ -n ${GITHUB_TOKEN=} ]]; then
358364
echo "${GITHUB_TOKEN}" | docker login \
359365
--username "${GITHUB_USERNAME:-apache}" \
360366
--password-stdin \
361367
"${GITHUB_REGISTRY}"
362368
fi
363-
# GitHub Registry names must be lowercase :(
364-
github_repository_lowercase="$(echo "${GITHUB_REPOSITORY}" |tr '[:upper:]' '[:lower:]')"
365-
export GITHUB_REGISTRY_AIRFLOW_CI_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/${AIRFLOW_CI_BASE_TAG}"
366-
export GITHUB_REGISTRY_PYTHON_BASE_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster"
367369
fi
370+
371+
}
372+
373+
# Prepares all variables needed by the CI build. Depending on the configuration used (python version
374+
# DockerHub user etc. the variables are set so that other functions can use those variables.
375+
function build_images::prepare_ci_build() {
376+
export AIRFLOW_CI_LOCAL_MANIFEST_IMAGE="local/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}-manifest"
377+
export AIRFLOW_CI_REMOTE_MANIFEST_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}-manifest"
368378
export THE_IMAGE_TYPE="CI"
369379
export IMAGE_DESCRIPTION="Airflow CI"
370380

@@ -375,6 +385,7 @@ function build_images::prepare_ci_build() {
375385
export AIRFLOW_IMAGE="${AIRFLOW_CI_IMAGE}"
376386
readonly AIRFLOW_IMAGE
377387

388+
build_image::login_to_github_registry_if_needed
378389
sanity_checks::go_to_airflow_sources
379390
permissions::fix_group_permissions
380391
}
@@ -662,19 +673,7 @@ function build_images::prepare_prod_build() {
662673
export AIRFLOW_IMAGE="${AIRFLOW_PROD_IMAGE}"
663674
readonly AIRFLOW_IMAGE
664675

665-
if [[ ${USE_GITHUB_REGISTRY="false"} == "true" ]]; then
666-
if [[ -n ${GITHUB_TOKEN=} ]]; then
667-
echo "${GITHUB_TOKEN}" | docker login \
668-
--username "${GITHUB_USERNAME}" \
669-
--password-stdin \
670-
"${GITHUB_REGISTRY}"
671-
fi
672-
# GitHub Registry names must be lowercase :(
673-
github_repository_lowercase="$(echo "${GITHUB_REPOSITORY}" |tr '[:upper:]' '[:lower:]')"
674-
export GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/${AIRFLOW_PROD_BASE_TAG}"
675-
export GITHUB_REGISTRY_AIRFLOW_PROD_BUILD_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/${AIRFLOW_PROD_BASE_TAG}-build"
676-
export GITHUB_REGISTRY_PYTHON_BASE_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster"
677-
fi
676+
build_image::login_to_github_registry_if_needed
678677

679678
AIRFLOW_BRANCH_FOR_PYPI_PRELOADING="${BRANCH_NAME}"
680679
sanity_checks::go_to_airflow_sources

0 commit comments

Comments
 (0)