Skip to content

Commit df608fe

Browse files
potiukpotiuk
authored
Move Integration tests to separate package under tests (#28170)
This is the first stage of improving the way how integration tests are run in our CI - first we want to separate them in a separate packages, and then we want to run them separately - one integration each in the CI.
1 parent 0d90c62 commit df608fe

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+1069
-440
lines changed

‎.pre-commit-config.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -467,6 +467,7 @@ repos:
467467
^airflow/www/static/|
468468
^airflow/providers/|
469469
^tests/providers/apache/cassandra/hooks/test_cassandra.py$|
470+
^tests/integration/providers/apache/cassandra/hooks/test_cassandra.py$|
470471
^tests/system/providers/apache/spark/example_spark_dag.py$|
471472
^docs/apache-airflow-providers-apache-cassandra/connections/cassandra.rst$|
472473
^docs/apache-airflow-providers-apache-hive/commits.rst$|

‎Dockerfile.ci‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -890,11 +890,13 @@ else
890890
)
891891
WWW_TESTS=("tests/www")
892892
HELM_CHART_TESTS=("tests/charts")
893+
INTEGRATION_TESTS=("tests/integration")
893894
ALL_TESTS=("tests")
894895
ALL_PRESELECTED_TESTS=(
895896
"${CLI_TESTS[@]}"
896897
"${API_TESTS[@]}"
897898
"${HELM_CHART_TESTS[@]}"
899+
"${INTEGRATION_TESTS[@]}"
898900
"${PROVIDERS_TESTS[@]}"
899901
"${CORE_TESTS[@]}"
900902
"${ALWAYS_TESTS[@]}"
@@ -915,14 +917,15 @@ else
915917
SELECTED_TESTS=("${WWW_TESTS[@]}")
916918
elif [[ ${TEST_TYPE:=""} == "Helm" ]]; then
917919
SELECTED_TESTS=("${HELM_CHART_TESTS[@]}")
920+
elif [[ ${TEST_TYPE:=""} == "Integration" ]]; then
921+
SELECTED_TESTS=("${INTEGRATION_TESTS[@]}")
918922
elif [[ ${TEST_TYPE:=""} == "Other" ]]; then
919923
find_all_other_tests
920924
SELECTED_TESTS=("${ALL_OTHER_TESTS[@]}")
921925
elif [[ ${TEST_TYPE:=""} == "All" || ${TEST_TYPE} == "Quarantined" || \
922926
${TEST_TYPE} == "Always" || \
923927
${TEST_TYPE} == "Postgres" || ${TEST_TYPE} == "MySQL" || \
924-
${TEST_TYPE} == "Long" || \
925-
${TEST_TYPE} == "Integration" ]]; then
928+
${TEST_TYPE} == "Long" ]]; then
926929
SELECTED_TESTS=("${ALL_TESTS[@]}")
927930
elif [[ ${TEST_TYPE} =~ Providers\[(.*)\] ]]; then
928931
SELECTED_TESTS=()

‎dev/breeze/src/airflow_breeze/commands/testing_commands.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,7 @@ def _run_tests_in_pool(
226226
progress_matcher=GenericRegexpProgressMatcher(
227227
regexp=TEST_PROGRESS_REGEXP,
228228
regexp_for_joined_line=PERCENT_TEST_PROGRESS_REGEXP,
229-
lines_to_search=40,
229+
lines_to_search=200,
230230
),
231231
) as (pool, outputs):
232232
results = [

‎scripts/docker/entrypoint_ci.sh‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -335,11 +335,13 @@ else
335335
)
336336
WWW_TESTS=("tests/www")
337337
HELM_CHART_TESTS=("tests/charts")
338+
INTEGRATION_TESTS=("tests/integration")
338339
ALL_TESTS=("tests")
339340
ALL_PRESELECTED_TESTS=(
340341
"${CLI_TESTS[@]}"
341342
"${API_TESTS[@]}"
342343
"${HELM_CHART_TESTS[@]}"
344+
"${INTEGRATION_TESTS[@]}"
343345
"${PROVIDERS_TESTS[@]}"
344346
"${CORE_TESTS[@]}"
345347
"${ALWAYS_TESTS[@]}"
@@ -360,14 +362,15 @@ else
360362
SELECTED_TESTS=("${WWW_TESTS[@]}")
361363
elif [[ ${TEST_TYPE:=""} == "Helm" ]]; then
362364
SELECTED_TESTS=("${HELM_CHART_TESTS[@]}")
365+
elif [[ ${TEST_TYPE:=""} == "Integration" ]]; then
366+
SELECTED_TESTS=("${INTEGRATION_TESTS[@]}")
363367
elif [[ ${TEST_TYPE:=""} == "Other" ]]; then
364368
find_all_other_tests
365369
SELECTED_TESTS=("${ALL_OTHER_TESTS[@]}")
366370
elif [[ ${TEST_TYPE:=""} == "All" || ${TEST_TYPE} == "Quarantined" || \
367371
${TEST_TYPE} == "Always" || \
368372
${TEST_TYPE} == "Postgres" || ${TEST_TYPE} == "MySQL" || \
369-
${TEST_TYPE} == "Long" || \
370-
${TEST_TYPE} == "Integration" ]]; then
373+
${TEST_TYPE} == "Long" ]]; then
371374
SELECTED_TESTS=("${ALL_TESTS[@]}")
372375
elif [[ ${TEST_TYPE} =~ Providers\[(.*)\] ]]; then
373376
SELECTED_TESTS=()

‎tests/cli/commands/test_celery_command.py‎

Lines changed: 0 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -61,37 +61,6 @@ def test_validate_session_dbapi_exception(self, mock_session):
6161
assert airflow.settings.validate_session() is False
6262

6363

64-
@pytest.mark.integration("redis")
65-
@pytest.mark.integration("rabbitmq")
66-
@pytest.mark.backend("mysql", "postgres")
67-
class TestWorkerServeLogs:
68-
@classmethod
69-
def setup_class(cls):
70-
cls.parser = cli_parser.get_parser()
71-
72-
@mock.patch("airflow.cli.commands.celery_command.celery_app")
73-
@conf_vars({("core", "executor"): "CeleryExecutor"})
74-
def test_serve_logs_on_worker_start(self, mock_celery_app):
75-
with mock.patch("airflow.cli.commands.celery_command.Process") as mock_process:
76-
args = self.parser.parse_args(["celery", "worker", "--concurrency", "1"])
77-
78-
with mock.patch("celery.platforms.check_privileges") as mock_privil:
79-
mock_privil.return_value = 0
80-
celery_command.worker(args)
81-
mock_process.assert_called()
82-
83-
@mock.patch("airflow.cli.commands.celery_command.celery_app")
84-
@conf_vars({("core", "executor"): "CeleryExecutor"})
85-
def test_skip_serve_logs_on_worker_start(self, mock_celery_app):
86-
with mock.patch("airflow.cli.commands.celery_command.Process") as mock_popen:
87-
args = self.parser.parse_args(["celery", "worker", "--concurrency", "1", "--skip-serve-logs"])
88-
89-
with mock.patch("celery.platforms.check_privileges") as mock_privil:
90-
mock_privil.return_value = 0
91-
celery_command.worker(args)
92-
mock_popen.assert_not_called()
93-
94-
9564
@pytest.mark.backend("mysql", "postgres")
9665
class TestCeleryStopCommand:
9766
@classmethod

0 commit comments

Comments
 (0)