Skip to content

Commit 41fefa1

Browse files
potiukpotiuk
authored
Add CI-friendly progress output for tests (apache#24236)
This is the first step to run breeze tests in parallel in CI. This flag adds "limited progress" output when running tests which means that the runnig tests will just print few lines with percent progress and color status indication from last few progress lines of Pytest output, but when it completes, the whole output is printed in a CI group - colored depending on status. The final version (wnen we implement parallel test execution) should also defer writing the output to until all tests are completed, but this should be a follow-up PR.
1 parent cfbcd31 commit 41fefa1

File tree

8 files changed

+319
-113
lines changed

8 files changed

+319
-113
lines changed

‎TESTING.rst‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,21 @@ You can also specify individual tests or a group of tests:
182182
183183
breeze tests --db-reset tests/core/test_core.py::TestCore
184184
185+
You can also limit the tests to execute to specific group of tests
186+
187+
.. code-block:: bash
188+
189+
breeze tests --test-type Core
190+
191+
192+
You can also write tests in "limited progress" mode (useful in the future to run CI). In this mode each
193+
test just prints "percentage" summary of the run as single line and only dumps full output of the test
194+
after it completes.
195+
196+
.. code-block:: bash
197+
198+
breeze tests --test-type Core --limit-progress-output
199+
185200
186201
Running Tests of a specified type from the Host
187202
-----------------------------------------------

‎dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -157,9 +157,9 @@ def cleanup(verbose: bool, dry_run: bool, github_repository: str, all: bool, ans
157157
)
158158
images = command_result.stdout.splitlines() if command_result and command_result.stdout else []
159159
if images:
160-
get_console().print("[light_blue]Removing images:[/]")
160+
get_console().print("[info]Removing images:[/]")
161161
for image in images:
162-
get_console().print(f"[light_blue] * {image}[/]")
162+
get_console().print(f"[info] * {image}[/]")
163163
get_console().print()
164164
docker_rmi_command_to_execute = [
165165
'docker',
@@ -173,7 +173,7 @@ def cleanup(verbose: bool, dry_run: bool, github_repository: str, all: bool, ans
173173
elif given_answer == Answer.QUIT:
174174
sys.exit(0)
175175
else:
176-
get_console().print("[light_blue]No locally downloaded images to remove[/]\n")
176+
get_console().print("[info]No locally downloaded images to remove[/]\n")
177177
get_console().print("Pruning docker images")
178178
given_answer = user_confirm("Are you sure with the removal?")
179179
if given_answer == Answer.YES:

‎dev/breeze/src/airflow_breeze/commands/testing_commands.py‎

Lines changed: 154 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -14,35 +14,46 @@
1414
# KIND, either express or implied. See the License for the
1515
# specific language governing permissions and limitations
1616
# under the License.
17-
17+
import errno
1818
import os
19+
import re
20+
import shutil
21+
import subprocess
1922
import sys
20-
from typing import Tuple
23+
import tempfile
24+
from threading import Event, Thread
25+
from time import sleep
26+
from typing import Dict, List, Tuple
2127

2228
import click
2329

2430
from airflow_breeze.commands.main_command import main
2531
from airflow_breeze.global_constants import ALLOWED_TEST_TYPES
2632
from airflow_breeze.params.build_prod_params import BuildProdParams
2733
from airflow_breeze.params.shell_params import ShellParams
34+
from airflow_breeze.utils.ci_group import ci_group
2835
from airflow_breeze.utils.common_options import (
36+
option_backend,
2937
option_db_reset,
3038
option_dry_run,
3139
option_github_repository,
3240
option_image_name,
3341
option_image_tag,
3442
option_integration,
43+
option_mssql_version,
44+
option_mysql_version,
45+
option_postgres_version,
3546
option_python,
3647
option_verbose,
3748
)
38-
from airflow_breeze.utils.console import get_console
49+
from airflow_breeze.utils.console import get_console, message_type_from_return_code
3950
from airflow_breeze.utils.custom_param_types import BetterChoice
4051
from airflow_breeze.utils.docker_command_utils import (
4152
get_env_variables_for_docker_commands,
4253
perform_environment_checks,
4354
)
4455
from airflow_breeze.utils.run_tests import run_docker_compose_tests
45-
from airflow_breeze.utils.run_utils import run_command
56+
from airflow_breeze.utils.run_utils import RunCommandResult, run_command
4657

4758
TESTING_COMMANDS = {
4859
"name": "Testing",
@@ -55,8 +66,8 @@
5566
"name": "Docker-compose tests flag",
5667
"options": [
5768
"--image-name",
58-
"--python",
5969
"--image-tag",
70+
"--python",
6071
],
6172
}
6273
],
@@ -66,7 +77,13 @@
6677
"options": [
6778
"--integration",
6879
"--test-type",
80+
"--limit-progress-output",
6981
"--db-reset",
82+
"--backend",
83+
"--python",
84+
"--postgres-version",
85+
"--mysql-version",
86+
"--mssql-version",
7087
],
7188
}
7289
],
@@ -112,6 +129,91 @@ def docker_compose_tests(
112129
sys.exit(return_code)
113130

114131

132+
class MonitoringThread(Thread):
133+
"""Thread class with a stop() method. The thread itself has to check
134+
regularly for the stopped() condition."""
135+
136+
def __init__(self, title: str, file_name: str):
137+
super().__init__(target=self.peek_percent_at_last_lines_of_file, daemon=True)
138+
self._stop_event = Event()
139+
self.title = title
140+
self.file_name = file_name
141+
142+
def peek_percent_at_last_lines_of_file(self) -> None:
143+
max_line_length = 400
144+
matcher = re.compile(r"^.*\[([^\]]*)\]$")
145+
while not self.stopped():
146+
if os.path.exists(self.file_name):
147+
try:
148+
with open(self.file_name, 'rb') as temp_f:
149+
temp_f.seek(-(max_line_length * 2), os.SEEK_END)
150+
tail = temp_f.read().decode()
151+
try:
152+
two_last_lines = tail.splitlines()[-2:]
153+
previous_no_ansi_line = escape_ansi(two_last_lines[0])
154+
m = matcher.match(previous_no_ansi_line)
155+
if m:
156+
get_console().print(f"[info]{self.title}:[/] {m.group(1).strip()}")
157+
print(f"\r{two_last_lines[0]}\r")
158+
print(f"\r{two_last_lines[1]}\r")
159+
except IndexError:
160+
pass
161+
except OSError as e:
162+
if e.errno == errno.EINVAL:
163+
pass
164+
else:
165+
raise
166+
sleep(5)
167+
168+
def stop(self):
169+
self._stop_event.set()
170+
171+
def stopped(self):
172+
return self._stop_event.is_set()
173+
174+
175+
def escape_ansi(line):
176+
ansi_escape = re.compile(r'(?:\x1B[@-_]|[\x80-\x9F])[0-?]*[ -/]*[@-~]')
177+
return ansi_escape.sub('', line)
178+
179+
180+
def run_with_progress(
181+
cmd: List[str],
182+
env_variables: Dict[str, str],
183+
test_type: str,
184+
python: str,
185+
backend: str,
186+
version: str,
187+
verbose: bool,
188+
dry_run: bool,
189+
) -> RunCommandResult:
190+
title = f"Running tests: {test_type}, Python: {python}, Backend: {backend}:{version}"
191+
try:
192+
with tempfile.NamedTemporaryFile(mode='w+t', delete=False) as f:
193+
get_console().print(f"[info]Starting test = {title}[/]")
194+
thread = MonitoringThread(title=title, file_name=f.name)
195+
thread.start()
196+
try:
197+
result = run_command(
198+
cmd,
199+
verbose=verbose,
200+
dry_run=dry_run,
201+
env=env_variables,
202+
check=False,
203+
stdout=f,
204+
stderr=subprocess.STDOUT,
205+
)
206+
finally:
207+
thread.stop()
208+
thread.join()
209+
with ci_group(f"Result of {title}", message_type=message_type_from_return_code(result.returncode)):
210+
with open(f.name) as f:
211+
shutil.copyfileobj(f, sys.stdout)
212+
finally:
213+
os.unlink(f.name)
214+
return result
215+
216+
115217
@main.command(
116218
name='tests',
117219
help="Run the specified unit test targets. Multiple targets may be specified separated by spaces.",
@@ -122,10 +224,19 @@ def docker_compose_tests(
122224
)
123225
@option_dry_run
124226
@option_verbose
227+
@option_python
228+
@option_backend
229+
@option_postgres_version
230+
@option_mysql_version
231+
@option_mssql_version
125232
@option_integration
233+
@click.option(
234+
'--limit-progress-output',
235+
help="Limit progress to percentage only and just show the summary when tests complete.",
236+
is_flag=True,
237+
)
126238
@click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED)
127239
@click.option(
128-
"-tt",
129240
"--test-type",
130241
help="Type of test to run.",
131242
default="All",
@@ -135,6 +246,12 @@ def docker_compose_tests(
135246
def tests(
136247
dry_run: bool,
137248
verbose: bool,
249+
python: str,
250+
backend: str,
251+
postgres_version: str,
252+
mysql_version: str,
253+
mssql_version: str,
254+
limit_progress_output: bool,
138255
integration: Tuple,
139256
extra_pytest_args: Tuple,
140257
test_type: str,
@@ -149,11 +266,39 @@ def tests(
149266
os.environ["LIST_OF_INTEGRATION_TESTS_TO_RUN"] = ' '.join(list(integration))
150267
if db_reset:
151268
os.environ["DB_RESET"] = "true"
152-
153-
exec_shell_params = ShellParams(verbose=verbose, dry_run=dry_run)
269+
exec_shell_params = ShellParams(
270+
verbose=verbose,
271+
dry_run=dry_run,
272+
python=python,
273+
backend=backend,
274+
postgres_version=postgres_version,
275+
mysql_version=mysql_version,
276+
mssql_version=mssql_version,
277+
)
154278
env_variables = get_env_variables_for_docker_commands(exec_shell_params)
155279
perform_environment_checks(verbose=verbose)
156280
cmd = ['docker-compose', 'run', '--service-ports', '--rm', 'airflow']
157281
cmd.extend(list(extra_pytest_args))
158-
result = run_command(cmd, verbose=verbose, dry_run=dry_run, env=env_variables, check=False)
282+
version = (
283+
mssql_version
284+
if backend == "mssql"
285+
else mysql_version
286+
if backend == "mysql"
287+
else postgres_version
288+
if backend == "postgres"
289+
else "none"
290+
)
291+
if limit_progress_output:
292+
result = run_with_progress(
293+
cmd=cmd,
294+
env_variables=env_variables,
295+
test_type=test_type,
296+
python=python,
297+
backend=backend,
298+
version=version,
299+
verbose=verbose,
300+
dry_run=dry_run,
301+
)
302+
else:
303+
result = run_command(cmd, verbose=verbose, dry_run=dry_run, env=env_variables, check=False)
159304
sys.exit(result.returncode)

‎dev/breeze/src/airflow_breeze/utils/ci_group.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,11 @@
1818
import os
1919
from contextlib import contextmanager
2020

21-
from airflow_breeze.utils.console import get_console
21+
from airflow_breeze.utils.console import MessageType, get_console
2222

2323

2424
@contextmanager
25-
def ci_group(title: str, enabled: bool = True):
25+
def ci_group(title: str, enabled: bool = True, message_type: MessageType = MessageType.INFO):
2626
"""
2727
If used in GitHub Action, creates an expandable group in the GitHub Action log.
2828
Otherwise, display simple text groups.
@@ -34,9 +34,9 @@ def ci_group(title: str, enabled: bool = True):
3434
yield
3535
return
3636
if os.environ.get('GITHUB_ACTIONS', 'false') != "true":
37-
get_console().print(f"[info]{title}[/]")
37+
get_console().print(f"[{message_type.value}]{title}[/]")
3838
yield
3939
return
40-
get_console().print(f"::group::<CLICK_TO_EXPAND>: [info]{title}[/]")
40+
get_console().print(f"::group::<CLICK_TO_EXPAND>: [{message_type.value}]{title}[/]")
4141
yield
4242
get_console().print("::endgroup::")

‎dev/breeze/src/airflow_breeze/utils/console.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
to be only run in CI or real development terminal - in both cases we want to have colors on.
2020
"""
2121
import os
22+
from enum import Enum
2223
from functools import lru_cache
2324

2425
from rich.console import Console
@@ -56,6 +57,19 @@ def get_theme() -> Theme:
5657
)
5758

5859

60+
class MessageType(Enum):
61+
SUCCESS = "success"
62+
INFO = "info"
63+
WARNING = "warning"
64+
ERROR = "error"
65+
66+
67+
def message_type_from_return_code(return_code: int) -> MessageType:
68+
if return_code == 0:
69+
return MessageType.SUCCESS
70+
return MessageType.ERROR
71+
72+
5973
@lru_cache(maxsize=None)
6074
def get_console() -> Console:
6175
return Console(

‎images/breeze/output-commands-hash.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
7f2019004f86eeab48332eb0ea11114d
1+
2942c0bca323521e3e9af5922d527201

0 commit comments

Comments
 (0)