Skip to content

Commit 7212c88

Browse files
authored
[v3-0-test] Remove gunicorn daemonize for api-server (apache#52860) (apache#53372)
* [v3-0-test] Remove gunicorn daemonize for api-server (apache#52860) * Remove gunicorn daemonize for api-server * Rename _CommonCLIUvicornTestClass * Add test for api-server with daemon option * Fix setproctitle on MacOS * Replace api/access_logfile with api/log_config * Fix [api/log_config] cli_config * Remove logging and unsed option in api_server_command - the logging before daemonization is not logging the correct PID so remove it - should_setup_logging is default as False * Fix test_run_command_daemon * Add significant rst (cherry picked from commit ab1ee72) * Fix spelling wordlist * Fix configuration
1 parent 192e662 commit 7212c88

8 files changed

Lines changed: 269 additions & 73 deletions

File tree

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
Replace API server ``access_logfile`` configuration with ``log_config``
2+
3+
The API server configuration option ``[api] access_logfile`` has been replaced with ``[api] log_config`` to align with uvicorn's logging configuration instead of the legacy gunicorn approach.
4+
The new ``log_config`` option accepts a path to a logging configuration file compatible with ``logging.config.fileConfig``, providing more flexible logging configuration for the API server.
5+
6+
This change also removes the dependency on gunicorn for daemonization, making the API server ``--daemon`` option consistent with other Airflow components like scheduler and triggerer.
7+
8+
* Types of change
9+
10+
* [ ] Dag changes
11+
* [x] Config changes
12+
* [ ] API changes
13+
* [ ] CLI changes
14+
* [ ] Behaviour changes
15+
* [ ] Plugin changes
16+
* [ ] Dependency changes
17+
* [ ] Code interface changes

‎airflow-core/src/airflow/cli/cli_config.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -618,10 +618,10 @@ def string_lower_type(val):
618618
default=conf.get("api", "host"),
619619
help="Set the host on which to run the API server",
620620
)
621-
ARG_API_SERVER_ACCESS_LOGFILE = Arg(
622-
("-A", "--access-logfile"),
623-
default=conf.get("api", "access_logfile"),
624-
help="The logfile to store the access log. Use '-' to print to stdout",
621+
ARG_API_SERVER_LOG_CONFIG = Arg(
622+
("--log-config",),
623+
default=conf.get("api", "log_config", fallback=None),
624+
help="(Optional) Path to the logging configuration file for the uvicorn server. If not set, the default uvicorn logging configuration will be used.",
625625
)
626626
ARG_API_SERVER_APPS = Arg(
627627
("--apps",),
@@ -1795,7 +1795,7 @@ class GroupCommand(NamedTuple):
17951795
ARG_DAEMON,
17961796
ARG_STDOUT,
17971797
ARG_STDERR,
1798-
ARG_API_SERVER_ACCESS_LOGFILE,
1798+
ARG_API_SERVER_LOG_CONFIG,
17991799
ARG_API_SERVER_APPS,
18001800
ARG_LOG_FILE,
18011801
ARG_SSL_CERT,

‎airflow-core/src/airflow/cli/commands/api_server_command.py‎

Lines changed: 64 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,13 @@
2121
import logging
2222
import os
2323
import subprocess
24+
import sys
2425
import textwrap
2526

2627
import uvicorn
27-
from gunicorn.util import daemonize
28-
from setproctitle import setproctitle
2928

3029
from airflow import settings
30+
from airflow.cli.commands.daemon_utils import run_command_with_daemon_option
3131
from airflow.exceptions import AirflowConfigException
3232
from airflow.utils import cli as cli_utils
3333
from airflow.utils.providers_configuration_loader import providers_configuration_loaded
@@ -40,14 +40,62 @@
4040
# more info here: https://github.com/benoitc/gunicorn/issues/1877#issuecomment-1911136399
4141

4242

43+
def _run_api_server(args, apps: str, num_workers: int, worker_timeout: int, proxy_headers: bool):
44+
"""Run the API server."""
45+
log.info(
46+
textwrap.dedent(
47+
f"""\
48+
Running the uvicorn with:
49+
Apps: {apps}
50+
Workers: {num_workers}
51+
Host: {args.host}:{args.port}
52+
Timeout: {worker_timeout}
53+
Logfiles: {args.log_file or "-"}
54+
================================================================="""
55+
)
56+
)
57+
# get ssl cert and key filepaths here instead of passing them as arguments to reduce the number of arguments
58+
ssl_cert, ssl_key = _get_ssl_cert_and_key_filepaths(args)
59+
60+
# setproctitle causes issue on Mac OS: https://github.com/benoitc/gunicorn/issues/3021
61+
os_type = sys.platform
62+
if os_type == "darwin":
63+
log.debug("Mac OS detected, skipping setproctitle")
64+
else:
65+
from setproctitle import setproctitle
66+
67+
setproctitle(f"airflow api_server -- host:{args.host} port:{args.port}")
68+
69+
uvicorn_kwargs = {
70+
"host": args.host,
71+
"port": args.port,
72+
"workers": num_workers,
73+
"timeout_keep_alive": worker_timeout,
74+
"timeout_graceful_shutdown": worker_timeout,
75+
"ssl_keyfile": ssl_key,
76+
"ssl_certfile": ssl_cert,
77+
"access_log": True,
78+
"proxy_headers": proxy_headers,
79+
}
80+
# Only set the log_config if it is provided, otherwise use the default uvicorn logging configuration.
81+
if args.log_config and args.log_config != "-":
82+
# The [api/log_config] is migrated from [api/access_logfile] and [api/access_logfile] defaults to "-" for stdout for Gunicorn.
83+
# So we need to check if the log_config is set to "-" or not; if it is set to "-", we regard it as not set.
84+
uvicorn_kwargs["log_config"] = args.log_config
85+
86+
uvicorn.run(
87+
"airflow.api_fastapi.main:app",
88+
**uvicorn_kwargs,
89+
)
90+
91+
4392
@cli_utils.action_cli
4493
@providers_configuration_loaded
4594
def api_server(args):
4695
"""Start Airflow API server."""
4796
print(settings.HEADER)
4897

4998
apps = args.apps
50-
access_logfile = args.access_logfile or "-"
5199
num_workers = args.workers
52100
worker_timeout = args.worker_timeout
53101
proxy_headers = args.proxy_headers
@@ -74,6 +122,9 @@ def api_server(args):
74122
if args.proxy_headers:
75123
run_args.append("--proxy-headers")
76124

125+
if args.log_config and args.log_config != "-":
126+
run_args.extend(["--log-config", args.log_config])
127+
77128
# There is no way to pass the apps to airflow/api_fastapi/main.py in the development mode
78129
# because fastapi dev command does not accept any additional arguments
79130
# so environment variable is being used to pass it
@@ -85,35 +136,16 @@ def api_server(args):
85136
process.wait()
86137
os.environ.pop("AIRFLOW_API_APPS")
87138
else:
88-
if args.daemon:
89-
daemonize()
90-
log.info("Daemonized the API server process PID: %s", os.getpid())
91-
92-
log.info(
93-
textwrap.dedent(
94-
f"""\
95-
Running the uvicorn with:
96-
Apps: {apps}
97-
Workers: {num_workers}
98-
Host: {args.host}:{args.port}
99-
Timeout: {worker_timeout}
100-
Logfiles: {access_logfile}
101-
================================================================="""
102-
)
103-
)
104-
ssl_cert, ssl_key = _get_ssl_cert_and_key_filepaths(args)
105-
setproctitle(f"airflow api_server -- host:{args.host} port:{args.port}")
106-
uvicorn.run(
107-
"airflow.api_fastapi.main:app",
108-
host=args.host,
109-
port=args.port,
110-
workers=num_workers,
111-
timeout_keep_alive=worker_timeout,
112-
timeout_graceful_shutdown=worker_timeout,
113-
ssl_keyfile=ssl_key,
114-
ssl_certfile=ssl_cert,
115-
access_log=access_logfile,
116-
proxy_headers=proxy_headers,
139+
run_command_with_daemon_option(
140+
args=args,
141+
process_name="api_server",
142+
callback=lambda: _run_api_server(
143+
args=args,
144+
apps=apps,
145+
num_workers=num_workers,
146+
worker_timeout=worker_timeout,
147+
proxy_headers=proxy_headers,
148+
),
117149
)
118150

119151

‎airflow-core/src/airflow/config_templates/config.yml‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1391,13 +1391,14 @@ api:
13911391
type: integer
13921392
example: ~
13931393
default: "120"
1394-
access_logfile:
1394+
log_config:
13951395
description: |
1396-
Log files for the api server. '-' means log to stderr.
1396+
Path to the logging configuration file for the uvicorn server.
1397+
If not set, the default uvicorn logging configuration will be used.
13971398
version_added: ~
13981399
type: string
1399-
example: ~
1400-
default: "-"
1400+
example: path/to/logging_config.yaml
1401+
default: ~
14011402
ssl_cert:
14021403
description: |
14031404
Paths to the SSL certificate and key for the api server. When both are

‎airflow-core/src/airflow/configuration.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -365,6 +365,7 @@ def sensitive_config_values(self) -> set[tuple[str, str]]:
365365
("api", "secret_key"): ("webserver", "secret_key", "3.0.2"),
366366
("api", "enable_swagger_ui"): ("webserver", "enable_swagger_ui", "3.0.2"),
367367
("dag_processor", "parsing_pre_import_modules"): ("scheduler", "parsing_pre_import_modules", "3.0.3"),
368+
("api", "log_config"): ("api", "access_logfile", "3.0.4"),
368369
}
369370

370371
# A mapping of new section -> (old section, since_version).

‎airflow-core/tests/unit/cli/commands/_common_cli_classes.py‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
console = Console(width=400, color_system="standard")
3434

3535

36-
class _CommonCLIGunicornTestClass:
36+
class _CommonCLIUvicornTestClass:
3737
main_process_regexp: str = "process_to_look_for"
3838

3939
@pytest.fixture(autouse=True)
@@ -49,12 +49,12 @@ def _check_processes(self, ignore_running: bool):
4949
# Confirm that nmain procss hasn't been launched.
5050
# pgrep returns exit status 1 if no process matched.
5151
# Use more specific regexps (^) to avoid matching pytest run when running specific method.
52-
# For instance, we want to be able to do: pytest -k 'gunicorn'
52+
# For instance, we want to be able to do: pytest -k 'uvicorn'
5353
airflow_internal_api_pids = self._find_all_processes(self.main_process_regexp)
54-
gunicorn_pids = self._find_all_processes(r"gunicorn: ")
55-
if airflow_internal_api_pids or gunicorn_pids:
54+
uvicorn_pids = self._find_all_processes(r"uvicorn: ")
55+
if airflow_internal_api_pids or uvicorn_pids:
5656
console.print("[blue]Some processes are still running")
57-
for pid in gunicorn_pids + airflow_internal_api_pids:
57+
for pid in uvicorn_pids + airflow_internal_api_pids:
5858
with suppress(NoSuchProcess):
5959
console.print(psutil.Process(pid).as_dict(attrs=["pid", "name", "cmdline"]))
6060
console.print("[blue]Here list of processes ends")
@@ -63,9 +63,9 @@ def _check_processes(self, ignore_running: bool):
6363
for pid in airflow_internal_api_pids:
6464
with suppress(NoSuchProcess):
6565
psutil.Process(pid).kill()
66-
if gunicorn_pids:
67-
console.print("[yellow]Forcefully killing all gunicorn processes")
68-
for pid in gunicorn_pids:
66+
if uvicorn_pids:
67+
console.print("[yellow]Forcefully killing all uvicorn processes")
68+
for pid in uvicorn_pids:
6969
with suppress(NoSuchProcess):
7070
psutil.Process(pid).kill()
7171
if not ignore_running:

0 commit comments

Comments
 (0)