Skip to content

Commit aad0c15

Browse files
authored
feat: Add --gc-freeze option to feast serve (#7089)
1 parent dec2f24 commit aad0c15

7 files changed

Lines changed: 109 additions & 2 deletions

File tree

‎docs/how-to-guides/online-server-performance-tuning.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,10 @@ feast serve \
185185
**Sizing rule of thumb:** Start with `workers = 2 × vCPU + 1` and allocate ~256–512 MB memory per worker. Monitor RSS per process and adjust. If p99 latency is high under load, add more workers or scale horizontally rather than increasing connections.
186186
{% endhint %}
187187

188+
### Garbage collection pauses
189+
190+
Every full Python garbage collection walks all live objects, including the registry and feature definitions that live for the whole process. In a worker this blocks the event loop, and while it runs every waiting request stalls. The stall shows up as a slow online store read. With `--gc-freeze` (or `feature_server.gc_freeze: true` in `feature_store.yaml`) the server runs `gc.freeze()` after it has loaded the registry and before the workers start, so full collections skip these objects and stay short. Try it if p99.9 latency has spikes that the online store does not explain.
191+
188192
---
189193

190194
## Registry cache tuning

‎docs/reference/feature-servers/python-feature-server.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ Key performance options:
3030
- `--max-requests-jitter`: Jitter to prevent thundering herd on worker restart (default: 50)
3131
- `--registry_ttl_sec, -r`: Registry refresh interval in seconds. Higher values reduce overhead but increase staleness (default: 60)
3232
- `--keep-alive-timeout`: Keep-alive connection timeout in seconds (default: 30)
33+
- `--gc-freeze`: Run `gc.freeze()` after startup to shorten full garbage collection pauses. See [Garbage collection pauses](../../how-to-guides/online-server-performance-tuning.md#garbage-collection-pauses) (default: off)
3334

3435
### Performance Best Practices
3536

‎sdk/python/feast/cli/serve.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,12 @@
116116
show_default=True,
117117
help="Enable the Metrics Server",
118118
)
119+
@click.option(
120+
"--gc-freeze",
121+
is_flag=True,
122+
show_default=True,
123+
help="Run gc.freeze() after startup so full garbage collections skip the registry and other startup objects",
124+
)
119125
@click.pass_context
120126
def serve_command(
121127
ctx: click.Context,
@@ -132,6 +138,7 @@ def serve_command(
132138
tls_key_path: str,
133139
tls_cert_path: str,
134140
metrics: bool,
141+
gc_freeze: bool,
135142
):
136143
"""Start a feature server locally on a given port."""
137144
if (tls_key_path and not tls_cert_path) or (not tls_key_path and tls_cert_path):
@@ -158,6 +165,7 @@ def serve_command(
158165
tls_key_path=tls_key_path,
159166
tls_cert_path=tls_cert_path,
160167
registry_ttl_sec=registry_ttl_sec,
168+
gc_freeze=gc_freeze,
161169
)
162170

163171

‎sdk/python/feast/feature_server.py‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
import asyncio
1616
import functools
17+
import gc
1718
import os
1819
import sys
1920
import threading
@@ -1306,6 +1307,21 @@ def _gunicorn_child_exit(server, worker):
13061307
feast_metrics.mark_process_dead(worker.pid)
13071308

13081309

1310+
def _freeze_gc() -> None:
1311+
"""Move the objects that exist now out of the cyclic garbage collector.
1312+
1313+
The registry, feature definitions and protobuf classes are built once and
1314+
live until the process exits, yet every full collection walks all of them.
1315+
In a worker that walk blocks the event loop, so the requests waiting on it
1316+
look like slow online store reads. gc.collect() runs first so no garbage is
1317+
frozen. Called in the gunicorn master before the workers are forked, so
1318+
every worker starts frozen, including one restarted by --max-requests.
1319+
"""
1320+
gc.collect()
1321+
gc.freeze()
1322+
logger.info(f"gc.freeze(): {gc.get_freeze_count()} objects frozen")
1323+
1324+
13091325
def start_server(
13101326
store: "feast.FeatureStore",
13111327
host: str,
@@ -1320,6 +1336,7 @@ def start_server(
13201336
tls_key_path: str,
13211337
tls_cert_path: str,
13221338
metrics: bool,
1339+
gc_freeze: bool = False,
13231340
):
13241341
if (tls_key_path and not tls_cert_path) or (not tls_key_path and tls_cert_path):
13251342
raise ValueError(
@@ -1331,6 +1348,7 @@ def start_server(
13311348

13321349
metrics_from_config = getattr(metrics_cfg, "enabled", False)
13331350
metrics_active = metrics or metrics_from_config
1351+
gc_freeze_active = gc_freeze or getattr(fs_cfg, "gc_freeze", False)
13341352
uses_gunicorn = sys.platform != "win32"
13351353
if metrics_active:
13361354
flags = feast_metrics.build_metrics_flags(metrics_cfg)
@@ -1369,13 +1387,18 @@ def start_server(
13691387
if tls_key_path and tls_cert_path:
13701388
options["keyfile"] = tls_key_path
13711389
options["certfile"] = tls_cert_path
1372-
FeastServeApplication(
1390+
application = FeastServeApplication(
13731391
store=store, metrics_enabled=metrics_active, **options
1374-
).run()
1392+
)
1393+
if gc_freeze_active:
1394+
_freeze_gc()
1395+
application.run()
13751396
else:
13761397
import uvicorn
13771398

13781399
app = get_app(store, registry_ttl_sec)
1400+
if gc_freeze_active:
1401+
_freeze_gc()
13791402
if tls_key_path and tls_cert_path:
13801403
uvicorn.run(
13811404
app,

‎sdk/python/feast/feature_store.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4778,6 +4778,7 @@ def serve(
47784778
tls_key_path: str = "",
47794779
tls_cert_path: str = "",
47804780
registry_ttl_sec: int = 60,
4781+
gc_freeze: bool = False,
47814782
) -> None:
47824783
"""Start the feature consumption server locally on a given port."""
47834784
type_ = type_.lower()
@@ -4800,6 +4801,7 @@ def serve(
48004801
tls_key_path=tls_key_path,
48014802
tls_cert_path=tls_cert_path,
48024803
registry_ttl_sec=registry_ttl_sec,
4804+
gc_freeze=gc_freeze,
48034805
)
48044806

48054807
def get_feature_server_endpoint(self) -> Optional[str]:

‎sdk/python/feast/infra/feature_servers/base_config.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,12 @@ class BaseFeatureServerConfig(FeastConfigBaseModel):
143143
"""Prometheus metrics configuration. Set ``metrics.enabled: true`` or
144144
pass the ``feast serve --metrics`` CLI flag to activate."""
145145

146+
gc_freeze: StrictBool = False
147+
"""Run ``gc.freeze()`` after the server has loaded the registry and before
148+
the workers start. Full garbage collections then skip these long-lived
149+
objects, which keeps the event loop pauses they cause short. Same as
150+
``feast serve --gc-freeze``."""
151+
146152
feature_logging: Optional[FeatureLoggingConfig] = None
147153
""" Feature logging configuration """
148154

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
import sys
2+
from types import SimpleNamespace
3+
from unittest.mock import MagicMock, patch
4+
5+
import pytest
6+
7+
pytestmark = pytest.mark.skipif(
8+
sys.platform == "win32", reason="gunicorn is not used on Windows"
9+
)
10+
11+
12+
def _start_server(feature_server_config, gc_freeze: bool):
13+
"""Run start_server with the gunicorn app mocked; return the call order."""
14+
from feast.feature_server import start_server
15+
16+
calls = MagicMock()
17+
store = MagicMock()
18+
store.config = SimpleNamespace(
19+
feature_server=feature_server_config, auth_config=MagicMock()
20+
)
21+
application = MagicMock()
22+
application.run.side_effect = lambda: calls.run()
23+
with (
24+
patch("feast.feature_server.feast_metrics"),
25+
patch("feast.feature_server.str_to_auth_manager_type"),
26+
patch("feast.feature_server.init_security_manager"),
27+
patch("feast.feature_server.init_auth_manager"),
28+
patch("feast.feature_server.FeastServeApplication", return_value=application),
29+
patch("feast.feature_server.gc.collect", side_effect=calls.collect),
30+
patch("feast.feature_server.gc.freeze", side_effect=calls.freeze),
31+
):
32+
start_server(
33+
store=store,
34+
host="127.0.0.1",
35+
port=6566,
36+
no_access_log=True,
37+
workers=1,
38+
worker_connections=1000,
39+
max_requests=1000,
40+
max_requests_jitter=50,
41+
keep_alive_timeout=30,
42+
registry_ttl_sec=60,
43+
tls_key_path="",
44+
tls_cert_path="",
45+
metrics=False,
46+
gc_freeze=gc_freeze,
47+
)
48+
return [name for name, _, _ in calls.mock_calls]
49+
50+
51+
def test_cli_flag_freezes_before_workers_start():
52+
assert _start_server(None, gc_freeze=True) == ["collect", "freeze", "run"]
53+
54+
55+
def test_config_enables_freeze():
56+
config = SimpleNamespace(metrics=None, gc_freeze=True)
57+
assert _start_server(config, gc_freeze=False) == ["collect", "freeze", "run"]
58+
59+
60+
def test_no_freeze_by_default():
61+
config = SimpleNamespace(metrics=None, gc_freeze=False)
62+
assert _start_server(config, gc_freeze=False) == ["run"]
63+
assert _start_server(None, gc_freeze=False) == ["run"]

0 commit comments

Comments
 (0)