Skip to content

Commit b891917

Browse files
Carlos Sánchezcasaar97
authored andcommitted
fix: Dispose SQL registry connection pools after Gunicorn fork
Gunicorn workers inherited the parent's pooled SQL registry connections, so several workers shared one socket and the server crashed on startup. A post_fork hook, set for every run, now disposes the pools with close=False, so the parent's connections are left open. Fixes feast-dev#5784 Signed-off-by: Carlos Sánchez <[email protected]>
1 parent 1e5e90c commit b891917

5 files changed

Lines changed: 122 additions & 2 deletions

File tree

‎sdk/python/feast/feature_server.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1286,6 +1286,9 @@ def load_config(self):
12861286
self.cfg.set(key.lower(), value)
12871287

12881288
self.cfg.set("worker_class", "uvicorn_worker.UvicornWorker")
1289+
self.cfg.set(
1290+
"post_fork", functools.partial(_gunicorn_post_fork, self._store)
1291+
)
12891292
if self._metrics_enabled:
12901293
self.cfg.set(
12911294
"post_worker_init",
@@ -1296,6 +1299,11 @@ def load_config(self):
12961299
def load(self):
12971300
return self._app
12981301

1302+
def _gunicorn_post_fork(store: "feast.FeatureStore", server, worker):
1303+
"""Give each Gunicorn worker its own registry connections after fork."""
1304+
if store._registry is not None:
1305+
store._registry.reset_after_fork()
1306+
12991307
def _gunicorn_post_worker_init(store: "feast.FeatureStore", worker):
13001308
"""Start per-worker resource and freshness monitoring after Gunicorn forks."""
13011309
feast_metrics.init_worker_monitoring()

‎sdk/python/feast/infra/registry/base_registry.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1016,6 +1016,13 @@ def is_cache_valid(self) -> bool:
10161016
"""
10171017
return False
10181018

1019+
def reset_after_fork(self) -> None:
1020+
"""Drop resources inherited from the parent process after os.fork().
1021+
1022+
Called in a forked child (e.g. a Gunicorn worker) before it uses the
1023+
registry. Registries that hold no connection pools need not override it.
1024+
"""
1025+
10191026
# Lineage operations
10201027
def get_registry_lineage(
10211028
self,

‎sdk/python/feast/infra/registry/sql.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -388,6 +388,14 @@ def __init__(
388388
if not self.purge_feast_metadata:
389389
self._maybe_init_project_metadata(project)
390390

391+
def reset_after_fork(self) -> None:
392+
# close=False because the parent still owns those connections; closing
393+
# them would send a disconnect on a socket the parent shares. See
394+
# https://docs.sqlalchemy.org/en/20/core/pooling.html#using-connection-pools-with-multiprocessing-or-os-fork
395+
self.write_engine.dispose(close=False)
396+
if self.read_engine is not self.write_engine:
397+
self.read_engine.dispose(close=False)
398+
391399
@staticmethod
392400
def _verify_schema(engine: Engine, registry_metadata: MetaData = metadata) -> None:
393401
"""Verify that all expected registry tables exist in the database.

‎sdk/python/tests/unit/infra/registry/test_sql_registry.py‎

Lines changed: 57 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,16 @@
1313
# limitations under the License.
1414

1515
import logging
16+
import os
1617
import sys
1718
import tempfile
1819
import types
1920
from datetime import datetime, timedelta, timezone
20-
from unittest.mock import MagicMock
21+
from unittest.mock import MagicMock, patch
2122

2223
import dill
2324
import pytest
25+
from sqlalchemy.pool import QueuePool
2426

2527
from feast import Field
2628
from feast.data_source import PushSource
@@ -331,6 +333,60 @@ def test_warn_if_narrow_blob_columns_swallows_query_errors(caplog):
331333
assert "still typed BLOB" not in caplog.text
332334

333335

336+
@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork")
337+
@pytest.mark.filterwarnings(
338+
"ignore:This process .* is multi-threaded:DeprecationWarning"
339+
)
340+
def test_forked_child_does_not_reuse_parent_pooled_connection(sqlite_registry):
341+
"""A forked child (e.g. a Gunicorn worker) must not share the parent's pooled connections."""
342+
assert isinstance(sqlite_registry.write_engine.pool, QueuePool)
343+
with sqlite_registry.write_engine.connect() as conn:
344+
parent_conn = conn.connection.dbapi_connection
345+
346+
pid = os.fork()
347+
if pid == 0:
348+
try:
349+
sqlite_registry.reset_after_fork()
350+
with sqlite_registry.write_engine.connect() as conn:
351+
child_conn = conn.connection.dbapi_connection
352+
os._exit(1 if child_conn is parent_conn else 0)
353+
except BaseException:
354+
os._exit(2)
355+
_, status = os.waitpid(pid, 0)
356+
357+
assert os.waitstatus_to_exitcode(status) == 0
358+
# The parent's pooled connection is still usable.
359+
assert parent_conn.execute("SELECT 1").fetchone() == (1,)
360+
361+
362+
def test_reset_after_fork_disposes_pool_without_closing(sqlite_registry):
363+
with patch.object(sqlite_registry.write_engine, "dispose") as dispose:
364+
sqlite_registry.reset_after_fork()
365+
366+
dispose.assert_called_once_with(close=False)
367+
368+
369+
def test_reset_after_fork_disposes_read_and_write_engines(tmp_path):
370+
config = SqlRegistryConfig(
371+
registry_type="sql",
372+
path=f"sqlite:///{tmp_path / 'registry.db'}",
373+
read_path=f"sqlite:///{tmp_path / 'registry.db'}",
374+
purge_feast_metadata=False,
375+
)
376+
registry = SqlRegistry(config, "test_project", None)
377+
try:
378+
with (
379+
patch.object(registry.write_engine, "dispose") as write_dispose,
380+
patch.object(registry.read_engine, "dispose") as read_dispose,
381+
):
382+
registry.reset_after_fork()
383+
384+
write_dispose.assert_called_once_with(close=False)
385+
read_dispose.assert_called_once_with(close=False)
386+
finally:
387+
registry.teardown()
388+
389+
334390
def test_sql_registry(sqlite_registry):
335391
"""
336392
Test the SQL registry

‎sdk/python/tests/unit/test_feature_server.py‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
import time
1717
from collections import Counter
1818
from types import SimpleNamespace
19-
from unittest.mock import AsyncMock, MagicMock, patch
19+
from unittest.mock import AsyncMock, MagicMock, PropertyMock, patch
2020

2121
import pytest
2222
from fastapi.testclient import TestClient
@@ -782,6 +782,47 @@ def test_gunicorn_post_worker_init_starts_resource_and_freshness_monitoring():
782782
mock_fm.init_worker_freshness_monitoring.assert_called_once_with(mock_store)
783783

784784

785+
@pytest.mark.skipif(sys.platform == "win32", reason="Gunicorn is not used on Windows")
786+
@pytest.mark.parametrize("metrics_enabled", [True, False])
787+
def test_gunicorn_post_fork_hook_is_registered(metrics_enabled):
788+
from feast.feature_server import FeastServeApplication, _gunicorn_post_fork
789+
790+
mock_store = MagicMock()
791+
with patch("feast.feature_server.get_app"):
792+
app = FeastServeApplication(
793+
store=mock_store,
794+
metrics_enabled=metrics_enabled,
795+
registry_ttl_sec=5,
796+
workers=2,
797+
)
798+
799+
assert app.cfg.post_fork.func is _gunicorn_post_fork
800+
assert app.cfg.post_fork.args == (mock_store,)
801+
802+
803+
@pytest.mark.skipif(sys.platform == "win32", reason="Gunicorn is not used on Windows")
804+
def test_gunicorn_post_fork_resets_registry():
805+
from feast.feature_server import _gunicorn_post_fork
806+
807+
mock_store = MagicMock()
808+
_gunicorn_post_fork(mock_store, server=MagicMock(), worker=MagicMock())
809+
810+
mock_store._registry.reset_after_fork.assert_called_once_with()
811+
812+
813+
@pytest.mark.skipif(sys.platform == "win32", reason="Gunicorn is not used on Windows")
814+
def test_gunicorn_post_fork_does_not_build_registry():
815+
from feast.feature_server import _gunicorn_post_fork
816+
817+
mock_store = MagicMock()
818+
mock_store._registry = None
819+
registry = PropertyMock()
820+
type(mock_store).registry = registry
821+
_gunicorn_post_fork(mock_store, server=MagicMock(), worker=MagicMock())
822+
823+
registry.assert_not_called()
824+
825+
785826
def test_get_online_features_returns_feature_view_version_metadata(test_client):
786827
"""``include_feature_view_version_metadata=True`` must surface the
787828
``feature_view_metadata`` block over REST, like it does on the proto."""

0 commit comments

Comments
 (0)