Skip to content

Commit 9237fb0

Browse files
aborgatinclaude
authored andcommitted
fix: Do not log /metrics requests from the pre-fork metrics server thread
The metrics HTTP server runs as a thread in the Gunicorn master, which forks the workers. Its default WSGIRequestHandler writes an access-log line to stderr for every scrape. A fork while that thread holds the stderr buffer lock leaves the new worker with the lock held forever: it blocks on its first log line ("Booting worker") and never serves. Use a request handler with a no-op log_message for both the IPv4 and the dual-stack server built by _make_metrics_httpd, like prometheus_client's own _SilentHandler. Fixes feast-dev#6928 Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]> Signed-off-by: Alexandr Borgatin <[email protected]>
1 parent a620f5c commit 9237fb0

2 files changed

Lines changed: 49 additions & 3 deletions

File tree

‎sdk/python/feast/metrics.py‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@
5454
from dataclasses import dataclass
5555
from datetime import datetime, timezone
5656
from typing import TYPE_CHECKING, Callable, Iterable, List, Optional
57-
from wsgiref.simple_server import WSGIServer, make_server
57+
from wsgiref.simple_server import WSGIRequestHandler, WSGIServer, make_server
5858

5959
import psutil
6060

@@ -585,6 +585,22 @@ def init_worker_freshness_monitoring(store: "FeatureStore"):
585585
t.start()
586586

587587

588+
class _QuietWSGIRequestHandler(WSGIRequestHandler):
589+
"""Request handler that does not log each scrape to stderr.
590+
591+
The metrics server runs as a thread in the Gunicorn master, which forks
592+
the workers. The default handler writes an access-log line to stderr for
593+
every request; a fork while that thread holds the stderr buffer lock
594+
leaves the new worker with the lock held forever, and the worker hangs
595+
on its first log line ("Booting worker") before it serves anything.
596+
Same class of problem as #6647. prometheus_client's own server uses a
597+
silent handler for the same reason.
598+
"""
599+
600+
def log_message(self, format, *args): # noqa: A002 - signature of the base class
601+
pass
602+
603+
588604
def _make_metrics_httpd(port: int, app: Callable[..., Iterable[bytes]]) -> WSGIServer:
589605
"""Build the metrics HTTP server, dual-stack ("::") when the host
590606
supports IPv6, or IPv4-only (falls back to `make_server`'s default
@@ -593,7 +609,7 @@ def _make_metrics_httpd(port: int, app: Callable[..., Iterable[bytes]]) -> WSGIS
593609
from feast import utils
594610

595611
if not utils._ipv6_available():
596-
return make_server("", port, app)
612+
return make_server("", port, app, handler_class=_QuietWSGIRequestHandler)
597613

598614
class DualStackWSGIServer(WSGIServer):
599615
address_family = socket.AF_INET6
@@ -602,7 +618,13 @@ def server_bind(self) -> None:
602618
self.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0)
603619
super().server_bind()
604620

605-
return make_server("::", port, app, server_class=DualStackWSGIServer)
621+
return make_server(
622+
"::",
623+
port,
624+
app,
625+
server_class=DualStackWSGIServer,
626+
handler_class=_QuietWSGIRequestHandler,
627+
)
606628

607629

608630
def start_metrics_server(

‎sdk/python/tests/unit/test_metrics.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import pytest
2323

2424
from feast.metrics import (
25+
_make_metrics_httpd,
2526
emit_offline_audit_log,
2627
emit_online_audit_log,
2728
feature_freshness_seconds,
@@ -2085,3 +2086,26 @@ def test_start_metrics_server_wires_make_metrics_httpd(mocker):
20852086
break
20862087
time.sleep(0.01)
20872088
assert mock_httpd.serve_forever.called
2089+
2090+
2091+
class TestMetricsHttpdDoesNotLog:
2092+
"""The metrics server must not write to stderr per request (fork safety, see _QuietWSGIRequestHandler)."""
2093+
2094+
def test_scrape_writes_nothing_to_stderr(self, capsys):
2095+
def app(environ, start_response):
2096+
start_response("200 OK", [("Content-Type", "text/plain")])
2097+
return [b"ok"]
2098+
2099+
httpd = _make_metrics_httpd(0, app)
2100+
port = httpd.server_address[1]
2101+
server = threading.Thread(target=httpd.handle_request, daemon=True)
2102+
server.start()
2103+
try:
2104+
conn = http.client.HTTPConnection("127.0.0.1", port, timeout=5)
2105+
conn.request("GET", "/metrics")
2106+
assert conn.getresponse().status == 200
2107+
server.join(timeout=5)
2108+
finally:
2109+
httpd.server_close()
2110+
2111+
assert "GET /metrics" not in capsys.readouterr().err

0 commit comments

Comments
 (0)