Skip to content

Commit 5d615ad

Browse files
mrembalskiclaude
andcommitted
fix(client): drain trailing bytes before closing streamed responses
Since 6132922 the stream helpers close the response right after `data: [DONE]`, before the HTTP/1.1 chunked terminator has been parsed, so httpcore never returns the connection to the pool and every streaming request opens a new TCP connection. Wrap the byte stream of streamed responses at the transport boundary so that closing it first re-reads the trailing bytes with a 100 ms bound, then closes as before. After `[DONE]` the only unread bytes are the terminator already in flight, so the drain completes from h11's buffer and the connection becomes reusable; a response abandoned mid-stream hits the bound and is closed exactly as today. Co-Authored-By: Claude Fable 5.1 <[email protected]>
1 parent 9301e31 commit 5d615ad

3 files changed

Lines changed: 420 additions & 2 deletions

File tree

‎src/openai/_base_client.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@
107107
from ._utils._json import openapi_dumps
108108
from ._utils._logs import get_http_method_for_logging
109109
from ._legacy_response import LegacyAPIResponse
110+
from ._connection_reuse import wrap_response_stream
110111

111112
log: logging.Logger = logging.getLogger(__name__)
112113
log.addFilter(SensitiveHeadersFilter())
@@ -1144,10 +1145,11 @@ def request(
11441145
log.debug("Sending HTTP Request: %s", get_http_method_for_logging(request.method))
11451146

11461147
response = None
1148+
stream_response_body = stream or self._should_stream_response_body(request=request)
11471149
try:
11481150
response = self._send_request(
11491151
request,
1150-
stream=stream or self._should_stream_response_body(request=request),
1152+
stream=stream_response_body,
11511153
**kwargs,
11521154
)
11531155
except timeout_exceptions() as err:
@@ -1182,6 +1184,9 @@ def request(
11821184
log.debug("Raising connection error")
11831185
raise APIConnectionError(request=request) from err
11841186

1187+
if stream_response_body:
1188+
wrap_response_stream(response)
1189+
11851190
log.debug(
11861191
"HTTP Response: %s %i",
11871192
get_http_method_for_logging(request.method),
@@ -1769,10 +1774,11 @@ async def request(
17691774
log.debug("Sending HTTP Request: %s", get_http_method_for_logging(request.method))
17701775

17711776
response = None
1777+
stream_response_body = stream or self._should_stream_response_body(request=request)
17721778
try:
17731779
response = await self._send_request(
17741780
request,
1775-
stream=stream or self._should_stream_response_body(request=request),
1781+
stream=stream_response_body,
17761782
**kwargs,
17771783
)
17781784
except timeout_exceptions() as err:
@@ -1807,6 +1813,9 @@ async def request(
18071813
log.debug("Raising connection error")
18081814
raise APIConnectionError(request=request) from err
18091815

1816+
if stream_response_body:
1817+
wrap_response_stream(response)
1818+
18101819
log.debug(
18111820
"HTTP Response: %s %i",
18121821
get_http_method_for_logging(request.method),

‎src/openai/_connection_reuse.py‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
from __future__ import annotations
2+
3+
import sys
4+
import time
5+
import logging
6+
from typing import Any, Iterator, AsyncIterator, cast
7+
8+
import httpx2
9+
10+
from ._utils import lru_cache
11+
from ._httpx2 import _loaded_legacy_httpx
12+
13+
log: logging.Logger = logging.getLogger(__name__)
14+
15+
# How long to wait for the bytes that trail a logically complete body before
16+
# giving up and closing the socket. After `data: [DONE]` the only thing left is
17+
# the HTTP/1.1 chunked terminator, which is already in flight when the consumer
18+
# stops reading, so this almost never waits at all.
19+
DRAIN_TIMEOUT = 0.1
20+
21+
22+
def wrap_response_stream(response: Any) -> None:
23+
"""Make closing a partially read streamed response drain its trailing bytes.
24+
25+
httpcore only returns a connection to the pool when the whole response body
26+
was parsed. The SDK stops reading at `data: [DONE]`, one chunk before the
27+
terminator, so without this every streaming request opens a new connection.
28+
"""
29+
stream = response.stream
30+
for name, module in _http_modules():
31+
sync_cls, async_cls = _wrapper_classes(name)
32+
if isinstance(stream, module.AsyncByteStream):
33+
response.stream = async_cls(stream, response.request)
34+
return
35+
if isinstance(stream, module.SyncByteStream):
36+
response.stream = sync_cls(stream, response.request)
37+
return
38+
39+
40+
def _http_modules() -> list[tuple[str, Any]]:
41+
modules: list[tuple[str, Any]] = [("httpx2", httpx2)]
42+
legacy = _loaded_legacy_httpx()
43+
if legacy is not None:
44+
modules.append(("httpx", cast(Any, legacy)))
45+
return modules
46+
47+
48+
class _ReadTimeout:
49+
"""Temporarily lower the read timeout httpcore picks up for a fresh body iteration."""
50+
51+
def __init__(self, request: Any, read_timeout: float) -> None:
52+
self._request = request
53+
self._read_timeout = read_timeout
54+
self._saved: Any = None
55+
56+
def __enter__(self) -> None:
57+
self._saved = self._request.extensions.get("timeout")
58+
self._request.extensions["timeout"] = {**(self._saved or {}), "read": self._read_timeout}
59+
60+
def __exit__(self, *_: object) -> None:
61+
if self._saved is None:
62+
self._request.extensions.pop("timeout", None)
63+
else:
64+
self._request.extensions["timeout"] = self._saved
65+
66+
67+
class _SyncDrain:
68+
_stream: Any
69+
_request: Any
70+
_exhausted: bool
71+
_closing: bool
72+
73+
def __init__(self, stream: Any, request: Any) -> None:
74+
self._stream = stream
75+
self._request = request
76+
self._exhausted = False
77+
self._closing = False
78+
79+
def __iter__(self) -> Iterator[bytes]:
80+
yield from self._stream
81+
self._exhausted = True
82+
83+
def close(self) -> None:
84+
# A body read to the end left h11 in DONE with an empty buffer; iterating
85+
# again would block on the socket for the whole drain timeout.
86+
if not self._exhausted and not self._closing:
87+
self._closing = True
88+
deadline = time.monotonic() + DRAIN_TIMEOUT
89+
try:
90+
with _ReadTimeout(self._request, DRAIN_TIMEOUT):
91+
for _ in self._stream:
92+
if time.monotonic() > deadline:
93+
break
94+
except Exception as exc:
95+
log.debug("Giving up on draining the response body: %s", type(exc).__name__)
96+
self._stream.close()
97+
98+
99+
class _AsyncDrain:
100+
_stream: Any
101+
_request: Any
102+
_exhausted: bool
103+
_closing: bool
104+
105+
def __init__(self, stream: Any, request: Any) -> None:
106+
self._stream = stream
107+
self._request = request
108+
self._exhausted = False
109+
self._closing = False
110+
111+
async def __aiter__(self) -> AsyncIterator[bytes]:
112+
async for chunk in self._stream:
113+
yield chunk
114+
self._exhausted = True
115+
116+
async def aclose(self) -> None:
117+
if not self._exhausted and not self._closing:
118+
self._closing = True
119+
deadline = time.monotonic() + DRAIN_TIMEOUT
120+
try:
121+
with _ReadTimeout(self._request, DRAIN_TIMEOUT):
122+
async for _ in self._stream:
123+
if time.monotonic() > deadline:
124+
break
125+
except Exception as exc:
126+
log.debug("Giving up on draining the response body: %s", type(exc).__name__)
127+
await self._stream.aclose()
128+
129+
130+
@lru_cache(maxsize=None)
131+
def _wrapper_classes(module_name: str) -> tuple[type[Any], type[Any]]:
132+
# `Response.close()` type-checks its stream against the ByteStream classes of
133+
# the module that produced it, so the wrappers must inherit from that module
134+
# (`httpx2`, or legacy `httpx` for clients built with it).
135+
module: Any = sys.modules[module_name]
136+
sync_cls = type("DrainingSyncByteStream", (_SyncDrain, module.SyncByteStream), {})
137+
async_cls = type("DrainingAsyncByteStream", (_AsyncDrain, module.AsyncByteStream), {})
138+
return sync_cls, async_cls

0 commit comments

Comments
 (0)