Skip to content

Commit b8826d8

Browse files
fix(streaming): scope bounded connection cleanup to async clients
1 parent 4466056 commit b8826d8

3 files changed

Lines changed: 25 additions & 99 deletions

File tree

‎src/openai/_streaming.py‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
from ._utils import is_mapping, extract_type_var_from_base
1414
from ._httpx2 import request_exceptions, timeout_exceptions
1515
from ._exceptions import APIError, APITimeoutError, APIConnectionError
16-
from ._streaming_cleanup import drain_response, drain_response_async
16+
from ._streaming_cleanup import drain_response_async
1717

1818
if TYPE_CHECKING:
1919
from ._client import OpenAI, AsyncOpenAI
@@ -69,11 +69,6 @@ def __stream__(self) -> Iterator[_T]:
6969
try:
7070
for sse in iterator:
7171
if sse.data.startswith("[DONE]"):
72-
# HTTP/1.1 needs the remaining body consumed for connection reuse.
73-
# HTTP/2 can close this stream and keep the connection, so draining
74-
# it would add unnecessary latency after [DONE].
75-
if response.http_version == "HTTP/1.1":
76-
drain_response(response)
7772
break
7873

7974
# we have to special case the Assistants `thread.` events since we won't have an "event" key in the data

‎src/openai/_streaming_cleanup.py‎

Lines changed: 2 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
from __future__ import annotations
22

3-
import time
4-
import socket
5-
import threading
6-
from typing import Iterable, AsyncIterator, cast
3+
from typing import AsyncIterator, cast
74

85
import anyio
96
import httpx2
@@ -14,50 +11,12 @@
1411
_DRAIN_TIMEOUT = 0.1
1512

1613

17-
def drain_response(response: httpx2.Response) -> None:
18-
# The standard HTTPX transports expose their socket through this extension.
19-
# Other sync transports cannot be interrupted safely; just close them.
20-
network_stream = response.extensions.get("network_stream")
21-
sock = network_stream.get_extra_info("socket") if network_stream is not None else None
22-
if not isinstance(sock, socket.socket) or response.is_closed:
23-
return
24-
25-
def interrupt_read() -> None:
26-
try:
27-
# Interrupt I/O without mutating SSLSocket's TLS state on this thread.
28-
socket.socket.shutdown(sock, socket.SHUT_RDWR)
29-
except OSError:
30-
pass # The peer or transport may already have closed the socket.
31-
32-
# A read timeout alone is insufficient: HTTP trailers can trickle in without
33-
# yielding body chunks. Interrupt the socket when the total budget expires.
34-
deadline = time.monotonic() + _DRAIN_TIMEOUT
35-
timer = threading.Timer(_DRAIN_TIMEOUT, interrupt_read)
36-
timer.daemon = True
37-
try:
38-
timer.start()
39-
except RuntimeError:
40-
return # Cleanup is optional if a watchdog thread cannot be started.
41-
try:
42-
# Drain below Response.iter_bytes() without decoding discarded bytes.
43-
# The watchdog supplies the deadline; the request's read timeout stays intact.
44-
for _ in cast(Iterable[bytes], response.stream):
45-
if time.monotonic() >= deadline:
46-
break
47-
except request_exceptions() + stream_consumed_exceptions():
48-
pass # Cleanup must not turn completed output into a request error.
49-
finally:
50-
timer.cancel()
51-
# The watchdog must finish before close() can return this connection to
52-
# the pool, or it could interrupt a subsequent request on the same socket.
53-
timer.join()
54-
55-
5614
async def drain_response_async(response: httpx2.Response) -> None:
5715
if response.is_closed:
5816
return
5917
try:
6018
with anyio.move_on_after(_DRAIN_TIMEOUT):
19+
# Discard raw transport bytes without decoding trailing SSE or content.
6120
async for _ in cast(AsyncIterator[bytes], response.stream):
6221
# Also enforce the deadline on transports yielding buffered data.
6322
await checkpoint()

‎tests/test_streaming.py‎

Lines changed: 22 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
import asyncio
88
import importlib
99
import threading
10-
from types import SimpleNamespace
1110
from typing import Any, Iterator, AsyncIterator
1211
from contextlib import aclosing, nullcontext
1312
from http.server import ThreadingHTTPServer, BaseHTTPRequestHandler
@@ -36,7 +35,7 @@ def http_module(request: pytest.FixtureRequest) -> Any:
3635
return importlib.import_module(request.param)
3736

3837

39-
@pytest.mark.parametrize("mode", ["sync", "async", "aiohttp"])
38+
@pytest.mark.parametrize("mode", ["async", "aiohttp"])
4039
@pytest.mark.parametrize("ending", ["complete", "truncate", "stall", "trickle"])
4140
async def test_done_http11_connection_reuse(mode: str, ending: str, http_module: Any) -> None:
4241
if mode == "aiohttp" and http_module.__name__ == "httpx":
@@ -68,10 +67,6 @@ def do_POST(self) -> None:
6867
b'"choices":[{"index":0,"delta":{"content":"hello"}}]}\n\ndata: [DONE]\n\n'
6968
)
7069
try:
71-
if requests == 2 and ending == "complete":
72-
# A cancelled watchdog must not interrupt the next request
73-
# after the connection has been returned to the pool.
74-
release.wait(timeout=0.15)
7570
self.wfile.write(f"{len(payload):x}\r\n".encode() + payload + b"\r\n")
7671
if requests == 1:
7772
if ending == "truncate":
@@ -108,30 +103,20 @@ def log_message(self, format: str, *args: object) -> None: # noqa: A002
108103
}
109104
transport_options: dict[str, Any] = {"trust_env": False, "limits": http_module.Limits(max_connections=1)}
110105
try:
111-
if mode == "sync":
112-
with OpenAI(**options, http_client=http_module.Client(**transport_options)) as client:
113-
for _ in range(2):
114-
with client.chat.completions.create(model="synthetic", messages=[], stream=True) as stream:
115-
assert next(stream).choices[0].delta.content == "hello"
116-
start = time.monotonic()
117-
assert list(stream) == []
118-
assert time.monotonic() - start < 0.4
119-
assert stream.response.request.extensions["timeout"]["read"] is None
120-
else:
121-
http_client = (
122-
DefaultAioHttpClient(**transport_options)
123-
if mode == "aiohttp"
124-
else http_module.AsyncClient(**transport_options)
125-
)
126-
async with AsyncOpenAI(**options, http_client=http_client) as async_client:
127-
for _ in range(2):
128-
async with await async_client.chat.completions.create(
129-
model="synthetic", messages=[], stream=True
130-
) as async_stream:
131-
assert (await async_stream.__anext__()).choices[0].delta.content == "hello"
132-
start = time.monotonic()
133-
assert [chunk async for chunk in async_stream] == []
134-
assert time.monotonic() - start < 0.4
106+
http_client = (
107+
DefaultAioHttpClient(**transport_options)
108+
if mode == "aiohttp"
109+
else http_module.AsyncClient(**transport_options)
110+
)
111+
async with AsyncOpenAI(**options, http_client=http_client) as async_client:
112+
for _ in range(2):
113+
async with await async_client.chat.completions.create(
114+
model="synthetic", messages=[], stream=True
115+
) as async_stream:
116+
assert (await async_stream.__anext__()).choices[0].delta.content == "hello"
117+
start = time.monotonic()
118+
assert [chunk async for chunk in async_stream] == []
119+
assert time.monotonic() - start < 0.4
135120
finally:
136121
release.set()
137122
server.shutdown()
@@ -146,7 +131,7 @@ def log_message(self, format: str, *args: object) -> None: # noqa: A002
146131

147132
@pytest.mark.parametrize("sync", [True, False], ids=["sync", "async"])
148133
@pytest.mark.parametrize("http_version", [b"HTTP/1.1", b"HTTP/2"])
149-
async def test_done_discards_bytes_only_for_http11(
134+
async def test_done_discards_bytes_only_for_async_http11(
150135
sync: bool, http_version: bytes, http_module: Any, client: OpenAI, async_client: AsyncOpenAI
151136
) -> None:
152137
read_tail = False
@@ -167,26 +152,13 @@ def __iter__(self) -> Iterator[bytes]:
167152
def __aiter__(self) -> AsyncIterator[bytes]:
168153
return async_iterator
169154

170-
with socket.socket() as sock:
171-
172-
def socket_info(_name: str) -> socket.socket:
173-
return sock
174-
175-
response = http_module.Response(
176-
200,
177-
stream=Body(),
178-
request=http_module.Request("GET", "http://example.test"),
179-
extensions={
180-
"http_version": http_version,
181-
"network_stream": SimpleNamespace(get_extra_info=socket_info),
182-
},
183-
)
184-
if sync:
185-
assert list(Stream(cast_to=object, response=response, client=client)) == []
186-
else:
187-
assert [event async for event in AsyncStream(cast_to=object, response=response, client=async_client)] == []
155+
response = http_module.Response(200, stream=Body(), extensions={"http_version": http_version})
156+
if sync:
157+
assert list(Stream(cast_to=object, response=response, client=client)) == []
158+
else:
159+
assert [event async for event in AsyncStream(cast_to=object, response=response, client=async_client)] == []
188160
assert response.is_closed
189-
assert read_tail == (http_version == b"HTTP/1.1")
161+
assert read_tail == (not sync and http_version == b"HTTP/1.1")
190162

191163

192164
@pytest.mark.parametrize("cancel", [True, False], ids=["caller-cancel", "deadline"])

0 commit comments

Comments
 (0)