Skip to content

Commit 60e57f5

Browse files
mdrxyopen-swe
andauthored
fix(fireworks): classify mid-stream read timeouts (#40874)
`ChatFireworks` now reports mid-stream read timeouts as retryable `ModelTimeoutError`s while remaining catchable as `httpx.ReadTimeout`. Partial streams are not automatically replayed. --- Fireworks stream read timeouts currently bypass LangChain's model-error classification after the first chunk. Classify them as retryable `ModelTimeoutError`s while preserving `httpx.ReadTimeout` compatibility, the original cause, and request context when available. - Covers synchronous and asynchronous stream consumption. - Keeps setup retries unchanged. Does **not** replay partial streams: doing so could duplicate text or corrupt tool-call arguments. Whole-generation recovery remains the caller's responsibility. - Built-in streaming retry and fallback behavior is unchanged: `.with_retry()` does not retry streaming, and `.with_fallbacks()` only switches models before the first chunk. Applications must explicitly handle recovery after output has started. Made by [Open SWE](https://github.com/langchain-ai/open-swe) · [view thread](https://openswe.vercel.app/agents/c2ba7088-1448-5e4d-9541-ae6f25b513c7) · openai:gpt-6-astra (medium) Co-authored-by: Mason Daugherty <[email protected]> Co-authored-by: open-swe[bot] <[email protected]>
1 parent 42f04f4 commit 60e57f5

2 files changed

Lines changed: 127 additions & 10 deletions

File tree

‎libs/partners/fireworks/langchain_fireworks/chat_models.py‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -653,6 +653,18 @@ class FireworksTimeoutError(APITimeoutError, ModelTimeoutError):
653653
"""Fireworks timeout error classified as a LangChain model error."""
654654

655655

656+
class FireworksReadTimeoutError(httpx.ReadTimeout, ModelTimeoutError):
657+
"""Fireworks stream read timeout classified as a retryable model error."""
658+
659+
660+
def _handle_stream_read_timeout(error: httpx.ReadTimeout) -> NoReturn:
661+
try:
662+
request = error.request
663+
except RuntimeError:
664+
request = None
665+
raise FireworksReadTimeoutError(str(error), request=request) from error
666+
667+
656668
def _handle_fireworks_invalid_request(e: BadRequestError) -> NoReturn:
657669
"""Promote prompt-too-long errors to `FireworksContextOverflowError`."""
658670
if "prompt is too long" in str(e):
@@ -815,13 +827,19 @@ async def _call() -> Any:
815827

816828
def _prepend_chunk(first: Any, rest: Iterator[Any]) -> Iterator[Any]:
817829
yield first
818-
yield from rest
830+
try:
831+
yield from rest
832+
except httpx.ReadTimeout as e:
833+
_handle_stream_read_timeout(e)
819834

820835

821836
async def _aprepend_chunk(first: Any, rest: AsyncIterator[Any]) -> AsyncIterator[Any]:
822837
yield first
823-
async for item in rest:
824-
yield item
838+
try:
839+
async for item in rest:
840+
yield item
841+
except httpx.ReadTimeout as e:
842+
_handle_stream_read_timeout(e)
825843

826844

827845
class ChatFireworks(BaseChatModel):

‎libs/partners/fireworks/tests/unit_tests/test_chat_models.py‎

Lines changed: 106 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,9 @@
55
import json
66
import logging
77
import os
8+
from collections.abc import AsyncIterator, Iterator
89
from typing import Any
9-
from unittest.mock import MagicMock
10+
from unittest.mock import AsyncMock, MagicMock
1011

1112
import httpx
1213
import pytest
@@ -841,7 +842,11 @@ def test_completion_with_retry_exhausts_and_raises() -> None:
841842
assert mock_client.create.call_count == 3
842843

843844

844-
def test_completion_with_retry_streaming_retries_on_setup() -> None:
845+
@pytest.mark.parametrize(
846+
"error",
847+
[_api_error(RateLimitError, "rate limited", 429), httpx.ReadTimeout("slow")],
848+
)
849+
def test_completion_with_retry_streaming_retries_on_setup(error: Exception) -> None:
845850
"""Streaming errors raised during the first-chunk pull are retried."""
846851
llm = _make_llm(max_retries=1)
847852

@@ -852,8 +857,7 @@ def _fail_then_stream(**_kwargs: Any) -> Any:
852857
if calls["n"] == 1:
853858

854859
def _failing_gen() -> Any:
855-
msg = "rate limited"
856-
raise _api_error(RateLimitError, msg, 429)
860+
raise error
857861
yield # pragma: no cover
858862

859863
return _failing_gen()
@@ -994,7 +998,13 @@ def test_chat_fireworks_invoke_routes_through_retry() -> None:
994998
assert mock_client.create.call_count == 2
995999

9961000

997-
async def test_acompletion_with_retry_streaming_retries_on_setup() -> None:
1001+
@pytest.mark.parametrize(
1002+
"error",
1003+
[_api_error(RateLimitError, "rate limited", 429), httpx.ReadTimeout("slow")],
1004+
)
1005+
async def test_acompletion_with_retry_streaming_retries_on_setup(
1006+
error: Exception,
1007+
) -> None:
9981008
"""Async streaming errors during the first-chunk pull are retried."""
9991009
llm = _make_llm(max_retries=1)
10001010
calls = {"n": 0}
@@ -1004,8 +1014,7 @@ async def _create(**_kwargs: Any) -> Any:
10041014
if calls["n"] == 1:
10051015

10061016
async def _failing_agen() -> Any:
1007-
msg = "rate limited"
1008-
raise _api_error(RateLimitError, msg, 429)
1017+
raise error
10091018
yield # pragma: no cover
10101019

10111020
return _failing_agen()
@@ -1546,6 +1555,96 @@ def test_extra_headers_forwarded_when_streaming(self) -> None:
15461555
assert "extra_headers" not in call_kwargs.get("extra_body", {})
15471556

15481557

1558+
@pytest.mark.parametrize("with_request", [False, True])
1559+
@pytest.mark.parametrize("async_mode", [False, True])
1560+
@pytest.mark.parametrize("tool_call", [False, True])
1561+
async def test_midstream_read_timeout_is_classified_without_replay(
1562+
*, with_request: bool, async_mode: bool, tool_call: bool
1563+
) -> None:
1564+
request = httpx.Request("POST", "https://api.fireworks.ai/inference/v1")
1565+
error = httpx.ReadTimeout(
1566+
"Timeout on reading data from socket",
1567+
request=request if with_request else None,
1568+
)
1569+
delta: dict[str, object] = (
1570+
{
1571+
"tool_calls": [
1572+
{
1573+
"index": 0,
1574+
"id": "call_1",
1575+
"type": "function",
1576+
"function": {"name": "lookup", "arguments": '{"query":'},
1577+
}
1578+
]
1579+
}
1580+
if tool_call
1581+
else {"content": "Hello"}
1582+
)
1583+
chunk: dict[str, object] = {"choices": [{"delta": delta, "index": 0}]}
1584+
1585+
def _stream() -> Iterator[dict[str, object]]:
1586+
yield chunk
1587+
raise error
1588+
1589+
async def _astream() -> AsyncIterator[dict[str, object]]:
1590+
yield chunk
1591+
raise error
1592+
1593+
model = _make_llm(max_retries=2)
1594+
received: list[AIMessageChunk] = []
1595+
with pytest.raises(httpx.ReadTimeout) as exc_info:
1596+
if async_mode:
1597+
model.async_client = MagicMock(create=AsyncMock(return_value=_astream()))
1598+
async for message in model.astream("Hello"):
1599+
received.append(message) # noqa: PERF401
1600+
else:
1601+
model.client = MagicMock(create=MagicMock(return_value=_stream()))
1602+
received.extend(model.stream("Hello"))
1603+
1604+
classified = exc_info.value
1605+
assert isinstance(classified, ModelTimeoutError)
1606+
assert classified.is_retryable
1607+
assert classified.__cause__ is error
1608+
assert str(classified) == str(error)
1609+
if with_request:
1610+
assert classified.request is request
1611+
else:
1612+
with pytest.raises(RuntimeError, match="request"):
1613+
_ = classified.request
1614+
assert len(received) == 1
1615+
if tool_call:
1616+
assert received[0].tool_call_chunks[0]["args"] == '{"query":'
1617+
else:
1618+
assert received[0].content == "Hello"
1619+
client = model.async_client if async_mode else model.client
1620+
client.create.assert_called_once()
1621+
1622+
1623+
@pytest.mark.parametrize("async_mode", [False, True])
1624+
async def test_midstream_non_timeout_propagates_unchanged(*, async_mode: bool) -> None:
1625+
error = ValueError("invalid stream")
1626+
1627+
def _stream() -> Iterator[dict[str, object]]:
1628+
yield {"choices": [{"delta": {"content": "Hello"}}]}
1629+
raise error
1630+
1631+
async def _astream() -> AsyncIterator[dict[str, object]]:
1632+
for chunk in _stream():
1633+
yield chunk
1634+
1635+
model = _make_llm(max_retries=2)
1636+
with pytest.raises(ValueError) as exc_info:
1637+
if async_mode:
1638+
model.async_client = MagicMock(create=AsyncMock(return_value=_astream()))
1639+
_ = [chunk async for chunk in model.astream("Hello")]
1640+
else:
1641+
model.client = MagicMock(create=MagicMock(return_value=_stream()))
1642+
list(model.stream("Hello"))
1643+
assert exc_info.value is error
1644+
client = model.async_client if async_mode else model.client
1645+
client.create.assert_called_once()
1646+
1647+
15491648
class TestStreamUsage:
15501649
"""Tests for the `stream_usage` field and `stream_options` plumbing."""
15511650

0 commit comments

Comments
 (0)