Skip to content

Commit 88b9727

Browse files
mdrxyopen-swe
andauthored
feat(fireworks): add prompt caching middleware (#38823)
Added `FireworksPromptCachingMiddleware` to improve prompt-cache reuse across calls in the same agent thread. Explicit affinity settings take precedence, and no affinity is generated without a thread ID. --- Fireworks agents need consistent routing to reuse a replica's prompt cache across turns. `FireworksPromptCachingMiddleware` supplies session affinity from a SHA-256 hash of `config.configurable.thread_id`, while respecting explicit `user`, `prompt_cache_key`, and `x-session-affinity` settings on the selected model or request. Affinity is scoped to the call and applied by `ChatFireworks` when invoking the API. Generated affinity stays out of shared request settings, and model-local headers remain scoped to their owning model, including during fallback. This works with either ordering of the caching and fallback middleware for a Fireworks primary model. Related fallback cleanup for explicitly supplied cache settings is in #40886, stacked on this PR. This PR works independently of that change. --------- Co-authored-by: open-swe[bot] <[email protected]>
1 parent 60e57f5 commit 88b9727

8 files changed

Lines changed: 1121 additions & 75 deletions

File tree

‎libs/partners/fireworks/langchain_fireworks/chat_models.py‎

Lines changed: 62 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import json
77
import logging
88
from collections.abc import AsyncIterator, Callable, Iterator, Mapping, Sequence
9+
from contextvars import ContextVar
910
from operator import itemgetter
1011
from typing import (
1112
Any,
@@ -123,6 +124,10 @@
123124

124125
logger = logging.getLogger(__name__)
125126

127+
_PROMPT_CACHE_AFFINITY: ContextVar[str | None] = ContextVar(
128+
"fireworks_prompt_cache_affinity", default=None
129+
)
130+
126131

127132
_MODEL_PROFILES = cast("ModelProfileRegistry", _PROFILES)
128133

@@ -759,14 +764,67 @@ def _prepare_sdk_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]:
759764
return kwargs
760765

761766

767+
def _merge_model_headers(llm: ChatFireworks, kwargs: dict[str, Any]) -> dict[str, Any]:
768+
"""Merge model-local headers only when invoking their owning model."""
769+
model_headers = llm.model_kwargs.get("extra_headers")
770+
request_headers = kwargs.get("extra_headers")
771+
if isinstance(model_headers, Mapping) and isinstance(request_headers, Mapping):
772+
# HTTP header names are case-insensitive; retain the request's spelling.
773+
headers = {
774+
key.lower(): (key, value)
775+
for source in (model_headers, request_headers)
776+
for key, value in source.items()
777+
}
778+
return {
779+
**kwargs,
780+
"extra_headers": dict(headers.values()),
781+
}
782+
return kwargs
783+
784+
785+
def _apply_prompt_cache_affinity(kwargs: dict[str, Any]) -> dict[str, Any]:
786+
"""Apply the middleware default after the selected model's settings are merged."""
787+
affinity = _PROMPT_CACHE_AFFINITY.get()
788+
if affinity is None:
789+
return kwargs
790+
791+
# The SDK merges extra_body over top-level fields before sending the request.
792+
extra_body = kwargs.get("extra_body")
793+
body = {**kwargs, **extra_body} if isinstance(extra_body, Mapping) else kwargs
794+
if any(body.get(key) for key in ("user", "prompt_cache_key")):
795+
return kwargs
796+
797+
headers = kwargs.get("extra_headers")
798+
if isinstance(headers, Mapping):
799+
if any(
800+
isinstance(key, str) and key.lower() == "x-session-affinity"
801+
for key in headers
802+
):
803+
return kwargs
804+
elif headers is not None:
805+
logger.warning(
806+
"Cannot set Fireworks session affinity because extra_headers is %s",
807+
type(headers).__name__,
808+
)
809+
return kwargs
810+
811+
return {
812+
**kwargs,
813+
"prompt_cache_key": affinity,
814+
"extra_headers": {**(headers or {}), "x-session-affinity": affinity},
815+
}
816+
817+
762818
def _completion_with_retry(
763819
llm: ChatFireworks,
764820
run_manager: CallbackManagerForLLMRun | None = None,
765821
**kwargs: Any,
766822
) -> Any:
767823
"""Retry the sync completion call, including stream setup."""
768824
retry_decorator = _create_retry_decorator(llm, run_manager=run_manager)
769-
kwargs = _prepare_sdk_kwargs(kwargs)
825+
kwargs = _prepare_sdk_kwargs(
826+
_apply_prompt_cache_affinity(_merge_model_headers(llm, kwargs))
827+
)
770828

771829
@retry_decorator
772830
def _call() -> Any:
@@ -799,7 +857,9 @@ async def _acompletion_with_retry(
799857
) -> Any:
800858
"""Retry the async completion call, including stream setup."""
801859
retry_decorator = _create_retry_decorator(llm, run_manager=run_manager)
802-
kwargs = _prepare_sdk_kwargs(kwargs)
860+
kwargs = _prepare_sdk_kwargs(
861+
_apply_prompt_cache_affinity(_merge_model_headers(llm, kwargs))
862+
)
803863

804864
@retry_decorator
805865
async def _call() -> Any:
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
"""Middleware for Fireworks models."""
2+
3+
from langchain_fireworks.middleware.prompt_caching import (
4+
FireworksPromptCachingMiddleware,
5+
)
6+
7+
__all__ = ["FireworksPromptCachingMiddleware"]
Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
"""Middleware for Fireworks prompt-cache session affinity.
2+
3+
Requires `langchain` for the agent middleware framework. It is imported lazily,
4+
and an `ImportError` with install guidance is raised if it is missing.
5+
`ChatFireworks` is provided by this package (`langchain-fireworks`).
6+
"""
7+
8+
from __future__ import annotations
9+
10+
import hashlib
11+
import logging
12+
from collections.abc import Awaitable, Callable, Iterator
13+
from contextlib import contextmanager
14+
from typing import Literal
15+
from warnings import warn
16+
17+
from langchain_fireworks.chat_models import _PROMPT_CACHE_AFFINITY, ChatFireworks
18+
19+
try:
20+
from langchain.agents.middleware.types import (
21+
AgentMiddleware,
22+
ModelCallResult,
23+
ModelRequest,
24+
ModelResponse,
25+
)
26+
from langgraph.config import get_config
27+
except ImportError as e:
28+
msg = (
29+
"FireworksPromptCachingMiddleware requires 'langchain' to be installed "
30+
f"(missing module: {e.name}). This middleware is designed for use with "
31+
"LangChain agents. Install it with: pip install langchain"
32+
)
33+
raise ImportError(msg) from e
34+
35+
logger = logging.getLogger(__name__)
36+
37+
_UNSUPPORTED_MODEL_BEHAVIORS = ("ignore", "warn", "raise")
38+
39+
40+
def _get_thread_id(request: ModelRequest) -> str | None:
41+
"""Return the thread ID from request runtime metadata or runnable config."""
42+
# Runtime metadata is available even without async config propagation on
43+
# Python 3.10. Older runtimes may not expose execution_info.
44+
execution_info = getattr(request.runtime, "execution_info", None)
45+
thread_id = getattr(execution_info, "thread_id", None)
46+
if isinstance(thread_id, str) and thread_id:
47+
return thread_id
48+
49+
try:
50+
config = get_config()
51+
except RuntimeError:
52+
return None
53+
thread_id = (config.get("configurable") or {}).get("thread_id")
54+
if isinstance(thread_id, str) and thread_id:
55+
return thread_id
56+
return None
57+
58+
59+
@contextmanager
60+
def _session_affinity(request: ModelRequest) -> Iterator[None]:
61+
"""Scope an affinity default to this call, including any fallback attempts."""
62+
thread_id = _get_thread_id(request)
63+
affinity = (
64+
hashlib.sha256(thread_id.encode("utf-8")).hexdigest()
65+
if thread_id is not None
66+
else None
67+
)
68+
if affinity is None:
69+
logger.debug("Fireworks session affinity not applied: no thread_id in config")
70+
token = _PROMPT_CACHE_AFFINITY.set(affinity)
71+
try:
72+
yield
73+
finally:
74+
_PROMPT_CACHE_AFFINITY.reset(token)
75+
76+
77+
class FireworksPromptCachingMiddleware(AgentMiddleware):
78+
"""Set Fireworks prompt-cache session affinity from the active thread ID.
79+
80+
Fireworks prompt caching is enabled by default. This middleware improves
81+
cache hit rate by pinning session affinity to a SHA-256 hash of
82+
`config.configurable.thread_id`, so related requests route to the same
83+
replica and reuse its warm cache. The hexadecimal hash keeps affinity values
84+
safe for HTTP headers, including when thread IDs contain Unicode.
85+
86+
The middleware supplies a scoped default that `ChatFireworks` applies to
87+
`prompt_cache_key` and `extra_headers["x-session-affinity"]` when invoking
88+
the API. Explicit `user` or `prompt_cache_key` body fields (including
89+
`extra_body` overrides), or `x-session-affinity` headers on the selected
90+
model or request take precedence, including on fallback models. No affinity
91+
is added when no thread ID is configured.
92+
93+
Generated affinity stays out of shared request settings, so it is never
94+
forwarded to another provider. This works with either ordering of this
95+
middleware and `ModelFallbackMiddleware` for a Fireworks primary model.
96+
"""
97+
98+
def __init__(
99+
self,
100+
*,
101+
unsupported_model_behavior: Literal["ignore", "warn", "raise"] = "warn",
102+
) -> None:
103+
"""Initialize the middleware.
104+
105+
Args:
106+
unsupported_model_behavior: Behavior when the request model is not
107+
`ChatFireworks`. `"ignore"` continues without affinity,
108+
`"warn"` emits a warning, and `"raise"` raises `ValueError`.
109+
110+
Raises:
111+
ValueError: If `unsupported_model_behavior` is not one of
112+
`"ignore"`, `"warn"`, or `"raise"`.
113+
"""
114+
if unsupported_model_behavior not in _UNSUPPORTED_MODEL_BEHAVIORS:
115+
msg = (
116+
"unsupported_model_behavior must be one of "
117+
f"{_UNSUPPORTED_MODEL_BEHAVIORS}, got {unsupported_model_behavior!r}."
118+
)
119+
raise ValueError(msg)
120+
self.unsupported_model_behavior = unsupported_model_behavior
121+
122+
def _should_apply_caching(self, request: ModelRequest) -> bool:
123+
"""Return whether session affinity should be applied to the request.
124+
125+
Args:
126+
request: The model request to check.
127+
128+
Returns:
129+
`True` if the request model is `ChatFireworks`, else `False`.
130+
131+
Raises:
132+
ValueError: If the model is unsupported and
133+
`unsupported_model_behavior` is `"raise"`.
134+
"""
135+
if isinstance(request.model, ChatFireworks):
136+
return True
137+
138+
msg = (
139+
"FireworksPromptCachingMiddleware only supports ChatFireworks, "
140+
f"not {type(request.model).__name__}."
141+
)
142+
if self.unsupported_model_behavior == "raise":
143+
raise ValueError(msg)
144+
if self.unsupported_model_behavior == "warn":
145+
warn(msg, stacklevel=3)
146+
return False
147+
148+
def wrap_model_call(
149+
self,
150+
request: ModelRequest,
151+
handler: Callable[[ModelRequest], ModelResponse],
152+
) -> ModelCallResult:
153+
"""Supply default Fireworks affinity while executing the handler.
154+
155+
Args:
156+
request: The outgoing model request.
157+
handler: Callable that executes the request.
158+
159+
Returns:
160+
The result produced by `handler`.
161+
"""
162+
if not self._should_apply_caching(request):
163+
return handler(request)
164+
with _session_affinity(request):
165+
return handler(request)
166+
167+
async def awrap_model_call(
168+
self,
169+
request: ModelRequest,
170+
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
171+
) -> ModelCallResult:
172+
"""Supply default Fireworks affinity while awaiting the handler.
173+
174+
Args:
175+
request: The outgoing model request.
176+
handler: Async callable that executes the request.
177+
178+
Returns:
179+
The result produced by `handler`.
180+
"""
181+
if not self._should_apply_caching(request):
182+
return await handler(request)
183+
with _session_affinity(request):
184+
return await handler(request)

‎libs/partners/fireworks/pyproject.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ test = [
5252
"pytest-socket>=0.7.0,<1.0.0",
5353
"pytest-xdist>=3.8.0,<4.0.0",
5454
"langchain-tests>=1.1.9,<2.0.0",
55+
"langchain>=1.0.0,<2.0.0",
5556
]
5657
test_integration = []
5758
lint = ["ruff>=0.13.1,<0.14.0"]
@@ -72,6 +73,7 @@ constraint-dependencies = [
7273
prerelease = "allow"
7374

7475
[tool.uv.sources]
76+
langchain = { path = "../../langchain_v1", editable = true }
7577
langchain-core = { path = "../../core", editable = true }
7678
langchain-tests = { path = "../../standard-tests", editable = true }
7779

‎libs/partners/fireworks/tests/unit_tests/middleware/__init__.py‎

Whitespace-only changes.

0 commit comments

Comments
 (0)