Skip to content

Commit e722615

Browse files
committed
perf(hosting): reuse composite session keys
Use tuple lookups in the session harness to avoid serialization and throwaway session construction on cache hits. Preserve existing JSON session identifiers by encoding only when a session is created. Refs #8251
1 parent 4451e8f commit e722615

4 files changed

Lines changed: 74 additions & 24 deletions

File tree

‎python/packages/foundry_hosting/README.md‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -86,10 +86,12 @@ Native Responses refusal parts are stored as text carrying
8686
`additional_properties["model_output_kind"] == "refusal"` and emitted as
8787
`response.refusal.*` events when streamed back to clients.
8888

89-
`InvocationsHostServer` keeps sessions in memory. When hosted, its session key and
90-
`AgentSession.session_id` are a compact JSON array string containing the platform session ID
91-
and user ID, preserving each identifier's boundaries. Locally, the platform session ID is
92-
used unchanged. Treat the hosted identifier as opaque; sessions do not survive host restarts.
89+
`InvocationsHostServer` keeps sessions in memory. When hosted, its internal lookup key is a tuple
90+
of the platform session ID and user ID, preserving each identifier's boundaries.
91+
`AgentSession.session_id` remains a compact JSON array string containing those identifiers,
92+
generated only when the session is created. Repeated requests reuse the session without
93+
serializing its identifiers again. Locally, the platform session ID is used unchanged.
94+
Treat the hosted identifier as opaque; sessions do not survive host restarts.
9395

9496
### Workflow checkpoints
9597

‎python/packages/foundry_hosting/agent_framework_foundry_hosting/_invocations.py‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -37,15 +37,15 @@ def __init__(
3737
super().__init__(openapi_spec=openapi_spec, **kwargs)
3838

3939
self._agent = agent
40-
self._sessions: dict[str, AgentSession] = {}
40+
self._sessions: dict[str | tuple[str, str], AgentSession] = {}
4141
self.invoke_handler(self._handle_invoke)
4242
mark_feature_used(FeatureIndex.FOUNDRY_HOSTING)
4343

44-
def _partition_key(self) -> str:
44+
def _partition_key(self) -> str | tuple[str, str]:
4545
"""Get the partition key for the current request.
4646
47-
A hosted partition key is a JSON array string containing the session ID and user ID,
48-
preserving their boundaries and escaping. Locally, the key is just the session ID. In the
47+
A hosted partition key is a tuple containing the session ID and user ID,
48+
preserving their boundaries. Locally, the key is just the session ID. In the
4949
Foundry hosted environment, the partition key is used to maintain isolation between
5050
different sessions and users, such that one user cannot access another user's sessions.
5151
@@ -63,7 +63,7 @@ def _partition_key(self) -> str:
6363
"The hosted environment is missing session_id or user_id in the request context. "
6464
"Please ensure that the request is coming from a valid Foundry platform service."
6565
)
66-
return json.dumps([context.session_id, context.user_id], separators=(",", ":"))
66+
return context.session_id, context.user_id
6767

6868
if not context.session_id:
6969
raise RuntimeError(
@@ -75,7 +75,7 @@ def _partition_key(self) -> str:
7575
async def _handle_invoke(self, request: Request) -> Response:
7676
"""Invoke the agent with the given request."""
7777
try:
78-
session_id = self._partition_key()
78+
partition_key = self._partition_key()
7979
except Exception as e:
8080
return Response(content=str(e), status_code=500)
8181

@@ -89,7 +89,13 @@ async def _handle_invoke(self, request: Request) -> Response:
8989
return StreamingResponse(content=error, status_code=400)
9090
return Response(content=error, status_code=400)
9191

92-
session = self._sessions.setdefault(session_id, AgentSession(session_id=session_id))
92+
session = self._sessions.get(partition_key)
93+
if session is None:
94+
session_id = (
95+
json.dumps(partition_key, separators=(",", ":")) if isinstance(partition_key, tuple) else partition_key
96+
)
97+
session = AgentSession(session_id=session_id)
98+
self._sessions[partition_key] = session
9399

94100
if stream:
95101

‎python/packages/foundry_hosting/tests/test_invocations.py‎

Lines changed: 42 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
from collections.abc import AsyncIterator, Iterator
1616
from contextlib import contextmanager
1717
from itertools import product
18-
from unittest.mock import AsyncMock, MagicMock
18+
from unittest.mock import AsyncMock, MagicMock, patch
1919

2020
import pytest
2121
from agent_framework import (
@@ -193,21 +193,31 @@ def test_hosted_returns_composite_key(self) -> None:
193193
server = InvocationsHostServer(_make_agent(response_text="hi"))
194194
server.config.is_hosted = True
195195
with _request_context(call_id="call-1", session_id="sess-1", user_id="user-1"):
196-
assert server._partition_key() == '["sess-1","user-1"]' # pyright: ignore[reportPrivateUsage]
196+
assert server._partition_key() == ("sess-1", "user-1") # pyright: ignore[reportPrivateUsage]
197197

198-
def test_hosted_keys_preserve_identifier_values(self) -> None:
199-
server = InvocationsHostServer(_make_agent(response_text="hi"))
198+
async def test_hosted_keys_and_session_ids_preserve_identifier_values(self) -> None:
199+
agent = _make_agent(response_text="hi")
200+
server = InvocationsHostServer(agent)
200201
server.config.is_hosted = True
201202
identifiers = ["part", "part:part", "part,part", "[part]", 'part"\\', "part\n\t", "\u00e9", r"\u00e9", " part "]
202-
keys: set[str] = set()
203+
keys: set[tuple[str, str]] = set()
204+
request = _make_request({"message": "Hi"})
203205

204206
for session_id, user_id in product(identifiers, repeat=2):
205207
with _request_context(call_id="call-1", session_id=session_id, user_id=user_id):
206208
key = server._partition_key() # pyright: ignore[reportPrivateUsage]
209+
response = await server._handle_invoke(request) # pyright: ignore[reportPrivateUsage]
207210

208-
assert json.loads(key) == [session_id, user_id]
211+
assert isinstance(key, tuple)
212+
assert key == (session_id, user_id)
209213
assert key not in keys
210214
keys.add(key)
215+
assert response.status_code == 200
216+
session = agent.calls[-1]["session"]
217+
assert isinstance(session, AgentSession)
218+
expected_id = json.dumps([session_id, user_id], separators=(",", ":"))
219+
assert session.session_id == expected_id
220+
assert session.to_dict()["session_id"] == expected_id
211221

212222

213223
# endregion
@@ -217,6 +227,32 @@ def test_hosted_keys_preserve_identifier_values(self) -> None:
217227

218228

219229
class TestHandleInvoke:
230+
@pytest.mark.parametrize("stream", [False, True])
231+
@pytest.mark.parametrize("hosted", [False, True])
232+
async def test_reusing_session_skips_serialization_and_construction(self, hosted: bool, stream: bool) -> None:
233+
agent = _make_agent(response_text="ok", stream_texts=["ok"])
234+
server = InvocationsHostServer(agent)
235+
server.config.is_hosted = hosted
236+
request = _make_request({"message": "Hi", "stream": stream})
237+
expected_id = '["sess-1","user-1"]' if hosted else "sess-1"
238+
239+
with (
240+
_request_context(call_id="call-1", session_id="sess-1", user_id="user-1"),
241+
patch("agent_framework_foundry_hosting._invocations.json", wraps=json) as serializer,
242+
patch("agent_framework_foundry_hosting._invocations.AgentSession", wraps=AgentSession) as session_factory,
243+
):
244+
for _ in range(2):
245+
response = await server._handle_invoke(request) # pyright: ignore[reportPrivateUsage]
246+
if isinstance(response, StreamingResponse):
247+
assert await _collect_stream(response) == "ok"
248+
else:
249+
assert bytes(response.body).decode() == "ok"
250+
251+
assert agent.calls[0]["session"] is agent.calls[1]["session"]
252+
assert agent.calls[0]["session"].session_id == expected_id
253+
assert serializer.dumps.call_count == (1 if hosted else 0)
254+
session_factory.assert_called_once_with(session_id=expected_id)
255+
220256
async def test_missing_message_returns_400(self) -> None:
221257
server = InvocationsHostServer(_make_agent(response_text="hi"))
222258
request = _make_request({"stream": False})

‎python/samples/04-hosting/foundry-hosted-agents/invocations/break_glass/main.py‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
# In-memory session store — keyed by session and user IDs when a user is present.
2121
# WARNING: This is lost on restart. Use durable storage in production.
22-
_sessions: dict[str, AgentSession] = {}
22+
_sessions: dict[str | tuple[str, str], AgentSession] = {}
2323

2424
# Create the agent
2525
client = FoundryChatClient(
@@ -40,11 +40,11 @@
4040
app = InvocationAgentServerHost()
4141

4242

43-
def get_session_partition_key() -> str:
43+
def get_session_partition_key() -> str | tuple[str, str]:
4444
"""Get the partition key for the current request.
4545
46-
A partition key is a JSON array string containing the session ID and user ID when a user ID
47-
is present, preserving their boundaries and escaping. Otherwise, the key is just the session ID. In the
46+
A partition key is a tuple containing the session ID and user ID when a user ID
47+
is present, preserving their boundaries. Otherwise, the key is just the session ID. In the
4848
Foundry hosted environment, the partition key is used to maintain isolation between
4949
different sessions and users, such that one user cannot access another user's sessions.
5050
@@ -57,7 +57,7 @@ def get_session_partition_key() -> str:
5757
"The request context is missing session_id. Please ensure that the request is a valid request."
5858
)
5959
if context.user_id is not None:
60-
return json.dumps([context.session_id, context.user_id], separators=(",", ":"))
60+
return context.session_id, context.user_id
6161
return context.session_id
6262

6363

@@ -73,8 +73,14 @@ async def handle_invoke(request: Request):
7373
return StreamingResponse(content=error, status_code=400)
7474
return Response(content=error, status_code=400)
7575

76-
session_id = get_session_partition_key()
77-
session = _sessions.setdefault(session_id, AgentSession(session_id=session_id))
76+
partition_key = get_session_partition_key()
77+
session = _sessions.get(partition_key)
78+
if session is None:
79+
session_id = (
80+
json.dumps(partition_key, separators=(",", ":")) if isinstance(partition_key, tuple) else partition_key
81+
)
82+
session = AgentSession(session_id=session_id)
83+
_sessions[partition_key] = session
7884

7985
if stream:
8086

0 commit comments

Comments
 (0)