Skip to content

Commit b2c4de3

Browse files
authored
Python: Preserve request metadata in hosted responses (#9745)
* fix(foundry-hosting): preserve response request metadata * fix(foundry-hosting): keep recovery metadata server-owned Drop a client-supplied _internal_metadata bag when a fresh response is seeded, and keep that bag only for a trusted persisted snapshot. When public metadata already holds 16 entries, still record a truncating finish reason so the response ends incomplete instead of failed. * test(foundry-hosting): cover 16-key truncation in both response modes A full public metadata map still ends length and content_filter turns incomplete, with the assistant text and all 16 public keys kept, for both streaming and non-streaming responses. * fix(foundry-hosting): persist recovery state outside public metadata
1 parent f4bdd1d commit b2c4de3

3 files changed

Lines changed: 501 additions & 26 deletions

File tree

‎python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py‎

Lines changed: 86 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -425,13 +425,70 @@ def _reject_busy_conversation(session: AgentSession) -> None:
425425
)
426426

427427

428-
def _create_response_event_stream(context: ResponseContext) -> ResponseEventStream:
429-
"""Create a response stream seeded from recovery state when available."""
428+
def _request_without_reserved_metadata(request: CreateResponse) -> CreateResponse:
429+
"""Copy a create request with any client-supplied reserved metadata bag removed.
430+
431+
``ResponseEventStream`` seeds the response from the request, and host recovery later trusts
432+
that bag for incomplete status, provider cursors, usage, and citations. Only a persisted
433+
response is a trusted source. The caller's request mapping is left unchanged.
434+
"""
435+
raw_metadata = request.get("metadata")
436+
if not isinstance(raw_metadata, Mapping):
437+
return request
438+
# Metadata is a TypedDict of known public keys. Cast so a client-supplied reserved key is visible.
439+
metadata = cast(Mapping[str, Any], raw_metadata)
440+
if _RESERVED_RESPONSE_METADATA_KEY not in metadata:
441+
return request
442+
public_metadata: dict[str, Any] = {
443+
key: value for key, value in metadata.items() if key != _RESERVED_RESPONSE_METADATA_KEY
444+
}
445+
seeded: dict[str, Any] = dict(request)
446+
if public_metadata:
447+
seeded["metadata"] = public_metadata
448+
else:
449+
seeded.pop("metadata", None)
450+
return cast(CreateResponse, seeded)
451+
452+
453+
def _create_response_event_stream(context: ResponseContext, request: CreateResponse) -> ResponseEventStream:
454+
"""Create a response stream from recovery state or the originating request."""
430455
if context.is_recovery:
431456
persisted_response = context.persisted_response
432457
if persisted_response is not None:
433458
return ResponseEventStream(response=persisted_response, response_id=context.response_id)
434-
return ResponseEventStream(response_id=context.response_id)
459+
return ResponseEventStream(
460+
request=_request_without_reserved_metadata(request),
461+
response_id=context.response_id,
462+
)
463+
464+
465+
def _hosted_internal_metadata(stream: ResponseEventStream) -> dict[str, Any]:
466+
"""Keep private state in the durable envelope, outside the public metadata quota.
467+
468+
The SDK strips its reserved ``internal_metadata`` key at every wire boundary. Older
469+
checkpoints used its quota-limited response metadata bag; copy that trusted state only
470+
when the envelope has no private bag of its own.
471+
"""
472+
metadata = stream.response.get("internal_metadata")
473+
if metadata is None:
474+
metadata = dict(stream.internal_metadata)
475+
stream.response["internal_metadata"] = metadata
476+
if not isinstance(metadata, dict):
477+
raise RuntimeError("The persisted host metadata is invalid.")
478+
return cast(dict[str, Any], metadata)
479+
480+
481+
def _read_persisted_incomplete_reason(stream: ResponseEventStream) -> str | None:
482+
"""Return a truncating finish reason restored from a trusted response snapshot."""
483+
persisted = _hosted_internal_metadata(stream).get(_INCOMPLETE_REASON_KEY)
484+
if isinstance(persisted, str):
485+
return persisted
486+
details = stream.response.get("incomplete_details")
487+
if isinstance(details, Mapping):
488+
reason = cast(Mapping[str, Any], details).get("reason")
489+
if isinstance(reason, str):
490+
return reason
491+
return None
435492

436493

437494
def _agent_response_updates(response: AgentResponse[Any], response_id: str) -> list[AgentResponseUpdate]:
@@ -635,6 +692,10 @@ async def aclose(self) -> None:
635692
# ``internal_metadata`` key carrying a truncating finish reason across resilient checkpoints, so a
636693
# turn cut short before a crash still ends ``incomplete`` after recovery.
637694
_INCOMPLETE_REASON_KEY = "_incomplete_reason"
695+
# SDK-reserved response metadata entry. It round-trips server-owned recovery state (this reason,
696+
# provider cursors, usage, and citations). Clients may send the same key; it is never trusted on a
697+
# fresh request. Foundry counts it toward the 16-entry public metadata limit.
698+
_RESERVED_RESPONSE_METADATA_KEY = "_internal_metadata"
638699

639700

640701
# Foundry Toolbox Auth integration
@@ -1213,7 +1274,7 @@ async def _handle_response(
12131274
async for event in events:
12141275
yield event
12151276
return
1216-
response_event_stream = _create_response_event_stream(context)
1277+
response_event_stream = _create_response_event_stream(context, request)
12171278
if context.is_steered_turn:
12181279
logger.debug("Serving steered turn (pending_input_count=%d)", context.pending_input_count)
12191280
yield response_event_stream.emit_created()
@@ -1831,7 +1892,9 @@ async def _handle_inner_agent(
18311892
if provider_background:
18321893
if session_storage is None or not isinstance(agent, RawAgent):
18331894
raise RuntimeError("Provider background requires a stored MAF agent session.")
1834-
output_count = response_event_stream.internal_metadata.get(_HOSTED_PROVIDER_OUTPUT_COUNT_KEY, 0)
1895+
output_count = _hosted_internal_metadata(response_event_stream).get(
1896+
_HOSTED_PROVIDER_OUTPUT_COUNT_KEY, 0
1897+
)
18351898
if type(output_count) is not int or output_count < 0:
18361899
raise RuntimeError("The persisted provider output cursor is invalid.")
18371900
responses = self._provider_background_responses(
@@ -1857,8 +1920,9 @@ async def _handle_inner_agent(
18571920
continue
18581921
for event in tracker.close():
18591922
yield event
1860-
response_event_stream.internal_metadata[_HOSTED_PROVIDER_OUTPUT_COUNT_KEY] = output_count
1861-
response_event_stream.internal_metadata[_HOSTED_PROVIDER_USAGE_KEY] = tracker.usage_details
1923+
private_metadata = _hosted_internal_metadata(response_event_stream)
1924+
private_metadata[_HOSTED_PROVIDER_OUTPUT_COUNT_KEY] = output_count
1925+
private_metadata[_HOSTED_PROVIDER_USAGE_KEY] = tracker.usage_details
18621926
if self._resilient_background:
18631927
yield response_event_stream.checkpoint()
18641928
else:
@@ -2234,7 +2298,7 @@ async def _handle_inner_workflow(
22342298
# snapshot (recorded in that snapshot's own metadata) -- NOT simply the latest workflow
22352299
# checkpoint in storage, which may be ahead of what response.output actually reflects if
22362300
# the crash happened between two response-stream checkpoint() calls.
2237-
checkpoint_id = response_event_stream.internal_metadata.get(_LATEST_CHECKPOINT_ID_KEY)
2301+
checkpoint_id = _hosted_internal_metadata(response_event_stream).get(_LATEST_CHECKPOINT_ID_KEY)
22382302
if checkpoint_id is not None:
22392303
logger.debug("Serving recovery request from workflow checkpoint %s", checkpoint_id)
22402304
run_stream = self._resume_workflow_from_checkpoint(
@@ -2350,12 +2414,12 @@ def snapshot_response(
23502414
) -> Generator[ResponseStreamEvent | ResponseCheckpointEvent]:
23512415
# Pair the response output emitted so far with the workflow checkpoint it corresponds
23522416
# to, so recovery from that checkpoint replays exactly the updates that came after it.
2353-
if checkpoint_id is None or checkpoint_id == response_event_stream.internal_metadata.get(
2417+
if checkpoint_id is None or checkpoint_id == _hosted_internal_metadata(response_event_stream).get(
23542418
_LATEST_CHECKPOINT_ID_KEY
23552419
):
23562420
return
23572421
yield from tracker.close()
2358-
response_event_stream.internal_metadata[_LATEST_CHECKPOINT_ID_KEY] = checkpoint_id
2422+
_hosted_internal_metadata(response_event_stream)[_LATEST_CHECKPOINT_ID_KEY] = checkpoint_id
23592423
yield response_event_stream.checkpoint()
23602424

23612425
main_iter = _SignalledIterator(
@@ -2483,7 +2547,7 @@ def __init__(
24832547
self._stream = stream
24842548
self._allowed_oauth_consent_origins = allowed_oauth_consent_origins
24852549
self._usage_details: UsageDetails | None = None
2486-
persisted_usage = stream.internal_metadata.get(_HOSTED_PROVIDER_USAGE_KEY)
2550+
persisted_usage = _hosted_internal_metadata(stream).get(_HOSTED_PROVIDER_USAGE_KEY)
24872551
if persisted_usage is not None:
24882552
if not isinstance(persisted_usage, Mapping) or any(
24892553
not isinstance(key, str) or (value is not None and type(value) is not int)
@@ -2516,15 +2580,15 @@ def __init__(
25162580
self._message_text_annotations: dict[int, list[ResponsesAnnotation]] = {}
25172581
# Set when an agent update reports the model stopped early (content filter, token
25182582
# limit); the response then ends as ``incomplete`` instead of ``completed`` so callers
2519-
# can tell a cut-short turn from a successful one. Mirrored into the stream's
2520-
# ``internal_metadata`` so it survives a resilient checkpoint/recovery cycle, which
2521-
# rebuilds this tracker from the persisted response.
2583+
# can tell a cut-short turn from a successful one. Mirrored onto the response so it
2584+
# survives a resilient checkpoint/recovery cycle, which rebuilds this tracker from the
2585+
# persisted response.
25222586
self._incomplete_reason: ResponseIncompleteReason | None = None
2523-
persisted_reason = stream.internal_metadata.get(_INCOMPLETE_REASON_KEY)
2524-
if isinstance(persisted_reason, str):
2587+
persisted_reason = _read_persisted_incomplete_reason(stream)
2588+
if persisted_reason is not None:
25252589
with suppress(ValueError):
25262590
self._incomplete_reason = ResponseIncompleteReason(persisted_reason)
2527-
persisted_citations = stream.internal_metadata.get(_PENDING_CONTAINER_FILE_CITATIONS_KEY)
2591+
persisted_citations = _hosted_internal_metadata(stream).get(_PENDING_CONTAINER_FILE_CITATIONS_KEY)
25282592
if persisted_citations is not None:
25292593
if not isinstance(persisted_citations, Sequence) or isinstance(
25302594
persisted_citations, (str, bytes, bytearray)
@@ -2612,15 +2676,16 @@ def record_finish_reason(self, finish_reason: str | None) -> None:
26122676
Only finish reasons that mean the model stopped early are retained, mapped onto the
26132677
Responses ``incomplete_details.reason`` vocabulary. A content filter is kept in
26142678
preference to a token limit if both are seen during a multi-step turn, since it is the
2615-
more actionable signal for the caller.
2679+
more actionable signal for the caller. The reason is recorded even when the caller's
2680+
metadata already fills the public map, and that map is left intact.
26162681
"""
26172682
if finish_reason == "content_filter":
26182683
self._incomplete_reason = ResponseIncompleteReason.CONTENT_FILTER
26192684
elif finish_reason == "length" and self._incomplete_reason is None:
26202685
self._incomplete_reason = ResponseIncompleteReason.MAX_OUTPUT_TOKENS
26212686
else:
26222687
return
2623-
self._stream.internal_metadata[_INCOMPLETE_REASON_KEY] = self._incomplete_reason.value
2688+
_hosted_internal_metadata(self._stream)[_INCOMPLETE_REASON_KEY] = self._incomplete_reason.value
26242689

26252690
def _consume_text_update(self, text: str, merge_mode: Literal["append", "replace"] | None) -> str:
26262691
"""Update the authoritative text buffer and return the wire delta still needed."""
@@ -3227,9 +3292,9 @@ def _container_file_annotations_for_text(self, text: str) -> list[ContainerFileC
32273292

32283293
def _persist_pending_container_file_citations(self) -> None:
32293294
if not self._pending_container_file_citations:
3230-
self._stream.internal_metadata.pop(_PENDING_CONTAINER_FILE_CITATIONS_KEY, None)
3295+
_hosted_internal_metadata(self._stream).pop(_PENDING_CONTAINER_FILE_CITATIONS_KEY, None)
32313296
return
3232-
self._stream.internal_metadata[_PENDING_CONTAINER_FILE_CITATIONS_KEY] = [
3297+
_hosted_internal_metadata(self._stream)[_PENDING_CONTAINER_FILE_CITATIONS_KEY] = [
32333298
{
32343299
"container_id": citation.container_id,
32353300
"file_id": citation.file_id,

‎python/packages/foundry_hosting/agent_framework_foundry_hosting/_workflow_responses.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,10 @@
5353
from ._responses import (
5454
_HOSTED_PROVIDER_USAGE_KEY, # pyright: ignore[reportPrivateUsage]
5555
_agent_response_updates, # pyright: ignore[reportPrivateUsage]
56+
_hosted_internal_metadata, # pyright: ignore[reportPrivateUsage]
5657
_item_to_message, # pyright: ignore[reportPrivateUsage]
5758
_OutputItemTracker, # pyright: ignore[reportPrivateUsage]
59+
_request_without_reserved_metadata, # pyright: ignore[reportPrivateUsage]
5860
_SignalledIterator, # pyright: ignore[reportPrivateUsage]
5961
)
6062
from ._scope import FoundryRequestScope
@@ -392,7 +394,10 @@ async def handle_response(
392394
cancellation_signal: asyncio.Event,
393395
) -> AsyncGenerator[ResponseStreamEvent | ResponseCheckpointEvent]:
394396
"""Validate, run, pair, close owned resources, and only then publish terminal success."""
395-
stream = ResponseEventStream(response_id=context.response_id, request=request)
397+
stream = ResponseEventStream(
398+
response_id=context.response_id,
399+
request=_request_without_reserved_metadata(request),
400+
)
396401
run: HostedWorkflowRun | None = None
397402
phase = "preparation"
398403
completed = False
@@ -515,7 +520,7 @@ async def flush(checkpoint_id: str | None) -> list[ResponseStreamEvent]:
515520
for event in buffered:
516521
events.extend(await _project(event, draft, next_tracker, approval_projection))
517522
events.extend(next_tracker.close())
518-
draft.internal_metadata[_HOSTED_PROVIDER_USAGE_KEY] = next_tracker.usage_details
523+
_hosted_internal_metadata(draft)[_HOSTED_PROVIDER_USAGE_KEY] = next_tracker.usage_details
519524
await run.stage(_snapshot(draft), approvals=next_aliases, checkpoint_id=checkpoint_id)
520525
stream, tracker, aliases = draft, next_tracker, next_aliases
521526
buffered.clear()

0 commit comments

Comments
 (0)