Skip to content

Commit 1200347

Browse files
anishmehta24claude
andcommitted
Handle updates through one tracker entry point and pair snapshots with the checkpoint an update follows
_OutputItemTracker.handle_update records the update's finish reason and then handles its contents, so the plain-agent and workflow loops share one ordering rule. The workflow loop snapshotted the response against the latest checkpoint in storage at the time an update was consumed. _SignalledIterator drives the workflow one update ahead, so by then the runner could already have checkpointed past the update; a crash after that snapshot resumed the workflow with the update never replayed and its output (and any incomplete reason) lost. _SignalledIterator now takes an optional stamp coroutine that the driver awaits right after each item is produced, before advancing the wrapped iterator again, and the loop pairs snapshots with that stamped checkpoint id. After the workflow completes, its final checkpoint is paired with the full output so recovery does not replay the last superstep. The real-crash recovery integration tests that were xfailed against #7809 pass with this. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
1 parent abfad69 commit 1200347

2 files changed

Lines changed: 174 additions & 37 deletions

File tree

‎python/packages/foundry_hosting/agent_framework_foundry_hosting/_responses.py‎

Lines changed: 92 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,16 @@
99
import logging
1010
import os
1111
import re
12-
from collections.abc import AsyncGenerator, AsyncIterable, AsyncIterator, Generator, Mapping, Sequence
12+
from collections.abc import (
13+
AsyncGenerator,
14+
AsyncIterable,
15+
AsyncIterator,
16+
Awaitable,
17+
Callable,
18+
Generator,
19+
Mapping,
20+
Sequence,
21+
)
1322
from contextlib import AbstractAsyncContextManager, AsyncExitStack, aclosing, suppress
1423
from dataclasses import asdict, dataclass, is_dataclass
1524
from typing import Generic, Literal, TypeGuard, TypeVar, cast
@@ -162,15 +171,27 @@ class _SignalledIterator(Generic[_T]):
162171
task -- and the real agent/workflow run it's pumping -- would otherwise be silently abandoned.
163172
"""
164173

165-
def __init__(self, iterator: AsyncIterator[_T], *events: asyncio.Event) -> None:
174+
def __init__(
175+
self,
176+
iterator: AsyncIterator[_T],
177+
*events: asyncio.Event,
178+
stamp: Callable[[], Awaitable[Any]] | None = None,
179+
) -> None:
166180
"""Wrap an async iterator, stopping early if any of ``events`` fires.
167181
168182
Args:
169183
iterator: The async iterator to wrap.
170184
events: One or more asyncio.Event objects to watch for. If any of them is set, iteration stops early.
185+
stamp: Optional coroutine function the driver awaits right after the wrapped iterator produces an
186+
item and before it is advanced again. Its result is exposed as :attr:`stamp` while that item
187+
is the current one, which lets a consumer observe state (e.g. the latest persisted workflow
188+
checkpoint) as it was when the item was produced rather than when it is consumed: the driver
189+
runs one item ahead, so by consumption time the wrapped iterator may already have moved on.
171190
"""
172191
self._iterator = iterator
173192
self._events = events
193+
self._stamp_fn = stamp
194+
self._stamp: Any = None
174195
self._signalled = False
175196
# The queue is used to communicate items from the background driver task to the main iteration loop.
176197
self._queue: asyncio.Queue[Any] = asyncio.Queue(maxsize=1)
@@ -186,6 +207,11 @@ def signalled(self) -> bool:
186207
"""
187208
return self._signalled
188209

210+
@property
211+
def stamp(self) -> Any:
212+
"""The ``stamp`` result taken when the current item was produced (``None`` without a ``stamp``)."""
213+
return self._stamp
214+
189215
def __aiter__(self) -> _SignalledIterator[_T]:
190216
return self
191217

@@ -194,13 +220,14 @@ async def _drive(self) -> None:
194220
while True:
195221
try:
196222
item: Any = await self._iterator.__anext__()
223+
stamp = await self._stamp_fn() if self._stamp_fn is not None else None
197224
except StopAsyncIteration:
198225
await self._queue.put(_STOP_SENTINEL)
199226
return
200227
except Exception as exc:
201228
await self._queue.put(exc)
202229
return
203-
await self._queue.put(item)
230+
await self._queue.put((item, stamp))
204231

205232
async def __anext__(self) -> _T:
206233
if self._driver is None:
@@ -233,6 +260,7 @@ async def __anext__(self) -> _T:
233260
raise StopAsyncIteration
234261
if isinstance(item, Exception):
235262
raise item
263+
item, self._stamp = item
236264
return cast(_T, item)
237265

238266
async def aclose(self) -> None:
@@ -857,12 +885,8 @@ async def _handle_inner_agent(
857885
)
858886
async with aclosing(agent_stream):
859887
async for update in agent_stream:
860-
tracker.record_finish_reason(update.finish_reason)
861-
for content in update.contents:
862-
async for event in tracker.handle(
863-
content, message_id=update.message_id, approval_storage=approval_storage
864-
):
865-
yield event
888+
async for event in tracker.handle_update(update, approval_storage=approval_storage):
889+
yield event
866890
except (asyncio.CancelledError, GeneratorExit):
867891
request_interrupted = True
868892
raise
@@ -1054,42 +1078,57 @@ async def _handle_inner_workflow(
10541078
checkpoint_storage=checkpoint_storage,
10551079
)
10561080

1057-
main_iter = _SignalledIterator(run_stream, context.shutdown, cancellation_signal)
1081+
workflow_name = self._agent.workflow.name
1082+
1083+
async def latest_checkpoint_id() -> str | None:
1084+
latest = await checkpoint_storage.get_latest(workflow_name=workflow_name)
1085+
return latest.checkpoint_id if latest is not None else None
1086+
1087+
def snapshot_response(
1088+
checkpoint_id: str | None,
1089+
) -> Generator[ResponseStreamEvent | ResponseCheckpointEvent, None, None]:
1090+
# Pair the response output emitted so far with the workflow checkpoint it corresponds
1091+
# to, so recovery from that checkpoint replays exactly the updates that came after it.
1092+
if checkpoint_id is None or checkpoint_id == response_event_stream.internal_metadata.get(
1093+
_LATEST_CHECKPOINT_ID_KEY
1094+
):
1095+
return
1096+
yield from tracker.close()
1097+
response_event_stream.internal_metadata[_LATEST_CHECKPOINT_ID_KEY] = checkpoint_id
1098+
yield response_event_stream.checkpoint()
1099+
1100+
main_iter = _SignalledIterator(
1101+
run_stream,
1102+
context.shutdown,
1103+
cancellation_signal,
1104+
# The runner creates a checkpoint at the end of each superstep, inside the generator
1105+
# that produces the updates (see RunnerImpl.run_until_convergence). Stamping each
1106+
# update with the latest checkpoint as it was produced tells which checkpoint the
1107+
# update follows; the driver runs one update ahead, so by the time an update is
1108+
# consumed the workflow may already have checkpointed past it.
1109+
stamp=latest_checkpoint_id if self._resilient_background else None,
1110+
)
10581111
async with aclosing(main_iter):
10591112
async for update in main_iter:
10601113
if self._resilient_background:
1061-
latest_checkpoint = await checkpoint_storage.get_latest(workflow_name=self._agent.workflow.name)
1062-
if (
1063-
latest_checkpoint is not None
1064-
and latest_checkpoint.checkpoint_id
1065-
!= response_event_stream.internal_metadata.get(_LATEST_CHECKPOINT_ID_KEY)
1066-
):
1067-
# A new checkpoint is created when we pull the next item from the stream
1068-
# (see RunnerImpl.run_until_convergence). We only take a snapshot of the
1069-
# response (response_event_stream.checkpoint()) once the checkpoint is
1070-
# durably persisted. This means all items from the previous superstep
1071-
# has been pulled thus we can safely close the tracker. The latest checkpoint
1072-
# now reflects the state of the workflow that matches the response output.
1073-
# Note that if a workflow crashes before any update is created, no response
1074-
# snapshot is taken. However, upon recovery the workflow will still be resumed
1075-
# from the latest checkpoint.
1076-
for event in tracker.close():
1077-
yield event
1078-
response_event_stream.internal_metadata[_LATEST_CHECKPOINT_ID_KEY] = (
1079-
latest_checkpoint.checkpoint_id
1080-
)
1081-
yield response_event_stream.checkpoint()
1082-
1083-
tracker.record_finish_reason(update.finish_reason)
1084-
for content in update.contents:
1085-
async for event in tracker.handle(
1086-
content, message_id=update.message_id, approval_storage=approval_storage
1087-
):
1114+
# Every update before this one belongs to the stamped checkpoint (or an
1115+
# earlier one), so the output so far can be snapshotted against it. If the
1116+
# workflow crashes before any update is produced, no snapshot is taken and
1117+
# recovery still resumes from the latest workflow checkpoint.
1118+
for event in snapshot_response(main_iter.stamp):
10881119
yield event
1120+
1121+
async for event in tracker.handle_update(update, approval_storage=approval_storage):
1122+
yield event
10891123
# Cancellation needs no extra action here (the loop above already stopped); shutdown
10901124
# does, but only if it's what actually stopped the loop, not a natural completion.
10911125
if main_iter.signalled and context.shutdown.is_set():
10921126
await context.exit_for_recovery()
1127+
elif self._resilient_background and not main_iter.signalled:
1128+
# The workflow ran to completion: pair its final checkpoint with the full output, so
1129+
# recovery after this point does not replay the last superstep.
1130+
for event in snapshot_response(await latest_checkpoint_id()):
1131+
yield event
10931132
except Exception:
10941133
logger.exception("Failed to produce response for workflow agent")
10951134
raise
@@ -1256,6 +1295,22 @@ def record_finish_reason(self, finish_reason: str | None) -> None:
12561295
return
12571296
self._stream.internal_metadata[_INCOMPLETE_REASON_KEY] = self._incomplete_reason.value
12581297

1298+
async def handle_update(
1299+
self,
1300+
update: AgentResponseUpdate,
1301+
*,
1302+
approval_storage: FunctionApprovalStore | None = None,
1303+
) -> AsyncGenerator[ResponseStreamEvent]:
1304+
"""Process one agent update: note its finish reason, then handle each of its contents.
1305+
1306+
This is the single entry point for both the plain-agent and the workflow loops, so the
1307+
finish reason cannot be forgotten on one of them.
1308+
"""
1309+
self.record_finish_reason(update.finish_reason)
1310+
for content in update.contents:
1311+
async for event in self.handle(content, message_id=update.message_id, approval_storage=approval_storage):
1312+
yield event
1313+
12591314
async def handle(
12601315
self,
12611316
content: Content,

‎python/packages/foundry_hosting/tests/test_responses.py‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,13 @@
1111
from __future__ import annotations
1212

1313
import asyncio
14+
import copy
1415
import json
1516
import logging
1617
import os
1718
import uuid
1819
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Generator, Mapping, Sequence
20+
from contextlib import aclosing
1921
from dataclasses import dataclass
2022
from importlib import import_module
2123
from pathlib import Path
@@ -71,12 +73,14 @@
7173
from agent_framework_foundry_hosting import ResponsesHostServer
7274
from agent_framework_foundry_hosting._responses import (
7375
_INCOMPLETE_REASON_KEY, # pyright: ignore[reportPrivateUsage]
76+
_LATEST_CHECKPOINT_ID_KEY, # pyright: ignore[reportPrivateUsage]
7477
CONSENT_ERROR_CODE,
7578
ConsentError,
7679
_item_to_message, # pyright: ignore[reportPrivateUsage]
7780
_json_safe_to_str, # pyright: ignore[reportPrivateUsage]
7881
_output_item_to_message, # pyright: ignore[reportPrivateUsage]
7982
_OutputItemTracker, # pyright: ignore[reportPrivateUsage]
83+
_SignalledIterator, # pyright: ignore[reportPrivateUsage]
8084
_stringify_mcp_output, # pyright: ignore[reportPrivateUsage]
8185
consent_url_from_error,
8286
)
@@ -6325,6 +6329,84 @@ async def test_workflow_yields_checkpoint_event_when_resilient_background(self,
63256329
checkpoint_events = [e for e in events if isinstance(e, ResponseCheckpointEvent)]
63266330
assert checkpoint_events, "expected at least one checkpoint event yielded for a resilient background run"
63276331

6332+
async def test_signalled_iterator_stamps_items_when_produced(self) -> None:
6333+
"""The stamp reflects state as of production, not consumption.
6334+
6335+
The driver runs one item ahead: once the consumer holds item k, the wrapped iterator may
6336+
already have resumed and created a checkpoint after it. The stamp taken right after item k
6337+
was produced must not see that later checkpoint.
6338+
"""
6339+
checkpoints: list[int] = []
6340+
6341+
async def produce() -> AsyncIterator[int]:
6342+
for k in range(1, 4):
6343+
yield k
6344+
# Runs when the iterator is resumed to produce the next item, i.e. after item k
6345+
# was handed over, mirroring the runner checkpointing at the end of a superstep.
6346+
checkpoints.append(k)
6347+
6348+
async def stamp() -> int:
6349+
return len(checkpoints)
6350+
6351+
seen: list[tuple[int, int, int]] = []
6352+
it = _SignalledIterator(produce(), asyncio.Event(), stamp=stamp)
6353+
async with aclosing(it):
6354+
async for item in it:
6355+
# Give the driver every chance to run ahead before we look at the stamp.
6356+
for _ in range(5):
6357+
await asyncio.sleep(0)
6358+
seen.append((item, it.stamp, len(checkpoints)))
6359+
6360+
assert [(item, stamped) for item, stamped, _ in seen] == [(1, 0), (2, 1), (3, 2)]
6361+
# Consumption-time state had already moved past the stamped one for every item.
6362+
assert all(consumed > stamped for _, stamped, consumed in seen)
6363+
6364+
async def test_snapshots_pair_output_with_the_checkpoint_it_follows(self, tmp_path: Path) -> None:
6365+
"""Every persisted snapshot must contain exactly the output emitted before it, and the final
6366+
snapshot must carry the full output and the incomplete reason.
6367+
"""
6368+
workflow_agent = _build_text_workflow_agent("filtered by workflow", finish_reason="content_filter")
6369+
server = _make_server(
6370+
workflow_agent,
6371+
response_store=FileResponseStore(storage_dir=tmp_path),
6372+
options=ResponsesServerOptions(resilient_background=True),
6373+
)
6374+
request = CreateResponse(model="m", input="hi", background=True, stream=True, store=True)
6375+
context = ResponseContext(response_id="response-current", mode_flags=MagicMock())
6376+
6377+
emitted_text = ""
6378+
snapshots: list[tuple[str, dict[str, Any]]] = []
6379+
async for event in server._handle_response( # pyright: ignore[reportPrivateUsage]
6380+
request, context, asyncio.Event()
6381+
):
6382+
if isinstance(event, ResponseCheckpointEvent):
6383+
# The event references the live response; copy it as it is at persistence time.
6384+
snapshots.append((emitted_text, copy.deepcopy(dict(event.response))))
6385+
elif isinstance(event, Mapping) and event.get("type") == "response.output_text.delta":
6386+
emitted_text += str(event["delta"])
6387+
6388+
assert emitted_text == "filtered by workflow"
6389+
assert snapshots, "expected the completed workflow to be snapshotted"
6390+
checkpoint_ids: list[str] = []
6391+
for text_before, response in snapshots:
6392+
internal = json.loads(response["metadata"]["_internal_metadata"])
6393+
checkpoint_ids.append(internal[_LATEST_CHECKPOINT_ID_KEY])
6394+
snapshot_text = "".join(
6395+
part["text"]
6396+
for item in response["output"]
6397+
if item["type"] == "message"
6398+
for part in item["content"]
6399+
if part["type"] == "output_text"
6400+
)
6401+
assert snapshot_text == text_before
6402+
assert len(set(checkpoint_ids)) == len(checkpoint_ids), "each checkpoint is snapshotted once"
6403+
6404+
# The last snapshot is paired with the workflow's final checkpoint and carries everything.
6405+
final_text, final_response = snapshots[-1]
6406+
assert final_text == "filtered by workflow"
6407+
assert json.loads(final_response["metadata"]["_internal_metadata"])[_INCOMPLETE_REASON_KEY] == "content_filter"
6408+
assert final_response["status"] == "in_progress"
6409+
63286410

63296411
# endregion
63306412

0 commit comments

Comments
 (0)