99import logging
1010import os
1111import re
12- from collections .abc import AsyncGenerator , AsyncIterable , AsyncIterator , Generator , Mapping , Sequence
12+ from collections .abc import (
13+ AsyncGenerator ,
14+ AsyncIterable ,
15+ AsyncIterator ,
16+ Awaitable ,
17+ Callable ,
18+ Generator ,
19+ Mapping ,
20+ Sequence ,
21+ )
1322from contextlib import AbstractAsyncContextManager , AsyncExitStack , aclosing , suppress
1423from dataclasses import asdict , dataclass , is_dataclass
1524from typing import Generic , Literal , TypeGuard , TypeVar , cast
@@ -162,15 +171,27 @@ class _SignalledIterator(Generic[_T]):
162171 task -- and the real agent/workflow run it's pumping -- would otherwise be silently abandoned.
163172 """
164173
165- def __init__ (self , iterator : AsyncIterator [_T ], * events : asyncio .Event ) -> None :
174+ def __init__ (
175+ self ,
176+ iterator : AsyncIterator [_T ],
177+ * events : asyncio .Event ,
178+ stamp : Callable [[], Awaitable [Any ]] | None = None ,
179+ ) -> None :
166180 """Wrap an async iterator, stopping early if any of ``events`` fires.
167181
168182 Args:
169183 iterator: The async iterator to wrap.
170184 events: One or more asyncio.Event objects to watch for. If any of them is set, iteration stops early.
185+ stamp: Optional coroutine function the driver awaits right after the wrapped iterator produces an
186+ item and before it is advanced again. Its result is exposed as :attr:`stamp` while that item
187+ is the current one, which lets a consumer observe state (e.g. the latest persisted workflow
188+ checkpoint) as it was when the item was produced rather than when it is consumed: the driver
189+ runs one item ahead, so by consumption time the wrapped iterator may already have moved on.
171190 """
172191 self ._iterator = iterator
173192 self ._events = events
193+ self ._stamp_fn = stamp
194+ self ._stamp : Any = None
174195 self ._signalled = False
175196 # The queue is used to communicate items from the background driver task to the main iteration loop.
176197 self ._queue : asyncio .Queue [Any ] = asyncio .Queue (maxsize = 1 )
@@ -186,6 +207,11 @@ def signalled(self) -> bool:
186207 """
187208 return self ._signalled
188209
210+ @property
211+ def stamp (self ) -> Any :
212+ """The ``stamp`` result taken when the current item was produced (``None`` without a ``stamp``)."""
213+ return self ._stamp
214+
189215 def __aiter__ (self ) -> _SignalledIterator [_T ]:
190216 return self
191217
@@ -194,13 +220,14 @@ async def _drive(self) -> None:
194220 while True :
195221 try :
196222 item : Any = await self ._iterator .__anext__ ()
223+ stamp = await self ._stamp_fn () if self ._stamp_fn is not None else None
197224 except StopAsyncIteration :
198225 await self ._queue .put (_STOP_SENTINEL )
199226 return
200227 except Exception as exc :
201228 await self ._queue .put (exc )
202229 return
203- await self ._queue .put (item )
230+ await self ._queue .put (( item , stamp ) )
204231
205232 async def __anext__ (self ) -> _T :
206233 if self ._driver is None :
@@ -233,6 +260,7 @@ async def __anext__(self) -> _T:
233260 raise StopAsyncIteration
234261 if isinstance (item , Exception ):
235262 raise item
263+ item , self ._stamp = item
236264 return cast (_T , item )
237265
238266 async def aclose (self ) -> None :
@@ -857,12 +885,8 @@ async def _handle_inner_agent(
857885 )
858886 async with aclosing (agent_stream ):
859887 async for update in agent_stream :
860- tracker .record_finish_reason (update .finish_reason )
861- for content in update .contents :
862- async for event in tracker .handle (
863- content , message_id = update .message_id , approval_storage = approval_storage
864- ):
865- yield event
888+ async for event in tracker .handle_update (update , approval_storage = approval_storage ):
889+ yield event
866890 except (asyncio .CancelledError , GeneratorExit ):
867891 request_interrupted = True
868892 raise
@@ -1054,42 +1078,57 @@ async def _handle_inner_workflow(
10541078 checkpoint_storage = checkpoint_storage ,
10551079 )
10561080
1057- main_iter = _SignalledIterator (run_stream , context .shutdown , cancellation_signal )
1081+ workflow_name = self ._agent .workflow .name
1082+
1083+ async def latest_checkpoint_id () -> str | None :
1084+ latest = await checkpoint_storage .get_latest (workflow_name = workflow_name )
1085+ return latest .checkpoint_id if latest is not None else None
1086+
1087+ def snapshot_response (
1088+ checkpoint_id : str | None ,
1089+ ) -> Generator [ResponseStreamEvent | ResponseCheckpointEvent , None , None ]:
1090+ # Pair the response output emitted so far with the workflow checkpoint it corresponds
1091+ # to, so recovery from that checkpoint replays exactly the updates that came after it.
1092+ if checkpoint_id is None or checkpoint_id == response_event_stream .internal_metadata .get (
1093+ _LATEST_CHECKPOINT_ID_KEY
1094+ ):
1095+ return
1096+ yield from tracker .close ()
1097+ response_event_stream .internal_metadata [_LATEST_CHECKPOINT_ID_KEY ] = checkpoint_id
1098+ yield response_event_stream .checkpoint ()
1099+
1100+ main_iter = _SignalledIterator (
1101+ run_stream ,
1102+ context .shutdown ,
1103+ cancellation_signal ,
1104+ # The runner creates a checkpoint at the end of each superstep, inside the generator
1105+ # that produces the updates (see RunnerImpl.run_until_convergence). Stamping each
1106+ # update with the latest checkpoint as it was produced tells which checkpoint the
1107+ # update follows; the driver runs one update ahead, so by the time an update is
1108+ # consumed the workflow may already have checkpointed past it.
1109+ stamp = latest_checkpoint_id if self ._resilient_background else None ,
1110+ )
10581111 async with aclosing (main_iter ):
10591112 async for update in main_iter :
10601113 if self ._resilient_background :
1061- latest_checkpoint = await checkpoint_storage .get_latest (workflow_name = self ._agent .workflow .name )
1062- if (
1063- latest_checkpoint is not None
1064- and latest_checkpoint .checkpoint_id
1065- != response_event_stream .internal_metadata .get (_LATEST_CHECKPOINT_ID_KEY )
1066- ):
1067- # A new checkpoint is created when we pull the next item from the stream
1068- # (see RunnerImpl.run_until_convergence). We only take a snapshot of the
1069- # response (response_event_stream.checkpoint()) once the checkpoint is
1070- # durably persisted. This means all items from the previous superstep
1071- # has been pulled thus we can safely close the tracker. The latest checkpoint
1072- # now reflects the state of the workflow that matches the response output.
1073- # Note that if a workflow crashes before any update is created, no response
1074- # snapshot is taken. However, upon recovery the workflow will still be resumed
1075- # from the latest checkpoint.
1076- for event in tracker .close ():
1077- yield event
1078- response_event_stream .internal_metadata [_LATEST_CHECKPOINT_ID_KEY ] = (
1079- latest_checkpoint .checkpoint_id
1080- )
1081- yield response_event_stream .checkpoint ()
1082-
1083- tracker .record_finish_reason (update .finish_reason )
1084- for content in update .contents :
1085- async for event in tracker .handle (
1086- content , message_id = update .message_id , approval_storage = approval_storage
1087- ):
1114+ # Every update before this one belongs to the stamped checkpoint (or an
1115+ # earlier one), so the output so far can be snapshotted against it. If the
1116+ # workflow crashes before any update is produced, no snapshot is taken and
1117+ # recovery still resumes from the latest workflow checkpoint.
1118+ for event in snapshot_response (main_iter .stamp ):
10881119 yield event
1120+
1121+ async for event in tracker .handle_update (update , approval_storage = approval_storage ):
1122+ yield event
10891123 # Cancellation needs no extra action here (the loop above already stopped); shutdown
10901124 # does, but only if it's what actually stopped the loop, not a natural completion.
10911125 if main_iter .signalled and context .shutdown .is_set ():
10921126 await context .exit_for_recovery ()
1127+ elif self ._resilient_background and not main_iter .signalled :
1128+ # The workflow ran to completion: pair its final checkpoint with the full output, so
1129+ # recovery after this point does not replay the last superstep.
1130+ for event in snapshot_response (await latest_checkpoint_id ()):
1131+ yield event
10931132 except Exception :
10941133 logger .exception ("Failed to produce response for workflow agent" )
10951134 raise
@@ -1256,6 +1295,22 @@ def record_finish_reason(self, finish_reason: str | None) -> None:
12561295 return
12571296 self ._stream .internal_metadata [_INCOMPLETE_REASON_KEY ] = self ._incomplete_reason .value
12581297
1298+ async def handle_update (
1299+ self ,
1300+ update : AgentResponseUpdate ,
1301+ * ,
1302+ approval_storage : FunctionApprovalStore | None = None ,
1303+ ) -> AsyncGenerator [ResponseStreamEvent ]:
1304+ """Process one agent update: note its finish reason, then handle each of its contents.
1305+
1306+ This is the single entry point for both the plain-agent and the workflow loops, so the
1307+ finish reason cannot be forgotten on one of them.
1308+ """
1309+ self .record_finish_reason (update .finish_reason )
1310+ for content in update .contents :
1311+ async for event in self .handle (content , message_id = update .message_id , approval_storage = approval_storage ):
1312+ yield event
1313+
12591314 async def handle (
12601315 self ,
12611316 content : Content ,
0 commit comments