1111
1212import asyncio
1313import logging
14+ import uuid
1415from collections .abc import Awaitable , MutableMapping , Sequence
1516from dataclasses import dataclass , field
1617from enum import Enum
@@ -55,7 +56,13 @@ class BackgroundTaskStatus(str, Enum):
5556
5657@experimental (feature_id = ExperimentalFeature .HARNESS )
5758class BackgroundTaskInfo (SerializationMixin ):
58- """Metadata for a single background task."""
59+ """Metadata for a single background task.
60+
61+ ``execution_id`` identifies the execution (initial start or continuation) that this record
62+ describes. A new, globally unique ID is assigned to every execution, so a record restored from
63+ an older session snapshot is never reconciled against a different execution, even when that
64+ execution reuses the same task ID.
65+ """
5966
6067 DEFAULT_EXCLUDE : ClassVar [set [str ]] = set ()
6168
@@ -65,7 +72,8 @@ class BackgroundTaskInfo(SerializationMixin):
6572 status : BackgroundTaskStatus
6673 result_text : str | None
6774 error_text : str | None
68- __slots__ = ("agent_name" , "description" , "error_text" , "id" , "result_text" , "status" )
75+ execution_id : str | None
76+ __slots__ = ("agent_name" , "description" , "error_text" , "execution_id" , "id" , "result_text" , "status" )
6977
7078 def __init__ (
7179 self ,
@@ -75,6 +83,7 @@ def __init__(
7583 status : BackgroundTaskStatus = BackgroundTaskStatus .RUNNING ,
7684 result_text : str | None = None ,
7785 error_text : str | None = None ,
86+ execution_id : str | None = None ,
7887 ) -> None :
7988 """Initialize a background task info entry."""
8089 self .id = id
@@ -83,6 +92,7 @@ def __init__(
8392 self .status = status
8493 self .result_text = result_text
8594 self .error_text = error_text
95+ self .execution_id = execution_id
8696
8797 def to_dict (self , * , exclude : set [str ] | None = None , exclude_none : bool = True ) -> dict [str , Any ]:
8898 """Serialize for session state persistence."""
@@ -97,6 +107,8 @@ def to_dict(self, *, exclude: set[str] | None = None, exclude_none: bool = True)
97107 data ["result_text" ] = self .result_text
98108 if not exclude_none or self .error_text is not None :
99109 data ["error_text" ] = self .error_text
110+ if not exclude_none or self .execution_id is not None :
111+ data ["execution_id" ] = self .execution_id
100112 return data
101113
102114 @classmethod
@@ -109,24 +121,61 @@ def from_dict(cls, data: MutableMapping[str, Any], **kwargs: Any) -> BackgroundT
109121 status = BackgroundTaskStatus (data ["status" ]),
110122 result_text = data .get ("result_text" ),
111123 error_text = data .get ("error_text" ),
124+ execution_id = data .get ("execution_id" ),
112125 )
113126
114127
115128@dataclass
116129class _RuntimeState :
117- """Non-serializable per-session runtime state for background tasks."""
130+ """Non-serializable per-session runtime state for background tasks.
131+
132+ ``in_flight_tasks`` holds the latest execution started for each task ID and ``execution_ids``
133+ records which execution that is. Lookups made on behalf of a persisted
134+ :class:`BackgroundTaskInfo` go through :meth:`get_execution`, so a record only ever sees the
135+ execution it was created for.
136+
137+ All reads and writes happen on the event loop thread (every background tool is invoked there),
138+ so each tool's check-and-update sequence is atomic with respect to the others.
139+ """
118140
119141 in_flight_tasks : dict [int , asyncio .Task [AgentResponse [Any ]]] = field (default_factory = lambda : {})
142+ execution_ids : dict [int , str ] = field (default_factory = lambda : {})
120143 background_sessions : dict [int , AgentSession ] = field (default_factory = lambda : {})
144+ next_task_id : int = 1
121145 closed : bool = False
122146
123- def track_task (self , task_id : int , task : asyncio .Task [AgentResponse [Any ]]) -> None :
124- """Track a background task if this runtime is still open."""
147+ def allocate_task_id (self , persisted_next_task_id : int ) -> int :
148+ """Return a task ID that was never handed out by this runtime.
149+
150+ A restored older session snapshot carries a stale ``next_task_id``; reusing it would let a
151+ new task overwrite the runtime entry of a task started after that snapshot was taken.
152+ """
153+ task_id = max (persisted_next_task_id , self .next_task_id )
154+ self .next_task_id = task_id + 1
155+ return task_id
156+
157+ def track_task (self , task_id : int , task : asyncio .Task [AgentResponse [Any ]]) -> str :
158+ """Track a new execution for a task ID if this runtime is still open and return its execution ID."""
125159 if self .closed :
126160 task .cancel ()
127161 raise RuntimeError ("Session runtime is closed; cannot start background task." )
128162
163+ execution_id = uuid .uuid4 ().hex
129164 self .in_flight_tasks [task_id ] = task
165+ self .execution_ids [task_id ] = execution_id
166+ return execution_id
167+
168+ def get_execution (self , task_id : int , execution_id : str | None ) -> asyncio .Task [AgentResponse [Any ]] | None :
169+ """Return the tracked execution for a task ID only if it is the given execution."""
170+ if execution_id is None or self .execution_ids .get (task_id ) != execution_id :
171+ return None
172+ return self .in_flight_tasks .get (task_id )
173+
174+ def forget_task (self , task_id : int ) -> None :
175+ """Drop all runtime references for a task ID."""
176+ self .in_flight_tasks .pop (task_id , None )
177+ self .execution_ids .pop (task_id , None )
178+ self .background_sessions .pop (task_id , None )
130179
131180
132181# ---------------------------------------------------------------------------
@@ -153,6 +202,27 @@ def _log_abandoned_background_task(task: asyncio.Task[Any]) -> None:
153202 logger .debug ("Abandoned background task raised: %s" , exception )
154203
155204
205+ # Strong references to tasks abandoned by a session release. The event loop only keeps weak
206+ # references to tasks, so without these an abandoned task could be garbage collected while pending.
207+ _abandoned_background_tasks : set [asyncio .Task [Any ]] = set ()
208+
209+
210+ def _abandon_background_tasks (tasks : Sequence [asyncio .Task [Any ]]) -> None :
211+ """Retrieve exceptions of finished tasks now and of unfinished tasks once they complete.
212+
213+ Used when a session release stops waiting (timeout or cancellation): the runtime is cleared
214+ afterwards, so any task whose exception is not retrieved here would be reported as "Task exception
215+ was never retrieved". Unfinished tasks are kept alive until they complete.
216+ """
217+ for task in tasks :
218+ if task .done ():
219+ _log_abandoned_background_task (task )
220+ elif task not in _abandoned_background_tasks :
221+ _abandoned_background_tasks .add (task )
222+ task .add_done_callback (_abandoned_background_tasks .discard )
223+ task .add_done_callback (_log_abandoned_background_task )
224+
225+
156226def _validate_and_build_agent_dict (agents : Sequence [SupportsAgentRun ]) -> dict [str , SupportsAgentRun ]:
157227 """Validate agents and build a case-insensitive lookup dict.
158228
@@ -215,9 +285,27 @@ def _save_tasks(state: dict[str, Any], tasks: list[BackgroundTaskInfo]) -> None:
215285def _finalize_task (
216286 task_info : BackgroundTaskInfo ,
217287 completed_task : asyncio .Task [AgentResponse [Any ]],
218- runtime : _RuntimeState ,
219288) -> None :
220- """Extract results from a completed asyncio task and update task info."""
289+ """Extract results from a completed asyncio task and update task info.
290+
291+ The finished :class:`asyncio.Task` is intentionally **not** removed from
292+ ``runtime.in_flight_tasks`` here. Session state is persisted by the host
293+ only after the agent run completes; if the run fails, is stopped early, or
294+ the host restores an older session snapshot, the persisted record will still
295+ say ``RUNNING`` while this in-memory task no longer exists. On the next
296+ call to :func:`_refresh_task_state` a missing entry would be misidentified
297+ as ``LOST``, discarding a result that is already available.
298+
299+ Because the retained entry is keyed by task ID, callers must only pass the
300+ execution matching ``task_info.execution_id`` (see
301+ :meth:`_RuntimeState.get_execution`); a different execution under the same
302+ task ID must never be applied to the record.
303+
304+ Tasks are removed from ``in_flight_tasks`` by two callers:
305+
306+ - :func:`background_agents_clear_completed_task` — explicit caller-driven cleanup.
307+ - :meth:`BackgroundAgentsProvider.release_session` — bulk cleanup on session teardown.
308+ """
221309 if completed_task .cancelled ():
222310 task_info .status = BackgroundTaskStatus .FAILED
223311 task_info .error_text = "Task was canceled."
@@ -229,9 +317,6 @@ def _finalize_task(
229317 else :
230318 task_info .status = BackgroundTaskStatus .COMPLETED
231319 task_info .result_text = completed_task .result ().text
232- # A stale waiter may observe a task ID already reused by a continuation.
233- if runtime .in_flight_tasks .get (task_info .id ) is completed_task :
234- runtime .in_flight_tasks .pop (task_info .id , None )
235320
236321
237322def _refresh_task_state (
@@ -245,14 +330,17 @@ def _refresh_task_state(
245330 if task_info .status != BackgroundTaskStatus .RUNNING :
246331 continue
247332
248- in_flight = runtime .in_flight_tasks .get (task_info .id )
333+ # A missing execution, or a different one under the same task ID (for example, the task was
334+ # continued from another snapshot of this session), means this record's execution is no
335+ # longer tracked.
336+ in_flight = runtime .get_execution (task_info .id , task_info .execution_id )
249337 if in_flight is None :
250338 task_info .status = BackgroundTaskStatus .LOST
251339 changed = True
252340 continue
253341
254342 if in_flight .done ():
255- _finalize_task (task_info , in_flight , runtime )
343+ _finalize_task (task_info , in_flight )
256344 changed = True
257345
258346 if changed :
@@ -399,6 +487,7 @@ async def release_session(
399487 await asyncio .gather (* completed , return_exceptions = True )
400488 finally :
401489 runtime .in_flight_tasks .clear ()
490+ runtime .execution_ids .clear ()
402491 runtime .background_sessions .clear ()
403492
404493 if self ._runtime .get (session_id ) is runtime :
@@ -411,7 +500,16 @@ async def _drain_runtime(
411500 cancel_running : bool ,
412501 timeout : float | None ,
413502 ) -> None :
414- """Cancel and await tracked tasks, bounded by timeout."""
503+ """Cancel and await tracked tasks, bounded by timeout.
504+
505+ Cancelled tasks are observed with :func:`asyncio.wait`, which returns at the deadline. Using
506+ ``asyncio.wait_for(asyncio.gather(...))`` instead would, on timeout, cancel the gather and then
507+ wait for that cancellation to complete, so a task that keeps suppressing ``CancelledError``
508+ could hold the release open indefinitely. Because :func:`asyncio.wait` does not retrieve
509+ exceptions, when the release stops waiting (timeout or cancellation) the exceptions of finished
510+ tasks are retrieved immediately and tasks still running are abandoned with a done-callback that
511+ retrieves their eventual exception.
512+ """
415513 loop = asyncio .get_running_loop ()
416514 deadline = None if timeout is None else loop .time () + float (timeout )
417515
@@ -439,24 +537,21 @@ async def _drain_runtime(
439537 "Session release timed out before all tasks finished. Abandoning %s task(s)." ,
440538 len (pending ),
441539 )
442- for task in pending :
443- if not task .done ():
444- task .add_done_callback (_log_abandoned_background_task )
540+ _abandon_background_tasks (tasks )
445541 return
446542
447543 try :
448- await asyncio .wait_for (
449- asyncio .gather ( * pending , return_exceptions = True ),
450- timeout = remaining ,
451- )
452- except TimeoutError :
453- not_done = [ task for task in pending if not task . done ()]
544+ _ , not_done = await asyncio .wait ( pending , timeout = remaining )
545+ except asyncio .CancelledError :
546+ # The release itself was cancelled; still observe the tasks it will no longer track.
547+ _abandon_background_tasks ( tasks )
548+ raise
549+ if not_done :
454550 logger .warning (
455551 "Session release timed out waiting for %s task(s). They will be abandoned." ,
456552 len (not_done ),
457553 )
458- for task in not_done :
459- task .add_done_callback (_log_abandoned_background_task )
554+ _abandon_background_tasks (tasks )
460555 return
461556
462557 async def before_run (
@@ -487,13 +582,13 @@ def background_agents_start_task(agent_name: str, input: str, description: str)
487582 return f"Error: No background agent found with name '{ agent_name } '. Available agents: { available } "
488583
489584 bg_agent = self ._agents [key ]
490- task_id = provider_state .get ("next_task_id" , 1 )
585+ task_id = runtime . allocate_task_id ( provider_state .get ("next_task_id" , 1 ) )
491586
492587 sub_session = bg_agent .create_session ()
493588
494589 async_task = asyncio .create_task (_run_agent (bg_agent .run (input , session = sub_session )))
495590 try :
496- runtime .track_task (task_id , async_task )
591+ execution_id = runtime .track_task (task_id , async_task )
497592 except RuntimeError as exc :
498593 return f"Error: { exc } "
499594
@@ -505,6 +600,7 @@ def background_agents_start_task(agent_name: str, input: str, description: str)
505600 id = task_id ,
506601 agent_name = agent_name ,
507602 description = description ,
603+ execution_id = execution_id ,
508604 )
509605 tasks = _get_tasks (provider_state )
510606 tasks .append (task_info )
@@ -528,12 +624,15 @@ async def background_agents_wait_for_first_completion(task_ids: list[int]) -> st
528624 if not task_ids :
529625 return "Error: No task IDs provided."
530626
531- # Collect in-flight tasks matching the requested IDs .
627+ # Collect the executions that the requested running records refer to .
532628 waitable : list [tuple [int , asyncio .Task [AgentResponse [Any ]]]] = []
629+ tasks = _get_tasks (provider_state )
630+ running = {t .id : t .execution_id for t in tasks if t .status == BackgroundTaskStatus .RUNNING }
533631 for tid in task_ids :
534- in_flight = runtime .in_flight_tasks .get (tid )
535- if in_flight is not None :
536- waitable .append ((tid , in_flight ))
632+ if tid in running :
633+ in_flight = runtime .get_execution (tid , running [tid ])
634+ if in_flight is not None :
635+ waitable .append ((tid , in_flight ))
537636
538637 if not waitable :
539638 # Refresh state to catch any that completed.
@@ -572,9 +671,10 @@ async def background_agents_wait_for_first_completion(task_ids: list[int]) -> st
572671 tasks = _get_tasks (provider_state )
573672 task_info = next ((t for t in tasks if t .id == completed_id ), None )
574673 if task_info is not None and completed_task is not None :
575- # Report the waited-on execution without overwriting a newer one.
576- is_current_task = runtime .in_flight_tasks .get (task_info .id ) is completed_task
577- _finalize_task (task_info , completed_task , runtime )
674+ # Report the waited-on execution without overwriting a newer one: persist only if the
675+ # record still refers to the execution this waiter observed.
676+ is_current_task = runtime .get_execution (task_info .id , task_info .execution_id ) is completed_task
677+ _finalize_task (task_info , completed_task )
578678 if is_current_task :
579679 _save_tasks (provider_state , tasks )
580680 _save_provider_state (session , provider_state , source_id = source_id )
@@ -601,6 +701,8 @@ def background_agents_get_task_results(task_id: int) -> str:
601701 return f"Task { task_id } is still running."
602702 return f"Task { task_id } has status: { task_info .status .value } ."
603703
704+ background_agents_get_task_results ._invoke_sync_on_event_loop = True # pyright: ignore[reportPrivateUsage]
705+
604706 @tool (name = "background_agents_get_all_tasks" , approval_mode = "never_require" )
605707 def background_agents_get_all_tasks () -> str :
606708 """List all background tasks with their IDs, statuses, agent names, and descriptions."""
@@ -614,6 +716,8 @@ def background_agents_get_all_tasks() -> str:
614716 lines .append (f"- Task { t .id } [{ t .status .value } ] ({ t .agent_name } ): { t .description } " )
615717 return "\n " .join (lines )
616718
719+ background_agents_get_all_tasks ._invoke_sync_on_event_loop = True # pyright: ignore[reportPrivateUsage]
720+
617721 @tool (name = "background_agents_continue_task" , approval_mode = "never_require" )
618722 def background_agents_continue_task (task_id : int , text : str ) -> str :
619723 """Send follow-up input to a completed or failed task to resume its work."""
@@ -642,13 +746,23 @@ def background_agents_continue_task(task_id: int, text: str) -> str:
642746 if sub_session is None :
643747 return f"Error: Session for task { task_id } is no longer available."
644748
749+ # Only the copy of the session that owns the tracked execution may continue it. Otherwise
750+ # the task was already continued from another snapshot of this session: continuing again
751+ # would run concurrently with, or discard the unread result of, that other execution.
752+ if runtime .execution_ids .get (task_id ) != task_info .execution_id :
753+ return (
754+ f"Error: Task { task_id } was continued from another copy of this session and cannot be "
755+ "continued here. Start a new task instead."
756+ )
757+
645758 bg_agent = self ._agents [key ]
646759
647760 async_task = asyncio .create_task (_run_agent (bg_agent .run (text , session = sub_session )))
648761 try :
649- runtime .track_task (task_id , async_task )
762+ execution_id = runtime .track_task (task_id , async_task )
650763 except RuntimeError as exc :
651764 return f"Error: { exc } "
765+ task_info .execution_id = execution_id
652766 task_info .status = BackgroundTaskStatus .RUNNING
653767 task_info .result_text = None
654768 task_info .error_text = None
@@ -678,13 +792,18 @@ def background_agents_clear_completed_task(task_id: int) -> str:
678792 tasks = [t for t in tasks if t .id != task_id ]
679793 _save_tasks (provider_state , tasks )
680794
681- # Clean up runtime references.
682- runtime .in_flight_tasks .pop (task_id , None )
683- runtime .background_sessions .pop (task_id , None )
795+ # Release runtime references only when they belong to this record's execution. A different
796+ # execution under this task ID (started from another snapshot of this session) must stay
797+ # tracked so it can still be retrieved, cancelled, or drained by release_session.
798+ tracked_execution_id = runtime .execution_ids .get (task_id )
799+ if tracked_execution_id is None or tracked_execution_id == task_info .execution_id :
800+ runtime .forget_task (task_id )
684801
685802 _save_provider_state (session , provider_state , source_id = source_id )
686803 return f"Task { task_id } cleared."
687804
805+ background_agents_clear_completed_task ._invoke_sync_on_event_loop = True # pyright: ignore[reportPrivateUsage]
806+
688807 # Inject instructions and current task status.
689808 context .extend_instructions (self .source_id , [self ._instructions ])
690809 context .extend_tools (
0 commit comments