Skip to content

Commit 83c7190

Browse files
authored
Python: fix(core): keep finished background tasks until cleared or released (microsoft#9387)
* Python: fix(core): keep finished background tasks until cleared or released Part of microsoft#8760 ### Description Keep finished asyncio.Task objects in _RuntimeState.in_flight_tasks inside _finalize_task. They remain in memory until explicitly removed by background_agents_clear_completed_task or torn down via BackgroundAgentsProvider.release_session. ### Bug & Repro In the single-process implementation, _finalize_task extracted results and immediately removed the task from in_flight_tasks. However, host session state is persisted only after the agent run completes. If the host restores a session snapshot taken while a task was RUNNING (e.g. after a run failure, turn rollback, or host desync), subsequent calls to _refresh_task_state found no entry in in_flight_tasks and marked the task as LOST, silently discarding a completed or failed result. ### Trade-off Finished asyncio.Task objects remain referenced in in_flight_tasks until explicitly cleared via clear_completed_task or released via release_session, instead of being discarded immediately at finalization. ### Validation - pytest: 52 passed, 0 failed in 0.77s - ruff check: All checks passed - ruff format: 2 files formatted clean - pyright: 0 errors, 0 warnings, 0 informations - mypy: Success: no issues found * Python: Record generation token on background tasks to prevent wrong execution attribution (microsoft#8760) Address review finding on reusable task IDs in continuation flows: - Track a monotonic generation token on BackgroundTaskInfo and serialize it in to_dict/from_dict. - Dual-key _InFlightTasks with (task_id, generation) so each execution of a continued task is stored discretely. - _refresh_task_state reconciles RUNNING tasks against their specific generation, preventing a restored pre-continuation snapshot from erroneously receiving the result of a later continuation. - Clear and release paths clean up all executions associated with the task ID. - Add regression test test_reused_task_id_does_not_associate_stale_snapshot_with_continuation. * Python: Simplify background task retention without generation tracking (microsoft#8760) * Python: Track execution generation on background tasks (microsoft#8760) Persist a per-task generation so a restored older session snapshot is never reconciled against a later continuation that reused the same task ID. Continue/clear are generation-aware. * Python: Use unique execution IDs and run background tools on the event loop (microsoft#8760) * Python: Bound background task release with asyncio.wait * Python: Observe all task outcomes when session release stops waiting * Python: Move background agent test helpers into a helper module
1 parent 0c47936 commit 83c7190

3 files changed

Lines changed: 1081 additions & 39 deletions

File tree

‎python/packages/core/agent_framework/_harness/_background_agents.py‎

Lines changed: 156 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
import asyncio
1313
import logging
14+
import uuid
1415
from collections.abc import Awaitable, MutableMapping, Sequence
1516
from dataclasses import dataclass, field
1617
from enum import Enum
@@ -55,7 +56,13 @@ class BackgroundTaskStatus(str, Enum):
5556

5657
@experimental(feature_id=ExperimentalFeature.HARNESS)
5758
class BackgroundTaskInfo(SerializationMixin):
58-
"""Metadata for a single background task."""
59+
"""Metadata for a single background task.
60+
61+
``execution_id`` identifies the execution (initial start or continuation) that this record
62+
describes. A new, globally unique ID is assigned to every execution, so a record restored from
63+
an older session snapshot is never reconciled against a different execution, even when that
64+
execution reuses the same task ID.
65+
"""
5966

6067
DEFAULT_EXCLUDE: ClassVar[set[str]] = set()
6168

@@ -65,7 +72,8 @@ class BackgroundTaskInfo(SerializationMixin):
6572
status: BackgroundTaskStatus
6673
result_text: str | None
6774
error_text: str | None
68-
__slots__ = ("agent_name", "description", "error_text", "id", "result_text", "status")
75+
execution_id: str | None
76+
__slots__ = ("agent_name", "description", "error_text", "execution_id", "id", "result_text", "status")
6977

7078
def __init__(
7179
self,
@@ -75,6 +83,7 @@ def __init__(
7583
status: BackgroundTaskStatus = BackgroundTaskStatus.RUNNING,
7684
result_text: str | None = None,
7785
error_text: str | None = None,
86+
execution_id: str | None = None,
7887
) -> None:
7988
"""Initialize a background task info entry."""
8089
self.id = id
@@ -83,6 +92,7 @@ def __init__(
8392
self.status = status
8493
self.result_text = result_text
8594
self.error_text = error_text
95+
self.execution_id = execution_id
8696

8797
def to_dict(self, *, exclude: set[str] | None = None, exclude_none: bool = True) -> dict[str, Any]:
8898
"""Serialize for session state persistence."""
@@ -97,6 +107,8 @@ def to_dict(self, *, exclude: set[str] | None = None, exclude_none: bool = True)
97107
data["result_text"] = self.result_text
98108
if not exclude_none or self.error_text is not None:
99109
data["error_text"] = self.error_text
110+
if not exclude_none or self.execution_id is not None:
111+
data["execution_id"] = self.execution_id
100112
return data
101113

102114
@classmethod
@@ -109,24 +121,61 @@ def from_dict(cls, data: MutableMapping[str, Any], **kwargs: Any) -> BackgroundT
109121
status=BackgroundTaskStatus(data["status"]),
110122
result_text=data.get("result_text"),
111123
error_text=data.get("error_text"),
124+
execution_id=data.get("execution_id"),
112125
)
113126

114127

115128
@dataclass
116129
class _RuntimeState:
117-
"""Non-serializable per-session runtime state for background tasks."""
130+
"""Non-serializable per-session runtime state for background tasks.
131+
132+
``in_flight_tasks`` holds the latest execution started for each task ID and ``execution_ids``
133+
records which execution that is. Lookups made on behalf of a persisted
134+
:class:`BackgroundTaskInfo` go through :meth:`get_execution`, so a record only ever sees the
135+
execution it was created for.
136+
137+
All reads and writes happen on the event loop thread (every background tool is invoked there),
138+
so each tool's check-and-update sequence is atomic with respect to the others.
139+
"""
118140

119141
in_flight_tasks: dict[int, asyncio.Task[AgentResponse[Any]]] = field(default_factory=lambda: {})
142+
execution_ids: dict[int, str] = field(default_factory=lambda: {})
120143
background_sessions: dict[int, AgentSession] = field(default_factory=lambda: {})
144+
next_task_id: int = 1
121145
closed: bool = False
122146

123-
def track_task(self, task_id: int, task: asyncio.Task[AgentResponse[Any]]) -> None:
124-
"""Track a background task if this runtime is still open."""
147+
def allocate_task_id(self, persisted_next_task_id: int) -> int:
148+
"""Return a task ID that was never handed out by this runtime.
149+
150+
A restored older session snapshot carries a stale ``next_task_id``; reusing it would let a
151+
new task overwrite the runtime entry of a task started after that snapshot was taken.
152+
"""
153+
task_id = max(persisted_next_task_id, self.next_task_id)
154+
self.next_task_id = task_id + 1
155+
return task_id
156+
157+
def track_task(self, task_id: int, task: asyncio.Task[AgentResponse[Any]]) -> str:
158+
"""Track a new execution for a task ID if this runtime is still open and return its execution ID."""
125159
if self.closed:
126160
task.cancel()
127161
raise RuntimeError("Session runtime is closed; cannot start background task.")
128162

163+
execution_id = uuid.uuid4().hex
129164
self.in_flight_tasks[task_id] = task
165+
self.execution_ids[task_id] = execution_id
166+
return execution_id
167+
168+
def get_execution(self, task_id: int, execution_id: str | None) -> asyncio.Task[AgentResponse[Any]] | None:
169+
"""Return the tracked execution for a task ID only if it is the given execution."""
170+
if execution_id is None or self.execution_ids.get(task_id) != execution_id:
171+
return None
172+
return self.in_flight_tasks.get(task_id)
173+
174+
def forget_task(self, task_id: int) -> None:
175+
"""Drop all runtime references for a task ID."""
176+
self.in_flight_tasks.pop(task_id, None)
177+
self.execution_ids.pop(task_id, None)
178+
self.background_sessions.pop(task_id, None)
130179

131180

132181
# ---------------------------------------------------------------------------
@@ -153,6 +202,27 @@ def _log_abandoned_background_task(task: asyncio.Task[Any]) -> None:
153202
logger.debug("Abandoned background task raised: %s", exception)
154203

155204

205+
# Strong references to tasks abandoned by a session release. The event loop only keeps weak
206+
# references to tasks, so without these an abandoned task could be garbage collected while pending.
207+
_abandoned_background_tasks: set[asyncio.Task[Any]] = set()
208+
209+
210+
def _abandon_background_tasks(tasks: Sequence[asyncio.Task[Any]]) -> None:
211+
"""Retrieve exceptions of finished tasks now and of unfinished tasks once they complete.
212+
213+
Used when a session release stops waiting (timeout or cancellation): the runtime is cleared
214+
afterwards, so any task whose exception is not retrieved here would be reported as "Task exception
215+
was never retrieved". Unfinished tasks are kept alive until they complete.
216+
"""
217+
for task in tasks:
218+
if task.done():
219+
_log_abandoned_background_task(task)
220+
elif task not in _abandoned_background_tasks:
221+
_abandoned_background_tasks.add(task)
222+
task.add_done_callback(_abandoned_background_tasks.discard)
223+
task.add_done_callback(_log_abandoned_background_task)
224+
225+
156226
def _validate_and_build_agent_dict(agents: Sequence[SupportsAgentRun]) -> dict[str, SupportsAgentRun]:
157227
"""Validate agents and build a case-insensitive lookup dict.
158228
@@ -215,9 +285,27 @@ def _save_tasks(state: dict[str, Any], tasks: list[BackgroundTaskInfo]) -> None:
215285
def _finalize_task(
216286
task_info: BackgroundTaskInfo,
217287
completed_task: asyncio.Task[AgentResponse[Any]],
218-
runtime: _RuntimeState,
219288
) -> None:
220-
"""Extract results from a completed asyncio task and update task info."""
289+
"""Extract results from a completed asyncio task and update task info.
290+
291+
The finished :class:`asyncio.Task` is intentionally **not** removed from
292+
``runtime.in_flight_tasks`` here. Session state is persisted by the host
293+
only after the agent run completes; if the run fails, is stopped early, or
294+
the host restores an older session snapshot, the persisted record will still
295+
say ``RUNNING`` while this in-memory task no longer exists. On the next
296+
call to :func:`_refresh_task_state` a missing entry would be misidentified
297+
as ``LOST``, discarding a result that is already available.
298+
299+
Because the retained entry is keyed by task ID, callers must only pass the
300+
execution matching ``task_info.execution_id`` (see
301+
:meth:`_RuntimeState.get_execution`); a different execution under the same
302+
task ID must never be applied to the record.
303+
304+
Tasks are removed from ``in_flight_tasks`` by two callers:
305+
306+
- :func:`background_agents_clear_completed_task` — explicit caller-driven cleanup.
307+
- :meth:`BackgroundAgentsProvider.release_session` — bulk cleanup on session teardown.
308+
"""
221309
if completed_task.cancelled():
222310
task_info.status = BackgroundTaskStatus.FAILED
223311
task_info.error_text = "Task was canceled."
@@ -229,9 +317,6 @@ def _finalize_task(
229317
else:
230318
task_info.status = BackgroundTaskStatus.COMPLETED
231319
task_info.result_text = completed_task.result().text
232-
# A stale waiter may observe a task ID already reused by a continuation.
233-
if runtime.in_flight_tasks.get(task_info.id) is completed_task:
234-
runtime.in_flight_tasks.pop(task_info.id, None)
235320

236321

237322
def _refresh_task_state(
@@ -245,14 +330,17 @@ def _refresh_task_state(
245330
if task_info.status != BackgroundTaskStatus.RUNNING:
246331
continue
247332

248-
in_flight = runtime.in_flight_tasks.get(task_info.id)
333+
# A missing execution, or a different one under the same task ID (for example, the task was
334+
# continued from another snapshot of this session), means this record's execution is no
335+
# longer tracked.
336+
in_flight = runtime.get_execution(task_info.id, task_info.execution_id)
249337
if in_flight is None:
250338
task_info.status = BackgroundTaskStatus.LOST
251339
changed = True
252340
continue
253341

254342
if in_flight.done():
255-
_finalize_task(task_info, in_flight, runtime)
343+
_finalize_task(task_info, in_flight)
256344
changed = True
257345

258346
if changed:
@@ -399,6 +487,7 @@ async def release_session(
399487
await asyncio.gather(*completed, return_exceptions=True)
400488
finally:
401489
runtime.in_flight_tasks.clear()
490+
runtime.execution_ids.clear()
402491
runtime.background_sessions.clear()
403492

404493
if self._runtime.get(session_id) is runtime:
@@ -411,7 +500,16 @@ async def _drain_runtime(
411500
cancel_running: bool,
412501
timeout: float | None,
413502
) -> None:
414-
"""Cancel and await tracked tasks, bounded by timeout."""
503+
"""Cancel and await tracked tasks, bounded by timeout.
504+
505+
Cancelled tasks are observed with :func:`asyncio.wait`, which returns at the deadline. Using
506+
``asyncio.wait_for(asyncio.gather(...))`` instead would, on timeout, cancel the gather and then
507+
wait for that cancellation to complete, so a task that keeps suppressing ``CancelledError``
508+
could hold the release open indefinitely. Because :func:`asyncio.wait` does not retrieve
509+
exceptions, when the release stops waiting (timeout or cancellation) the exceptions of finished
510+
tasks are retrieved immediately and tasks still running are abandoned with a done-callback that
511+
retrieves their eventual exception.
512+
"""
415513
loop = asyncio.get_running_loop()
416514
deadline = None if timeout is None else loop.time() + float(timeout)
417515

@@ -439,24 +537,21 @@ async def _drain_runtime(
439537
"Session release timed out before all tasks finished. Abandoning %s task(s).",
440538
len(pending),
441539
)
442-
for task in pending:
443-
if not task.done():
444-
task.add_done_callback(_log_abandoned_background_task)
540+
_abandon_background_tasks(tasks)
445541
return
446542

447543
try:
448-
await asyncio.wait_for(
449-
asyncio.gather(*pending, return_exceptions=True),
450-
timeout=remaining,
451-
)
452-
except TimeoutError:
453-
not_done = [task for task in pending if not task.done()]
544+
_, not_done = await asyncio.wait(pending, timeout=remaining)
545+
except asyncio.CancelledError:
546+
# The release itself was cancelled; still observe the tasks it will no longer track.
547+
_abandon_background_tasks(tasks)
548+
raise
549+
if not_done:
454550
logger.warning(
455551
"Session release timed out waiting for %s task(s). They will be abandoned.",
456552
len(not_done),
457553
)
458-
for task in not_done:
459-
task.add_done_callback(_log_abandoned_background_task)
554+
_abandon_background_tasks(tasks)
460555
return
461556

462557
async def before_run(
@@ -487,13 +582,13 @@ def background_agents_start_task(agent_name: str, input: str, description: str)
487582
return f"Error: No background agent found with name '{agent_name}'. Available agents: {available}"
488583

489584
bg_agent = self._agents[key]
490-
task_id = provider_state.get("next_task_id", 1)
585+
task_id = runtime.allocate_task_id(provider_state.get("next_task_id", 1))
491586

492587
sub_session = bg_agent.create_session()
493588

494589
async_task = asyncio.create_task(_run_agent(bg_agent.run(input, session=sub_session)))
495590
try:
496-
runtime.track_task(task_id, async_task)
591+
execution_id = runtime.track_task(task_id, async_task)
497592
except RuntimeError as exc:
498593
return f"Error: {exc}"
499594

@@ -505,6 +600,7 @@ def background_agents_start_task(agent_name: str, input: str, description: str)
505600
id=task_id,
506601
agent_name=agent_name,
507602
description=description,
603+
execution_id=execution_id,
508604
)
509605
tasks = _get_tasks(provider_state)
510606
tasks.append(task_info)
@@ -528,12 +624,15 @@ async def background_agents_wait_for_first_completion(task_ids: list[int]) -> st
528624
if not task_ids:
529625
return "Error: No task IDs provided."
530626

531-
# Collect in-flight tasks matching the requested IDs.
627+
# Collect the executions that the requested running records refer to.
532628
waitable: list[tuple[int, asyncio.Task[AgentResponse[Any]]]] = []
629+
tasks = _get_tasks(provider_state)
630+
running = {t.id: t.execution_id for t in tasks if t.status == BackgroundTaskStatus.RUNNING}
533631
for tid in task_ids:
534-
in_flight = runtime.in_flight_tasks.get(tid)
535-
if in_flight is not None:
536-
waitable.append((tid, in_flight))
632+
if tid in running:
633+
in_flight = runtime.get_execution(tid, running[tid])
634+
if in_flight is not None:
635+
waitable.append((tid, in_flight))
537636

538637
if not waitable:
539638
# Refresh state to catch any that completed.
@@ -572,9 +671,10 @@ async def background_agents_wait_for_first_completion(task_ids: list[int]) -> st
572671
tasks = _get_tasks(provider_state)
573672
task_info = next((t for t in tasks if t.id == completed_id), None)
574673
if task_info is not None and completed_task is not None:
575-
# Report the waited-on execution without overwriting a newer one.
576-
is_current_task = runtime.in_flight_tasks.get(task_info.id) is completed_task
577-
_finalize_task(task_info, completed_task, runtime)
674+
# Report the waited-on execution without overwriting a newer one: persist only if the
675+
# record still refers to the execution this waiter observed.
676+
is_current_task = runtime.get_execution(task_info.id, task_info.execution_id) is completed_task
677+
_finalize_task(task_info, completed_task)
578678
if is_current_task:
579679
_save_tasks(provider_state, tasks)
580680
_save_provider_state(session, provider_state, source_id=source_id)
@@ -601,6 +701,8 @@ def background_agents_get_task_results(task_id: int) -> str:
601701
return f"Task {task_id} is still running."
602702
return f"Task {task_id} has status: {task_info.status.value}."
603703

704+
background_agents_get_task_results._invoke_sync_on_event_loop = True # pyright: ignore[reportPrivateUsage]
705+
604706
@tool(name="background_agents_get_all_tasks", approval_mode="never_require")
605707
def background_agents_get_all_tasks() -> str:
606708
"""List all background tasks with their IDs, statuses, agent names, and descriptions."""
@@ -614,6 +716,8 @@ def background_agents_get_all_tasks() -> str:
614716
lines.append(f"- Task {t.id} [{t.status.value}] ({t.agent_name}): {t.description}")
615717
return "\n".join(lines)
616718

719+
background_agents_get_all_tasks._invoke_sync_on_event_loop = True # pyright: ignore[reportPrivateUsage]
720+
617721
@tool(name="background_agents_continue_task", approval_mode="never_require")
618722
def background_agents_continue_task(task_id: int, text: str) -> str:
619723
"""Send follow-up input to a completed or failed task to resume its work."""
@@ -642,13 +746,23 @@ def background_agents_continue_task(task_id: int, text: str) -> str:
642746
if sub_session is None:
643747
return f"Error: Session for task {task_id} is no longer available."
644748

749+
# Only the copy of the session that owns the tracked execution may continue it. Otherwise
750+
# the task was already continued from another snapshot of this session: continuing again
751+
# would run concurrently with, or discard the unread result of, that other execution.
752+
if runtime.execution_ids.get(task_id) != task_info.execution_id:
753+
return (
754+
f"Error: Task {task_id} was continued from another copy of this session and cannot be "
755+
"continued here. Start a new task instead."
756+
)
757+
645758
bg_agent = self._agents[key]
646759

647760
async_task = asyncio.create_task(_run_agent(bg_agent.run(text, session=sub_session)))
648761
try:
649-
runtime.track_task(task_id, async_task)
762+
execution_id = runtime.track_task(task_id, async_task)
650763
except RuntimeError as exc:
651764
return f"Error: {exc}"
765+
task_info.execution_id = execution_id
652766
task_info.status = BackgroundTaskStatus.RUNNING
653767
task_info.result_text = None
654768
task_info.error_text = None
@@ -678,13 +792,18 @@ def background_agents_clear_completed_task(task_id: int) -> str:
678792
tasks = [t for t in tasks if t.id != task_id]
679793
_save_tasks(provider_state, tasks)
680794

681-
# Clean up runtime references.
682-
runtime.in_flight_tasks.pop(task_id, None)
683-
runtime.background_sessions.pop(task_id, None)
795+
# Release runtime references only when they belong to this record's execution. A different
796+
# execution under this task ID (started from another snapshot of this session) must stay
797+
# tracked so it can still be retrieved, cancelled, or drained by release_session.
798+
tracked_execution_id = runtime.execution_ids.get(task_id)
799+
if tracked_execution_id is None or tracked_execution_id == task_info.execution_id:
800+
runtime.forget_task(task_id)
684801

685802
_save_provider_state(session, provider_state, source_id=source_id)
686803
return f"Task {task_id} cleared."
687804

805+
background_agents_clear_completed_task._invoke_sync_on_event_loop = True # pyright: ignore[reportPrivateUsage]
806+
688807
# Inject instructions and current task status.
689808
context.extend_instructions(self.source_id, [self._instructions])
690809
context.extend_tools(

0 commit comments

Comments
 (0)