Skip to content

Commit c7453af

Browse files
committed
Python: cancel WorkflowExecutor's output-forwarding siblings too
Retroactive check on our own claim, per mistake #12: this PR's review reply said the other gather() sites in _workflow.py/_workflow_executor.py "hold no cross-invocation buffered state a restore can race with." Tested rather than re-argued it. _workflow.py:1034 (send_request_info_response) survives: it pops a still- pending event before writing anything, and a restore clears pending events too, so an orphaned write fails closed (raises) instead of landing. Reproduced directly against InProcRunnerContext to confirm. _workflow_executor.py's four gather() sites do not survive. ctx.send_message and ctx.yield_output/add_event write into RunnerContext state (_messages, the event queue) with no equivalent precondition check. Reproduced the same shape as the fan-in/fan-out bugs: a sub-workflow output still being forwarded when a sibling output's send fails calls ctx.send_message after a checkpoint restore has already cleared the parent's message queue, landing stale output in it. Fixed all four sites with the same gather_cancelling_siblings_on_error helper already used in _run_iteration and FanOutEdgeRunner - moved to _edge_runner.py in the prior commit specifically so _workflow_executor.py could import it without a cycle.
1 parent da23d14 commit c7453af

2 files changed

Lines changed: 90 additions & 8 deletions

File tree

‎python/packages/core/agent_framework/_workflows/_workflow_executor.py‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
# Copyright (c) Microsoft. All rights reserved.
22

3-
import asyncio
43
import logging
54
import sys
65
import types
@@ -11,6 +10,7 @@
1110
from ._workflow import Workflow
1211

1312
from ._const import GLOBAL_KWARGS_KEY, WORKFLOW_RUN_KWARGS_KEY
13+
from ._edge_runner import gather_cancelling_siblings_on_error
1414
from ._events import (
1515
WorkflowEvent,
1616
WorkflowRunState,
@@ -490,10 +490,12 @@ async def on_checkpoint_restore(self, state: dict[str, Any]) -> None:
490490
self.id,
491491
execution_context.execution_id,
492492
)
493-
await asyncio.gather(*[
494-
self.workflow._runner_context.add_request_info_event(event) # pyright: ignore[reportPrivateUsage]
495-
for event in request_info_events
496-
])
493+
await gather_cancelling_siblings_on_error(
494+
*(
495+
self.workflow._runner_context.add_request_info_event(event) # pyright: ignore[reportPrivateUsage]
496+
for event in request_info_events
497+
)
498+
)
497499

498500
async def _process_workflow_result(
499501
self,
@@ -524,9 +526,9 @@ async def _process_workflow_result(
524526
# Process outputs
525527
if self.allow_direct_output:
526528
# Note that the executor is allowed to continue its own execution after yielding outputs.
527-
await asyncio.gather(*[ctx.yield_output(output) for output in outputs])
529+
await gather_cancelling_siblings_on_error(*(ctx.yield_output(output) for output in outputs))
528530
else:
529-
await asyncio.gather(*[ctx.send_message(output) for output in outputs])
531+
await gather_cancelling_siblings_on_error(*(ctx.send_message(output) for output in outputs))
530532

531533
# Pipe sub-workflow intermediate emissions up through the parent's event stream.
532534
# Bypasses the parent's yield-output classifier so the 'intermediate' label is preserved
@@ -539,7 +541,9 @@ async def _forward_intermediate_output(output: Any) -> None:
539541
event = WorkflowEvent("intermediate", executor_id=self.id, data=output)
540542
await ctx.add_event(event)
541543

542-
await asyncio.gather(*[_forward_intermediate_output(output) for output in intermediate_outputs])
544+
await gather_cancelling_siblings_on_error(
545+
*(_forward_intermediate_output(output) for output in intermediate_outputs)
546+
)
543547

544548
# Process request info events
545549
for event in request_info_events:

‎python/packages/core/tests/workflow/test_sub_workflow.py‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# Copyright (c) Microsoft. All rights reserved.
22

3+
import asyncio
34
import logging
45
from dataclasses import dataclass, field
56
from typing import Any
@@ -14,14 +15,18 @@
1415
SubWorkflowResponseMessage,
1516
Workflow,
1617
WorkflowBuilder,
18+
WorkflowCheckpoint,
1719
WorkflowContext,
1820
WorkflowEvent,
1921
WorkflowExecutor,
22+
WorkflowRunResult,
2023
WorkflowRunState,
2124
handler,
2225
response_handler,
2326
)
2427
from agent_framework._workflows._checkpoint import InMemoryCheckpointStorage
28+
from agent_framework._workflows._runner_context import InProcRunnerContext, WorkflowMessage
29+
from agent_framework._workflows._state import State
2530

2631

2732
# Test message types
@@ -943,3 +948,76 @@ async def run(self, message: str, ctx: WorkflowContext[Never, str]) -> None: #
943948

944949
# The parent's own terminal output is unaffected.
945950
assert any(e.executor_id == "parent_sink" and e.data == "final: hello" for e in output_events)
951+
952+
953+
async def test_workflow_executor_orphaned_output_forward_cannot_repopulate_a_restored_message_queue() -> None:
954+
"""The orphaned-task race from PR #7948 applies to WorkflowExecutor's own output-forwarding gather too.
955+
956+
That PR's review excluded ``_workflow_executor.py``'s ``gather()`` sites with "none of them hold
957+
cross-invocation buffered state a restore can race with" - that reasoning was wrong. When
958+
``allow_direct_output`` is false (the default), ``_process_workflow_result`` forwards every
959+
sub-workflow output to the parent by gathering ``ctx.send_message(output)`` per output.
960+
``WorkflowContext.send_message`` reaches straight into ``RunnerContext._messages`` with no
961+
precondition check (unlike, say, ``send_request_info_response``'s pop-and-validate, which turned
962+
out to protect a sibling gather site in ``_workflow.py``). A sibling output still being sent when
963+
another fails is therefore not cancelled by plain ``asyncio.gather``, and can append into the
964+
parent's message queue after a checkpoint restore has already cleared it for the resumed run.
965+
"""
966+
967+
class _Inner(Executor):
968+
@handler
969+
async def handle(self, message: str, ctx: WorkflowContext[str]) -> None:
970+
pass
971+
972+
inner_workflow = WorkflowBuilder(start_executor=_Inner(id="inner")).build()
973+
workflow_executor = WorkflowExecutor(inner_workflow, id="wrapped")
974+
975+
ctx_impl = InProcRunnerContext()
976+
state = State()
977+
wctx: WorkflowContext[Any] = WorkflowContext(workflow_executor, ["parent_source"], state, ctx_impl)
978+
979+
entered_delay = asyncio.Event()
980+
release = asyncio.Event()
981+
real_send_message = ctx_impl.send_message
982+
983+
async def patched_send_message(message: WorkflowMessage) -> None:
984+
if message.data == "bad-output":
985+
raise RuntimeError("output send failed")
986+
if message.data == "stale-output-from-failed-run":
987+
entered_delay.set()
988+
await release.wait()
989+
await real_send_message(message)
990+
991+
ctx_impl.send_message = patched_send_message # type: ignore[method-assign] # ty: ignore[invalid-assignment]
992+
993+
# allow_direct_output defaults to False, so outputs go through the ctx.send_message branch.
994+
# A status event is required for get_final_state(), which _process_workflow_result reads before
995+
# the gather; without it the method raises immediately and entered_delay is never set.
996+
result = WorkflowRunResult(
997+
[
998+
WorkflowEvent("output", data="bad-output"),
999+
WorkflowEvent("output", data="stale-output-from-failed-run"),
1000+
],
1001+
status_events=[WorkflowEvent.status(WorkflowRunState.IDLE)],
1002+
)
1003+
1004+
task = asyncio.create_task(workflow_executor._process_workflow_result(result, wctx)) # pyright: ignore[reportPrivateUsage]
1005+
1006+
# The second output's send is now parked mid-flight when the first one raises.
1007+
await asyncio.wait_for(entered_delay.wait(), timeout=5)
1008+
1009+
with pytest.raises(RuntimeError, match="output send failed"):
1010+
await task
1011+
1012+
# The parent restores the same runner context from its last good checkpoint - what a caller does
1013+
# after catching a failed superstep, per Workflow._execute_with_message_or_checkpoint.
1014+
checkpoint = WorkflowCheckpoint(workflow_name="test_name", graph_signature_hash="test_hash")
1015+
await ctx_impl.apply_checkpoint(checkpoint)
1016+
assert not await ctx_impl.has_messages()
1017+
1018+
# Release the parked send. If the fix reached this gather, its task was already cancelled and
1019+
# this is a no-op; if it did not, the stale output lands in the just-restored queue.
1020+
release.set()
1021+
await asyncio.sleep(0)
1022+
1023+
assert not await ctx_impl.has_messages()

0 commit comments

Comments
 (0)