Skip to content

Commit 4a50020

Browse files
committed
fix(workflow): exclude replayed nodes from max_concurrency count
A replayed node fast-forwarded from history can block on the replay sequence barrier until an earlier-recorded sibling is scheduled. Counting it against max_concurrency let it hold the only slot, so with max_concurrency=1 the sibling was never started and resume failed with a replay divergence timeout. Only genuinely executing nodes now count. Fixes google#7618
1 parent 2084efb commit 4a50020

2 files changed

Lines changed: 133 additions & 5 deletions

File tree

‎src/google/adk/workflow/_workflow.py‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -559,12 +559,20 @@ async def _schedule_ready_nodes(
559559
await self._emit_node_checkpoint(loop_state, ctx)
560560

561561
def _at_concurrency_limit(self, loop_state: _LoopState) -> bool:
562-
"""Check if max_concurrency has been reached."""
563-
return (
564-
self.max_concurrency is not None
565-
and self.max_concurrency > 0
566-
and len(loop_state.pending_tasks) >= self.max_concurrency
562+
"""Check if max_concurrency has been reached.
563+
564+
Replayed nodes are history fast-forwards that may block on the replay
565+
sequence barrier waiting for another node to be scheduled, so they must
566+
not hold a concurrency slot.
567+
"""
568+
if self.max_concurrency is None or self.max_concurrency <= 0:
569+
return False
570+
active = sum(
571+
1
572+
for name in loop_state.pending_tasks
573+
if name not in loop_state.replayed_nodes
567574
)
575+
return active >= self.max_concurrency
568576

569577
def _pop_trigger(
570578
self, loop_state: _LoopState, node_name: str

‎tests/unittests/workflow/test_workflow_concurrency.py‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,16 +13,23 @@
1313
# limitations under the License.
1414

1515
import asyncio
16+
import functools
1617
from typing import Any
1718
from typing import AsyncGenerator
1819

1920
from google.adk.agents.context import Context
2021
from google.adk.apps.app import App
2122
from google.adk.events.event import Event
23+
from google.adk.events.request_input import RequestInput
24+
from google.adk.runners import Runner
25+
from google.adk.sessions.in_memory_session_service import InMemorySessionService
2226
from google.adk.workflow import BaseNode
27+
from google.adk.workflow import FunctionNode
28+
from google.adk.workflow import JoinNode
2329
from google.adk.workflow import START
2430
from google.adk.workflow._parallel_worker import _ParallelWorker as ParallelWorker
2531
from google.adk.workflow._workflow import Workflow
32+
from google.genai import types
2633
from pydantic import Field
2734
import pytest
2835
from typing_extensions import override
@@ -172,3 +179,116 @@ async def _run_impl(
172179
events = await runner.run_async(testing_utils.get_user_content('start'))
173180
outputs = [e.output for e in events if e.output is not None]
174181
assert 'workflow_done' in outputs
182+
183+
184+
@pytest.mark.parametrize('max_concurrency', [1, 2, None])
185+
@pytest.mark.asyncio
186+
async def test_max_concurrency_resume_does_not_deadlock_on_replay(
187+
max_concurrency: int | None,
188+
):
189+
"""Replayed nodes waiting on the replay barrier must not hold a slot.
190+
191+
Setup:
192+
A loop that requests input twice, beside a three-node sibling chain, both
193+
meeting in a JoinNode.
194+
Act:
195+
Run, then resume twice by answering each RequestInput.
196+
Assert:
197+
- The workflow completes instead of timing out on the replay barrier.
198+
- Nodes that really execute never exceed max_concurrency.
199+
"""
200+
rounds = 2
201+
running = 0
202+
peak = 0
203+
204+
def tracked(func):
205+
@functools.wraps(func)
206+
async def wrapper(*args, **kwargs):
207+
nonlocal running, peak
208+
running += 1
209+
peak = max(peak, running)
210+
try:
211+
await asyncio.sleep(0.01)
212+
return func(*args, **kwargs)
213+
finally:
214+
running -= 1
215+
216+
return wrapper
217+
218+
def start(node_input: dict):
219+
return {'round': 0}
220+
221+
async def ask(ctx, node_input: dict):
222+
ctx.state['round'] = node_input['round']
223+
yield RequestInput(
224+
interrupt_id=f"ask-{node_input['round']}", message='approve?'
225+
)
226+
227+
def answered(ctx, node_input):
228+
return {'round': ctx.state['round']}
229+
230+
def again(node_input: dict):
231+
state = {'round': node_input['round'] + 1}
232+
return Event(route=state['round'] < rounds, output=state)
233+
234+
def other(node_input: dict):
235+
return node_input
236+
237+
start_node = FunctionNode(func=tracked(start), name='start')
238+
answered_node = FunctionNode(func=tracked(answered), name='answered')
239+
again_node = FunctionNode(func=tracked(again), name='again')
240+
chain = [
241+
FunctionNode(func=tracked(other), name=f'other{i}') for i in range(3)
242+
]
243+
asking = FunctionNode(func=ask, name='ask', rerun_on_resume=False)
244+
join = JoinNode(name='join')
245+
workflow = Workflow(
246+
name='root',
247+
edges=[
248+
(START, (start_node, chain[0])),
249+
(start_node, asking),
250+
(asking, answered_node),
251+
(answered_node, again_node),
252+
(again_node, {True: asking, False: join}),
253+
*zip(chain, chain[1:]),
254+
(chain[-1], join),
255+
],
256+
max_concurrency=max_concurrency,
257+
)
258+
259+
sessions = InMemorySessionService()
260+
runner = Runner(app_name='repro', node=workflow, session_service=sessions)
261+
session = await sessions.create_session(app_name='repro', user_id='user')
262+
message = types.Content(role='user', parts=[types.Part(text='{}')])
263+
answered_ids = []
264+
while True:
265+
pending = []
266+
async for event in runner.run_async(
267+
user_id='user', session_id=session.id, new_message=message
268+
):
269+
if event.content:
270+
pending += [
271+
part.function_call.id
272+
for part in event.content.parts or []
273+
if part.function_call
274+
]
275+
if not pending:
276+
break
277+
answered_ids += pending
278+
message = types.Content(
279+
role='user',
280+
parts=[
281+
types.Part(
282+
function_response=types.FunctionResponse(
283+
id=call, name='adk_request_input', response={'ok': True}
284+
)
285+
)
286+
for call in pending
287+
],
288+
)
289+
290+
assert answered_ids == ['ask-0', 'ask-1']
291+
if max_concurrency is not None:
292+
assert peak <= max_concurrency
293+
if max_concurrency == 1:
294+
assert peak == 1

0 commit comments

Comments
 (0)