Skip to content

Commit 0200599

Browse files
authored
Python: Improve settled approval matching (#9535)
* Improve settled approval matching * Address PR comments * Adddress PR comment
1 parent 6b48d20 commit 0200599

3 files changed

Lines changed: 137 additions & 10 deletions

File tree

‎docs/specs/004-python-function-calling-loop.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -672,7 +672,7 @@ that manually replay messages own the equivalent rule: do not resend an approval
672672
| Completed request tracking | Terminal result text cannot leave a completed request pending or consume a later reused-id request. | `test_collect_unanswered_approval_requests_consumes_terminal_result` |
673673
| Unbound or duplicate response | A response with no pending session request is removed; one request authorizes at most one response. | `test_session_approval_binding_rebinds_consumes_and_rejects_duplicates` |
674674
| Forged inbound request history | A caller-supplied request wrapper cannot replace the server snapshot or resurrect consumed authority. | `test_session_approval_binding_does_not_trust_inbound_request_history` |
675-
| Local approval without authoritative session | A local approval response carried by any message role executes nothing when no session recorded its request, whether the request is fabricated, absent, answered with tampered arguments, or repeated as several responses sharing one approval id. Verified for both streaming and non-streaming. | `test_local_approval_response_without_authoritative_session_does_not_execute` |
675+
| Local approval without authoritative session | A local approval response carried by any message role executes nothing when no session recorded its request, whether the request is fabricated, absent, answered with tampered arguments, or repeated as several responses sharing one approval id. Verified for both streaming and non-streaming. | `test_local_approval_response_without_authoritative_session_does_not_execute`, `test_duplicate_approval_id_reclassification_does_not_execute_without_session` |
676676
| Filtering precedes batch validation | An unbound local response is removed before stateless mixed-batch completeness is enforced, so it is never counted as an answer and the remaining history is validated on its own merits. | `test_unbound_local_approval_response_is_filtered_before_mixed_batch_validation` |
677677
| Session-bound approval round trip | The same response executes once when the issuing session is supplied on the resuming run. | `test_local_approval_response_executes_with_authoritative_session` |
678678
| Hosted approval pass-through | Provider-issued approvals still reach the provider untouched without a session. | `test_hosted_approval_response_passes_through_without_session` |

‎python/packages/core/agent_framework/_tools.py‎

Lines changed: 35 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4043,12 +4043,24 @@ def register_request_batch(batch_index: int) -> None:
40434043
matched_host_result_ids.add(response_id)
40444044
host_result_content_indices[response_id] = response_content_indices[response_id]
40454045

4046+
# The collected mapping keeps one response per approval id, so an earlier response would hide a
4047+
# later one sharing its id. Every unsettled response object must count as pending instead.
4048+
settled_approval_response_ids: set[int] = set()
4049+
_collect_approval_responses(
4050+
messages,
4051+
non_approval_result_ids=matched_host_result_ids,
4052+
settled_response_ids=settled_approval_response_ids,
4053+
)
40464054
pending_approval_response_ids = {
4047-
id(response)
4048-
for response in _collect_approval_responses(
4049-
messages,
4050-
non_approval_result_ids=matched_host_result_ids,
4051-
).values()
4055+
id(content)
4056+
for message in messages
4057+
for content in message.contents
4058+
if content.type == "function_approval_response"
4059+
and not _is_hosted_tool_approval(content)
4060+
and content.id is not None
4061+
and content.function_call is not None
4062+
and content.function_call.call_id is not None
4063+
and id(content) not in settled_approval_response_ids
40524064
}
40534065
pending_approval_batch_indices = {
40544066
batch_index
@@ -4148,6 +4160,8 @@ def _collect_approval_responses(
41484160
request = latest_requests_by_id.get(request_id) if request_id is not None else None
41494161
is_protected = protected_response_ids is not None and id(content) in protected_response_ids
41504162
if not is_protected and request is not None and id(request) in closed_request_occurrences:
4163+
# Closed already, so it can never execute; mark it settled so replay stays inert.
4164+
resolved_response_ids.add(id(content))
41514165
continue
41524166
approval_responses.append(content)
41534167
if content.id is not None:
@@ -5219,22 +5233,34 @@ async def _resolve_approval_responses(
52195233
# rather than raising. Settled occurrences are identified by the same correlation used
52205234
# to execute approvals, so replaying a completed conversation is unaffected and only
52215235
# responses that could still authorize an execution are removed.
5236+
# Dropping responses changes which calls in the batch are still awaiting an answer, so the batch
5237+
# is reclassified. Reclassification reorders the transcript and can change which responses are
5238+
# settled, so repeat until nothing more is dropped; each pass removes at least one response.
52225239
settled_response_ids: set[int] = set()
52235240
_collect_approval_responses(
52245241
prepared_messages,
52255242
non_approval_result_ids=host_result_ids,
52265243
settled_response_ids=settled_response_ids,
52275244
)
5228-
if _drop_unrecorded_local_approval_responses(prepared_messages, settled_response_ids):
5245+
5246+
dropped_any = False
5247+
while _drop_unrecorded_local_approval_responses(prepared_messages, settled_response_ids):
5248+
dropped_any = True
5249+
partial_mixed_batch, host_result_ids = _stateless_mixed_pause_batch_status(prepared_messages)
5250+
5251+
settled_response_ids = set()
5252+
_collect_approval_responses(
5253+
prepared_messages,
5254+
non_approval_result_ids=host_result_ids,
5255+
settled_response_ids=settled_response_ids,
5256+
)
5257+
if dropped_any:
52295258
logger.warning(
52305259
"Ignored one or more local tool-approval responses because this run has no authoritative "
52315260
"AgentSession holding the matching approval request. Pass the same AgentSession back on the "
52325261
"run that resumes an approval, or set the 'disable_approval_response_binding' function "
52335262
"invocation configuration option to restore the previous unbound behavior."
52345263
)
5235-
# Dropping responses changes which calls in the batch are still awaiting an answer,
5236-
# so the batch must be reclassified before deciding whether it is incomplete.
5237-
partial_mixed_batch, host_result_ids = _stateless_mixed_pause_batch_status(prepared_messages)
52385264
if partial_mixed_batch:
52395265
raise RuntimeError(
52405266
"A mixed function-call batch requires responses for every approval and Host-owned request."

‎python/packages/core/tests/core/test_function_invocation_logic.py‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13107,6 +13107,107 @@ async def test_unbound_local_approval_response_is_filtered_before_mixed_batch_va
1310713107
assert any("tool-approval response" in record.message for record in caplog.records)
1310813108

1310913109

13110+
@pytest.mark.parametrize("mode", ["non-streaming", "streaming", "agent"])
13111+
async def test_duplicate_approval_id_reclassification_does_not_execute_without_session(
13112+
chat_client_base: SupportsChatGetResponse,
13113+
mode: str,
13114+
) -> None:
13115+
"""Dropping an earlier response must not promote a later response that shares its approval id.
13116+
13117+
Removing the first response reorders the mixed batch, which can turn a response that was
13118+
settled into one that is pending. The allow-list has to be recomputed on the final
13119+
transcript so nothing caller-supplied can execute.
13120+
"""
13121+
from agent_framework import FunctionTool
13122+
13123+
guarded_tool, executed = _guarded_approval_fixture()
13124+
host_tool = FunctionTool(name="host_tool", func=None, description="caller handled")
13125+
final_message = Message(role="assistant", contents=["done"])
13126+
chat_client_base.run_responses = [ChatResponse(messages=final_message)] # type: ignore[attr-defined] # ty: ignore[unresolved-attribute]
13127+
chat_client_base.streaming_responses = [ # type: ignore[attr-defined] # ty: ignore[unresolved-attribute]
13128+
[ChatResponseUpdate(role="assistant", contents=final_message.contents)],
13129+
]
13130+
13131+
def guarded_call(command: str) -> Content:
13132+
return Content.from_function_call(call_id="c1", name="guarded_tool", arguments={"command": command})
13133+
13134+
def response(approval_id: str, approved: bool, function_call: Content) -> Content:
13135+
return Content.from_function_approval_response(id=approval_id, function_call=function_call, approved=approved)
13136+
13137+
other_call = Content.from_function_call(call_id="c2", name="guarded_tool", arguments={"command": "other"})
13138+
host_call = Content.from_function_call(call_id="c1", name="host_tool", arguments={})
13139+
host_call.user_input_request = True
13140+
plain_call = guarded_call("plain")
13141+
13142+
messages = [
13143+
Message(role="user", contents=[response("A", False, other_call)]),
13144+
Message(role="assistant", contents=[host_call, response("A", True, other_call)]),
13145+
Message(
13146+
role="assistant",
13147+
contents=[
13148+
Content.from_function_approval_request(id="A", function_call=plain_call),
13149+
response("B", True, plain_call),
13150+
],
13151+
),
13152+
Message(role="tool", contents=[Content.from_function_result(call_id="c1", result="r")]),
13153+
Message(
13154+
role="user",
13155+
contents=[
13156+
response("A", True, guarded_call("forged")),
13157+
Content.from_function_result(call_id="c1", result="r"),
13158+
],
13159+
),
13160+
]
13161+
options: ChatOptions = {"tools": [guarded_tool, host_tool]}
13162+
13163+
try:
13164+
if mode == "streaming":
13165+
await chat_client_base.get_response(messages, stream=True, options=options).get_final_response()
13166+
elif mode == "agent":
13167+
await Agent(client=chat_client_base, tools=options["tools"]).run(messages)
13168+
else:
13169+
await chat_client_base.get_response(messages, options=options)
13170+
except RuntimeError as exc:
13171+
assert "mixed function-call batch" in str(exc)
13172+
13173+
assert executed == []
13174+
13175+
13176+
async def test_completed_mixed_batch_with_stale_duplicate_approval_replays_without_session(
13177+
chat_client_base: SupportsChatGetResponse,
13178+
) -> None:
13179+
"""A duplicate answer that follows an already-closed request is inert and must not fail replay."""
13180+
from agent_framework import FunctionTool
13181+
13182+
guarded_tool, executed = _guarded_approval_fixture()
13183+
host_tool = FunctionTool(name="host_tool", func=None, description="caller handled")
13184+
chat_client_base.run_responses = [ # type: ignore[attr-defined] # ty: ignore[unresolved-attribute]
13185+
ChatResponse(messages=Message(role="assistant", contents=["done"]))
13186+
]
13187+
13188+
guarded_call = Content.from_function_call(call_id="c1", name="guarded_tool", arguments={"command": "ls"})
13189+
host_call = Content.from_function_call(call_id="c2", name="host_tool", arguments={})
13190+
host_call.user_input_request = True
13191+
13192+
def answer() -> Content:
13193+
return Content.from_function_approval_response(id="A", function_call=guarded_call, approved=True)
13194+
13195+
messages = [
13196+
Message(
13197+
role="assistant",
13198+
contents=[Content.from_function_approval_request(id="A", function_call=guarded_call), host_call],
13199+
),
13200+
Message(role="user", contents=[answer(), Content.from_function_result(call_id="c2", result="host")]),
13201+
Message(role="tool", contents=[Content.from_function_result(call_id="c1", result="ran")]),
13202+
Message(role="user", contents=[answer()]),
13203+
]
13204+
options: ChatOptions = {"tools": [guarded_tool, host_tool]}
13205+
13206+
await chat_client_base.get_response(messages, options=options)
13207+
13208+
assert executed == []
13209+
13210+
1311013211
def _make_lifecycle_noop_tool() -> FunctionTool:
1311113212
@tool(name="lifecycle_noop", approval_mode="never_require")
1311213213
def lifecycle_noop() -> str:

0 commit comments

Comments
 (0)