Skip to content

Commit 81f0ccf

Browse files
fix(examples): render realtime tool events as text (#5283)
1 parent c4a1d04 commit 81f0ccf

3 files changed

Lines changed: 193 additions & 9 deletions

File tree

‎.github/workflows/tests.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,11 @@ jobs:
150150
- name: Run tests with coverage
151151
if: steps.changes.outputs.run == 'true' && matrix.python-version == '3.14'
152152
run: make coverage
153+
- name: Test realtime demo in Chromium
154+
if: steps.changes.outputs.run == 'true' && matrix.python-version == '3.14'
155+
run: |
156+
uv run playwright install --with-deps chromium
157+
uv run pytest -o pythonpath=. examples/realtime/app/test_frontend.py
153158
- name: Run tests
154159
if: steps.changes.outputs.run == 'true' && matrix.python-version != '3.14'
155160
run: make tests

‎examples/realtime/app/static/app.js‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -543,15 +543,27 @@ class RealtimeDemo {
543543
eventClass = 'tool';
544544
}
545545

546-
eventDiv.innerHTML = `
547-
<div class="event-header ${eventClass}">
548-
<div>
549-
<div style="font-weight: 600; margin-bottom: 2px;">${title}</div>
550-
<div style="font-size: 0.8rem; opacity: 0.8;">${description}</div>
551-
</div>
552-
<span style="font-size: 0.7rem; opacity: 0.6;">${new Date().toLocaleTimeString()}</span>
553-
</div>
554-
`;
546+
const headerDiv = document.createElement('div');
547+
headerDiv.className = `event-header ${eventClass}`;
548+
549+
const detailsDiv = document.createElement('div');
550+
const titleDiv = document.createElement('div');
551+
titleDiv.style.cssText = 'font-weight: 600; margin-bottom: 2px;';
552+
titleDiv.textContent = title;
553+
554+
const descriptionDiv = document.createElement('div');
555+
descriptionDiv.style.cssText = 'font-size: 0.8rem; opacity: 0.8;';
556+
descriptionDiv.textContent = description;
557+
558+
const timeSpan = document.createElement('span');
559+
timeSpan.style.cssText = 'font-size: 0.7rem; opacity: 0.6;';
560+
timeSpan.textContent = new Date().toLocaleTimeString();
561+
562+
detailsDiv.appendChild(titleDiv);
563+
detailsDiv.appendChild(descriptionDiv);
564+
headerDiv.appendChild(detailsDiv);
565+
headerDiv.appendChild(timeSpan);
566+
eventDiv.appendChild(headerDiv);
555567

556568
this.toolsContent.appendChild(eventDiv);
557569

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
"""Run after `uv run playwright install chromium` with
2+
`uv run pytest -o pythonpath=. examples/realtime/app/test_frontend.py`.
3+
"""
4+
5+
import json
6+
from collections.abc import AsyncIterator
7+
from pathlib import Path
8+
from typing import Any
9+
10+
import pytest
11+
import pytest_asyncio
12+
from playwright.async_api import Page, async_playwright, expect
13+
14+
from agents import RunContextWrapper
15+
from agents.realtime.events import RealtimeEventInfo, RealtimeToolEnd
16+
from agents.tool_context import ToolContext
17+
from examples.realtime.app.agent import get_weather, triage_agent, update_seat
18+
from examples.realtime.app.server import RealtimeWebSocketManager
19+
20+
STATIC = Path(__file__).parent / "static"
21+
22+
23+
@pytest_asyncio.fixture(loop_scope="function")
24+
async def page() -> AsyncIterator[Page]:
25+
async with async_playwright() as playwright:
26+
browser = await playwright.chromium.launch()
27+
context = await browser.new_context()
28+
29+
# Serve only the demo's static files; no server or model connection is opened.
30+
async def serve(route):
31+
files = {
32+
"http://demo.test/": ("index.html", "text/html"),
33+
"http://demo.test/app.js": ("app.js", "application/javascript"),
34+
}
35+
if route.request.url in files:
36+
name, content_type = files[route.request.url]
37+
await route.fulfill(path=STATIC / name, content_type=content_type)
38+
else:
39+
await route.abort()
40+
41+
await context.route("**/*", serve)
42+
await context.add_init_script("""
43+
window.sentMessages = [];
44+
window.WebSocket = class {
45+
static OPEN = 1;
46+
constructor() { this.readyState = 1; window.demoSocket = this; }
47+
send(message) { window.sentMessages.push(JSON.parse(message)); }
48+
};
49+
""")
50+
page = await context.new_page()
51+
await page.goto("http://demo.test/")
52+
await page.locator("#connectBtn").click()
53+
try:
54+
yield page
55+
finally:
56+
await context.close()
57+
await browser.close()
58+
59+
60+
async def receive(page: Page, event: dict[str, Any]) -> None:
61+
await page.evaluate("data => window.demoSocket.onmessage({data})", json.dumps(event))
62+
63+
64+
async def weather_event(city: str) -> dict[str, Any]:
65+
arguments = json.dumps({"city": city})
66+
context = ToolContext(
67+
context=None, tool_name="get_weather", tool_call_id="weather-1", tool_arguments=arguments
68+
)
69+
output = await get_weather.on_invoke_tool(context, arguments)
70+
return await RealtimeWebSocketManager()._serialize_event(
71+
RealtimeToolEnd(
72+
agent=triage_agent,
73+
tool=get_weather,
74+
arguments=arguments,
75+
output=output,
76+
info=RealtimeEventInfo(context=RunContextWrapper(context=None)),
77+
)
78+
)
79+
80+
81+
@pytest.mark.parametrize(
82+
"city",
83+
[
84+
'<img src=x onerror="window.injected=true;window.confirm=()=>true">',
85+
'<svg onload="window.injected=true;window.confirm=()=>true"></svg>',
86+
'A & B <district> "north"',
87+
],
88+
ids=["image-handler", "svg-handler", "literal-punctuation"],
89+
)
90+
async def test_weather_output_is_text_and_cannot_replace_approval(page: Page, city: str) -> None:
91+
assert get_weather.needs_approval is False
92+
assert update_seat.needs_approval is True
93+
await receive(page, await weather_event(city))
94+
95+
panel = page.locator("#toolsContent")
96+
await expect(panel).to_contain_text(f"get_weather: The weather in {city} is sunny.")
97+
await expect(panel.locator("img, svg")).to_have_count(0)
98+
assert await page.evaluate("window.injected === undefined")
99+
100+
# A completion must not replace the browser's approval dialog or its decision.
101+
dialogs = []
102+
103+
async def reject(dialog):
104+
dialogs.append(dialog.message)
105+
await dialog.dismiss()
106+
107+
page.on("dialog", reject)
108+
await receive(
109+
page, {"type": "tool_approval_required", "tool": "update_seat", "call_id": "seat-1"}
110+
)
111+
assert dialogs == ['Allow tool "update_seat" to run?']
112+
assert await page.evaluate("window.sentMessages") == [
113+
{"type": "tool_approval_decision", "call_id": "seat-1", "approve": False}
114+
]
115+
116+
117+
async def test_normal_events_and_approval_remain_usable(page: Page) -> None:
118+
await receive(page, {"type": "tool_start", "tool": "get_weather"})
119+
await receive(page, await weather_event("San Francisco"))
120+
await receive(page, {"type": "tool_end", "tool": "empty", "output": ""})
121+
await receive(page, {"type": "handoff", "from": "Triage Agent", "to": "Seat Booking Agent"})
122+
123+
async def accept(dialog):
124+
await dialog.accept()
125+
126+
page.on("dialog", accept)
127+
await receive(
128+
page, {"type": "tool_approval_required", "tool": "update_seat", "call_id": "seat-2"}
129+
)
130+
panel = page.locator("#toolsContent")
131+
for text in [
132+
"Running get_weather",
133+
"get_weather: The weather in San Francisco is sunny.",
134+
"empty: No output",
135+
"From Triage Agent to Seat Booking Agent",
136+
"Waiting on update_seat",
137+
"✅ Approved",
138+
"update_seat (seat-2)",
139+
]:
140+
await expect(panel).to_contain_text(text)
141+
await expect(panel.locator(".event-header.tool")).to_have_count(5)
142+
await expect(panel.locator(".event-header.handoff")).to_have_count(1)
143+
assert await page.evaluate("window.sentMessages") == [
144+
{"type": "tool_approval_decision", "call_id": "seat-2", "approve": True}
145+
]
146+
147+
148+
async def test_shared_event_descriptions_are_literal(page: Page) -> None:
149+
name = '<img src=x onerror="window.injected=true">'
150+
await receive(page, {"type": "tool_start", "tool": name})
151+
await receive(page, {"type": "handoff", "from": name, "to": name})
152+
153+
async def reject(dialog):
154+
await dialog.dismiss()
155+
156+
page.on("dialog", reject)
157+
await receive(page, {"type": "tool_approval_required", "tool": name, "call_id": name})
158+
panel = page.locator("#toolsContent")
159+
for text in [
160+
f"Running {name}",
161+
f"From {name} to {name}",
162+
f"Waiting on {name}",
163+
f"{name} ({name})",
164+
]:
165+
await expect(panel).to_contain_text(text)
166+
await expect(panel.locator("img")).to_have_count(0)
167+
assert await page.evaluate("window.injected === undefined")

0 commit comments

Comments
 (0)