Skip to content

Commit 4adad5e

Browse files
authored
fix(examples): restrict the Realtime demo to local bounded sessions (#5111)
1 parent b1213bf commit 4adad5e

4 files changed

Lines changed: 452 additions & 23 deletions

File tree

‎examples/realtime/app/README.md‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,27 @@ cd examples/realtime/app && uv run python server.py
2020

2121
Then open your browser to: http://localhost:8000
2222

23+
### Local-use boundary and limits
24+
25+
Run this demo only on your own trusted machine. The launch command binds to
26+
`127.0.0.1`. WebSocket connections require a `localhost` or `127.0.0.1` Host and a
27+
matching HTTP Origin, so open the page from this server. Missing or unrelated
28+
browser origins are rejected before the server opens an OpenAI session.
29+
30+
These checks do not authenticate local processes, which can forge HTTP headers.
31+
Do not expose this demo through a public bind address, proxy, or tunnel. A deployed
32+
service needs its own authentication, authorization, rate limits, account quotas,
33+
and transport security before opening sessions with server credentials.
34+
35+
The demo allows four simultaneous sessions, including sessions still connecting.
36+
Each client text message is limited to 1 MiB and each audio message to 24,000 int16
37+
samples (one second at 24 kHz). Each connection can assemble one image at a time,
38+
up to 4 MiB of ASCII data URL content in at most 128 chunks; the UI sends images in 60,000-character
39+
chunks. A direct image message must also fit the text-message limit. These are
40+
demo limits, not OpenAI API limits. Invalid or excessive input closes the socket
41+
and releases its session. Keep the launch command's WebSocket backend and queue
42+
limits when running the example.
43+
2344
### Debugging Realtime usage
2445

2546
Set `LOG_LEVEL=DEBUG` to log the raw `response.done` usage, the typed per-response usage with modality details, and the cumulative session usage:

‎examples/realtime/app/server.py‎

Lines changed: 105 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,9 @@
88
from dataclasses import asdict
99
from pathlib import Path
1010
from typing import TYPE_CHECKING, Any
11+
from uuid import uuid4
1112

12-
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
13+
from fastapi import FastAPI, WebSocket, WebSocketDisconnect, WebSocketException
1314
from fastapi.responses import FileResponse
1415
from fastapi.staticfiles import StaticFiles
1516
from typing_extensions import assert_never
@@ -46,6 +47,59 @@
4647
logger.setLevel(_log_level)
4748

4849
STATIC_DIR = Path(__file__).with_name("static")
50+
# Local demo limits, not Realtime API limits.
51+
MAX_SESSIONS = 4
52+
MAX_MESSAGE_BYTES = 1024 * 1024
53+
MAX_AUDIO_SAMPLES = 24_000
54+
MAX_IMAGE_CHARS = 4 * 1024 * 1024
55+
MAX_IMAGE_CHUNKS = 128
56+
57+
58+
def validate_client_message(data: str) -> dict[str, Any]:
59+
"""Validate the demo's untrusted browser protocol before forwarding or buffering."""
60+
if len(data.encode("utf-8")) > MAX_MESSAGE_BYTES:
61+
raise WebSocketException(code=1009, reason="Message too large")
62+
message: Any = None
63+
try:
64+
message = json.loads(data)
65+
except (ValueError, RecursionError):
66+
logger.debug("Invalid JSON payload received from client", exc_info=True)
67+
message = None
68+
if not isinstance(message, dict):
69+
raise WebSocketException(code=1008, reason="Expected a JSON object")
70+
71+
message_type = message.get("type")
72+
if message_type == "audio":
73+
samples = message.get("data")
74+
if not isinstance(samples, list) or not samples or len(samples) > MAX_AUDIO_SAMPLES:
75+
raise WebSocketException(code=1008, reason="Invalid audio sample count")
76+
if any(type(sample) is not int or not -32768 <= sample <= 32767 for sample in samples):
77+
raise WebSocketException(code=1008, reason="Expected int16 audio samples")
78+
elif message_type in ("image", "image_start", "image_chunk", "image_end"):
79+
if message_type != "image":
80+
image_id = message.get("id")
81+
if not isinstance(image_id, str) or not image_id or len(image_id) > 128:
82+
raise WebSocketException(code=1008, reason="Invalid image ID")
83+
if message_type in ("image", "image_start"):
84+
if "text" in message and not isinstance(message["text"], str):
85+
raise WebSocketException(code=1008, reason="Invalid image prompt")
86+
if message_type in ("image", "image_chunk"):
87+
image_data = message.get("data_url" if message_type == "image" else "chunk")
88+
if not isinstance(image_data, str) or not image_data or not image_data.isascii():
89+
raise WebSocketException(code=1008, reason="Invalid image data")
90+
if len(image_data) > MAX_IMAGE_CHARS:
91+
raise WebSocketException(code=1009, reason="Image too large")
92+
elif message_type == "tool_approval_decision":
93+
if not isinstance(message.get("call_id"), str) or not message["call_id"]:
94+
raise WebSocketException(code=1008, reason="Invalid tool call ID")
95+
if (
96+
type(message.get("approve")) is not bool
97+
or type(message.get("always", False)) is not bool
98+
):
99+
raise WebSocketException(code=1008, reason="Invalid approval decision")
100+
elif message_type not in ("commit_audio", "interrupt"):
101+
raise WebSocketException(code=1008, reason="Unknown message type")
102+
return message
49103

50104

51105
class RealtimeWebSocketManager:
@@ -56,8 +110,11 @@ def __init__(self):
56110
self.event_tasks: dict[str, asyncio.Task[None]] = {}
57111

58112
async def connect(self, websocket: WebSocket, session_id: str):
59-
await websocket.accept()
113+
if len(self.websockets) >= MAX_SESSIONS:
114+
raise WebSocketException(code=1008, reason="Demo session limit reached")
115+
# Reserve before the first await, including sessions still connecting upstream.
60116
self.websockets[session_id] = websocket
117+
await websocket.accept()
61118

62119
agent = get_starting_agent()
63120
runner = RealtimeRunner(agent)
@@ -429,12 +486,26 @@ async def lifespan(app: FastAPI):
429486

430487
@app.websocket("/ws/{session_id}")
431488
async def websocket_endpoint(websocket: WebSocket, session_id: str):
489+
# Origin is a browser boundary, not authentication for a public service.
490+
if (
491+
len(websocket.headers.getlist("host")) != 1
492+
or websocket.url.hostname not in {"localhost", "127.0.0.1"}
493+
or websocket.headers.getlist("origin") != [f"http://{websocket.url.netloc}"]
494+
):
495+
raise WebSocketException(code=1008, reason="Use the local demo page")
496+
# The URL label is client-controlled and must not select another socket's resources.
497+
session_id = uuid4().hex
432498
try:
433499
await manager.connect(websocket, session_id)
434500
image_buffers: dict[str, dict[str, Any]] = {}
435501
while True:
436-
data = await websocket.receive_text()
437-
message = json.loads(data)
502+
frame = await websocket.receive()
503+
if frame["type"] == "websocket.disconnect":
504+
break
505+
data = frame.get("text")
506+
if data is None:
507+
raise WebSocketException(code=1003, reason="Expected a text message")
508+
message = validate_client_message(data)
438509

439510
if message["type"] == "audio":
440511
# Convert int16 array to bytes
@@ -487,30 +558,40 @@ async def websocket_endpoint(websocket: WebSocket, session_id: str):
487558
# Force close the current input audio turn
488559
await manager.send_client_event(session_id, {"type": "input_audio_buffer.commit"})
489560
elif message["type"] == "image_start":
490-
img_id = str(message.get("id"))
561+
if image_buffers:
562+
raise WebSocketException(code=1008, reason="Finish the pending image first")
563+
img_id = message["id"]
491564
image_buffers[img_id] = {
492565
"text": message.get("text") or "Please describe this image.",
493566
"chunks": [],
567+
"size": 0,
494568
}
495569
await websocket.send_text(
496570
json.dumps({"type": "client_info", "info": "image_start_ack", "id": img_id})
497571
)
498572
elif message["type"] == "image_chunk":
499-
img_id = str(message.get("id"))
500-
chunk = message.get("chunk", "")
501-
if img_id in image_buffers:
502-
image_buffers[img_id]["chunks"].append(chunk)
503-
if len(image_buffers[img_id]["chunks"]) % 10 == 0:
504-
await websocket.send_text(
505-
json.dumps(
506-
{
507-
"type": "client_info",
508-
"info": "image_chunk_ack",
509-
"id": img_id,
510-
"count": len(image_buffers[img_id]["chunks"]),
511-
}
512-
)
573+
img_id = message["id"]
574+
chunk = message["chunk"]
575+
if img_id not in image_buffers:
576+
raise WebSocketException(code=1008, reason="Unknown image ID")
577+
buffer = image_buffers[img_id]
578+
if buffer["size"] + len(chunk) > MAX_IMAGE_CHARS:
579+
raise WebSocketException(code=1009, reason="Image too large")
580+
if len(buffer["chunks"]) >= MAX_IMAGE_CHUNKS:
581+
raise WebSocketException(code=1009, reason="Too many image chunks")
582+
buffer["size"] += len(chunk)
583+
image_buffers[img_id]["chunks"].append(chunk)
584+
if len(image_buffers[img_id]["chunks"]) % 10 == 0:
585+
await websocket.send_text(
586+
json.dumps(
587+
{
588+
"type": "client_info",
589+
"info": "image_chunk_ack",
590+
"id": img_id,
591+
"count": len(image_buffers[img_id]["chunks"]),
592+
}
513593
)
594+
)
514595
elif message["type"] == "image_end":
515596
img_id = str(message.get("id"))
516597
buf = image_buffers.pop(img_id, None)
@@ -599,8 +680,10 @@ async def read_index():
599680

600681
uvicorn.run(
601682
app,
602-
host="0.0.0.0",
683+
host="127.0.0.1",
603684
port=8000,
604-
# Increased WebSocket frame size to comfortably handle image data URLs.
605-
ws_max_size=16 * 1024 * 1024,
685+
# Select the backend that enforces these limits before application parsing.
686+
ws="websockets",
687+
ws_max_size=MAX_MESSAGE_BYTES,
688+
ws_max_queue=4,
606689
)

‎examples/realtime/app/static/app.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ class RealtimeDemo {
102102

103103
async connect() {
104104
try {
105-
this.ws = new WebSocket(`ws://localhost:8000/ws/${this.sessionId}`);
105+
this.ws = new WebSocket(`ws://${window.location.host}/ws/${this.sessionId}`);
106106

107107
this.ws.onopen = () => {
108108
this.isConnected = true;

0 commit comments

Comments
 (0)