Skip to content

Commit f38b326

Browse files
feat: Add native Session voice workflows
1 parent 96c59ef commit f38b326

11 files changed

Lines changed: 1006 additions & 18 deletions

File tree

‎examples/codex/realtime_voice.py‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
"""Let a Realtime speech agent delegate work to a native Session.
2+
3+
Install the matching SDK's voice and classic-realtime extras and set
4+
OPENAI_API_KEY. Run: python examples/codex/realtime_voice.py 'Your question' answer.wav
5+
A text prompt starts the speech conversation; Realtime generates PCM audio.
6+
The two conversations stay separate. Application code must handle any native
7+
Session interactions; this example grants no approvals. Realtime barge-in does
8+
not automatically cancel native work. Scope exit explicitly cancels that work. Record for a bounded --seconds window;
9+
a spoken preamble does not end the example before delegation completes.
10+
"""
11+
12+
import argparse
13+
import asyncio
14+
import wave
15+
from contextlib import aclosing
16+
from pathlib import Path
17+
18+
from agents import Agent
19+
from agents.classic import function_tool as realtime_function_tool
20+
from agents.classic.realtime import RealtimeAgent, RealtimeRunner
21+
from agents.voice import NativeSessionVoiceWorkflow
22+
23+
24+
async def main(prompt: str, destination: Path, seconds: float) -> None:
25+
if seconds <= 0:
26+
raise ValueError("seconds must be positive")
27+
native = Agent(instructions="Solve the user's request. Return a concise answer for speech.")
28+
async with native.session() as session:
29+
workflow = NativeSessionVoiceWorkflow(session)
30+
31+
@realtime_function_tool(failure_error_function=None)
32+
async def ask_native(input: str) -> str:
33+
"""Delegate substantive work to the native Session."""
34+
async with aclosing(workflow.run(input)) as chunks:
35+
return "".join([chunk async for chunk in chunks])
36+
37+
speech = RealtimeAgent(
38+
name="Voice concierge",
39+
instructions="Always use ask_native for the user's request, then speak its answer.",
40+
tools=[ask_native],
41+
)
42+
try:
43+
async with await RealtimeRunner(speech).run() as realtime:
44+
with wave.open(str(destination), "wb") as output:
45+
output.setparams((1, 2, 24000, 0, "NONE", "not compressed"))
46+
await realtime.send_message(prompt)
47+
48+
async def record() -> None:
49+
async for event in realtime:
50+
if event.type == "audio":
51+
output.writeframes(event.audio.data)
52+
elif event.type == "tool_end":
53+
print("Native answer:", event.output)
54+
elif event.type == "error":
55+
raise RuntimeError("Realtime speech failed")
56+
57+
try:
58+
await asyncio.wait_for(record(), timeout=seconds)
59+
except asyncio.TimeoutError:
60+
print("Recording window ended; stopping delegated work.")
61+
finally:
62+
await workflow.cancel()
63+
64+
65+
if __name__ == "__main__":
66+
parser = argparse.ArgumentParser(description=__doc__)
67+
parser.add_argument("prompt")
68+
parser.add_argument("output", type=Path)
69+
parser.add_argument("--seconds", type=float, default=30)
70+
options = parser.parse_args()
71+
asyncio.run(main(options.prompt, options.output, options.seconds))

‎examples/codex/voice.py‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
"""Transcribe a WAV file, run a native Session, and write spoken output.
2+
3+
Install the matching SDK's voice extra. Set OPENAI_API_KEY for speech and native
4+
API requests. Run: python examples/codex/voice.py input.wav output.wav
5+
Input must be mono PCM16 at 24 kHz. No microphone or speaker is opened.
6+
"""
7+
8+
import argparse
9+
import asyncio
10+
import wave
11+
from contextlib import aclosing
12+
from pathlib import Path
13+
14+
import numpy as np
15+
16+
from agents import Agent
17+
from agents.voice import AudioInput, NativeSessionVoiceWorkflow, VoicePipeline
18+
19+
20+
async def main(source: Path, destination: Path) -> None:
21+
with wave.open(str(source), "rb") as audio:
22+
if (audio.getnchannels(), audio.getsampwidth(), audio.getframerate()) != (1, 2, 24000):
23+
raise ValueError("Input must be mono PCM16 WAV at 24 kHz")
24+
samples = np.frombuffer(audio.readframes(audio.getnframes()), dtype="<i2")
25+
26+
agent = Agent(instructions="Answer spoken questions briefly and clearly.")
27+
async with agent.session() as session:
28+
workflow = NativeSessionVoiceWorkflow(session)
29+
pipeline = VoicePipeline(workflow=workflow)
30+
result = await pipeline.run(AudioInput(buffer=samples))
31+
with wave.open(str(destination), "wb") as output:
32+
output.setparams((1, 2, 24000, 0, "NONE", "not compressed"))
33+
async with aclosing(result.stream()) as events:
34+
async for event in events:
35+
if event.type == "voice_stream_event_audio" and event.data is not None:
36+
output.writeframes(event.data.astype("<i2", copy=False).tobytes())
37+
elif event.type == "voice_stream_event_lifecycle":
38+
print(event.event)
39+
40+
41+
if __name__ == "__main__":
42+
parser = argparse.ArgumentParser(description=__doc__)
43+
parser.add_argument("input", type=Path)
44+
parser.add_argument("output", type=Path)
45+
options = parser.parse_args()
46+
asyncio.run(main(options.input, options.output))
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"""Native speech workflow execution without speech-provider dependencies."""
Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
"""Run speech transcriptions through a caller-owned native Session."""
2+
3+
from __future__ import annotations
4+
5+
import asyncio
6+
from collections.abc import AsyncGenerator
7+
from dataclasses import dataclass
8+
from typing import Any
9+
10+
from .._public import preserve_module_identity as _preserve_module_identity
11+
from ..observation import EventGap, OutputTextDelta, TurnStarted
12+
from ..session import Session
13+
from ..turn import Turn
14+
15+
16+
class VoiceEventGapError(RuntimeError):
17+
"""Native output observations were lost before they could be spoken."""
18+
19+
20+
@dataclass
21+
class _Invocation:
22+
submission: asyncio.Task[Turn[str]]
23+
cancellation: asyncio.Task[None] | None = None
24+
finished: bool = False
25+
26+
27+
async def _join(task: asyncio.Task[Any]) -> None:
28+
"""Join owned cleanup even if the caller receives another cancellation."""
29+
while not task.done():
30+
try:
31+
await asyncio.shield(task)
32+
except asyncio.CancelledError:
33+
continue
34+
task.result()
35+
36+
37+
class NativeSessionVoiceWorkflow:
38+
"""Speak native output while retaining the Session's history and policies.
39+
40+
The application opens and closes the Session and handles its interactions.
41+
Each transcription submits a new root turn. Concurrent runs are rejected;
42+
an already busy Session follows its normal rejection policy. Close a run
43+
iterator explicitly when stopping early, for example with contextlib.aclosing.
44+
Closing or cancelling a run stops and joins only its own native turn.
45+
"""
46+
47+
def __init__(self, session: Session[Any, Any]) -> None:
48+
self._session = session
49+
self._invocation: _Invocation | None = None
50+
51+
async def on_start(self) -> AsyncGenerator[str, None]:
52+
"""Start without adding a greeting to the caller's conversation."""
53+
return
54+
yield
55+
56+
async def run(self, transcription: str) -> AsyncGenerator[str, None]:
57+
"""Yield this transcription's native text; propagate terminal failures."""
58+
if not isinstance(transcription, str) or not transcription.strip():
59+
raise ValueError("transcription must be a nonempty string")
60+
if self._invocation is not None:
61+
raise RuntimeError("NativeSessionVoiceWorkflow already has an active run")
62+
63+
# Reserve before awaiting admission. Shielding retains the exact Turn if
64+
# cancellation races with submit returning its locally issued handle.
65+
invocation = _Invocation(
66+
asyncio.create_task(self._session.submit(transcription, output_type=str))
67+
)
68+
self._invocation = invocation
69+
completed = False
70+
primary_error: BaseException | None = None
71+
try:
72+
turn = await asyncio.shield(invocation.submission)
73+
started = False
74+
preceding_gap = False
75+
async for event in turn.events():
76+
if invocation.cancellation is not None:
77+
raise asyncio.CancelledError
78+
if isinstance(event, TurnStarted) and event.submission_id == turn.submission_id:
79+
started = True
80+
preceding_gap = False
81+
elif isinstance(event, EventGap):
82+
if started:
83+
raise VoiceEventGapError("Native voice output observations were lost")
84+
preceding_gap = True
85+
elif isinstance(event, OutputTextDelta):
86+
if preceding_gap:
87+
raise VoiceEventGapError("Native voice turn start or output was lost")
88+
yield event.delta
89+
if preceding_gap:
90+
raise VoiceEventGapError("Native voice turn observations were lost")
91+
await turn.get_result()
92+
if invocation.cancellation is not None:
93+
raise asyncio.CancelledError
94+
completed = True
95+
except BaseException as error:
96+
primary_error = error
97+
raise
98+
finally:
99+
invocation.finished = True
100+
try:
101+
if not completed:
102+
await self._cancel(invocation)
103+
except BaseException:
104+
# Keep the reservation after failed cleanup. A later cancel can
105+
# retry the same exact turn before any new work is admitted.
106+
if primary_error is None:
107+
raise
108+
else:
109+
if self._invocation is invocation:
110+
self._invocation = None
111+
112+
async def _cancel(self, invocation: _Invocation) -> None:
113+
cancellation = invocation.cancellation
114+
if cancellation is None or (
115+
cancellation.done()
116+
and not cancellation.cancelled()
117+
and cancellation.exception() is not None
118+
):
119+
120+
async def stop() -> None:
121+
try:
122+
turn = await invocation.submission
123+
except Exception:
124+
# Failed admission issued no handle and owns no native work.
125+
return
126+
await turn.cancel()
127+
128+
invocation.cancellation = asyncio.create_task(stop())
129+
assert invocation.cancellation is not None
130+
await _join(invocation.cancellation)
131+
132+
async def cancel(self) -> None:
133+
"""Stop and join the current owned turn without closing the Session.
134+
135+
A suspended run iterator must still be closed by its consumer before a
136+
new run can begin. Calling cancel again is safe and joins the same stop.
137+
"""
138+
invocation = self._invocation
139+
if invocation is not None:
140+
await self._cancel(invocation)
141+
if invocation.finished and self._invocation is invocation:
142+
self._invocation = None
143+
144+
145+
_preserve_module_identity(globals(), "agents.voice.workflow")

0 commit comments

Comments
 (0)