Skip to content

Commit d4a6223

Browse files
authored
feat(server): add experimental voice session endpoint (#55951)
1 parent 180fd9d commit d4a6223

20 files changed

Lines changed: 1419 additions & 0 deletions

File tree

‎packages/ai/src/experimental.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,3 +27,10 @@ export type {
2727
EvaluationResponseFor,
2828
EvaluationRoute,
2929
} from "./experimental/evaluation.js"
30+
export { Realtime } from "./experimental/realtime.js"
31+
export type {
32+
RealtimeConnectInput,
33+
RealtimeSession,
34+
RealtimeTransport,
35+
RealtimeTransportInput,
36+
} from "./experimental/realtime.js"
Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
import { Effect, type Redacted, Schema } from "effect"
2+
import { HttpClientRequest } from "effect/http"
3+
import { RequestExecutor } from "../route/executor.js"
4+
import { AIError, HttpContext, InvalidProviderOutputError, InvalidRequestError } from "../schema/index.js"
5+
import { encodeJson } from "../utils/json.js"
6+
7+
const OPENAI_BASE_URL = "https://api.openai.com/v1"
8+
const XAI_BASE_URL = "https://api.x.ai/v1"
9+
10+
export type RealtimeTransportInput =
11+
| { readonly type: "webrtc"; readonly sdp: string; readonly dcid?: number }
12+
| { readonly type: "websocket" }
13+
14+
export type RealtimeTransport =
15+
| { readonly type: "webrtc"; readonly sdp: string }
16+
| { readonly type: "websocket"; readonly url: string; readonly token: string; readonly expiresAt: number }
17+
18+
export type RealtimeSession = Readonly<Record<string, unknown>>
19+
20+
export interface RealtimeConnectInput {
21+
readonly model: string
22+
readonly apiKey: Redacted.Redacted
23+
readonly session?: RealtimeSession
24+
readonly transport: RealtimeTransportInput
25+
}
26+
27+
const engines = [
28+
{ family: "gpt-live", id: "openai-live", provider: "openai" },
29+
{ family: "gpt-realtime", id: "openai-realtime", provider: "openai" },
30+
{ family: "grok-voice", id: "xai-realtime", provider: "xai" },
31+
] as const
32+
33+
// Matches the bare family name (`gpt-realtime`) and its versions (`gpt-realtime-2.1`, `gpt-realtime-mini`).
34+
export const engine = (model: string) =>
35+
engines.find((item) => model === item.family || model.startsWith(`${item.family}-`))
36+
37+
export const connect = Effect.fn("Realtime.connect")(function* (input: RealtimeConnectInput) {
38+
const resolved = engine(input.model)
39+
if (!resolved) return yield* invalidRequest(`Unsupported voice model: ${input.model}`)
40+
const session = { ...input.session, model: input.model }
41+
42+
switch (resolved.id) {
43+
case "openai-live": {
44+
if (input.transport.type !== "webrtc")
45+
return yield* invalidRequest("OpenAI GPT-Live only supports WebRTC transport")
46+
const data = yield* post(
47+
resolved.id,
48+
`${OPENAI_BASE_URL}/live/sessions`,
49+
input.apiKey,
50+
{ session, transport: input.transport },
51+
LiveSessionResponse,
52+
)
53+
return { type: "webrtc" as const, sdp: data.transport.sdp }
54+
}
55+
case "openai-realtime": {
56+
if (input.transport.type === "websocket")
57+
return yield* clientSecret(resolved.id, OPENAI_BASE_URL, input.model, input.apiKey, session)
58+
const form = new FormData()
59+
form.append("sdp", input.transport.sdp)
60+
form.append("session", encodeJson(session))
61+
const query = input.transport.dcid === undefined ? "" : `?dcid=${input.transport.dcid}`
62+
const sdp = yield* post(
63+
resolved.id,
64+
`${OPENAI_BASE_URL}/realtime/calls${query}`,
65+
input.apiKey,
66+
form,
67+
Schema.String,
68+
)
69+
return { type: "webrtc" as const, sdp }
70+
}
71+
case "xai-realtime": {
72+
if (input.transport.type !== "websocket")
73+
return yield* invalidRequest("xAI Grok Voice only supports WebSocket transport")
74+
return yield* clientSecret(resolved.id, XAI_BASE_URL, input.model, input.apiKey)
75+
}
76+
}
77+
})
78+
79+
export const Realtime = {
80+
engine,
81+
connect,
82+
} as const
83+
84+
const LiveSessionResponse = Schema.fromJsonString(
85+
Schema.Struct({
86+
transport: Schema.Struct({ sdp: Schema.String }),
87+
}),
88+
)
89+
90+
// xAI's ephemeral-token docs (https://docs.x.ai/developers/model-capabilities/audio/ephemeral-tokens)
91+
// document the request body (`{ expires_after: { seconds } }`) but not the response payload; we assume
92+
// it mirrors OpenAI's `{ value, expires_at }` response shape.
93+
const ClientSecretResponse = Schema.fromJsonString(
94+
Schema.Struct({
95+
value: Schema.String,
96+
expires_at: Schema.Number,
97+
}),
98+
)
99+
100+
const invalidRequest = (message: string) => new AIError({ reason: new InvalidRequestError({ message }) })
101+
102+
const post = <A>(
103+
route: string,
104+
url: string,
105+
apiKey: Redacted.Redacted,
106+
body: Record<string, unknown> | FormData,
107+
response: Schema.Codec<A, string>,
108+
) =>
109+
Effect.gen(function* () {
110+
const executor = yield* RequestExecutor.Service
111+
const res = yield* executor.execute(
112+
HttpClientRequest.post(url).pipe(
113+
HttpClientRequest.bearerToken(apiKey),
114+
body instanceof FormData
115+
? HttpClientRequest.bodyFormData(body)
116+
: HttpClientRequest.bodyText(encodeJson(body), "application/json"),
117+
),
118+
)
119+
const http = new HttpContext({ url: res.request.url, status: res.status, headers: res.headers })
120+
const fail = (message: string, cause?: unknown, raw?: string) =>
121+
new AIError({ reason: new InvalidProviderOutputError({ route, message, body: raw, http, cause }) })
122+
const text = yield* res.text.pipe(Effect.mapError((cause) => fail(`Failed to read ${route} response`, cause)))
123+
return yield* Schema.decodeEffect(response)(text).pipe(
124+
Effect.mapError((cause) => fail(`${route} returned an invalid response`, cause, text)),
125+
)
126+
})
127+
128+
const clientSecret = (
129+
route: string,
130+
baseURL: string,
131+
model: string,
132+
apiKey: Redacted.Redacted,
133+
session?: Record<string, unknown>,
134+
) =>
135+
post(
136+
route,
137+
`${baseURL}/realtime/client_secrets`,
138+
apiKey,
139+
{ expires_after: { seconds: 60 }, session },
140+
ClientSecretResponse,
141+
).pipe(
142+
Effect.map(
143+
(data): RealtimeTransport => ({
144+
type: "websocket",
145+
url: `${baseURL.replace(/^http/, "ws")}/realtime?model=${encodeURIComponent(model)}`,
146+
token: data.value,
147+
expiresAt: data.expires_at,
148+
}),
149+
),
150+
)

‎packages/ai/test/realtime.test.ts‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
import { describe, expect, test } from "bun:test"
2+
import { Effect, Redacted } from "effect"
3+
import { HttpClientRequest } from "effect/http"
4+
import { Realtime } from "../src/experimental.js"
5+
import { it } from "./lib/effect.js"
6+
import { dynamicResponse } from "./lib/http.js"
7+
8+
describe("experimental Realtime", () => {
9+
it.effect("connects OpenAI GPT-Live, OpenAI Realtime (WebRTC + WebSocket), and xAI Grok Voice", () =>
10+
Effect.gen(function* () {
11+
expect(
12+
yield* Realtime.connect({
13+
model: "gpt-live-1",
14+
apiKey: Redacted.make("sk-openai"),
15+
session: { instructions: "Be concise.", voice: "marin" },
16+
transport: { type: "webrtc", sdp: "v=0\r\no=offer\r\n", dcid: 4 },
17+
}),
18+
).toEqual({ type: "webrtc", sdp: "v=0\r\no=live-answer\r\n" })
19+
20+
expect(
21+
yield* Realtime.connect({
22+
model: "gpt-realtime-2.1",
23+
apiKey: Redacted.make("sk-openai"),
24+
session: { type: "realtime", instructions: "Brief." },
25+
transport: { type: "webrtc", sdp: "v=0\r\no=offer\r\n", dcid: 4 },
26+
}),
27+
).toEqual({ type: "webrtc", sdp: "v=0\r\no=rtc-answer\r\n" })
28+
29+
expect(
30+
yield* Realtime.connect({
31+
model: "gpt-realtime-2.1",
32+
apiKey: Redacted.make("sk-openai"),
33+
session: { type: "realtime", instructions: "Brief." },
34+
transport: { type: "websocket" },
35+
}),
36+
).toEqual({
37+
type: "websocket",
38+
url: "wss://api.openai.com/v1/realtime?model=gpt-realtime-2.1",
39+
token: "ek_1",
40+
expiresAt: 1760000060,
41+
})
42+
43+
expect(
44+
yield* Realtime.connect({
45+
model: "grok-voice-3.0",
46+
apiKey: Redacted.make("xai-key"),
47+
session: { voice: "ara" },
48+
transport: { type: "websocket" },
49+
}),
50+
).toEqual({
51+
type: "websocket",
52+
url: "wss://api.x.ai/v1/realtime?model=grok-voice-3.0",
53+
token: "ek_2",
54+
expiresAt: 1760000120,
55+
})
56+
}).pipe(
57+
Effect.provide(
58+
dynamicResponse((input) =>
59+
Effect.gen(function* () {
60+
const req = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
61+
if (req.url === "https://api.openai.com/v1/live/sessions") {
62+
expect(req.headers.get("authorization")).toBe("Bearer sk-openai")
63+
expect(JSON.parse(input.text)).toEqual({
64+
session: { instructions: "Be concise.", voice: "marin", model: "gpt-live-1" },
65+
transport: { type: "webrtc", sdp: "v=0\r\no=offer\r\n", dcid: 4 },
66+
})
67+
return input.respond(
68+
JSON.stringify({ session: { id: "s1" }, transport: { sdp: "v=0\r\no=live-answer\r\n" } }),
69+
)
70+
}
71+
if (req.url === "https://api.openai.com/v1/realtime/calls?dcid=4") {
72+
expect(req.headers.get("authorization")).toBe("Bearer sk-openai")
73+
const boundary = input.text.slice(2, input.text.indexOf("\r\n"))
74+
const form = yield* Effect.promise(() =>
75+
new Response(input.text, {
76+
headers: { "content-type": `multipart/form-data; boundary=${boundary}` },
77+
}).formData(),
78+
)
79+
expect(form.get("sdp")).toBe("v=0\r\no=offer\r\n")
80+
expect(JSON.parse(String(form.get("session")))).toEqual({
81+
type: "realtime",
82+
instructions: "Brief.",
83+
model: "gpt-realtime-2.1",
84+
})
85+
return input.respond("v=0\r\no=rtc-answer\r\n", { headers: { "content-type": "application/sdp" } })
86+
}
87+
if (req.url === "https://api.openai.com/v1/realtime/client_secrets") {
88+
expect(req.headers.get("authorization")).toBe("Bearer sk-openai")
89+
expect(JSON.parse(input.text)).toEqual({
90+
expires_after: { seconds: 60 },
91+
session: { type: "realtime", instructions: "Brief.", model: "gpt-realtime-2.1" },
92+
})
93+
return input.respond(JSON.stringify({ value: "ek_1", expires_at: 1760000060 }))
94+
}
95+
expect(req.url).toBe("https://api.x.ai/v1/realtime/client_secrets")
96+
expect(req.headers.get("authorization")).toBe("Bearer xai-key")
97+
expect(JSON.parse(input.text)).toEqual({ expires_after: { seconds: 60 } })
98+
return input.respond(JSON.stringify({ value: "ek_2", expires_at: 1760000120 }))
99+
}),
100+
),
101+
),
102+
),
103+
)
104+
105+
test("routes bare and versioned model names to their engine", () => {
106+
expect(
107+
["gpt-realtime", "gpt-realtime-2.1", "gpt-live", "grok-voice-latest"].map((model) => Realtime.engine(model)?.id),
108+
).toEqual(["openai-realtime", "openai-realtime", "openai-live", "xai-realtime"])
109+
expect(Realtime.engine("gpt-realtimex")).toBeUndefined()
110+
})
111+
112+
it.effect("rejects unsupported transports before network I/O", () =>
113+
Effect.gen(function* () {
114+
const liveWs = yield* Realtime.connect({
115+
model: "gpt-live-1",
116+
apiKey: Redacted.make("k"),
117+
transport: { type: "websocket" },
118+
}).pipe(Effect.flip)
119+
const grokRtc = yield* Realtime.connect({
120+
model: "grok-voice-3.0",
121+
apiKey: Redacted.make("k"),
122+
transport: { type: "webrtc", sdp: "v=0" },
123+
}).pipe(Effect.flip)
124+
expect([liveWs.reason._tag, grokRtc.reason._tag]).toEqual(["InvalidRequest", "InvalidRequest"])
125+
}).pipe(Effect.provide(dynamicResponse(() => Effect.die("unsupported transport reached the network")))),
126+
)
127+
})

‎packages/client/src/effect/api/api.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1402,6 +1402,24 @@ export interface GenerateApi<E = never> {
14021402
readonly text: GenerateTextOperation<E>
14031403
}
14041404

1405+
export type VoiceSessionInput = {
1406+
readonly model: string
1407+
readonly session?: { readonly [x: string]: unknown } | undefined
1408+
readonly transport:
1409+
| { readonly type: "webrtc"; readonly sdp: string; readonly dcid?: number | undefined }
1410+
| { readonly type: "websocket" }
1411+
}
1412+
export type VoiceSessionOutput = {
1413+
readonly transport:
1414+
| { readonly type: "webrtc"; readonly sdp: string }
1415+
| { readonly type: "websocket"; readonly url: string; readonly token: string; readonly expiresAt: number }
1416+
}
1417+
export type VoiceSessionOperation<E = never> = (input: VoiceSessionInput) => Effect.Effect<VoiceSessionOutput, E>
1418+
1419+
export interface VoiceApi<E = never> {
1420+
readonly session: VoiceSessionOperation<E>
1421+
}
1422+
14051423
export type ProviderListInput = { readonly location?: { readonly directory?: string | undefined } | undefined }
14061424
export type ProviderListOutput = { readonly location: Location.PublicRef; readonly data: ReadonlyArray<Provider.Info> }
14071425
export type ProviderListOperation<E = never> = (input?: ProviderListInput) => Effect.Effect<ProviderListOutput, E>
@@ -2272,6 +2290,7 @@ export interface AppApi<E = never> {
22722290
readonly message: MessageApi<E>
22732291
readonly model: ModelApi<E>
22742292
readonly generate: GenerateApi<E>
2293+
readonly voice: VoiceApi<E>
22752294
readonly provider: ProviderApi<E>
22762295
readonly integration: IntegrationApi<E>
22772296
readonly mcp: McpApi<E>

‎packages/client/src/effect/generated/client.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,8 @@ import type {
115115
ModelDefaultOutput,
116116
GenerateTextInput,
117117
GenerateTextOutput,
118+
VoiceSessionInput,
119+
VoiceSessionOutput,
118120
ProviderListInput,
119121
ProviderListOutput,
120122
ProviderGetInput,
@@ -863,6 +865,18 @@ const EndpointGenerateText = (raw: RawClient["server.generate"]) => (input: Gene
863865

864866
const adaptGroupGenerate = (raw: RawClient["server.generate"]) => ({ text: EndpointGenerateText(raw) })
865867

868+
const EndpointVoiceSession = (raw: RawClient["server.voice"]) => (input: VoiceSessionInput) =>
869+
preserveEffect<VoiceSessionOutput>()(
870+
raw["voice.session"]({
871+
payload: { model: input["model"], session: input["session"], transport: input["transport"] },
872+
}).pipe(
873+
Effect.mapError(mapClientError),
874+
Effect.map((value) => value.data),
875+
),
876+
)
877+
878+
const adaptGroupVoice = (raw: RawClient["server.voice"]) => ({ session: EndpointVoiceSession(raw) })
879+
866880
const EndpointProviderList = (raw: RawClient["server.provider"]) => (input?: ProviderListInput) =>
867881
preserveEffect<ProviderListOutput>()(
868882
raw["provider.list"]({ query: { location: input?.["location"] } }).pipe(Effect.mapError(mapClientError)),
@@ -1616,6 +1630,7 @@ const adaptClient = (raw: RawClient) => ({
16161630
message: adaptGroupMessage(raw["server.message"]),
16171631
model: adaptGroupModel(raw["server.model"]),
16181632
generate: adaptGroupGenerate(raw["server.generate"]),
1633+
voice: adaptGroupVoice(raw["server.voice"]),
16191634
provider: adaptGroupProvider(raw["server.provider"]),
16201635
integration: adaptGroupIntegration(raw["server.integration"]),
16211636
mcp: adaptGroupMcp(raw["server.mcp"]),

‎packages/client/src/promise/generated/client.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,8 @@ import type {
109109
ModelDefaultOutput,
110110
GenerateTextInput,
111111
GenerateTextOutput,
112+
VoiceSessionInput,
113+
VoiceSessionOutput,
112114
ProviderListInput,
113115
ProviderListOutput,
114116
ProviderGetInput,
@@ -1151,6 +1153,20 @@ export function make(options: ClientOptions) {
11511153
requestOptions,
11521154
).then((value) => value.data),
11531155
},
1156+
voice: {
1157+
session: (input: VoiceSessionInput, requestOptions?: RequestOptions) =>
1158+
request<{ readonly data: VoiceSessionOutput }>(
1159+
{
1160+
method: "POST",
1161+
path: `/api/experimental/voice/session`,
1162+
body: { model: input["model"], session: input["session"], transport: input["transport"] },
1163+
successStatus: 200,
1164+
declaredStatuses: [400, 401, 404, 503],
1165+
empty: false,
1166+
},
1167+
requestOptions,
1168+
).then((value) => value.data),
1169+
},
11541170
provider: {
11551171
list: (input?: ProviderListInput, requestOptions?: RequestOptions) =>
11561172
request<ProviderListOutput>(

0 commit comments

Comments
 (0)