Skip to content

Commit bb1b075

Browse files
dynamite-budclaude
authored and
obviyus
committed
feat: add Telnyx Media Streaming for voice-call realtime
Wires bidirectional PCMU WebSocket audio for Telnyx so realtime providers (OpenAI Realtime, etc.) can drive Telnyx calls the same way they drive Twilio. Telnyx attaches Media Streaming at dial time and answer-action time per the documented canonical patterns (no actions/streaming_start call needed). New StreamFrameAdapter abstraction owns provider-shaped frame parsing and outbound serialization, so realtime-handler.ts stays carrier-agnostic. RealtimeAudioPacer is generalized to accept any serializer. The provider-twilio realtime gate widens to accept telnyx. Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
1 parent a517434 commit bb1b075

12 files changed

Lines changed: 542 additions & 81 deletions

File tree

‎extensions/voice-call/src/config.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -868,9 +868,14 @@ export function validateProviderConfig(config: VoiceCallConfig): {
868868
);
869869
}
870870

871-
if (config.realtime.enabled && config.provider && config.provider !== "twilio") {
871+
if (
872+
config.realtime.enabled &&
873+
config.provider &&
874+
config.provider !== "twilio" &&
875+
config.provider !== "telnyx"
876+
) {
872877
errors.push(
873-
'plugins.entries.voice-call.config.provider must be "twilio" when realtime.enabled is true',
878+
'plugins.entries.voice-call.config.provider must be "twilio" or "telnyx" when realtime.enabled is true',
874879
);
875880
}
876881

‎extensions/voice-call/src/manager.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import path from "node:path";
44
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
55
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
66
import type { VoiceCallConfig } from "./config.js";
7-
import type { CallManagerContext } from "./manager/context.js";
7+
import type { CallManagerContext, StreamSessionIssuer } from "./manager/context.js";
88
import { processEvent as processManagerEvent } from "./manager/events.js";
99
import { getCallByProviderCallId as getCallByProviderCallIdFromMaps } from "./manager/lookup.js";
1010
import {
@@ -87,6 +87,13 @@ export class CallManager {
8787
private maxDurationTimers = new Map<CallId, NodeJS.Timeout>();
8888
private initialMessageInFlight = new Set<CallId>();
8989

90+
/**
91+
* Carrier-side stream session issuer. Wired by the runtime when realtime is
92+
* enabled so the manager can pre-issue stream URLs for providers (e.g.
93+
* Telnyx) that attach Media Streaming at dial or answer time.
94+
*/
95+
streamSessionIssuer: StreamSessionIssuer | undefined;
96+
9097
constructor(config: VoiceCallConfig, storePath?: string) {
9198
this.config = config;
9299
this.storePath = resolveDefaultStoreBase(config, storePath);
@@ -339,6 +346,7 @@ export class CallManager {
339346
onCallAnswered: (call) => {
340347
this.maybeSpeakInitialMessageOnAnswered(call);
341348
},
349+
streamSessionIssuer: this.streamSessionIssuer,
342350
};
343351
}
344352

‎extensions/voice-call/src/manager/context.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,27 @@ type CallManagerTransientState = {
3131
initialMessageInFlight: Set<CallId>;
3232
};
3333

34+
/**
35+
* Lazily issue a per-call stream session (token + WSS URL) for carriers that
36+
* attach Media Streaming at dial or answer time (e.g. Telnyx). The manager
37+
* calls this just before delegating to the provider's initiate/answer so the
38+
* streaming params can be embedded in the carrier API payload.
39+
*
40+
* Returns `undefined` when realtime is not configured.
41+
*/
42+
export type StreamSessionIssuer = (request: {
43+
providerName: "twilio" | "telnyx";
44+
callId: CallId;
45+
from?: string;
46+
to?: string;
47+
direction: "inbound" | "outbound";
48+
}) => { token: string; streamUrl: string } | undefined;
49+
3450
type CallManagerHooks = {
3551
/** Optional runtime hook invoked after an event transitions a call into answered state. */
3652
onCallAnswered?: (call: CallRecord) => void;
53+
/** Carrier-side stream session issuer; supplied by runtime when realtime is enabled. */
54+
streamSessionIssuer?: StreamSessionIssuer;
3755
};
3856

3957
export type CallManagerContext = CallManagerRuntimeState &

‎extensions/voice-call/src/manager/events.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ type EventContext = Pick<
2323
| "transcriptWaiters"
2424
| "maxDurationTimers"
2525
| "onCallAnswered"
26+
| "streamSessionIssuer"
2627
>;
2728

2829
function shouldAcceptInbound(config: EventContext["config"], from: string | undefined): boolean {
@@ -195,10 +196,26 @@ export function processEvent(ctx: EventContext, event: NormalizedEvent): void {
195196
case "call.initiated":
196197
transitionState(call, "initiated");
197198
if (call.direction === "inbound" && call.providerCallId && ctx.provider?.answerCall) {
199+
const inboundStreamSession =
200+
ctx.config.realtime?.enabled && ctx.provider.name === "telnyx" && ctx.streamSessionIssuer
201+
? ctx.streamSessionIssuer({
202+
providerName: "telnyx",
203+
callId: call.callId,
204+
from: call.from,
205+
to: call.to,
206+
direction: "inbound",
207+
})
208+
: undefined;
198209
void ctx.provider
199210
.answerCall({
200211
callId: call.callId,
201212
providerCallId: call.providerCallId,
213+
...(inboundStreamSession
214+
? {
215+
streamUrl: inboundStreamSession.streamUrl,
216+
streamAuthToken: inboundStreamSession.token,
217+
}
218+
: {}),
202219
})
203220
.catch((err) => {
204221
const message = formatErrorMessage(err);

‎extensions/voice-call/src/manager/outbound.ts‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,13 @@ import { generateDtmfRedirectTwiml, generateNotifyTwiml } from "./twiml.js";
2424

2525
type InitiateContext = Pick<
2626
CallManagerContext,
27-
"activeCalls" | "providerCallIdMap" | "provider" | "config" | "storePath" | "webhookUrl"
27+
| "activeCalls"
28+
| "providerCallIdMap"
29+
| "provider"
30+
| "config"
31+
| "storePath"
32+
| "webhookUrl"
33+
| "streamSessionIssuer"
2834
>;
2935

3036
type SpeakContext = Pick<
@@ -201,13 +207,27 @@ export async function initiateCall(
201207
);
202208
}
203209

210+
const streamSession =
211+
ctx.config.realtime?.enabled && ctx.provider.name === "telnyx" && ctx.streamSessionIssuer
212+
? ctx.streamSessionIssuer({
213+
providerName: "telnyx",
214+
callId,
215+
from,
216+
to,
217+
direction: "outbound",
218+
})
219+
: undefined;
220+
204221
const result = await ctx.provider.initiateCall({
205222
callId,
206223
from,
207224
to,
208225
webhookUrl: ctx.webhookUrl,
209226
inlineTwiml,
210227
preConnectTwiml,
228+
...(streamSession
229+
? { streamUrl: streamSession.streamUrl, streamAuthToken: streamSession.token }
230+
: {}),
211231
});
212232

213233
callRecord.providerCallId = result.providerCallId;

‎extensions/voice-call/src/providers/base.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,14 @@ export interface VoiceCallProvider {
3131
/** Provider identifier */
3232
readonly name: ProviderName;
3333

34+
/**
35+
* Update the public origin (`https://host[:port]`) the gateway is reachable
36+
* at. Providers that build stream URLs at dial/answer time (e.g. Telnyx)
37+
* use it; Twilio derives stream URLs from the request Host header inside
38+
* TwiML rendering and may ignore.
39+
*/
40+
setPublicUrl?(url: string): void;
41+
3442
/**
3543
* Verify webhook signature/HMAC before processing.
3644
* Must be called before parseWebhookEvent.

‎extensions/voice-call/src/providers/telnyx.ts‎

Lines changed: 57 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,23 @@ export class TelnyxProvider implements VoiceCallProvider {
208208
digits: data.payload?.digit || "",
209209
};
210210

211+
case "call.streaming.started":
212+
case "call.streaming.stopped":
213+
// Informational. The realtime bridge tracks its own lifecycle via the
214+
// WebSocket; we acknowledge the webhook (200) but skip event emission
215+
// to avoid duplicate signal at the manager.
216+
return null;
217+
218+
case "call.streaming.failed": {
219+
const reason = typeof data.payload?.reason === "string" ? data.payload.reason : undefined;
220+
return {
221+
...baseEvent,
222+
type: "call.error",
223+
error: `Telnyx streaming failed${reason ? `: ${reason}` : ""}`,
224+
retryable: false,
225+
};
226+
}
227+
211228
default:
212229
return null;
213230
}
@@ -251,18 +268,24 @@ export class TelnyxProvider implements VoiceCallProvider {
251268
}
252269

253270
/**
254-
* Initiate an outbound call via Telnyx API.
271+
* Initiate an outbound call via Telnyx API. When `input.streamUrl` is set,
272+
* the dial payload also opens a bidirectional Media Streaming session on
273+
* answer (PCMU 8 kHz), per the Telnyx documented "AI agent" pattern.
255274
*/
256275
async initiateCall(input: InitiateCallInput): Promise<InitiateCallResult> {
257-
const result = await this.apiRequest<TelnyxCallResponse>("/calls", {
276+
const body: Record<string, unknown> = {
258277
connection_id: this.connectionId,
259278
to: input.to,
260279
from: input.from,
261280
webhook_url: input.webhookUrl,
262281
webhook_url_method: "POST",
263282
client_state: Buffer.from(input.callId).toString("base64"),
264283
timeout_secs: 30,
265-
});
284+
...(input.streamUrl
285+
? buildTelnyxStreamingFields(input.streamUrl, input.streamAuthToken)
286+
: {}),
287+
};
288+
const result = await this.apiRequest<TelnyxCallResponse>("/calls", body);
266289

267290
return {
268291
providerCallId: result.data.call_control_id,
@@ -282,12 +305,18 @@ export class TelnyxProvider implements VoiceCallProvider {
282305
}
283306

284307
/**
285-
* Answer an inbound Telnyx Call Control leg.
308+
* Answer an inbound Telnyx Call Control leg. When `input.streamUrl` is set,
309+
* the answer action also attaches a bidirectional Media Streaming session
310+
* (PCMU 8 kHz), per the Telnyx canonical "answer-action inline" pattern.
286311
*/
287312
async answerCall(input: AnswerCallInput): Promise<void> {
288-
await this.apiRequest(`/calls/${input.providerCallId}/actions/answer`, {
313+
const body: Record<string, unknown> = {
289314
command_id: `openclaw-answer-${input.callId}`,
290-
});
315+
...(input.streamUrl
316+
? buildTelnyxStreamingFields(input.streamUrl, input.streamAuthToken)
317+
: {}),
318+
};
319+
await this.apiRequest(`/calls/${input.providerCallId}/actions/answer`, body);
291320
}
292321

293322
/**
@@ -355,6 +384,28 @@ export class TelnyxProvider implements VoiceCallProvider {
355384
}
356385
}
357386

387+
/**
388+
* Build the streaming-related fields for a Telnyx dial or answer-action
389+
* payload. PCMU 8 kHz mono only; bidirectional via RTP; target legs `"self"`
390+
* so the bot receives both inbound and outbound audio without the routing
391+
* gotcha that drops the call leg when `"opposite"` is configured.
392+
*/
393+
function buildTelnyxStreamingFields(
394+
streamUrl: string,
395+
streamAuthToken: string | undefined,
396+
): Record<string, unknown> {
397+
return {
398+
stream_url: streamUrl,
399+
stream_track: "inbound_track",
400+
stream_codec: "PCMU",
401+
stream_bidirectional_mode: "rtp",
402+
stream_bidirectional_codec: "PCMU",
403+
stream_bidirectional_sampling_rate: 8000,
404+
stream_bidirectional_target_legs: "self",
405+
...(streamAuthToken ? { stream_auth_token: streamAuthToken } : {}),
406+
};
407+
}
408+
358409
// -----------------------------------------------------------------------------
359410
// Telnyx-specific types
360411
// -----------------------------------------------------------------------------

‎extensions/voice-call/src/runtime.ts‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -457,13 +457,23 @@ export async function createVoiceCallRuntime(params: {
457457
);
458458
}
459459

460-
if (publicUrl && provider.name === "twilio") {
461-
(provider as TwilioProvider).setPublicUrl(publicUrl);
460+
if (publicUrl) {
461+
provider.setPublicUrl?.(publicUrl);
462462
}
463463
if (publicUrl && realtimeProvider) {
464464
webhookServer.getRealtimeHandler()?.setPublicUrl(publicUrl);
465465
}
466466

467+
// Once the realtime handler has its public URL, expose its session issuer
468+
// to the manager so carriers that attach Media Streaming at dial / answer
469+
// time (e.g. Telnyx) can embed the streaming params in their carrier API
470+
// payloads. Twilio learns the stream URL from TwiML so this is a no-op
471+
// for the Twilio path.
472+
const realtimeHandler = webhookServer.getRealtimeHandler();
473+
if (realtimeHandler) {
474+
manager.streamSessionIssuer = (request) => realtimeHandler.issueStreamSession(request);
475+
}
476+
467477
if (provider.name === "twilio" && config.streaming?.enabled) {
468478
const twilioProvider = provider as TwilioProvider;
469479
if (ttsRuntime?.textToSpeechTelephony) {

‎extensions/voice-call/src/types.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,15 @@ export type InitiateCallInput = {
215215
inlineTwiml?: string;
216216
/** TwiML to serve once before normal webhook-driven call handling resumes. */
217217
preConnectTwiml?: string;
218+
/**
219+
* Optional `wss://` URL the carrier should open for bidirectional Media
220+
* Streaming on call connect. Used by carriers (e.g. Telnyx) that attach
221+
* streaming at dial time. Twilio learns the URL from TwiML so it ignores
222+
* this field.
223+
*/
224+
streamUrl?: string;
225+
/** Per-call auth token the carrier echoes back on the WS upgrade. */
226+
streamAuthToken?: string;
218227
};
219228

220229
export type InitiateCallResult = {
@@ -231,6 +240,15 @@ export type HangupCallInput = {
231240
export type AnswerCallInput = {
232241
callId: CallId;
233242
providerCallId: ProviderCallId;
243+
/**
244+
* Optional `wss://` URL the carrier should open for bidirectional Media
245+
* Streaming on answer. Used by carriers (e.g. Telnyx) that attach
246+
* streaming at answer time. Twilio learns the URL from TwiML so it ignores
247+
* this field.
248+
*/
249+
streamUrl?: string;
250+
/** Per-call auth token the carrier echoes back on the WS upgrade. */
251+
streamAuthToken?: string;
234252
};
235253

236254
export type PlayTtsInput = {

0 commit comments

Comments
 (0)