Skip to content

Commit 464e488

Browse files
ifthikar-razikifthikar-razik
committed
feat(voice-call): add Telnyx realtime media streams
1 parent 9094c80 commit 464e488

13 files changed

Lines changed: 310 additions & 18 deletions

File tree

‎extensions/voice-call/README.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ Put under `plugins.entries.voice-call.config`:
3737

3838
```json5
3939
{
40-
provider: "twilio", // or "telnyx" | "plivo" | "mock"
40+
provider: "telnyx", // or "twilio" | "plivo" | "mock"
4141
fromNumber: "+15550001234",
4242
toNumber: "+15550005678",
4343
sessionScope: "per-phone", // or "per-call"
@@ -99,6 +99,7 @@ Put under `plugins.entries.voice-call.config`:
9999

100100
Notes:
101101

102+
- Telnyx is the recommended production provider for Voice Call. Twilio and Plivo remain supported for existing installs.
102103
- Twilio/Telnyx/Plivo require a **publicly reachable** webhook URL.
103104
- `mock` is a local dev provider (no network calls).
104105
- Telnyx requires `telnyx.publicKey` (or `TELNYX_PUBLIC_KEY`) unless `skipSignatureVerification` is true.
@@ -161,5 +162,6 @@ Actions:
161162
- While a Twilio stream is active, playback does not fall back to TwiML `<Say>`; stream-TTS failures fail the playback request.
162163
- Outbound conversation calls suppress barge-in only while the initial greeting is actively speaking, then re-enable normal interruption.
163164
- Twilio stream disconnect auto-end uses a short grace window so quick reconnects do not end the call.
165+
- Realtime voice supports Twilio `<Connect><Stream>` and Telnyx Call Control `streaming_start` media streams. Telnyx realtime uses bidirectional PCMU/RTP streaming into the same realtime voice bridge.
164166
- Realtime provider selection is generic. Configure `streaming.provider` / `realtime.provider` and put provider-owned options under `providers.<id>`.
165167
- Runtime fallback still accepts the old voice-call keys for now, but migration is a doctor step and the compat shim is scheduled to go away in a future release.

‎extensions/voice-call/index.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,20 +25,27 @@ import { createVoiceCallContinueOperationStore } from "./src/gateway-continue-op
2525
const VOICE_CALL_WRITE_METHOD_SCOPE = { scope: "operator.write" as const };
2626
const VOICE_CALL_READ_METHOD_SCOPE = { scope: "operator.read" as const };
2727

28+
function resolveDefaultVoiceCallProvider(enabled: boolean): "telnyx" | "mock" | undefined {
29+
if (!enabled) {
30+
return undefined;
31+
}
32+
return process.env.TELNYX_API_KEY && process.env.TELNYX_CONNECTION_ID ? "telnyx" : "mock";
33+
}
34+
2835
const voiceCallConfigSchema = {
2936
parse(value: unknown): VoiceCallConfig {
3037
const normalized = normalizeVoiceCallLegacyConfigInput(value);
3138
const enabled = typeof normalized.enabled === "boolean" ? normalized.enabled : true;
3239
return parseVoiceCallPluginConfig({
3340
...normalized,
3441
enabled,
35-
provider: normalized.provider ?? (enabled ? "mock" : undefined),
42+
provider: normalized.provider ?? resolveDefaultVoiceCallProvider(enabled),
3643
});
3744
},
3845
uiHints: {
3946
provider: {
4047
label: "Provider",
41-
help: "Use twilio, telnyx, or mock for dev/no-network.",
48+
help: "Use telnyx for production voice AI; twilio/plivo remain supported; mock is for dev/no-network.",
4249
},
4350
fromNumber: { label: "From Number", placeholder: "+15550001234" },
4451
toNumber: { label: "Default To Number", placeholder: "+15550001234" },

‎extensions/voice-call/src/config.test.ts‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,36 @@ describe("validateProviderConfig", () => {
250250
"plugins.entries.voice-call.config.realtime.enabled and plugins.entries.voice-call.config.streaming.enabled cannot both be true",
251251
);
252252
});
253+
254+
it("allows Telnyx realtime mode", () => {
255+
const config = createBaseConfig("telnyx");
256+
config.realtime.enabled = true;
257+
config.inboundPolicy = "allowlist";
258+
config.telnyx = {
259+
apiKey: "KEY123",
260+
connectionId: "CONN456",
261+
publicKey: "public-key",
262+
};
263+
264+
const result = validateProviderConfig(config);
265+
266+
expect(result.valid).toBe(true);
267+
expect(result.errors).toEqual([]);
268+
});
269+
270+
it("rejects realtime mode for providers without realtime bridges", () => {
271+
const config = createBaseConfig("plivo");
272+
config.realtime.enabled = true;
273+
config.inboundPolicy = "allowlist";
274+
config.plivo = { authId: "MA123", authToken: "secret" };
275+
276+
const result = validateProviderConfig(config);
277+
278+
expect(result.valid).toBe(false);
279+
expect(result.errors).toContain(
280+
'plugins.entries.voice-call.config.provider must be "twilio" or "telnyx" when realtime.enabled is true',
281+
);
282+
});
253283
});
254284
});
255285

‎extensions/voice-call/src/config.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -846,9 +846,14 @@ export function validateProviderConfig(config: VoiceCallConfig): {
846846
);
847847
}
848848

849-
if (config.realtime.enabled && config.provider && config.provider !== "twilio") {
849+
if (
850+
config.realtime.enabled &&
851+
config.provider &&
852+
config.provider !== "twilio" &&
853+
config.provider !== "telnyx"
854+
) {
850855
errors.push(
851-
'plugins.entries.voice-call.config.provider must be "twilio" when realtime.enabled is true',
856+
'plugins.entries.voice-call.config.provider must be "twilio" or "telnyx" when realtime.enabled is true',
852857
);
853858
}
854859

‎extensions/voice-call/src/manager.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -377,6 +377,7 @@ export class CallManager {
377377
const mode = (call.metadata?.mode as string | undefined) ?? "conversation";
378378
if (mode === "conversation") {
379379
if (this.config.realtime.enabled) {
380+
this.maybeStartProviderRealtimeStream(call);
380381
return;
381382
}
382383
const shouldWaitForStreamConnect =
@@ -399,6 +400,37 @@ export class CallManager {
399400
});
400401
}
401402

403+
private maybeStartProviderRealtimeStream(call: CallRecord): void {
404+
if (!this.provider || this.provider.name === "twilio" || !call.providerCallId) {
405+
return;
406+
}
407+
if (typeof this.provider.startRealtimeStream !== "function") {
408+
return;
409+
}
410+
const metadata = call.metadata ?? {};
411+
if (metadata.realtimeStreamStartedAt || metadata.realtimeStreamStartPending) {
412+
return;
413+
}
414+
metadata.realtimeStreamStartPending = true;
415+
call.metadata = metadata;
416+
void this.provider
417+
.startRealtimeStream({ callId: call.callId, providerCallId: call.providerCallId })
418+
.then(() => {
419+
const nextMetadata = call.metadata ?? {};
420+
delete nextMetadata.realtimeStreamStartPending;
421+
nextMetadata.realtimeStreamStartedAt = Date.now();
422+
call.metadata = nextMetadata;
423+
})
424+
.catch((err) => {
425+
const nextMetadata = call.metadata ?? {};
426+
delete nextMetadata.realtimeStreamStartPending;
427+
call.metadata = nextMetadata;
428+
console.warn(
429+
`[voice-call] Failed to start realtime stream for call ${call.callId}: ${formatErrorMessage(err)}`,
430+
);
431+
});
432+
}
433+
402434
/**
403435
* Get an active call by ID.
404436
*/

‎extensions/voice-call/src/providers/base.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import type {
1111
WebhookParseOptions,
1212
ProviderWebhookParseResult,
1313
StartListeningInput,
14+
StartRealtimeStreamInput,
1415
StopListeningInput,
1516
WebhookContext,
1617
WebhookVerificationResult,
@@ -72,6 +73,13 @@ export interface VoiceCallProvider {
7273
*/
7374
playTts(input: PlayTtsInput): Promise<void>;
7475

76+
/**
77+
* Start a provider media stream for realtime voice bridges.
78+
* Providers that connect streams via webhook response (for example Twilio
79+
* <Connect><Stream>) do not need to implement this hook.
80+
*/
81+
startRealtimeStream?: (input: StartRealtimeStreamInput) => Promise<void>;
82+
7583
/**
7684
* Send DTMF digits to an active call.
7785
*/

‎extensions/voice-call/src/providers/telnyx.test.ts‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -302,6 +302,51 @@ describe("TelnyxProvider answer control", () => {
302302
});
303303
});
304304

305+
describe("TelnyxProvider realtime streaming", () => {
306+
it("starts bidirectional PCMU media streaming with the configured realtime URL", async () => {
307+
const release = vi.fn(async () => {});
308+
apiMocks.fetchWithSsrFGuard.mockResolvedValue({
309+
response: new Response(JSON.stringify({ data: {} }), { status: 200 }),
310+
release,
311+
});
312+
const provider = new TelnyxProvider({
313+
apiKey: "KEY123",
314+
connectionId: "CONN456",
315+
publicKey: undefined,
316+
});
317+
provider.setRealtimeStreamUrlFactory(
318+
(input) => `wss://voice.example.com/voice/stream/realtime/${input.providerCallId}`,
319+
);
320+
321+
await provider.startRealtimeStream({
322+
callId: "call-1",
323+
providerCallId: "call-control-1",
324+
});
325+
326+
expect(apiMocks.fetchWithSsrFGuard).toHaveBeenCalledWith(
327+
expect.objectContaining({
328+
url: "https://api.telnyx.com/v2/calls/call-control-1/actions/streaming_start",
329+
auditContext: "voice-call.telnyx.api",
330+
policy: { allowedHostnames: ["api.telnyx.com"] },
331+
init: expect.objectContaining({
332+
method: "POST",
333+
body: JSON.stringify({
334+
command_id: "openclaw-realtime-stream-call-1",
335+
stream_url: "wss://voice.example.com/voice/stream/realtime/call-control-1",
336+
stream_track: "both_tracks",
337+
stream_codec: "PCMU",
338+
stream_bidirectional_mode: "rtp",
339+
stream_bidirectional_codec: "PCMU",
340+
stream_bidirectional_sampling_rate: 8000,
341+
stream_bidirectional_target_legs: "both",
342+
}),
343+
}),
344+
}),
345+
);
346+
expect(release).toHaveBeenCalledTimes(1);
347+
});
348+
});
349+
305350
describe("TelnyxProvider speak control", () => {
306351
it("passes custom Telnyx voice ids to the speak action", async () => {
307352
const release = vi.fn(async () => {});

‎extensions/voice-call/src/providers/telnyx.ts‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import type {
1212
PlayTtsInput,
1313
ProviderWebhookParseResult,
1414
StartListeningInput,
15+
StartRealtimeStreamInput,
1516
StopListeningInput,
1617
WebhookContext,
1718
WebhookParseOptions,
@@ -56,6 +57,7 @@ export class TelnyxProvider implements VoiceCallProvider {
5657
private readonly options: TelnyxProviderOptions;
5758
private readonly baseUrl = "https://api.telnyx.com/v2";
5859
private readonly apiHost = "api.telnyx.com";
60+
private realtimeStreamUrlFactory: ((input: StartRealtimeStreamInput) => string) | undefined;
5961

6062
constructor(config: TelnyxConfig, options: TelnyxProviderOptions = {}) {
6163
if (!config.apiKey) {
@@ -302,6 +304,30 @@ export class TelnyxProvider implements VoiceCallProvider {
302304
});
303305
}
304306

307+
setRealtimeStreamUrlFactory(factory: (input: StartRealtimeStreamInput) => string): void {
308+
this.realtimeStreamUrlFactory = factory;
309+
}
310+
311+
/**
312+
* Start a bidirectional Telnyx media stream for realtime voice.
313+
*/
314+
async startRealtimeStream(input: StartRealtimeStreamInput): Promise<void> {
315+
if (!this.realtimeStreamUrlFactory) {
316+
throw new Error("Telnyx realtime stream URL factory is not configured");
317+
}
318+
const streamUrl = this.realtimeStreamUrlFactory(input);
319+
await this.apiRequest(`/calls/${input.providerCallId}/actions/streaming_start`, {
320+
command_id: `openclaw-realtime-stream-${input.callId}`,
321+
stream_url: streamUrl,
322+
stream_track: "both_tracks",
323+
stream_codec: "PCMU",
324+
stream_bidirectional_mode: "rtp",
325+
stream_bidirectional_codec: "PCMU",
326+
stream_bidirectional_sampling_rate: 8000,
327+
stream_bidirectional_target_legs: "both",
328+
});
329+
}
330+
305331
/**
306332
* Start transcription (STT) via Telnyx.
307333
*/

‎extensions/voice-call/src/runtime.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import {
2020
import type { CoreAgentDeps, CoreConfig } from "./core-bridge.js";
2121
import { CallManager } from "./manager.js";
2222
import type { VoiceCallProvider } from "./providers/base.js";
23+
import type { TelnyxProvider } from "./providers/telnyx.js";
2324
import type { TwilioProvider } from "./providers/twilio.js";
2425
import { buildRealtimeVoiceInstructions } from "./realtime-agent-context.js";
2526
import { resolveRealtimeFastContextConsult } from "./realtime-fast-context.js";
@@ -459,6 +460,17 @@ export async function createVoiceCallRuntime(params: {
459460
if (publicUrl && realtimeProvider) {
460461
webhookServer.getRealtimeHandler()?.setPublicUrl(publicUrl);
461462
}
463+
if (provider.name === "telnyx" && realtimeProvider) {
464+
const realtimeHandler = webhookServer.getRealtimeHandler();
465+
if (realtimeHandler) {
466+
(provider as TelnyxProvider).setRealtimeStreamUrlFactory((input) =>
467+
realtimeHandler.buildProviderStreamUrl({
468+
provider: "telnyx",
469+
providerCallId: input.providerCallId,
470+
}),
471+
);
472+
}
473+
}
462474

463475
if (provider.name === "twilio" && config.streaming?.enabled) {
464476
const twilioProvider = provider as TwilioProvider;

‎extensions/voice-call/src/types.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -260,6 +260,11 @@ export type StopListeningInput = {
260260
providerCallId: ProviderCallId;
261261
};
262262

263+
export type StartRealtimeStreamInput = {
264+
callId: CallId;
265+
providerCallId: ProviderCallId;
266+
};
267+
263268
// -----------------------------------------------------------------------------
264269
// Call Status Verification (used on restart to verify persisted calls)
265270
// -----------------------------------------------------------------------------

0 commit comments

Comments
 (0)