Skip to content

Commit 524a4c1

Browse files
steipetedvy
steipete
and
dvy
committed
fix(voice-call): auto-respond to webhook transcripts
Co-authored-by: dvy <[email protected]>
1 parent 5f2df24 commit 524a4c1

11 files changed

Lines changed: 281 additions & 40 deletions

File tree

‎extensions/voice-call/src/manager.closed-loop.test.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ describe("CallManager closed-loop turns", () => {
130130

131131
const expectedTurnToken = requireTurnToken(provider);
132132

133-
manager.processEvent({
133+
const staleResult = manager.processEvent({
134134
id: "evt-turn-token-bad",
135135
type: "call.speech",
136136
callId: started.callId,
@@ -140,10 +140,11 @@ describe("CallManager closed-loop turns", () => {
140140
isFinal: true,
141141
turnToken: "wrong-token",
142142
});
143+
expect(staleResult).toEqual({ kind: "ignored" });
143144

144145
expectTranscriptWaiter(manager, started.callId);
145146

146-
manager.processEvent({
147+
const finalResult = manager.processEvent({
147148
id: "evt-turn-token-good",
148149
type: "call.speech",
149150
callId: started.callId,
@@ -153,6 +154,11 @@ describe("CallManager closed-loop turns", () => {
153154
isFinal: true,
154155
turnToken: expectedTurnToken,
155156
});
157+
expect(finalResult).toMatchObject({
158+
kind: "final-speech",
159+
transcript: "final answer",
160+
waiterResolved: true,
161+
});
156162

157163
const turnResult = await turnPromise;
158164
expect(turnResult.success).toBe(true);

‎extensions/voice-call/src/manager.ts‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
66
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
77
import type { VoiceCallConfig, VoiceCallCoreSessionConfig } from "./config.js";
88
import type { CallManagerContext, StreamSessionIssuer } from "./manager/context.js";
9-
import { processEvent as processManagerEvent } from "./manager/events.js";
9+
import { processEvent as processManagerEvent, type ProcessEventResult } from "./manager/events.js";
1010
import { getCallByProviderCallId as getCallByProviderCallIdFromMaps } from "./manager/lookup.js";
1111
import {
1212
continueCall as continueCallWithContext,
@@ -15,6 +15,7 @@ import {
1515
sendDtmf as sendDtmfWithContext,
1616
speak as speakWithContext,
1717
speakInitialMessage as speakInitialMessageWithContext,
18+
type SpeakOptions,
1819
} from "./manager/outbound.js";
1920
import {
2021
getCallHistoryFromStore,
@@ -316,8 +317,12 @@ export class CallManager {
316317
/**
317318
* Speak to user in an active call.
318319
*/
319-
async speak(callId: CallId, text: string): Promise<{ success: boolean; error?: string }> {
320-
return speakWithContext(this.getContext(), callId, text);
320+
async speak(
321+
callId: CallId,
322+
text: string,
323+
options?: SpeakOptions,
324+
): Promise<{ success: boolean; error?: string }> {
325+
return speakWithContext(this.getContext(), callId, text, options);
321326
}
322327

323328
/**
@@ -376,8 +381,8 @@ export class CallManager {
376381
/**
377382
* Process a webhook event.
378383
*/
379-
processEvent(event: NormalizedEvent): void {
380-
processManagerEvent(this.getContext(), event);
384+
processEvent(event: NormalizedEvent): ProcessEventResult {
385+
return processManagerEvent(this.getContext(), event);
381386
}
382387

383388
private shouldDeferConversationInitialMessageUntilStreamConnect(): boolean {

‎extensions/voice-call/src/manager/events.test.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -687,7 +687,7 @@ describe("processEvent (functional)", () => {
687687
});
688688
ctx.providerCallIdMap.set("provider-dedupe", "call-dedupe");
689689

690-
processEvent(ctx, {
690+
const firstResult = processEvent(ctx, {
691691
id: "evt-1",
692692
dedupeKey: "stable-key-1",
693693
type: "call.speech",
@@ -698,7 +698,7 @@ describe("processEvent (functional)", () => {
698698
isFinal: true,
699699
});
700700

701-
processEvent(ctx, {
701+
const replayResult = processEvent(ctx, {
702702
id: "evt-2",
703703
dedupeKey: "stable-key-1",
704704
type: "call.speech",
@@ -715,6 +715,12 @@ describe("processEvent (functional)", () => {
715715
}
716716
expect(call.transcript).toHaveLength(1);
717717
expect(Array.from(ctx.processedEventIds)).toEqual(["stable-key-1"]);
718+
expect(firstResult).toMatchObject({
719+
kind: "final-speech",
720+
transcript: "hello",
721+
waiterResolved: false,
722+
});
723+
expect(replayResult).toEqual({ kind: "ignored" });
718724
});
719725

720726
it("keeps retryable call.error events replayable", () => {

‎extensions/voice-call/src/manager/events.ts‎

Lines changed: 27 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,16 @@ type EventContext = Pick<
3131
| "streamSessionIssuer"
3232
>;
3333

34+
export type ProcessEventResult =
35+
| { kind: "ignored" }
36+
| { kind: "processed" }
37+
| {
38+
kind: "final-speech";
39+
call: CallRecord;
40+
transcript: string;
41+
waiterResolved: boolean;
42+
};
43+
3444
function shouldAcceptInbound(config: EventContext["config"], from: string | undefined): boolean {
3545
const { inboundPolicy: policy, allowFrom } = config;
3646

@@ -138,10 +148,10 @@ function persistRejectedInboundCall(params: {
138148
persistCallRecord(params.ctx.storePath, rejectedCall);
139149
}
140150

141-
export function processEvent(ctx: EventContext, event: NormalizedEvent): void {
151+
export function processEvent(ctx: EventContext, event: NormalizedEvent): ProcessEventResult {
142152
const dedupeKey = event.dedupeKey || event.id;
143153
if (ctx.processedEventIds.has(dedupeKey)) {
144-
return;
154+
return { kind: "ignored" };
145155
}
146156

147157
let call = findCall({
@@ -166,11 +176,11 @@ export function processEvent(ctx: EventContext, event: NormalizedEvent): void {
166176
console.warn(
167177
`[voice-call] Inbound call rejected by policy but no provider to hang up (providerCallId: ${pid}, from: ${event.from}); call will time out on provider side.`,
168178
);
169-
return;
179+
return { kind: "ignored" };
170180
}
171181
ctx.processedEventIds.add(dedupeKey);
172182
if (ctx.rejectedProviderCallIds.has(pid)) {
173-
return;
183+
return { kind: "ignored" };
174184
}
175185
ctx.rejectedProviderCallIds.add(pid);
176186
const callId = event.callId ?? pid;
@@ -187,7 +197,7 @@ export function processEvent(ctx: EventContext, event: NormalizedEvent): void {
187197
const message = formatErrorMessage(err);
188198
console.warn(`[voice-call] Failed to reject inbound call ${pid}:`, message);
189199
});
190-
return;
200+
return { kind: "processed" };
191201
}
192202

193203
call = createWebhookCall({
@@ -203,7 +213,7 @@ export function processEvent(ctx: EventContext, event: NormalizedEvent): void {
203213
}
204214

205215
if (!call) {
206-
return;
216+
return { kind: "ignored" };
207217
}
208218

209219
if (event.providerCallId && event.providerCallId !== call.providerCallId) {
@@ -224,6 +234,7 @@ export function processEvent(ctx: EventContext, event: NormalizedEvent): void {
224234
call.processedEventIds.push(dedupeKey);
225235
}
226236

237+
let result: ProcessEventResult = { kind: "processed" };
227238
switch (event.type) {
228239
case "call.initiated":
229240
transitionState(call, "initiated");
@@ -305,9 +316,16 @@ export function processEvent(ctx: EventContext, event: NormalizedEvent): void {
305316
console.warn(
306317
`[voice-call] Ignoring speech event with mismatched turn token for ${call.callId}`,
307318
);
319+
result = { kind: "ignored" };
308320
break;
309321
}
310322
addTranscriptEntry(call, "user", event.transcript);
323+
result = {
324+
kind: "final-speech",
325+
call,
326+
transcript: event.transcript,
327+
waiterResolved: resolved,
328+
};
311329
}
312330
ensureMaxDurationTimerForLiveCall({
313331
ctx,
@@ -331,7 +349,7 @@ export function processEvent(ctx: EventContext, event: NormalizedEvent): void {
331349
endReason: event.reason,
332350
endedAt: event.timestamp,
333351
});
334-
return;
352+
return { kind: "processed" };
335353

336354
case "call.error":
337355
if (!event.retryable) {
@@ -342,12 +360,13 @@ export function processEvent(ctx: EventContext, event: NormalizedEvent): void {
342360
endedAt: event.timestamp,
343361
transcriptRejectReason: `Call error: ${event.error}`,
344362
});
345-
return;
363+
return { kind: "processed" };
346364
}
347365
// Keep retryable provider errors replayable so a redelivery can still
348366
// drive later recovery or terminal handling for the same event key.
349367
break;
350368
}
351369

352370
persistCallRecord(ctx.storePath, call);
371+
return result;
353372
}

‎extensions/voice-call/src/manager/outbound.test.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -321,13 +321,16 @@ describe("voice-call outbound helpers", () => {
321321
storePath: "/tmp/voice-call.json",
322322
};
323323

324-
await expect(speak(ctx as never, "call-1", "hello")).resolves.toEqual({ success: true });
324+
await expect(
325+
speak(ctx as never, "call-1", "hello", { listenAfterPlayback: true }),
326+
).resolves.toEqual({ success: true });
325327
expect(transitionStateMock).toHaveBeenCalledWith(call, "speaking");
326328
expect(playTts).toHaveBeenCalledWith({
327329
callId: "call-1",
328330
providerCallId: "provider-1",
329331
text: "hello",
330332
voice: "alloy",
333+
listenAfterPlayback: true,
331334
});
332335
expect(addTranscriptEntryMock).toHaveBeenCalledWith(call, "bot", "hello");
333336

‎extensions/voice-call/src/manager/outbound.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -268,10 +268,15 @@ export async function initiateCall(
268268
}
269269
}
270270

271+
export type SpeakOptions = {
272+
listenAfterPlayback?: boolean;
273+
};
274+
271275
export async function speak(
272276
ctx: SpeakContext,
273277
callId: CallId,
274278
text: string,
279+
options?: SpeakOptions,
275280
): Promise<{ success: boolean; error?: string }> {
276281
const connected = requireConnectedCall(ctx, callId);
277282
if (!connected.ok) {
@@ -295,11 +300,13 @@ export async function speak(
295300
const voice = resolvePreferredTtsVoice(
296301
resolveVoiceCallEffectiveConfig(ctx.config, numberRouteKey).config,
297302
);
303+
const playbackOptions = options?.listenAfterPlayback ? { listenAfterPlayback: true } : {};
298304
await provider.playTts({
299305
callId,
300306
providerCallId,
301307
text,
302308
voice,
309+
...playbackOptions,
303310
});
304311

305312
addTranscriptEntry(call, "bot", text);

‎extensions/voice-call/src/providers/plivo.test.ts‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
// Voice Call tests cover plivo plugin behavior.
2-
import { describe, expect, it } from "vitest";
2+
import { describe, expect, it, vi } from "vitest";
33
import { PlivoProvider } from "./plivo.js";
44

55
function requireEvent<T>(event: T | undefined, message: string): T {
@@ -91,4 +91,54 @@ describe("PlivoProvider", () => {
9191

9292
expect(callbackMap.get("call-uuid")).toBe("https://voice.openclaw.ai/voice/webhook");
9393
});
94+
95+
it("renders an auto-response as the prompt for the next speech input", async () => {
96+
const provider = new PlivoProvider({
97+
authId: "MA000000000000000000",
98+
authToken: "test-token",
99+
});
100+
const apiRequest = vi.fn(async (_params: unknown) => ({}));
101+
(
102+
provider as unknown as {
103+
apiRequest: (params: unknown) => Promise<unknown>;
104+
}
105+
).apiRequest = apiRequest;
106+
(
107+
provider as unknown as {
108+
callIdToWebhookUrl: Map<string, string>;
109+
}
110+
).callIdToWebhookUrl.set("internal-call-id", "https://example.com/voice/webhook");
111+
112+
await provider.playTts({
113+
callId: "internal-call-id",
114+
providerCallId: "call-uuid",
115+
text: "How can I help?",
116+
locale: "en-US",
117+
listenAfterPlayback: true,
118+
});
119+
120+
expect(apiRequest).toHaveBeenCalledWith(
121+
expect.objectContaining({
122+
method: "POST",
123+
endpoint: "/Call/call-uuid/",
124+
body: expect.objectContaining({
125+
aleg_url: expect.stringContaining("flow=xml-speak"),
126+
}),
127+
}),
128+
);
129+
130+
const result = provider.parseWebhookEvent({
131+
headers: { host: "example.com" },
132+
rawBody: "CallUUID=call-uuid",
133+
url: "https://example.com/voice/webhook?provider=plivo&flow=xml-speak&callId=internal-call-id",
134+
method: "POST",
135+
query: { provider: "plivo", flow: "xml-speak", callId: "internal-call-id" },
136+
});
137+
const responseBody = requireResponseBody(result.providerResponseBody);
138+
expect(responseBody).toContain('<GetInput inputType="speech"');
139+
expect(responseBody).toContain('speechEndTimeout="2"');
140+
expect(responseBody).toContain("flow=getinput");
141+
expect(responseBody).toContain('<Speak language="en-US">How can I help?</Speak>');
142+
expect(responseBody.indexOf("<GetInput")).toBeLessThan(responseBody.indexOf("<Speak"));
143+
});
94144
});

‎extensions/voice-call/src/providers/plivo.ts‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ export interface PlivoProviderOptions {
3737
webhookSecurity?: WebhookSecurityConfig;
3838
}
3939

40-
type PendingSpeak = { text: string; locale?: string };
40+
type PendingSpeak = { text: string; locale?: string; listenAfterPlayback?: boolean };
4141
type PendingListen = { language?: string };
4242

4343
function createPlivoRequestDedupeKey(ctx: WebhookContext): string {
@@ -158,8 +158,18 @@ export class PlivoProvider implements VoiceCallProvider {
158158
this.pendingSpeakByCallId.delete(callId);
159159
}
160160

161+
const actionUrl =
162+
pending?.listenAfterPlayback && callId
163+
? this.buildActionUrl(ctx, { flow: "getinput", callId })
164+
: null;
161165
const xml = pending
162-
? PlivoProvider.xmlSpeak(pending.text, pending.locale)
166+
? actionUrl
167+
? PlivoProvider.xmlSpeakAndListen({
168+
text: pending.text,
169+
language: pending.locale,
170+
actionUrl,
171+
})
172+
: PlivoProvider.xmlSpeak(pending.text, pending.locale)
163173
: PlivoProvider.xmlKeepAlive();
164174
return {
165175
events: [],
@@ -415,6 +425,7 @@ export class PlivoProvider implements VoiceCallProvider {
415425
this.pendingSpeakByCallId.set(input.callId, {
416426
text: input.text,
417427
locale: input.locale,
428+
listenAfterPlayback: input.listenAfterPlayback,
418429
});
419430

420431
await this.transferCallLeg({
@@ -515,7 +526,22 @@ export class PlivoProvider implements VoiceCallProvider {
515526
const language = params.language || "en-US";
516527
return `<?xml version="1.0" encoding="UTF-8"?>
517528
<Response>
518-
<GetInput inputType="speech" method="POST" action="${escapeXml(params.actionUrl)}" language="${escapeXml(language)}" executionTimeout="30" speechEndTimeout="1" redirect="false">
529+
<GetInput inputType="speech" method="POST" action="${escapeXml(params.actionUrl)}" language="${escapeXml(language)}" executionTimeout="30" speechEndTimeout="2" redirect="false">
530+
</GetInput>
531+
<Wait length="300" />
532+
</Response>`;
533+
}
534+
535+
private static xmlSpeakAndListen(params: {
536+
text: string;
537+
actionUrl: string;
538+
language?: string;
539+
}): string {
540+
const language = params.language || "en-US";
541+
return `<?xml version="1.0" encoding="UTF-8"?>
542+
<Response>
543+
<GetInput inputType="speech" method="POST" action="${escapeXml(params.actionUrl)}" language="${escapeXml(language)}" executionTimeout="30" speechEndTimeout="2" redirect="false">
544+
<Speak language="${escapeXml(language)}">${escapeXml(params.text)}</Speak>
519545
</GetInput>
520546
<Wait length="300" />
521547
</Response>`;

‎extensions/voice-call/src/types.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -258,6 +258,8 @@ export type PlayTtsInput = {
258258
text: string;
259259
voice?: string;
260260
locale?: string;
261+
/** Keep collecting speech after playback when the provider owns the listening XML. */
262+
listenAfterPlayback?: boolean;
261263
};
262264

263265
export type SendDtmfInput = {

0 commit comments

Comments
 (0)