Skip to content

Commit 7fc9a82

Browse files
scoootscooobsteipete
scoootscooob
authored and
steipete
committed
fix(voice-call): pace realtime Twilio audio
1 parent 19f948a commit 7fc9a82

8 files changed

Lines changed: 431 additions & 45 deletions

File tree

‎docs/plugins/google-meet.md‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -445,7 +445,7 @@ Enable the Voice Call plugin on the Gateway host, not on the Chrome node:
445445
```json5
446446
{
447447
plugins: {
448-
allow: ["google-meet", "voice-call"],
448+
allow: ["google-meet", "voice-call", "google"],
449449
entries: {
450450
"google-meet": {
451451
enabled: true,
@@ -458,8 +458,24 @@ Enable the Voice Call plugin on the Gateway host, not on the Chrome node:
458458
enabled: true,
459459
config: {
460460
provider: "twilio",
461+
inboundPolicy: "allowlist",
462+
realtime: {
463+
enabled: true,
464+
provider: "google",
465+
instructions: "Join this Google Meet as an OpenClaw agent. Be brief.",
466+
toolPolicy: "safe-read-only",
467+
providers: {
468+
google: {
469+
silenceDurationMs: 500,
470+
startSensitivity: "high",
471+
},
472+
},
473+
},
461474
},
462475
},
476+
google: {
477+
enabled: true,
478+
},
463479
},
464480
},
465481
}
@@ -472,8 +488,12 @@ secrets out of `openclaw.json`:
472488
export TWILIO_ACCOUNT_SID=AC...
473489
export TWILIO_AUTH_TOKEN=...
474490
export TWILIO_FROM_NUMBER=+15550001234
491+
export GEMINI_API_KEY=...
475492
```
476493

494+
Use `realtime.provider: "openai"` with the OpenAI provider plugin and
495+
`OPENAI_API_KEY` instead if that is your realtime voice provider.
496+
477497
Restart or reload the Gateway after enabling `voice-call`; plugin config changes
478498
do not appear in an already running Gateway process until it reloads.
479499

‎docs/plugins/voice-call.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,9 @@ Current runtime behaviour:
250250
Defaults: API key from `realtime.providers.google.apiKey`,
251251
`GEMINI_API_KEY`, or `GOOGLE_GENERATIVE_AI_API_KEY`; model
252252
`gemini-2.5-flash-native-audio-preview-12-2025`; voice `Kore`.
253+
`sessionResumption` and `contextWindowCompression` default on for longer,
254+
reconnectable calls. Use `silenceDurationMs`, `startSensitivity`, and
255+
`endSensitivity` to tune faster turn-taking on telephony audio.
253256

254257
```json5
255258
{
@@ -270,6 +273,8 @@ Current runtime behaviour:
270273
apiKey: "${GEMINI_API_KEY}",
271274
model: "gemini-2.5-flash-native-audio-preview-12-2025",
272275
voice: "Kore",
276+
silenceDurationMs: 500,
277+
startSensitivity: "high",
273278
},
274279
},
275280
},

‎extensions/google/realtime-voice-provider.test.ts‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,8 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
107107
turnCoverage: "only-activity",
108108
automaticActivityDetectionDisabled: false,
109109
enableAffectiveDialog: undefined,
110+
sessionResumption: undefined,
111+
contextWindowCompression: undefined,
110112
thinkingLevel: undefined,
111113
thinkingBudget: undefined,
112114
});
@@ -181,6 +183,8 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
181183
},
182184
turnCoverage: "TURN_INCLUDES_ONLY_ACTIVITY",
183185
},
186+
sessionResumption: {},
187+
contextWindowCompression: { slidingWindow: {} },
184188
tools: [
185189
{
186190
functionDeclarations: [
@@ -312,6 +316,42 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
312316
});
313317
});
314318

319+
it("can opt out of Google Live session resumption and context compression", async () => {
320+
const provider = buildGoogleRealtimeVoiceProvider();
321+
const bridge = provider.createBridge({
322+
providerConfig: {
323+
apiKey: "gemini-key",
324+
contextWindowCompression: false,
325+
sessionResumption: false,
326+
},
327+
onAudio: vi.fn(),
328+
onClearAudio: vi.fn(),
329+
});
330+
331+
await bridge.connect();
332+
333+
expect(lastConnectParams().config).not.toHaveProperty("contextWindowCompression");
334+
expect(lastConnectParams().config).not.toHaveProperty("sessionResumption");
335+
});
336+
337+
it("captures Google Live resumption handles and reuses them on reconnect", async () => {
338+
const provider = buildGoogleRealtimeVoiceProvider();
339+
const bridge = provider.createBridge({
340+
providerConfig: { apiKey: "gemini-key" },
341+
onAudio: vi.fn(),
342+
onClearAudio: vi.fn(),
343+
});
344+
345+
await bridge.connect();
346+
lastConnectParams().callbacks.onmessage({
347+
sessionResumptionUpdate: { resumable: true, newHandle: "resume-1" },
348+
});
349+
350+
await bridge.connect();
351+
352+
expect(lastConnectParams().config.sessionResumption).toEqual({ handle: "resume-1" });
353+
});
354+
315355
it("waits for setup completion before draining audio and firing ready", async () => {
316356
const provider = buildGoogleRealtimeVoiceProvider();
317357
const onReady = vi.fn();

‎extensions/google/realtime-voice-provider.ts‎

Lines changed: 58 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,20 @@
11
import { randomUUID } from "node:crypto";
2-
import {
2+
import type {
33
ActivityHandling,
44
Behavior,
55
EndSensitivity,
6+
FunctionDeclaration,
7+
FunctionResponse,
68
FunctionResponseScheduling,
9+
LiveConnectConfig,
10+
LiveServerContent,
11+
LiveServerMessage,
12+
LiveServerToolCall,
713
Modality,
14+
RealtimeInputConfig,
815
StartSensitivity,
16+
ThinkingConfig,
917
TurnCoverage,
10-
type FunctionDeclaration,
11-
type FunctionResponse,
12-
type LiveConnectConfig,
13-
type LiveServerContent,
14-
type LiveServerMessage,
15-
type LiveServerToolCall,
16-
type RealtimeInputConfig,
17-
type ThinkingConfig,
1818
} from "@google/genai";
1919
import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-onboard";
2020
import type {
@@ -47,7 +47,7 @@ const GOOGLE_REALTIME_BROWSER_API_VERSION = "v1alpha";
4747
const GOOGLE_REALTIME_BROWSER_WEBSOCKET_URL =
4848
"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained";
4949
const MAX_PENDING_AUDIO_CHUNKS = 320;
50-
const DEFAULT_AUDIO_STREAM_END_SILENCE_MS = 700;
50+
const DEFAULT_AUDIO_STREAM_END_SILENCE_MS = 500;
5151
const GOOGLE_REALTIME_BROWSER_SESSION_TTL_MS = 30 * 60 * 1000;
5252
const GOOGLE_REALTIME_BROWSER_NEW_SESSION_TTL_MS = 60 * 1000;
5353

@@ -70,6 +70,8 @@ type GoogleRealtimeVoiceProviderConfig = {
7070
turnCoverage?: GoogleRealtimeTurnCoverage;
7171
automaticActivityDetectionDisabled?: boolean;
7272
enableAffectiveDialog?: boolean;
73+
sessionResumption?: boolean;
74+
contextWindowCompression?: boolean;
7375
thinkingLevel?: GoogleRealtimeThinkingLevel;
7476
thinkingBudget?: number;
7577
};
@@ -90,6 +92,8 @@ type GoogleRealtimeLiveConfig = {
9092
turnCoverage?: GoogleRealtimeTurnCoverage;
9193
automaticActivityDetectionDisabled?: boolean;
9294
enableAffectiveDialog?: boolean;
95+
sessionResumption?: boolean;
96+
contextWindowCompression?: boolean;
9397
thinkingLevel?: GoogleRealtimeThinkingLevel;
9498
thinkingBudget?: number;
9599
};
@@ -209,6 +213,8 @@ function normalizeProviderConfig(
209213
turnCoverage: asTurnCoverage(raw?.turnCoverage),
210214
automaticActivityDetectionDisabled: asBoolean(raw?.automaticActivityDetectionDisabled),
211215
enableAffectiveDialog: asBoolean(raw?.enableAffectiveDialog),
216+
sessionResumption: asBoolean(raw?.sessionResumption),
217+
contextWindowCompression: asBoolean(raw?.contextWindowCompression),
212218
thinkingLevel: asThinkingLevel(raw?.thinkingLevel),
213219
thinkingBudget: asFiniteNumber(raw?.thinkingBudget),
214220
};
@@ -223,9 +229,9 @@ function mapStartSensitivity(
223229
): StartSensitivity | undefined {
224230
switch (value) {
225231
case "high":
226-
return StartSensitivity.START_SENSITIVITY_HIGH;
232+
return "START_SENSITIVITY_HIGH" as StartSensitivity;
227233
case "low":
228-
return StartSensitivity.START_SENSITIVITY_LOW;
234+
return "START_SENSITIVITY_LOW" as StartSensitivity;
229235
default:
230236
return undefined;
231237
}
@@ -236,9 +242,9 @@ function mapEndSensitivity(
236242
): EndSensitivity | undefined {
237243
switch (value) {
238244
case "high":
239-
return EndSensitivity.END_SENSITIVITY_HIGH;
245+
return "END_SENSITIVITY_HIGH" as EndSensitivity;
240246
case "low":
241-
return EndSensitivity.END_SENSITIVITY_LOW;
247+
return "END_SENSITIVITY_LOW" as EndSensitivity;
242248
default:
243249
return undefined;
244250
}
@@ -249,9 +255,9 @@ function mapActivityHandling(
249255
): ActivityHandling | undefined {
250256
switch (value) {
251257
case "no-interruption":
252-
return ActivityHandling.NO_INTERRUPTION;
258+
return "NO_INTERRUPTION" as ActivityHandling;
253259
case "start-of-activity-interrupts":
254-
return ActivityHandling.START_OF_ACTIVITY_INTERRUPTS;
260+
return "START_OF_ACTIVITY_INTERRUPTS" as ActivityHandling;
255261
default:
256262
return undefined;
257263
}
@@ -260,11 +266,11 @@ function mapActivityHandling(
260266
function mapTurnCoverage(value: GoogleRealtimeTurnCoverage | undefined): TurnCoverage | undefined {
261267
switch (value) {
262268
case "only-activity":
263-
return TurnCoverage.TURN_INCLUDES_ONLY_ACTIVITY;
269+
return "TURN_INCLUDES_ONLY_ACTIVITY" as TurnCoverage;
264270
case "all-input":
265-
return TurnCoverage.TURN_INCLUDES_ALL_INPUT;
271+
return "TURN_INCLUDES_ALL_INPUT" as TurnCoverage;
266272
case "audio-activity-and-all-video":
267-
return TurnCoverage.TURN_INCLUDES_AUDIO_ACTIVITY_AND_ALL_VIDEO;
273+
return "TURN_INCLUDES_AUDIO_ACTIVITY_AND_ALL_VIDEO" as TurnCoverage;
268274
default:
269275
return undefined;
270276
}
@@ -316,7 +322,7 @@ function buildFunctionDeclarations(tools: RealtimeVoiceTool[] | undefined): Func
316322
parametersJsonSchema: tool.parameters,
317323
};
318324
if (tool.name === REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) {
319-
declaration.behavior = Behavior.NON_BLOCKING;
325+
declaration.behavior = "NON_BLOCKING" as Behavior;
320326
}
321327
return declaration;
322328
});
@@ -325,7 +331,7 @@ function buildFunctionDeclarations(tools: RealtimeVoiceTool[] | undefined): Func
325331
function buildGoogleLiveConnectConfig(config: GoogleRealtimeLiveConfig): LiveConnectConfig {
326332
const functionDeclarations = buildFunctionDeclarations(config.tools);
327333
return {
328-
responseModalities: [Modality.AUDIO],
334+
responseModalities: ["AUDIO" as Modality],
329335
...(typeof config.temperature === "number" && config.temperature > 0
330336
? { temperature: config.temperature }
331337
: {}),
@@ -359,7 +365,7 @@ function buildBrowserInitialSetup(model: string) {
359365
setup: {
360366
model: toGoogleModelResource(model),
361367
generationConfig: {
362-
responseModalities: [Modality.AUDIO],
368+
responseModalities: ["AUDIO" as Modality],
363369
},
364370
inputAudioTranscription: {},
365371
outputAudioTranscription: {},
@@ -403,6 +409,7 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge {
403409
private audioStreamEnded = false;
404410
private pendingFunctionNames = new Map<string, string>();
405411
private readonly audioFormat: RealtimeVoiceAudioFormat;
412+
private resumptionHandle: string | undefined;
406413

407414
constructor(private readonly config: GoogleRealtimeVoiceBridgeConfig) {
408415
this.audioFormat = config.audioFormat ?? REALTIME_VOICE_AUDIO_FORMAT_G711_ULAW_8KHZ;
@@ -425,7 +432,17 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge {
425432

426433
this.session = (await ai.live.connect({
427434
model: this.config.model ?? GOOGLE_REALTIME_DEFAULT_MODEL,
428-
config: buildGoogleLiveConnectConfig(this.config),
435+
config: {
436+
...buildGoogleLiveConnectConfig(this.config),
437+
...(this.config.sessionResumption === false
438+
? {}
439+
: {
440+
sessionResumption: this.resumptionHandle ? { handle: this.resumptionHandle } : {},
441+
}),
442+
...(this.config.contextWindowCompression === false
443+
? {}
444+
: { contextWindowCompression: { slidingWindow: {} } }),
445+
},
429446
callbacks: {
430447
onopen: () => {
431448
this.connected = true;
@@ -548,7 +565,7 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge {
548565
: { output: result },
549566
};
550567
if (isConsultTool) {
551-
functionResponse.scheduling = FunctionResponseScheduling.WHEN_IDLE;
568+
functionResponse.scheduling = "WHEN_IDLE" as FunctionResponseScheduling;
552569
if (options?.willContinue === true) {
553570
functionResponse.willContinue = true;
554571
}
@@ -607,6 +624,7 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge {
607624
}
608625

609626
private handleMessage(message: LiveServerMessage): void {
627+
this.captureSessionLifecycle(message);
610628
if (message.setupComplete) {
611629
this.handleSetupComplete();
612630
}
@@ -618,6 +636,20 @@ class GoogleRealtimeVoiceBridge implements RealtimeVoiceBridge {
618636
}
619637
}
620638

639+
private captureSessionLifecycle(message: LiveServerMessage): void {
640+
const raw = message as unknown as {
641+
goAway?: { timeLeft?: string };
642+
sessionResumptionUpdate?: { newHandle?: string; resumable?: boolean };
643+
};
644+
const update = raw.sessionResumptionUpdate;
645+
if (update?.resumable && update.newHandle) {
646+
this.resumptionHandle = update.newHandle;
647+
}
648+
if (raw.goAway?.timeLeft) {
649+
this.config.onError?.(new Error(`Google Live session goAway: ${raw.goAway.timeLeft}`));
650+
}
651+
}
652+
621653
private handleSetupComplete(): void {
622654
this.sessionConfigured = true;
623655
for (const chunk of this.pendingAudio.splice(0)) {
@@ -784,6 +816,8 @@ export function buildGoogleRealtimeVoiceProvider(): RealtimeVoiceProviderPlugin
784816
turnCoverage: config.turnCoverage,
785817
automaticActivityDetectionDisabled: config.automaticActivityDetectionDisabled,
786818
enableAffectiveDialog: config.enableAffectiveDialog,
819+
sessionResumption: config.sessionResumption,
820+
contextWindowCompression: config.contextWindowCompression,
787821
thinkingLevel: config.thinkingLevel,
788822
thinkingBudget: config.thinkingBudget,
789823
});

‎extensions/voice-call/index.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -589,9 +589,9 @@ export default definePluginEntry({
589589
respondError(respond, "to required", ErrorCodes.INVALID_REQUEST);
590590
return;
591591
}
592-
const rt = await ensureRuntime();
593592
const mode =
594593
params?.mode === "notify" || params?.mode === "conversation" ? params.mode : undefined;
594+
const rt = await ensureRuntime();
595595
await initiateCallAndRespond({
596596
rt,
597597
respond,

0 commit comments

Comments
 (0)