Skip to content

Commit c434d77

Browse files
steipetesteipete
committed
feat: add unified talk gateway sessions
1 parent 7225a26 commit c434d77

40 files changed

Lines changed: 7057 additions & 206 deletions

‎src/commands/doctor/shared/legacy-config-migrate.provider-shapes.test.ts‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { describe, expect, it } from "vitest";
22
import type { OpenClawConfig } from "../../../config/types.js";
33
import { LEGACY_CONFIG_MIGRATIONS_RUNTIME_TTS } from "./legacy-config-migrations.runtime.tts.js";
4+
import { normalizeLegacyTalkConfig } from "./legacy-talk-config-normalizer.js";
45

56
function migrateLegacyConfig(raw: unknown): {
67
config: OpenClawConfig | null;
@@ -21,6 +22,83 @@ function migrateLegacyConfig(raw: unknown): {
2122
}
2223

2324
describe("legacy migrate provider-shaped config", () => {
25+
it("moves legacy realtime Talk selectors into talk.realtime without treating speech config as runtime fallback", () => {
26+
const changes: string[] = [];
27+
const migrated = normalizeLegacyTalkConfig(
28+
{
29+
talk: {
30+
provider: "openai",
31+
providers: {
32+
openai: {
33+
apiKey: "test-key",
34+
custom: true,
35+
},
36+
},
37+
mode: "realtime",
38+
transport: "gateway-relay",
39+
brain: "agent-consult",
40+
model: "gpt-realtime",
41+
voice: "alloy",
42+
} as never,
43+
},
44+
changes,
45+
);
46+
47+
expect(changes).toContain(
48+
"Moved legacy realtime Talk provider/model fields into talk.realtime.",
49+
);
50+
expect(migrated.talk).toEqual({
51+
provider: "openai",
52+
providers: {
53+
openai: {
54+
apiKey: "test-key",
55+
custom: true,
56+
},
57+
},
58+
realtime: {
59+
provider: "openai",
60+
providers: {
61+
openai: {
62+
apiKey: "test-key",
63+
custom: true,
64+
},
65+
},
66+
mode: "realtime",
67+
transport: "gateway-relay",
68+
brain: "agent-consult",
69+
model: "gpt-realtime",
70+
voice: "alloy",
71+
},
72+
});
73+
});
74+
75+
it("does not copy plain Talk speech provider config into talk.realtime", () => {
76+
const changes: string[] = [];
77+
const migrated = normalizeLegacyTalkConfig(
78+
{
79+
talk: {
80+
provider: "elevenlabs",
81+
providers: {
82+
elevenlabs: {
83+
voiceId: "voice-1",
84+
},
85+
},
86+
},
87+
},
88+
changes,
89+
);
90+
91+
expect(changes).toEqual([]);
92+
expect(migrated.talk).toEqual({
93+
provider: "elevenlabs",
94+
providers: {
95+
elevenlabs: {
96+
voiceId: "voice-1",
97+
},
98+
},
99+
});
100+
});
101+
24102
it("moves messages.tts.<provider> keys into messages.tts.providers", () => {
25103
const res = migrateLegacyConfig({
26104
messages: {

‎src/commands/doctor/shared/legacy-talk-config-normalizer.ts‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,31 @@ function buildLegacyTalkProviderCompat(
1414
return Object.keys(compat).length > 0 ? compat : undefined;
1515
}
1616

17+
function buildLegacyRealtimeTalkCompat(
18+
talk: Record<string, unknown>,
19+
normalizedTalk: NonNullable<OpenClawConfig["talk"]>,
20+
): Record<string, unknown> | undefined {
21+
if (talk.realtime !== undefined) {
22+
return undefined;
23+
}
24+
const compat: Record<string, unknown> = {};
25+
for (const key of ["model", "voice", "mode", "transport", "brain"] as const) {
26+
if (talk[key] !== undefined) {
27+
compat[key] = talk[key];
28+
}
29+
}
30+
if (Object.keys(compat).length === 0) {
31+
return undefined;
32+
}
33+
if (normalizedTalk.provider !== undefined) {
34+
compat.provider = normalizedTalk.provider;
35+
}
36+
if (normalizedTalk.providers !== undefined) {
37+
compat.providers = normalizedTalk.providers;
38+
}
39+
return normalizeTalkSection({ realtime: compat } as OpenClawConfig["talk"])?.realtime;
40+
}
41+
1742
function isRecord(value: unknown): value is Record<string, unknown> {
1843
return Boolean(value && typeof value === "object" && !Array.isArray(value));
1944
}
@@ -35,13 +60,23 @@ export function normalizeLegacyTalkConfig(cfg: OpenClawConfig, changes: string[]
3560
},
3661
};
3762
}
63+
const legacyRealtimeCompat = buildLegacyRealtimeTalkCompat(rawTalk, normalizedTalk);
64+
if (legacyRealtimeCompat) {
65+
normalizedTalk.realtime = {
66+
...legacyRealtimeCompat,
67+
...normalizedTalk.realtime,
68+
};
69+
}
3870
if (Object.keys(normalizedTalk).length === 0 || isDeepStrictEqual(normalizedTalk, rawTalk)) {
3971
return cfg;
4072
}
4173

4274
changes.push(
4375
"Normalized talk.provider/providers shape (trimmed provider ids and merged missing compatibility fields).",
4476
);
77+
if (legacyRealtimeCompat) {
78+
changes.push("Moved legacy realtime Talk provider/model fields into talk.realtime.");
79+
}
4580
return {
4681
...cfg,
4782
talk: normalizedTalk,

‎src/config/schema.help.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,21 @@ export const FIELD_HELP: Record<string, string> = {
150150
"Provider-specific Talk settings keyed by provider id. During migration, prefer this over legacy talk.* keys.",
151151
"talk.providers.*": "Provider-owned Talk config fields for the matching provider id.",
152152
"talk.providers.*.apiKey": "Provider API key for Talk mode.", // pragma: allowlist secret
153+
"talk.realtime":
154+
"Realtime Talk provider, model, voice, mode, transport, and brain strategy. Keep speech/TTS provider config in talk.provider and talk.providers.",
155+
"talk.realtime.provider": "Active realtime voice provider id, such as openai or google.",
156+
"talk.realtime.providers": "Provider-specific realtime voice settings keyed by provider id.",
157+
"talk.realtime.providers.*": "Provider-owned realtime voice config for the matching provider id.",
158+
"talk.realtime.providers.*.apiKey": "Provider API key for realtime Talk.", // pragma: allowlist secret
159+
"talk.realtime.model":
160+
"Realtime provider model id override for browser or Gateway-owned Talk sessions.",
161+
"talk.realtime.voice":
162+
"Realtime provider voice id override for browser or Gateway-owned Talk sessions.",
163+
"talk.realtime.mode": "Talk execution mode: realtime, stt-tts, or transcription.",
164+
"talk.realtime.transport":
165+
"Talk byte/session transport: webrtc, provider-websocket, gateway-relay, or managed-room.",
166+
"talk.realtime.brain":
167+
"Talk reasoning strategy: agent-consult for Gateway-mediated agent help, direct-tools for owner-only tool calls, or none.",
153168
"talk.speechLocale":
154169
'BCP 47 locale id for Talk speech recognition on device nodes, for example "ru-RU". Leave unset to use each device default.',
155170
"talk.interruptOnSpeech":

‎src/config/schema.labels.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -875,6 +875,16 @@ export const FIELD_LABELS: Record<string, string> = {
875875
"talk.providers": "Talk Provider Settings",
876876
"talk.providers.*": "Talk Provider Config",
877877
"talk.providers.*.apiKey": "Talk Provider API Key", // pragma: allowlist secret
878+
"talk.realtime": "Talk Realtime",
879+
"talk.realtime.provider": "Talk Realtime Provider",
880+
"talk.realtime.providers": "Talk Realtime Provider Settings",
881+
"talk.realtime.providers.*": "Talk Realtime Provider Config",
882+
"talk.realtime.providers.*.apiKey": "Talk Realtime Provider API Key", // pragma: allowlist secret
883+
"talk.realtime.model": "Talk Realtime Model",
884+
"talk.realtime.voice": "Talk Realtime Voice",
885+
"talk.realtime.mode": "Talk Realtime Mode",
886+
"talk.realtime.transport": "Talk Realtime Transport",
887+
"talk.realtime.brain": "Talk Realtime Brain",
878888
channels: "Channels",
879889
"channels.defaults": "Channel Defaults",
880890
"channels.defaults.groupPolicy": "Default Group Policy",

‎src/config/talk.normalize.test.ts‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,19 @@ describe("talk normalization", () => {
3131
custom: true,
3232
},
3333
},
34+
realtime: {
35+
provider: "openai",
36+
providers: {
37+
openai: {
38+
model: "gpt-realtime",
39+
},
40+
},
41+
model: "gpt-realtime",
42+
voice: "alloy",
43+
mode: "realtime",
44+
transport: "webrtc",
45+
brain: "agent-consult",
46+
},
3447
interruptOnSpeech: true,
3548
});
3649

@@ -42,6 +55,19 @@ describe("talk normalization", () => {
4255
custom: true,
4356
},
4457
},
58+
realtime: {
59+
provider: "openai",
60+
providers: {
61+
openai: {
62+
model: "gpt-realtime",
63+
},
64+
},
65+
model: "gpt-realtime",
66+
voice: "alloy",
67+
mode: "realtime",
68+
transport: "webrtc",
69+
brain: "agent-consult",
70+
},
4571
interruptOnSpeech: true,
4672
});
4773
});

‎src/config/talk.ts‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import type {
55
TalkConfig,
66
TalkConfigResponse,
77
TalkProviderConfig,
8+
TalkRealtimeConfig,
89
} from "./types.gateway.js";
910
import type { OpenClawConfig } from "./types.openclaw.js";
1011
import { coerceSecretRef } from "./types.secrets.js";
@@ -85,6 +86,50 @@ function normalizeTalkProviders(value: unknown): Record<string, TalkProviderConf
8586
return Object.keys(providers).length > 0 ? providers : undefined;
8687
}
8788

89+
function normalizeTalkRealtimeConfig(value: unknown): TalkRealtimeConfig | undefined {
90+
if (!isRecord(value)) {
91+
return undefined;
92+
}
93+
const source = value;
94+
const normalized: TalkRealtimeConfig = {};
95+
96+
const provider = normalizeOptionalString(source.provider);
97+
if (provider) {
98+
normalized.provider = provider;
99+
}
100+
const providers = normalizeTalkProviders(source.providers);
101+
if (providers) {
102+
normalized.providers = providers;
103+
}
104+
const model = normalizeOptionalString(source.model);
105+
if (model) {
106+
normalized.model = model;
107+
}
108+
const voice = normalizeOptionalString(source.voice);
109+
if (voice) {
110+
normalized.voice = voice;
111+
}
112+
if (source.mode === "realtime" || source.mode === "stt-tts" || source.mode === "transcription") {
113+
normalized.mode = source.mode;
114+
}
115+
if (
116+
source.transport === "webrtc" ||
117+
source.transport === "provider-websocket" ||
118+
source.transport === "gateway-relay" ||
119+
source.transport === "managed-room"
120+
) {
121+
normalized.transport = source.transport;
122+
}
123+
if (
124+
source.brain === "agent-consult" ||
125+
source.brain === "direct-tools" ||
126+
source.brain === "none"
127+
) {
128+
normalized.brain = source.brain;
129+
}
130+
return Object.keys(normalized).length > 0 ? normalized : undefined;
131+
}
132+
88133
function activeProviderFromTalk(talk: TalkConfig): string | undefined {
89134
const provider = normalizeOptionalString(talk.provider);
90135
const providers = talk.providers;
@@ -118,10 +163,14 @@ export function normalizeTalkSection(value: TalkConfig | undefined): TalkConfig
118163
}
119164

120165
const providers = normalizeTalkProviders(source.providers);
166+
const realtime = normalizeTalkRealtimeConfig(source.realtime);
121167
const provider = normalizeOptionalString(source.provider);
122168
if (providers) {
123169
normalized.providers = providers;
124170
}
171+
if (realtime) {
172+
normalized.realtime = realtime;
173+
}
125174
if (provider) {
126175
normalized.provider = provider;
127176
}
@@ -182,6 +231,9 @@ export function buildTalkConfigResponse(value: unknown): TalkConfigResponse | un
182231
if (normalized?.providers && Object.keys(normalized.providers).length > 0) {
183232
payload.providers = normalized.providers;
184233
}
234+
if (normalized?.realtime && Object.keys(normalized.realtime).length > 0) {
235+
payload.realtime = normalized.realtime;
236+
}
185237

186238
const resolved =
187239
resolveActiveTalkProviderConfig(normalized) ??

‎src/config/types.gateway.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,23 @@ export type TalkProviderConfig = {
5555
[key: string]: unknown;
5656
};
5757

58+
export type TalkRealtimeConfig = {
59+
/** Active realtime voice provider. */
60+
provider?: string;
61+
/** Provider-specific realtime voice config keyed by provider id. */
62+
providers?: Record<string, TalkProviderConfig>;
63+
/** Provider model override for realtime sessions. */
64+
model?: string;
65+
/** Provider voice override for realtime sessions. */
66+
voice?: string;
67+
/** Realtime execution mode. */
68+
mode?: "realtime" | "stt-tts" | "transcription";
69+
/** Byte/session transport. */
70+
transport?: "webrtc" | "provider-websocket" | "gateway-relay" | "managed-room";
71+
/** Tool/agent strategy for realtime sessions. */
72+
brain?: "agent-consult" | "direct-tools" | "none";
73+
};
74+
5875
export type ResolvedTalkConfig = {
5976
/** Active Talk TTS provider resolved from the current config payload. */
6077
provider: string;
@@ -67,6 +84,8 @@ export type TalkConfig = {
6784
provider?: string;
6885
/** Provider-specific Talk config keyed by provider id. */
6986
providers?: Record<string, TalkProviderConfig>;
87+
/** Realtime Talk provider, model, voice, mode, transport, and brain config. */
88+
realtime?: TalkRealtimeConfig;
7089
/** BCP 47 locale id used for Talk speech recognition on device nodes. */
7190
speechLocale?: string;
7291
/** Stop speaking when user starts talking (default: true). */

‎src/config/zod-schema.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,10 +212,44 @@ const TalkProviderEntrySchema = z
212212
})
213213
.catchall(z.unknown());
214214

215+
const TalkRealtimeSchema = z
216+
.object({
217+
provider: z.string().optional(),
218+
providers: z.record(z.string(), TalkProviderEntrySchema).optional(),
219+
model: z.string().optional(),
220+
voice: z.string().optional(),
221+
mode: z.enum(["realtime", "stt-tts", "transcription"]).optional(),
222+
transport: z.enum(["webrtc", "provider-websocket", "gateway-relay", "managed-room"]).optional(),
223+
brain: z.enum(["agent-consult", "direct-tools", "none"]).optional(),
224+
})
225+
.strict()
226+
.superRefine((realtime, ctx) => {
227+
const provider = normalizeLowercaseStringOrEmpty(realtime.provider ?? "");
228+
const providers = realtime.providers ? Object.keys(realtime.providers) : [];
229+
230+
if (provider && providers.length > 0 && !(provider in realtime.providers!)) {
231+
ctx.addIssue({
232+
code: z.ZodIssueCode.custom,
233+
path: ["provider"],
234+
message: `talk.realtime.provider must match a key in talk.realtime.providers (missing "${provider}")`,
235+
});
236+
}
237+
238+
if (!provider && providers.length > 1) {
239+
ctx.addIssue({
240+
code: z.ZodIssueCode.custom,
241+
path: ["provider"],
242+
message:
243+
"talk.realtime.provider is required when talk.realtime.providers defines multiple providers",
244+
});
245+
}
246+
});
247+
215248
const TalkSchema = z
216249
.object({
217250
provider: z.string().optional(),
218251
providers: z.record(z.string(), TalkProviderEntrySchema).optional(),
252+
realtime: TalkRealtimeSchema.optional(),
219253
speechLocale: z.string().optional(),
220254
interruptOnSpeech: z.boolean().optional(),
221255
silenceTimeoutMs: z.number().int().positive().optional(),

‎src/gateway/gateway-misc.test.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -322,7 +322,9 @@ describe("gateway broadcaster", () => {
322322
expect(readSocket.send).toHaveBeenCalledTimes(0);
323323

324324
broadcastToConnIds("tick", { ts: 1 }, new Set(["c-read"]));
325-
expect(readSocket.send).toHaveBeenCalledTimes(1);
325+
broadcastToConnIds("talk.realtime.relay", { type: "ready" }, new Set(["c-read"]));
326+
broadcastToConnIds("talk.transcription.relay", { type: "session.ready" }, new Set(["c-read"]));
327+
expect(readSocket.send).toHaveBeenCalledTimes(3);
326328
expect(approvalsSocket.send).toHaveBeenCalledTimes(1);
327329
expect(pairingSocket.send).toHaveBeenCalledTimes(1);
328330
});

0 commit comments

Comments
 (0)