Skip to content

Commit 0f16555

Browse files
author
Homer
committed
Merge remote-tracking branch 'origin/main' into codex/operator-overview
2 parents 9fb619b + 7efbaf7 commit 0f16555

4 files changed

Lines changed: 414 additions & 105 deletions

File tree

‎scripts/dev/realtime-talk-live-smoke.ts‎

Lines changed: 195 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,16 @@ import { GoogleGenAI, Modality } from "@google/genai";
66
import { chromium, type Browser } from "playwright";
77
import { createServer } from "vite";
88
import { buildOpenAIRealtimeVoiceProvider } from "../../extensions/openai/realtime-voice-provider.ts";
9-
import { previewForDevToolLog, redactJsonValueForDevToolLog } from "../lib/dev-tooling-safety.ts";
9+
import {
10+
parseStrictIntegerOption,
11+
previewForDevToolLog,
12+
redactJsonValueForDevToolLog,
13+
} from "../lib/dev-tooling-safety.ts";
1014

1115
const OPENAI_REALTIME_MODEL =
1216
process.env.OPENCLAW_REALTIME_OPENAI_MODEL?.trim() || "gpt-realtime-2";
1317
const OPENAI_REALTIME_VOICE = process.env.OPENCLAW_REALTIME_OPENAI_VOICE?.trim() || "alloy";
18+
const DEFAULT_OPENAI_HTTP_TIMEOUT_MS = 30_000;
1419
const GOOGLE_REALTIME_MODEL =
1520
process.env.OPENCLAW_REALTIME_GOOGLE_MODEL?.trim() ||
1621
"gemini-2.5-flash-native-audio-preview-12-2025";
@@ -24,6 +29,17 @@ type SmokeResult = {
2429
details?: Record<string, unknown>;
2530
};
2631

32+
type TimeoutOptions<T> = {
33+
label: string;
34+
timeoutMs: number;
35+
run: (signal: AbortSignal) => Promise<T>;
36+
};
37+
38+
type OpenAIHttpOptions = {
39+
fetchImpl?: typeof fetch;
40+
timeoutMs?: number;
41+
};
42+
2743
function getEnv(name: string): string | undefined {
2844
const value = process.env[name]?.trim();
2945
return value ? value : undefined;
@@ -38,6 +54,36 @@ async function readBoundedText(response: Response): Promise<string> {
3854
return previewForDevToolLog(text, 600);
3955
}
4056

57+
function resolveOpenAIHttpTimeoutMs(
58+
raw = process.env.OPENCLAW_REALTIME_OPENAI_HTTP_TIMEOUT_MS,
59+
): number {
60+
return parseStrictIntegerOption({
61+
fallback: DEFAULT_OPENAI_HTTP_TIMEOUT_MS,
62+
label: "OPENCLAW_REALTIME_OPENAI_HTTP_TIMEOUT_MS",
63+
min: 1,
64+
raw,
65+
});
66+
}
67+
68+
async function withTimeout<T>(options: TimeoutOptions<T>): Promise<T> {
69+
const controller = new AbortController();
70+
let timeout: ReturnType<typeof setTimeout> | undefined;
71+
const timeoutPromise = new Promise<T>((_resolve, reject) => {
72+
timeout = setTimeout(() => {
73+
const error = new Error(`${options.label} exceeded timeout of ${options.timeoutMs}ms`);
74+
reject(error);
75+
controller.abort(error);
76+
}, options.timeoutMs);
77+
});
78+
try {
79+
return await Promise.race([options.run(controller.signal), timeoutPromise]);
80+
} finally {
81+
if (timeout) {
82+
clearTimeout(timeout);
83+
}
84+
}
85+
}
86+
4187
function printResult(result: SmokeResult): void {
4288
console.log(
4389
`${result.name}: ${result.ok ? "ok" : "failed"}`,
@@ -49,31 +95,43 @@ function compareStrings(left: string | undefined, right: string | undefined): nu
4995
return (left ?? "").localeCompare(right ?? "");
5096
}
5197

52-
async function createOpenAIClientSecret(apiKey: string): Promise<string> {
53-
const response = await fetch("https://api.openai.com/v1/realtime/client_secrets", {
54-
method: "POST",
55-
headers: {
56-
Authorization: `Bearer ${apiKey}`,
57-
"Content-Type": "application/json",
58-
},
59-
body: JSON.stringify({
60-
session: {
61-
type: "realtime",
62-
model: OPENAI_REALTIME_MODEL,
63-
audio: {
64-
output: { voice: OPENAI_REALTIME_VOICE },
98+
async function createOpenAIClientSecret(
99+
apiKey: string,
100+
options: OpenAIHttpOptions = {},
101+
): Promise<string> {
102+
const fetchImpl = options.fetchImpl ?? fetch;
103+
const timeoutMs = options.timeoutMs ?? resolveOpenAIHttpTimeoutMs();
104+
const payload = await withTimeout({
105+
label: "OpenAI Realtime client secret request",
106+
timeoutMs,
107+
run: async (signal) => {
108+
const response = await fetchImpl("https://api.openai.com/v1/realtime/client_secrets", {
109+
method: "POST",
110+
headers: {
111+
Authorization: `Bearer ${apiKey}`,
112+
"Content-Type": "application/json",
65113
},
66-
},
67-
}),
114+
body: JSON.stringify({
115+
session: {
116+
type: "realtime",
117+
model: OPENAI_REALTIME_MODEL,
118+
audio: {
119+
output: { voice: OPENAI_REALTIME_VOICE },
120+
},
121+
},
122+
}),
123+
signal,
124+
});
125+
if (!response.ok) {
126+
throw new Error(
127+
`OpenAI Realtime client secret failed (${response.status}): ${await readBoundedText(
128+
response,
129+
)}`,
130+
);
131+
}
132+
return (await response.json()) as Record<string, unknown>;
133+
},
68134
});
69-
if (!response.ok) {
70-
throw new Error(
71-
`OpenAI Realtime client secret failed (${response.status}): ${await readBoundedText(
72-
response,
73-
)}`,
74-
);
75-
}
76-
const payload = (await response.json()) as Record<string, unknown>;
77135
const nested =
78136
payload.client_secret && typeof payload.client_secret === "object"
79137
? (payload.client_secret as Record<string, unknown>)
@@ -128,79 +186,118 @@ async function smokeOpenAIBackendBridge(apiKey: string): Promise<SmokeResult> {
128186

129187
async function smokeOpenAIWebRtc(browser: Browser, apiKey: string): Promise<SmokeResult> {
130188
try {
131-
const clientSecret = await createOpenAIClientSecret(apiKey);
189+
const openAIHttpTimeoutMs = resolveOpenAIHttpTimeoutMs();
190+
const clientSecret = await createOpenAIClientSecret(apiKey, { timeoutMs: openAIHttpTimeoutMs });
132191
const context = await browser.newContext({
133192
permissions: ["microphone"],
134193
});
135-
const page = await context.newPage();
136-
const result = await page.evaluate(
137-
async ({ clientSecret: secret }) => {
138-
let media: MediaStream;
139-
if (navigator.mediaDevices?.getUserMedia) {
140-
media = await navigator.mediaDevices.getUserMedia({ audio: true });
141-
} else {
142-
const audioContext = new AudioContext();
143-
const destination = audioContext.createMediaStreamDestination();
144-
const oscillator = audioContext.createOscillator();
145-
oscillator.connect(destination);
146-
oscillator.start();
147-
media = destination.stream;
148-
}
149-
const peer = new RTCPeerConnection();
150-
for (const track of media.getAudioTracks()) {
151-
peer.addTrack(track, media);
152-
}
153-
const channel = peer.createDataChannel("oai-events");
154-
const connectionState = new Promise<string>((resolve) => {
155-
const timeout = window.setTimeout(() => resolve(peer.connectionState), 12_000);
156-
peer.addEventListener("connectionstatechange", () => {
157-
if (peer.connectionState === "connected" || peer.connectionState === "failed") {
158-
window.clearTimeout(timeout);
159-
resolve(peer.connectionState);
194+
try {
195+
const page = await context.newPage();
196+
await page.evaluate("globalThis.__name = (fn) => fn");
197+
const result = await page.evaluate(
198+
async ({ clientSecret: secret, timeoutMs }) => {
199+
const withBrowserTimeout = async <T>(
200+
label: string,
201+
run: (signal: AbortSignal) => Promise<T>,
202+
): Promise<T> => {
203+
const controller = new AbortController();
204+
let timeout: number | undefined;
205+
const timeoutPromise = new Promise<T>((_resolve, reject) => {
206+
timeout = window.setTimeout(() => {
207+
const error = new Error(`${label} exceeded timeout of ${timeoutMs}ms`);
208+
reject(error);
209+
controller.abort(error);
210+
}, timeoutMs);
211+
});
212+
try {
213+
return await Promise.race([run(controller.signal), timeoutPromise]);
214+
} finally {
215+
if (timeout !== undefined) {
216+
window.clearTimeout(timeout);
217+
}
160218
}
161-
});
162-
channel.addEventListener("open", () => {
163-
window.clearTimeout(timeout);
164-
resolve(peer.connectionState || "data-channel-open");
165-
});
166-
});
167-
const offer = await peer.createOffer();
168-
await peer.setLocalDescription(offer);
169-
const response = await fetch("https://api.openai.com/v1/realtime/calls", {
170-
method: "POST",
171-
body: offer.sdp,
172-
headers: {
173-
Authorization: `Bearer ${secret}`,
174-
"Content-Type": "application/sdp",
175-
},
176-
});
177-
if (!response.ok) {
178-
throw new Error(`OpenAI Realtime SDP offer failed (${response.status})`);
179-
}
180-
const answer = await response.text();
181-
await peer.setRemoteDescription({ type: "answer", sdp: answer });
182-
const state = await connectionState;
183-
peer.close();
184-
media.getTracks().forEach((track) => track.stop());
185-
return {
186-
answerHasAudio: answer.includes("m=audio"),
187-
remoteDescriptionApplied: peer.remoteDescription?.type === "answer",
188-
connectionState: state,
189-
};
190-
},
191-
{ clientSecret },
192-
);
193-
await context.close();
194-
return {
195-
name: "openai-webrtc-browser",
196-
ok: result.answerHasAudio && result.remoteDescriptionApplied,
197-
details: {
198-
model: OPENAI_REALTIME_MODEL,
199-
answerHasAudio: result.answerHasAudio,
200-
remoteDescriptionApplied: result.remoteDescriptionApplied,
201-
connectionState: result.connectionState,
202-
},
203-
};
219+
};
220+
let media: MediaStream | undefined;
221+
let peer: RTCPeerConnection | undefined;
222+
try {
223+
if (navigator.mediaDevices?.getUserMedia) {
224+
media = await navigator.mediaDevices.getUserMedia({ audio: true });
225+
} else {
226+
const audioContext = new AudioContext();
227+
const destination = audioContext.createMediaStreamDestination();
228+
const oscillator = audioContext.createOscillator();
229+
oscillator.connect(destination);
230+
oscillator.start();
231+
media = destination.stream;
232+
}
233+
peer = new RTCPeerConnection();
234+
for (const track of media.getAudioTracks()) {
235+
peer.addTrack(track, media);
236+
}
237+
const channel = peer.createDataChannel("oai-events");
238+
const connectionState = new Promise<string>((resolve) => {
239+
const timeout = window.setTimeout(
240+
() => resolve(peer?.connectionState ?? "timeout"),
241+
12_000,
242+
);
243+
peer?.addEventListener("connectionstatechange", () => {
244+
if (peer?.connectionState === "connected" || peer?.connectionState === "failed") {
245+
window.clearTimeout(timeout);
246+
resolve(peer.connectionState);
247+
}
248+
});
249+
channel.addEventListener("open", () => {
250+
window.clearTimeout(timeout);
251+
resolve(peer?.connectionState || "data-channel-open");
252+
});
253+
});
254+
const offer = await peer.createOffer();
255+
await peer.setLocalDescription(offer);
256+
const answer = await withBrowserTimeout(
257+
"OpenAI Realtime SDP offer request",
258+
async (signal) => {
259+
const response = await fetch("https://api.openai.com/v1/realtime/calls", {
260+
method: "POST",
261+
body: offer.sdp,
262+
headers: {
263+
Authorization: `Bearer ${secret}`,
264+
"Content-Type": "application/sdp",
265+
},
266+
signal,
267+
});
268+
if (!response.ok) {
269+
throw new Error(`OpenAI Realtime SDP offer failed (${response.status})`);
270+
}
271+
return await response.text();
272+
},
273+
);
274+
await peer.setRemoteDescription({ type: "answer", sdp: answer });
275+
const state = await connectionState;
276+
return {
277+
answerHasAudio: answer.includes("m=audio"),
278+
remoteDescriptionApplied: peer.remoteDescription?.type === "answer",
279+
connectionState: state,
280+
};
281+
} finally {
282+
peer?.close();
283+
media?.getTracks().forEach((track) => track.stop());
284+
}
285+
},
286+
{ clientSecret, timeoutMs: openAIHttpTimeoutMs },
287+
);
288+
return {
289+
name: "openai-webrtc-browser",
290+
ok: result.answerHasAudio && result.remoteDescriptionApplied,
291+
details: {
292+
model: OPENAI_REALTIME_MODEL,
293+
answerHasAudio: result.answerHasAudio,
294+
remoteDescriptionApplied: result.remoteDescriptionApplied,
295+
connectionState: result.connectionState,
296+
},
297+
};
298+
} finally {
299+
await context.close();
300+
}
204301
} catch (error) {
205302
return { name: "openai-webrtc-browser", ok: false, details: { error: shortError(error) } };
206303
}
@@ -578,3 +675,8 @@ if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
578675
process.exitCode = 1;
579676
});
580677
}
678+
679+
export const testing = {
680+
createOpenAIClientSecret,
681+
resolveOpenAIHttpTimeoutMs,
682+
};

‎src/plugins/current-plugin-metadata-snapshot.test.ts‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,34 @@ describe("current plugin metadata snapshot", () => {
177177
expect(getCurrentPluginMetadataSnapshot({ config, env: requestedEnv })).toBeUndefined();
178178
});
179179

180+
it("rejects an exact cached config object after in-place policy changes", () => {
181+
const config = { plugins: { allow: ["demo"] } };
182+
const snapshot = createSnapshot({ config });
183+
setCurrentPluginMetadataSnapshot(snapshot, { config });
184+
185+
expect(getCurrentPluginMetadataSnapshot({ config })).toBe(snapshot);
186+
187+
config.plugins.allow = ["other"];
188+
189+
expect(getCurrentPluginMetadataSnapshot({ config })).toBeUndefined();
190+
});
191+
192+
it("rejects an exact cached env object after in-place root changes", () => {
193+
const config = {};
194+
const snapshot = createSnapshot({ config });
195+
const env = {
196+
HOME: "/home/snapshot",
197+
OPENCLAW_HOME: undefined,
198+
} as NodeJS.ProcessEnv;
199+
setCurrentPluginMetadataSnapshot(snapshot, { config, env });
200+
201+
expect(getCurrentPluginMetadataSnapshot({ config, env })).toBe(snapshot);
202+
203+
env.HOME = "/home/requested";
204+
205+
expect(getCurrentPluginMetadataSnapshot({ config, env })).toBeUndefined();
206+
});
207+
180208
it("keeps source-policy compatibility when storing an auto-enabled runtime config", () => {
181209
const sourceConfig = { channels: { telegram: { botToken: "token" } } };
182210
const autoEnabledConfig = {

0 commit comments

Comments
 (0)