Skip to content

Commit 53b9efa

Browse files
steipetesteipete
committed
fix(openai): correlate realtime cancellation errors
1 parent 8a05c24 commit 53b9efa

2 files changed

Lines changed: 109 additions & 4 deletions

File tree

‎extensions/openai/realtime-voice-provider.test.ts‎

Lines changed: 96 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,13 @@ function expectedResponseCreateEvent() {
153153
});
154154
}
155155

156+
function expectedResponseCancelEvent() {
157+
return expect.objectContaining({
158+
type: "response.cancel",
159+
event_id: expect.stringMatching(/^openclaw-response-cancel-/),
160+
});
161+
}
162+
156163
function createJsonResponse(body: unknown, init?: { status?: number }): Response {
157164
return new Response(JSON.stringify(body), {
158165
status: init?.status ?? 200,
@@ -1436,7 +1443,7 @@ describe("buildOpenAIRealtimeVoiceProvider", () => {
14361443
expect(onAudio).toHaveBeenCalledTimes(1);
14371444
expect(onClearAudio).toHaveBeenCalledTimes(1);
14381445
expect(parseSent(socket).slice(-2)).toEqual([
1439-
{ type: "response.cancel" },
1446+
expectedResponseCancelEvent(),
14401447
{
14411448
type: "conversation.item.truncate",
14421449
item_id: "item_1",
@@ -2266,7 +2273,7 @@ describe("buildOpenAIRealtimeVoiceProvider", () => {
22662273
bridge.handleBargeIn?.({ audioPlaybackActive: true, force: true });
22672274

22682275
expect(parseSent(socket).slice(-2)).toEqual([
2269-
{ type: "response.cancel" },
2276+
expectedResponseCancelEvent(),
22702277
{
22712278
type: "conversation.item.truncate",
22722279
item_id: "item_1",
@@ -2323,7 +2330,7 @@ describe("buildOpenAIRealtimeVoiceProvider", () => {
23232330

23242331
expect(onClearAudio).toHaveBeenCalledTimes(1);
23252332
expect(parseSent(socket).slice(-2)).toEqual([
2326-
{ type: "response.cancel" },
2333+
expectedResponseCancelEvent(),
23272334
{
23282335
type: "conversation.item.truncate",
23292336
item_id: "item_1",
@@ -2359,12 +2366,19 @@ describe("buildOpenAIRealtimeVoiceProvider", () => {
23592366

23602367
bridge.submitToolResult("call_1", { text: "done" });
23612368
bridge.handleBargeIn?.({ audioPlaybackActive: true });
2369+
const responseCancelEvent = parseSent(socket).findLast(
2370+
(event) => event.type === "response.cancel",
2371+
);
2372+
if (!responseCancelEvent?.event_id) {
2373+
throw new Error("expected response.cancel event id");
2374+
}
23622375
socket.emit(
23632376
"message",
23642377
Buffer.from(
23652378
JSON.stringify({
23662379
type: "error",
23672380
error: {
2381+
event_id: responseCancelEvent.event_id,
23682382
message: "Cancellation failed: no active response found",
23692383
},
23702384
}),
@@ -2375,6 +2389,85 @@ describe("buildOpenAIRealtimeVoiceProvider", () => {
23752389
expect(parseSent(socket).slice(-1)).toEqual([expectedResponseCreateEvent()]);
23762390
});
23772391

2392+
it("ignores a stale cancellation error after a newer manual response starts", async () => {
2393+
const provider = buildOpenAIRealtimeVoiceProvider();
2394+
const onError = vi.fn();
2395+
const bridge = provider.createBridge({
2396+
providerConfig: { apiKey: "sk-test" }, // pragma: allowlist secret
2397+
onAudio: vi.fn(),
2398+
onClearAudio: vi.fn(),
2399+
onError,
2400+
});
2401+
const connecting = bridge.connect();
2402+
const socket = FakeWebSocket.instances[0];
2403+
if (!socket) {
2404+
throw new Error("expected bridge to create a websocket");
2405+
}
2406+
2407+
socket.readyState = FakeWebSocket.OPEN;
2408+
socket.emit("open");
2409+
socket.emit("message", Buffer.from(JSON.stringify({ type: "session.updated" })));
2410+
await connecting;
2411+
bridge.setMediaTimestamp(1000);
2412+
socket.emit(
2413+
"message",
2414+
Buffer.from(JSON.stringify({ type: "response.created", response: { id: "resp_1" } })),
2415+
);
2416+
socket.emit(
2417+
"message",
2418+
Buffer.from(
2419+
JSON.stringify({
2420+
type: "response.audio.delta",
2421+
item_id: "item_1",
2422+
delta: Buffer.from("assistant audio").toString("base64"),
2423+
}),
2424+
),
2425+
);
2426+
bridge.setMediaTimestamp(1300);
2427+
2428+
bridge.handleBargeIn?.({ audioPlaybackActive: true });
2429+
const responseCancelEvent = parseSent(socket).findLast(
2430+
(event) => event.type === "response.cancel",
2431+
);
2432+
if (!responseCancelEvent?.event_id) {
2433+
throw new Error("expected response.cancel event id");
2434+
}
2435+
bridge.submitToolResult("call_1", { text: "done" });
2436+
socket.emit("message", Buffer.from(JSON.stringify({ type: "response.done" })));
2437+
const sessionUpdateCount = parseSent(socket).filter(
2438+
(event) => event.type === "session.update",
2439+
).length;
2440+
2441+
socket.emit(
2442+
"message",
2443+
Buffer.from(
2444+
JSON.stringify({
2445+
type: "error",
2446+
error: {
2447+
event_id: responseCancelEvent.event_id,
2448+
message: "Cancellation failed: no active response found",
2449+
},
2450+
}),
2451+
),
2452+
);
2453+
2454+
expect(onError).not.toHaveBeenCalled();
2455+
expect(parseSent(socket).filter((event) => event.type === "session.update")).toHaveLength(
2456+
sessionUpdateCount,
2457+
);
2458+
expect(parseSent(socket).at(-1)).toEqual(expectedResponseCreateEvent());
2459+
2460+
socket.emit("message", Buffer.from(JSON.stringify({ type: "response.done" })));
2461+
expectRecordFields(
2462+
requireNestedRecord(parseSent(socket).at(-1)?.session, ["audio", "input", "turn_detection"]),
2463+
"restored turn detection",
2464+
{
2465+
create_response: true,
2466+
interrupt_response: true,
2467+
},
2468+
);
2469+
});
2470+
23782471
it("resets deferred response guards after websocket reconnect", async () => {
23792472
vi.useFakeTimers();
23802473
const provider = buildOpenAIRealtimeVoiceProvider();

‎extensions/openai/realtime-voice-provider.ts‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -429,6 +429,7 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge {
429429
private responseCreateInFlight = false;
430430
private manualResponseCreateEventId: string | null = null;
431431
private responseCancelInFlight = false;
432+
private manualResponseCancelEventId: string | null = null;
432433
private responseCreatePending = false;
433434
private autoRespondSuppressedForManualResponse = false;
434435
private continuingToolCallIds = new Set<string>();
@@ -1067,6 +1068,7 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge {
10671068
this.responseCreateInFlight = false;
10681069
this.manualResponseCreateEventId = null;
10691070
this.responseCancelInFlight = false;
1071+
this.manualResponseCancelEventId = null;
10701072
if (this.responseCreatePending) {
10711073
this.flushPendingResponseCreate();
10721074
} else {
@@ -1130,9 +1132,16 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge {
11301132
this.responseCreatePending = true;
11311133
return;
11321134
}
1135+
const rejectsManualResponseCancel =
1136+
this.manualResponseCancelEventId !== null &&
1137+
readRealtimeErrorEventId(event.error) === this.manualResponseCancelEventId;
11331138
if (detail === OPENAI_REALTIME_NO_ACTIVE_RESPONSE_CANCEL_ERROR) {
1139+
if (!rejectsManualResponseCancel) {
1140+
return;
1141+
}
11341142
this.responseActive = false;
11351143
this.responseCancelInFlight = false;
1144+
this.manualResponseCancelEventId = null;
11361145
if (this.responseCreatePending) {
11371146
this.flushPendingResponseCreate();
11381147
} else {
@@ -1188,7 +1197,9 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge {
11881197
this.responseActive &&
11891198
!this.responseCancelInFlight
11901199
) {
1191-
this.sendEvent({ type: "response.cancel" }, "reason=barge-in");
1200+
const eventId = `openclaw-response-cancel-${randomUUID()}`;
1201+
this.manualResponseCancelEventId = eventId;
1202+
this.sendEvent({ type: "response.cancel", event_id: eventId }, "reason=barge-in");
11921203
this.responseCancelInFlight = true;
11931204
}
11941205
if (shouldInterruptProvider) {
@@ -1292,6 +1303,7 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge {
12921303
this.responseCreateInFlight = false;
12931304
this.manualResponseCreateEventId = null;
12941305
this.responseCancelInFlight = false;
1306+
this.manualResponseCancelEventId = null;
12951307
this.responseCreatePending = false;
12961308
this.autoRespondSuppressedForManualResponse = false;
12971309
this.continuingToolCallIds.clear();

0 commit comments

Comments
 (0)