Skip to content

Commit 9481b37

Browse files
committed
fix(responses): harden malformed stream repair
1 parent 659c9e4 commit 9481b37

4 files changed

Lines changed: 285 additions & 10 deletions

File tree

‎src/server/responses-custom-tool-repair.ts‎

Lines changed: 78 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -65,12 +65,21 @@ type OpenCustomCall = {
6565
retainedBytes: number;
6666
};
6767

68+
type PendingArgumentBlock = {
69+
block: string;
70+
itemId?: string;
71+
outputIndex?: number;
72+
retainedBytes: number;
73+
};
74+
6875
export function createRoutedCustomToolRestoreBlockRewrite(
6976
names: ReadonlySet<string>,
7077
budget?: TranslatorBudget,
7178
): SseBlockRewrite {
7279
const itemNames = new Map<string, string>();
80+
const ordinaryItemIds = new Set<string>();
7381
const openCalls = new Map<string, OpenCustomCall>();
82+
let pendingArguments: PendingArgumentBlock[] = [];
7483
let disposed = false;
7584

7685
const releaseCall = (itemId: string): void => {
@@ -86,7 +95,42 @@ export function createRoutedCustomToolRestoreBlockRewrite(
8695
if (disposed) return;
8796
disposed = true;
8897
for (const itemId of openCalls.keys()) releaseCall(itemId);
98+
const pendingBytes = pendingArguments.reduce((total, pending) => total + pending.retainedBytes, 0);
99+
if (pendingBytes > 0) {
100+
budget?.releaseRetained(pendingBytes, { kind: "retained_collectors" });
101+
}
102+
pendingArguments = [];
89103
itemNames.clear();
104+
ordinaryItemIds.clear();
105+
};
106+
107+
const retainPendingArgument = (
108+
block: string,
109+
itemId: string | undefined,
110+
outputIndex: number | undefined,
111+
): void => {
112+
const retainedBytes = Buffer.byteLength(block, "utf8");
113+
if (retainedBytes > 0) budget?.chargeRetained(retainedBytes, { kind: "retained_collectors" });
114+
pendingArguments.push({ block, itemId, outputIndex, retainedBytes });
115+
};
116+
117+
const takePendingArguments = (
118+
itemId: string | undefined,
119+
outputIndex: number | undefined,
120+
): string[] => {
121+
const matched: PendingArgumentBlock[] = [];
122+
const remaining: PendingArgumentBlock[] = [];
123+
for (const pending of pendingArguments) {
124+
const sameItem = itemId !== undefined && pending.itemId === itemId;
125+
const sameIndex = outputIndex !== undefined && pending.outputIndex === outputIndex;
126+
(sameItem || sameIndex ? matched : remaining).push(pending);
127+
}
128+
pendingArguments = remaining;
129+
const retainedBytes = matched.reduce((total, pending) => total + pending.retainedBytes, 0);
130+
if (retainedBytes > 0) {
131+
budget?.releaseRetained(retainedBytes, { kind: "retained_collectors" });
132+
}
133+
return matched.map(pending => pending.block);
90134
};
91135

92136
const rewrite: SseBlockRewrite = (block: string): readonly string[] => {
@@ -101,28 +145,56 @@ export function createRoutedCustomToolRestoreBlockRewrite(
101145
if (!isPlainObject(parsed)) return [block];
102146

103147
const type = typeof parsed.type === "string" ? parsed.type : "";
148+
const outputIndex = typeof parsed.output_index === "number"
149+
&& Number.isInteger(parsed.output_index)
150+
&& parsed.output_index >= 0
151+
? parsed.output_index
152+
: undefined;
104153
if (
105154
(type === "response.output_item.added" || type === "response.output_item.done")
106155
&& isPlainObject(parsed.item)
107156
&& parsed.item.type === "function_call"
108157
&& typeof parsed.item.name === "string"
109-
&& names.has(parsed.item.name)
110158
) {
111159
const upstreamItemId = typeof parsed.item.id === "string" ? parsed.item.id : undefined;
160+
const routed = names.has(parsed.item.name);
112161
if (upstreamItemId) {
113-
itemNames.set(upstreamItemId, parsed.item.name);
114-
if (type === "response.output_item.added") {
162+
if (routed) {
163+
itemNames.set(upstreamItemId, parsed.item.name);
164+
ordinaryItemIds.delete(upstreamItemId);
165+
} else {
166+
ordinaryItemIds.add(upstreamItemId);
167+
}
168+
if (routed && type === "response.output_item.added") {
115169
openCalls.set(upstreamItemId, { argumentsText: "", emittedInput: "", retainedBytes: 0 });
116170
}
117171
}
172+
const pending = takePendingArguments(upstreamItemId, outputIndex);
173+
if (!routed) {
174+
if (type === "response.output_item.done" && upstreamItemId) ordinaryItemIds.delete(upstreamItemId);
175+
return [...pending, block];
176+
}
177+
if (upstreamItemId && pending.length > 0 && !openCalls.has(upstreamItemId)) {
178+
openCalls.set(upstreamItemId, { argumentsText: "", emittedInput: "", retainedBytes: 0 });
179+
}
118180
const restored = restoreRoutedCustomCalls(parsed, names);
181+
const restoredBlock = restored.changed
182+
? replaceSseDataPayload(block, JSON.stringify(restored.value))
183+
: block;
184+
const replayed = pending.flatMap(pendingBlock => rewrite(pendingBlock));
119185
if (type === "response.output_item.done" && upstreamItemId) releaseCall(upstreamItemId);
120-
return restored.changed
121-
? [replaceSseDataPayload(block, JSON.stringify(restored.value))]
122-
: [block];
186+
return type === "response.output_item.added"
187+
? [restoredBlock, ...replayed]
188+
: [...replayed, restoredBlock];
123189
}
124190

125191
const upstreamItemId = typeof parsed.item_id === "string" ? parsed.item_id : undefined;
192+
const argumentEvent = type === "response.function_call_arguments.delta"
193+
|| type === "response.function_call_arguments.done";
194+
if (argumentEvent && (!upstreamItemId || (!itemNames.has(upstreamItemId) && !ordinaryItemIds.has(upstreamItemId)))) {
195+
retainPendingArgument(block, upstreamItemId, outputIndex);
196+
return [];
197+
}
126198
if (
127199
type === "response.function_call_arguments.delta"
128200
&& upstreamItemId

‎src/server/responses-terminal-repair.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -292,10 +292,10 @@ export function relayResponsesSseWithTerminalRepair(
292292
if (done) {
293293
appendBuffer(decoder.decode());
294294
if (buffer.length > 0) {
295-
const kind = inspectPayload(sseDataPayload(buffer));
296-
if (kind === "done" && !realTerminalSeen) {
297-
emitSynthetic(completeCandidate() ? "completed" : "incomplete", controller);
298-
}
295+
// A delimiter-less suffix is not a complete SSE event. Preserve the
296+
// upstream bytes for passthrough compatibility, but never let a
297+
// truncated lifecycle frame establish synthetic success.
298+
tainted = true;
299299
controller.enqueue(encoder.encode(buffer));
300300
}
301301
if (!realTerminalSeen) {

‎tests/responses-custom-tool-repair.test.ts‎

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
import { createRoutedCustomToolRestoreBlockRewrite } from "../src/server/responses-custom-tool-repair";
88
import { handleResponses } from "../src/server/responses";
99
import type { OcxConfig } from "../src/types";
10+
import { createTestTranslatorBudget } from "./helpers/translator-budget";
1011

1112
function dataPayload(block: string): Record<string, unknown> {
1213
const line = block.split(/\r?\n/).find(entry => entry.startsWith("data:"));
@@ -167,6 +168,90 @@ describe("routed Responses custom-tool compatibility", () => {
167168
rewrite.dispose?.();
168169
});
169170

171+
test("buffers argument events until a missing added event is identified by item done", () => {
172+
const budget = createTestTranslatorBudget();
173+
const rewrite = createRoutedCustomToolRestoreBlockRewrite(new Set(["exec"]), budget);
174+
const deltaBlock = frame("response.function_call_arguments.delta", {
175+
output_index: 0,
176+
item_id: "fc_exec",
177+
delta: "{\"input\":\"echo",
178+
});
179+
const argumentsDoneBlock = frame("response.function_call_arguments.done", {
180+
output_index: 0,
181+
item_id: "fc_exec",
182+
arguments: "{\"input\":\"echo ok\"}",
183+
});
184+
185+
expect(rewrite(deltaBlock)).toEqual([]);
186+
expect(rewrite(argumentsDoneBlock)).toEqual([]);
187+
expect(budget.snapshot().currentBytes).toBeGreaterThan(0);
188+
189+
const replayed = rewrite(frame("response.output_item.done", {
190+
output_index: 0,
191+
item: {
192+
type: "function_call",
193+
id: "fc_exec",
194+
call_id: "call_exec",
195+
name: "exec",
196+
arguments: "{\"input\":\"echo ok\"}",
197+
status: "completed",
198+
},
199+
}));
200+
expect(replayed.map(block => dataPayload(block).type)).toEqual([
201+
"response.custom_tool_call_input.delta",
202+
"response.custom_tool_call_input.done",
203+
"response.output_item.done",
204+
]);
205+
expect(dataPayload(replayed[0]!)).toMatchObject({ item_id: "ctc_exec", delta: "echo" });
206+
expect(dataPayload(replayed[1]!)).toMatchObject({ item_id: "ctc_exec", input: "echo ok" });
207+
expect(dataPayload(replayed[2]!).item).toMatchObject({
208+
type: "custom_tool_call",
209+
id: "ctc_exec",
210+
name: "exec",
211+
input: "echo ok",
212+
});
213+
expect(budget.snapshot().currentBytes).toBe(0);
214+
rewrite.dispose?.();
215+
});
216+
217+
test("replays buffered events unchanged when item done identifies an ordinary function", () => {
218+
const budget = createTestTranslatorBudget();
219+
const rewrite = createRoutedCustomToolRestoreBlockRewrite(new Set(["exec"]), budget);
220+
const deltaBlock = frame("response.function_call_arguments.delta", {
221+
output_index: 1,
222+
item_id: "fc_other",
223+
delta: "{}",
224+
});
225+
226+
expect(rewrite(deltaBlock)).toEqual([]);
227+
expect(budget.snapshot().currentBytes).toBeGreaterThan(0);
228+
229+
const replayed = rewrite(frame("response.output_item.done", {
230+
output_index: 1,
231+
item: {
232+
type: "function_call",
233+
id: "fc_other",
234+
call_id: "call_other",
235+
name: "ordinary",
236+
arguments: "{}",
237+
status: "completed",
238+
},
239+
}));
240+
expect(replayed).toEqual([deltaBlock, frame("response.output_item.done", {
241+
output_index: 1,
242+
item: {
243+
type: "function_call",
244+
id: "fc_other",
245+
call_id: "call_other",
246+
name: "ordinary",
247+
arguments: "{}",
248+
status: "completed",
249+
},
250+
})]);
251+
expect(budget.snapshot().currentBytes).toBe(0);
252+
rewrite.dispose?.();
253+
});
254+
170255
test("keeps progressive exec input consistent for escaped control characters", () => {
171256
const rewrite = createRoutedCustomToolRestoreBlockRewrite(new Set(["exec"]));
172257
rewrite(frame("response.output_item.added", {
@@ -250,9 +335,114 @@ describe("routed Responses custom-tool compatibility", () => {
250335
expect(clientSse).toContain('"type":"response.custom_tool_call_input.done"');
251336
expect(clientSse).toContain('"input":"const apps = await sky.list_apps();"');
252337
expect(clientSse).not.toContain("response.function_call_arguments.done");
338+
expect(clientSse).not.toContain('"type":"function_call"');
253339
expect(clientSse).toContain("data: [DONE]");
254340
} finally {
255341
globalThis.fetch = savedFetch;
256342
}
257343
});
344+
345+
test("handleResponses restores routed custom calls in non-streaming JSON", async () => {
346+
const savedFetch = globalThis.fetch;
347+
const upstreamItem = {
348+
type: "function_call",
349+
id: "fc_exec",
350+
call_id: "call_exec",
351+
name: "exec",
352+
arguments: "{\"input\":\"const apps = await sky.list_apps();\"}",
353+
status: "completed",
354+
};
355+
globalThis.fetch = (async () => new Response(JSON.stringify({
356+
id: "resp_json",
357+
status: "completed",
358+
output: [upstreamItem],
359+
}), { headers: { "content-type": "application/json" } })) as typeof fetch;
360+
const config = {
361+
port: 0,
362+
defaultProvider: "fixture",
363+
providers: {
364+
fixture: {
365+
adapter: "openai-responses",
366+
baseUrl: "https://fixture.test/v1",
367+
authMode: "key",
368+
apiKey: "fixture-key",
369+
},
370+
},
371+
} as OcxConfig;
372+
373+
try {
374+
const response = await handleResponses(new Request("http://localhost/v1/responses", {
375+
method: "POST",
376+
headers: { "content-type": "application/json" },
377+
body: JSON.stringify({
378+
model: "fixture/deepseek-v4-flash",
379+
stream: false,
380+
input: [{ role: "user", content: [{ type: "input_text", text: "list apps" }] }],
381+
tools: [{ type: "custom", name: "exec", description: "Run JavaScript", format: { type: "grammar", syntax: "lark" } }],
382+
}),
383+
}), config, { model: "", provider: "" });
384+
const body = await response.json() as { output: Array<Record<string, unknown>> };
385+
386+
expect(body.output[0]).toMatchObject({
387+
type: "custom_tool_call",
388+
id: "ctc_exec",
389+
name: "exec",
390+
input: "const apps = await sky.list_apps();",
391+
});
392+
expect(body.output[0]).not.toHaveProperty("arguments");
393+
} finally {
394+
globalThis.fetch = savedFetch;
395+
}
396+
});
397+
398+
test("handleResponses leaves custom tools native for forward-auth passthrough", async () => {
399+
const savedFetch = globalThis.fetch;
400+
let outboundBody: Record<string, unknown> | undefined;
401+
const upstreamItem = {
402+
type: "function_call",
403+
id: "fc_exec",
404+
call_id: "call_exec",
405+
name: "exec",
406+
arguments: "{\"input\":\"native\"}",
407+
status: "completed",
408+
};
409+
globalThis.fetch = (async (_input, init) => {
410+
outboundBody = JSON.parse(String(init?.body)) as Record<string, unknown>;
411+
return new Response(JSON.stringify({ id: "resp_forward", status: "completed", output: [upstreamItem] }), {
412+
headers: { "content-type": "application/json" },
413+
});
414+
}) as typeof fetch;
415+
const config = {
416+
port: 0,
417+
defaultProvider: "fixture",
418+
providers: {
419+
fixture: {
420+
adapter: "openai-responses",
421+
baseUrl: "https://forward.fixture.test",
422+
authMode: "forward",
423+
},
424+
},
425+
} as OcxConfig;
426+
427+
try {
428+
const response = await handleResponses(new Request("http://localhost/v1/responses", {
429+
method: "POST",
430+
headers: { "content-type": "application/json", authorization: "Bearer caller-token" },
431+
body: JSON.stringify({
432+
model: "fixture/native-model",
433+
stream: false,
434+
input: "run",
435+
tools: [{ type: "custom", name: "exec", description: "Run JavaScript", format: { type: "grammar", syntax: "lark" } }],
436+
}),
437+
}), config, { model: "", provider: "" });
438+
const clientBody = await response.json() as { output: Array<Record<string, unknown>> };
439+
const outboundTools = outboundBody?.tools as Array<Record<string, unknown>> | undefined;
440+
441+
expect(outboundTools?.[0]).toMatchObject({ type: "custom", name: "exec" });
442+
expect(clientBody.output[0]).toMatchObject({ type: "function_call", name: "exec" });
443+
expect(clientBody.output[0]).not.toHaveProperty("input");
444+
} finally {
445+
globalThis.fetch = savedFetch;
446+
}
447+
});
258448
});

‎tests/responses-terminal-repair.test.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -337,6 +337,19 @@ describe("DeepSeek Responses terminal repair", () => {
337337
expect(output.endsWith("data: [DONE]\n\n")).toBe(true);
338338
});
339339

340+
test("an unframed output item done suffix taints EOF completion", async () => {
341+
const framed = completedMessageLifecycle();
342+
const finalDelimiter = framed.lastIndexOf("\n\n");
343+
expect(finalDelimiter).toBeGreaterThan(0);
344+
const input = framed.slice(0, finalDelimiter);
345+
346+
const { output } = await repairClosedText(input);
347+
348+
expect(output).toContain('"type":"response.output_item.done"');
349+
expect(terminalTypes(output)).toEqual(["response.incomplete"]);
350+
expect(output).not.toContain('"type":"response.completed"');
351+
});
352+
340353
test("DONE is replaced by completed then one DONE only for a complete candidate", async () => {
341354
const { output } = await repairClosedText(completedMessageLifecycle() + "data: [DONE]\n\n");
342355
expect(terminalTypes(output)).toEqual(["response.completed"]);

0 commit comments

Comments
 (0)