Skip to content

Commit ab02eb6

Browse files
fix(chat): buffer streamed tool call until name arrives (#577)
Some OpenAI-compatible upstreams (e.g. DeepSeek / Kimi) stream a tool call whose FIRST `tool_calls` delta carries only `id` + partial `arguments`, delivering `function.name` in a later delta. The Chat Completions stream encoder forwarded each delta verbatim, so the first chunk a client saw had no `function.name`. Strict clients — the Vercel AI SDK `@ai-sdk/openai-compatible`, and thus opencode — reject the first streamed chunk for a new tool-call index unless `function.name` is a string, aborting with: AI_InvalidResponseDataError: Expected 'function.name' to be a string. Plain (non-tool) generation was unaffected. PR #552 hardened the Responses / Messages / Generate Content encoders and the chat decoder against the same upstream quirk but left the chat encoder naive. `ChatStreamEncoder` now holds a tool call's opening chunk back — buffering argument fragments per call — until a non-empty name arrives, then emits one named opening chunk and streams the remaining arguments. A call whose name never arrives is dropped, consistent with the server-tool loop's handling of nameless calls. Encoder-only; the decoder and the other encoders are untouched. Adds a regression test that replays the captured wire shape through the chat decode -> encode round trip and asserts the first emitted chunk names the function. Ref: anomalyco/opencode#24137 Co-authored-by: Claude Opus 4.8 <[email protected]>
1 parent 5ffe7b1 commit ab02eb6

2 files changed

Lines changed: 145 additions & 23 deletions

File tree

‎crates/bitrouter-sdk/src/language_model/protocol/chat_completions.rs‎

Lines changed: 74 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -724,7 +724,7 @@ impl InboundAdapter for ChatCompletionsAdapter {
724724
request_id: request_id.to_string(),
725725
model: model.to_string(),
726726
role_sent: false,
727-
tool_call_indices: Vec::new(),
727+
tool_calls: Vec::new(),
728728
})
729729
}
730730
}
@@ -1533,24 +1533,41 @@ impl StreamDecoder for ChatStreamDecoder {
15331533
}
15341534
}
15351535

1536+
/// Per tool-call streaming state for the Chat Completions encoder. The slot's
1537+
/// position in [`ChatStreamEncoder::tool_calls`] is its `tool_calls[].index`,
1538+
/// assigned in first-seen order so parallel tool calls stay distinct and stable.
1539+
struct ChatToolCallState {
1540+
/// Upstream tool-call id, the key used to group continuation deltas.
1541+
id: String,
1542+
/// Whether the opening chunk (the one carrying `function.name`) was emitted.
1543+
opened: bool,
1544+
/// Argument fragments seen before the name arrived, replayed on the opening
1545+
/// chunk so nothing is lost while the call is held back.
1546+
pending_args: String,
1547+
}
1548+
15361549
/// Encodes canonical stream parts into Chat Completions `data:` SSE chunks.
15371550
struct ChatStreamEncoder {
15381551
request_id: String,
15391552
model: String,
15401553
role_sent: bool,
1541-
/// Tool-call id → its `tool_calls[].index`, assigned in first-seen order so
1542-
/// parallel tool calls get distinct, stable indices.
1543-
tool_call_indices: Vec<String>,
1554+
/// Per tool-call state, keyed by id and ordered by first sight.
1555+
tool_calls: Vec<ChatToolCallState>,
15441556
}
15451557

15461558
impl ChatStreamEncoder {
1547-
/// Index for a tool-call id, assigning the next free slot on first sight.
1559+
/// Index (= `tool_calls[].index`) for a tool-call id, creating its state on
1560+
/// first sight.
15481561
fn tool_call_index(&mut self, id: &str) -> usize {
1549-
if let Some(pos) = self.tool_call_indices.iter().position(|x| x == id) {
1562+
if let Some(pos) = self.tool_calls.iter().position(|s| s.id == id) {
15501563
pos
15511564
} else {
1552-
self.tool_call_indices.push(id.to_string());
1553-
self.tool_call_indices.len() - 1
1565+
self.tool_calls.push(ChatToolCallState {
1566+
id: id.to_string(),
1567+
opened: false,
1568+
pending_args: String::new(),
1569+
});
1570+
self.tool_calls.len() - 1
15541571
}
15551572
}
15561573
}
@@ -1609,22 +1626,56 @@ impl StreamEncoder for ChatStreamEncoder {
16091626
arguments,
16101627
} => {
16111628
let index = self.tool_call_index(id);
1612-
let mut function = serde_json::Map::new();
1613-
if let Some(name) = name {
1614-
function.insert("name".into(), name.clone().into());
1629+
let name = name.as_deref().filter(|n| !n.is_empty());
1630+
1631+
// The Vercel AI SDK (`@ai-sdk/openai-compatible`) and other
1632+
// strict clients reject the FIRST streamed `tool_calls` chunk for
1633+
// a given index unless its `function.name` is a string. Some
1634+
// OpenAI-compatible upstreams (e.g. DeepSeek / Kimi) send that
1635+
// first delta with only `id` + partial `arguments` and deliver
1636+
// `name` in a later delta, so hold the call's opening chunk back
1637+
// — buffering argument fragments — until the name arrives, then
1638+
// emit one named opening chunk. Like the Generate Content encoder
1639+
// buffers a tool call until complete, and consistent with the
1640+
// Responses / Messages encoders only opening a call on a
1641+
// non-empty name (both added in #552).
1642+
// <https://github.com/anomalyco/opencode/issues/24137>
1643+
let mut opening_name: Option<String> = None;
1644+
let mut out_args: Option<String> = None;
1645+
if let Some(state) = self.tool_calls.get_mut(index) {
1646+
if state.opened {
1647+
if !arguments.is_empty() {
1648+
out_args = Some(arguments.clone());
1649+
}
1650+
} else if let Some(name) = name {
1651+
let mut args = std::mem::take(&mut state.pending_args);
1652+
args.push_str(arguments);
1653+
state.opened = true;
1654+
opening_name = Some(name.to_string());
1655+
out_args = Some(args);
1656+
} else {
1657+
state.pending_args.push_str(arguments);
1658+
}
1659+
}
1660+
1661+
if opening_name.is_some() || out_args.is_some() {
1662+
let mut function = serde_json::Map::new();
1663+
if let Some(name) = opening_name {
1664+
function.insert("name".into(), name.into());
1665+
}
1666+
function.insert("arguments".into(), out_args.unwrap_or_default().into());
1667+
let mut delta = self.open_delta();
1668+
delta.insert(
1669+
"tool_calls".into(),
1670+
serde_json::json!([{
1671+
"index": index,
1672+
"id": id,
1673+
"type": "function",
1674+
"function": serde_json::Value::Object(function),
1675+
}]),
1676+
);
1677+
frames.push(self.chunk(serde_json::Value::Object(delta), None));
16151678
}
1616-
function.insert("arguments".into(), arguments.clone().into());
1617-
let mut delta = self.open_delta();
1618-
delta.insert(
1619-
"tool_calls".into(),
1620-
serde_json::json!([{
1621-
"index": index,
1622-
"id": id,
1623-
"type": "function",
1624-
"function": serde_json::Value::Object(function),
1625-
}]),
1626-
);
1627-
frames.push(self.chunk(serde_json::Value::Object(delta), None));
16281679
}
16291680
StreamPart::Usage { .. } => {
16301681
// usage is attached to the Finish chunk below; nothing here.

‎crates/bitrouter-sdk/src/language_model/protocol/tests.rs‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9630,6 +9630,77 @@ fn responses_encodes_mcp_call_output_item() {
96309630
);
96319631
}
96329632

9633+
/// Regression #560: some OpenAI-compatible upstreams (e.g. DeepSeek / Kimi)
9634+
/// stream a tool call whose FIRST `tool_calls` delta carries only `id` +
9635+
/// partial `arguments`, delivering `function.name` in a LATER delta. The Chat
9636+
/// Completions stream encoder must hold the tool call's opening chunk back until
9637+
/// the name is known, so the first `tool_calls` chunk a client sees always
9638+
/// carries a string `function.name`. Otherwise strict clients (the Vercel AI
9639+
/// SDK `@ai-sdk/openai-compatible` / opencode) abort the stream with
9640+
/// `AI_InvalidResponseDataError: Expected 'function.name' to be a string`.
9641+
/// <https://github.com/anomalyco/opencode/issues/24137>
9642+
#[test]
9643+
fn chat_encode_late_tool_name_first_chunk_carries_name() {
9644+
let parts = decode_stream(
9645+
ApiProtocol::ChatCompletions,
9646+
&[
9647+
// First delta: id + empty arguments, NO function.name yet.
9648+
(
9649+
"",
9650+
serde_json::json!({"choices":[{"delta":{"tool_calls":[
9651+
{"index":0,"id":"call_123","type":"function",
9652+
"function":{"arguments":""}}]}}]}),
9653+
),
9654+
// The name arrives only on the second delta, alongside partial args.
9655+
(
9656+
"",
9657+
serde_json::json!({"choices":[{"delta":{"tool_calls":[
9658+
{"index":0,"id":"call_123","type":"function",
9659+
"function":{"name":"bash","arguments":"{\"cmd\":\""}}]}}]}),
9660+
),
9661+
// Continuation re-sends an empty name (normalized to None upstream).
9662+
(
9663+
"",
9664+
serde_json::json!({"choices":[{"delta":{"tool_calls":[
9665+
{"index":0,"id":"call_123","type":"function",
9666+
"function":{"name":"","arguments":"ls\"}"}}]}}]}),
9667+
),
9668+
(
9669+
"",
9670+
serde_json::json!({"choices":[{"delta":{},"finish_reason":"tool_calls"}]}),
9671+
),
9672+
],
9673+
);
9674+
let events = encode_stream_events(ApiProtocol::ChatCompletions, &parts);
9675+
9676+
// Every emitted `tool_calls[]` delta, in wire order.
9677+
let tool_deltas: Vec<serde_json::Value> = events
9678+
.iter()
9679+
.filter_map(|(_, chunk)| chunk["choices"][0]["delta"].get("tool_calls").cloned())
9680+
.flat_map(|tc| tc.as_array().cloned().unwrap_or_default())
9681+
.collect();
9682+
9683+
// The FIRST tool_calls chunk the client sees must name the function.
9684+
let first = tool_deltas
9685+
.first()
9686+
.expect("at least one tool_calls delta is emitted");
9687+
assert_eq!(
9688+
first["function"]["name"], "bash",
9689+
"first streamed tool_calls chunk must carry a string function.name; got {first}"
9690+
);
9691+
assert_eq!(first["id"], "call_123", "tool-call id is preserved");
9692+
9693+
// All argument fragments survive the buffering, in order.
9694+
let args: String = tool_deltas
9695+
.iter()
9696+
.filter_map(|d| d["function"]["arguments"].as_str())
9697+
.collect();
9698+
assert_eq!(
9699+
args, "{\"cmd\":\"ls\"}",
9700+
"argument fragments survive the hold-until-name buffering: {tool_deltas:?}"
9701+
);
9702+
}
9703+
96339704
#[test]
96349705
fn coarse_wires_drop_server_tool_activity() {
96359706
// Chat Completions and Generate Content have no server-tool / MCP stream

0 commit comments

Comments
 (0)