Skip to content

Commit f3b4982

Browse files
author
benys
committed
fix: parse Kimi K2 Thinking tool calls with special tokens
Kimi K2 Thinking model uses special tokens for tool calls instead of standard OpenAI format: <|tool_calls_section_begin|><|tool_call_begin|>functions.name:idx<|tool_call_argument_begin|>{...}<|tool_call_end|><|tool_calls_section_end|> This PR adds: - kimi.ts: Parser for Kimi special token format - Support for toolCallFormat flag in ZenData model schema - Auto-detection fallback when flag is not set - Both streaming and non-streaming response handling The parser activates when: 1. Model has toolCallFormat: "kimi" in config, OR 2. Response content contains <|tool_call tokens (auto-detect) Fixes anomalyco#9878
1 parent 0cc206a commit f3b4982

5 files changed

Lines changed: 362 additions & 12 deletions

File tree

‎packages/console/app/src/routes/zen/util/handler.ts‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,7 @@ export async function handler(
173173

174174
// Handle non-streaming response
175175
if (!isStream) {
176-
const responseConverter = createResponseConverter(providerInfo.format, opts.format)
176+
const responseConverter = createResponseConverter(providerInfo.format, opts.format, modelInfo.toolCallFormat)
177177
const json = await res.json()
178178
const body = JSON.stringify(responseConverter(json))
179179
logger.metric({ response_length: body.length })
@@ -193,7 +193,7 @@ export async function handler(
193193
}
194194

195195
// Handle streaming response
196-
const streamConverter = createStreamPartConverter(providerInfo.format, opts.format)
196+
const streamConverter = createStreamPartConverter(providerInfo.format, opts.format, modelInfo.toolCallFormat)
197197
const usageParser = providerInfo.createUsageParser()
198198
const binaryDecoder = providerInfo.createBinaryStreamDecoder()
199199
const stream = new ReadableStream({
@@ -251,8 +251,15 @@ export async function handler(
251251
usageParser.parse(part)
252252

253253
if (providerInfo.format !== opts.format) {
254-
part = streamConverter(part)
255-
c.enqueue(encoder.encode(part + "\n\n"))
254+
const converted = streamConverter(part)
255+
if (converted === null) continue
256+
if (Array.isArray(converted)) {
257+
for (const item of converted) {
258+
c.enqueue(encoder.encode(item + "\n\n"))
259+
}
260+
} else {
261+
c.enqueue(encoder.encode(converted + "\n\n"))
262+
}
256263
}
257264
}
258265

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
/**
2+
* Kimi K2 Thinking model tool call parser
3+
*
4+
* Kimi K2 Thinking uses special tokens for tool calls:
5+
* - <|tool_calls_section_begin|> / <|tool_calls_section_end|> wraps all tool calls
6+
* - <|tool_call_begin|> / <|tool_call_end|> wraps each tool call
7+
* - <|tool_call_argument_begin|> separates tool ID from arguments
8+
*
9+
* Tool ID format: "functions.{func_name}:{idx}"
10+
*
11+
* Example:
12+
* <|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{"city":"Beijing"}<|tool_call_end|><|tool_calls_section_end|>
13+
*/
14+
15+
export interface KimiToolCall {
16+
id: string
17+
type: "function"
18+
function: {
19+
name: string
20+
arguments: string
21+
}
22+
}
23+
24+
const TOOL_CALLS_SECTION_BEGIN = "<|tool_calls_section_begin|>"
25+
const TOOL_CALLS_SECTION_END = "<|tool_calls_section_end|>"
26+
27+
const TOOL_CALL_PATTERN =
28+
/<\|tool_call_begin\|>\s*(?<id>[\w\.:\-]+)\s*<\|tool_call_argument_begin\|>\s*(?<arguments>[\s\S]*?)\s*<\|tool_call_end\|>/g
29+
30+
/**
31+
* Check if content contains Kimi K2 special tool call tokens
32+
*/
33+
export function hasKimiToolCalls(content: string | null | undefined): boolean {
34+
if (!content) return false
35+
return content.includes(TOOL_CALLS_SECTION_BEGIN)
36+
}
37+
38+
/**
39+
* Extract tool calls from Kimi K2 Thinking format
40+
*
41+
* @param content - Raw content that may contain Kimi tool call tokens
42+
* @returns Array of parsed tool calls in OpenAI format
43+
*/
44+
export function extractKimiToolCalls(content: string): KimiToolCall[] {
45+
if (!hasKimiToolCalls(content)) return []
46+
47+
const toolCalls: KimiToolCall[] = []
48+
49+
// Reset regex state
50+
TOOL_CALL_PATTERN.lastIndex = 0
51+
52+
let match
53+
while ((match = TOOL_CALL_PATTERN.exec(content)) !== null) {
54+
const { id, arguments: args } = match.groups!
55+
56+
// Parse function name from ID format: "functions.get_weather:0"
57+
const name = parseKimiFunctionName(id)
58+
if (!name) continue
59+
60+
toolCalls.push({
61+
id,
62+
type: "function",
63+
function: {
64+
name,
65+
arguments: args.trim(),
66+
},
67+
})
68+
}
69+
70+
return toolCalls
71+
}
72+
73+
/**
74+
* Parse function name from Kimi tool call ID
75+
*
76+
* ID format: "functions.{func_name}:{idx}"
77+
* Example: "functions.get_weather:0" -> "get_weather"
78+
*/
79+
function parseKimiFunctionName(id: string): string | null {
80+
if (!id) return null
81+
82+
// Handle format: functions.name:idx
83+
const parts = id.split(".")
84+
if (parts.length < 2) return null
85+
86+
const nameWithIdx = parts.slice(1).join(".")
87+
const name = nameWithIdx.split(":")[0]
88+
89+
return name || null
90+
}
91+
92+
/**
93+
* Remove Kimi tool call tokens from content, leaving only regular text
94+
*
95+
* @param content - Raw content with potential Kimi tokens
96+
* @returns Content with tool call sections removed
97+
*/
98+
export function stripKimiToolCallTokens(content: string): string {
99+
if (!hasKimiToolCalls(content)) return content
100+
101+
// Remove entire tool_calls_section
102+
const sectionPattern = /<\|tool_calls_section_begin\|>[\s\S]*?<\|tool_calls_section_end\|>/g
103+
return content.replace(sectionPattern, "").trim()
104+
}
105+
106+
/**
107+
* Process Kimi K2 response content - extract tool calls and clean content
108+
*
109+
* @param content - Raw content from Kimi K2 Thinking model
110+
* @returns Object with cleaned content and extracted tool calls
111+
*/
112+
export function processKimiContent(content: string | null | undefined): {
113+
content: string
114+
toolCalls: KimiToolCall[]
115+
} {
116+
if (!content) {
117+
return { content: "", toolCalls: [] }
118+
}
119+
120+
const toolCalls = extractKimiToolCalls(content)
121+
const cleanedContent = stripKimiToolCallTokens(content)
122+
123+
return {
124+
content: cleanedContent,
125+
toolCalls,
126+
}
127+
}
128+
129+
export interface KimiStreamState {
130+
buffer: string
131+
emittedToolCallIndices: Set<number>
132+
isInToolCallSection: boolean
133+
}
134+
135+
export function createKimiStreamParser() {
136+
const state: KimiStreamState = {
137+
buffer: "",
138+
emittedToolCallIndices: new Set(),
139+
isInToolCallSection: false,
140+
}
141+
142+
return {
143+
append(content: string): {
144+
textToEmit: string
145+
toolCalls: Array<KimiToolCall & { index: number }>
146+
isComplete: boolean
147+
} {
148+
state.buffer += content
149+
150+
const result: {
151+
textToEmit: string
152+
toolCalls: Array<KimiToolCall & { index: number }>
153+
isComplete: boolean
154+
} = {
155+
textToEmit: "",
156+
toolCalls: [],
157+
isComplete: false,
158+
}
159+
160+
if (!state.isInToolCallSection && state.buffer.includes(TOOL_CALLS_SECTION_BEGIN)) {
161+
state.isInToolCallSection = true
162+
const idx = state.buffer.indexOf(TOOL_CALLS_SECTION_BEGIN)
163+
result.textToEmit = state.buffer.substring(0, idx)
164+
state.buffer = state.buffer.substring(idx)
165+
}
166+
167+
if (!state.isInToolCallSection) {
168+
result.textToEmit = state.buffer
169+
state.buffer = ""
170+
return result
171+
}
172+
173+
if (state.buffer.includes(TOOL_CALLS_SECTION_END)) {
174+
result.isComplete = true
175+
const toolCalls = extractKimiToolCalls(state.buffer)
176+
toolCalls.forEach((tc, idx) => {
177+
if (!state.emittedToolCallIndices.has(idx)) {
178+
state.emittedToolCallIndices.add(idx)
179+
result.toolCalls.push({ ...tc, index: idx })
180+
}
181+
})
182+
state.buffer = ""
183+
return result
184+
}
185+
186+
TOOL_CALL_PATTERN.lastIndex = 0
187+
const matches = [...state.buffer.matchAll(TOOL_CALL_PATTERN)]
188+
matches.forEach((match, idx) => {
189+
if (!state.emittedToolCallIndices.has(idx)) {
190+
const { id, arguments: args } = match.groups!
191+
const name = id.split(".")[1]?.split(":")[0]
192+
if (name) {
193+
state.emittedToolCallIndices.add(idx)
194+
result.toolCalls.push({
195+
id,
196+
type: "function",
197+
function: { name, arguments: args.trim() },
198+
index: idx,
199+
})
200+
}
201+
}
202+
})
203+
204+
return result
205+
},
206+
207+
reset() {
208+
state.buffer = ""
209+
state.emittedToolCallIndices.clear()
210+
state.isInToolCallSection = false
211+
},
212+
}
213+
}

‎packages/console/app/src/routes/zen/util/provider/openai-compatible.ts‎

Lines changed: 111 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { ProviderHelper, CommonRequest, CommonResponse, CommonChunk } from "./provider"
2+
import { hasKimiToolCalls, processKimiContent, createKimiStreamParser, type KimiToolCall } from "./kimi"
23

34
type Usage = {
45
prompt_tokens?: number
@@ -211,7 +212,7 @@ export function toOaCompatibleRequest(body: CommonRequest) {
211212
}
212213
}
213214

214-
export function fromOaCompatibleResponse(resp: any): CommonResponse {
215+
export function fromOaCompatibleResponse(resp: any, toolCallFormat?: "kimi"): CommonResponse {
215216
if (!resp || typeof resp !== "object") return resp
216217

217218
if (!Array.isArray((resp as any).choices)) return resp
@@ -223,9 +224,20 @@ export function fromOaCompatibleResponse(resp: any): CommonResponse {
223224
if (!message) return resp
224225

225226
const content: any[] = []
227+
let kimiToolCalls: KimiToolCall[] = []
226228

227229
if (typeof message.content === "string" && message.content.length > 0) {
228-
content.push({ type: "text", text: message.content })
230+
// Parse Kimi tool calls if explicitly configured OR auto-detected in content
231+
const shouldParseKimi = toolCallFormat === "kimi" || hasKimiToolCalls(message.content)
232+
if (shouldParseKimi && hasKimiToolCalls(message.content)) {
233+
const processed = processKimiContent(message.content)
234+
if (processed.content.length > 0) {
235+
content.push({ type: "text", text: processed.content })
236+
}
237+
kimiToolCalls = processed.toolCalls
238+
} else {
239+
content.push({ type: "text", text: message.content })
240+
}
229241
}
230242

231243
if (Array.isArray(message.tool_calls)) {
@@ -247,7 +259,23 @@ export function fromOaCompatibleResponse(resp: any): CommonResponse {
247259
}
248260
}
249261

262+
for (const tc of kimiToolCalls) {
263+
let input
264+
try {
265+
input = JSON.parse(tc.function.arguments)
266+
} catch {
267+
input = tc.function.arguments
268+
}
269+
content.push({
270+
type: "tool_use",
271+
id: tc.id,
272+
name: tc.function.name,
273+
input,
274+
})
275+
}
276+
250277
const stopReason = (() => {
278+
if (kimiToolCalls.length > 0) return "tool_calls"
251279
const reason = choice.finish_reason
252280
if (reason === "stop") return "stop"
253281
if (reason === "tool_calls") return "tool_calls"
@@ -544,3 +572,84 @@ export function toOaCompatibleChunk(chunk: CommonChunk): string {
544572

545573
return `data: ${JSON.stringify(result)}`
546574
}
575+
576+
export function createKimiAwareChunkParser() {
577+
const kimiParser = createKimiStreamParser()
578+
let pendingToolCallChunks: CommonChunk[] = []
579+
580+
return {
581+
parse(chunk: string): CommonChunk[] {
582+
const baseResult = fromOaCompatibleChunk(chunk)
583+
if (typeof baseResult === "string") return []
584+
585+
const results: CommonChunk[] = []
586+
const contentDelta = baseResult.choices[0]?.delta?.content
587+
588+
if (contentDelta) {
589+
const parsed = kimiParser.append(contentDelta)
590+
591+
if (parsed.textToEmit) {
592+
results.push({
593+
...baseResult,
594+
choices: [
595+
{
596+
index: 0,
597+
delta: { content: parsed.textToEmit },
598+
finish_reason: null,
599+
},
600+
],
601+
})
602+
}
603+
604+
for (const tc of parsed.toolCalls) {
605+
results.push({
606+
...baseResult,
607+
choices: [
608+
{
609+
index: 0,
610+
delta: {
611+
tool_calls: [
612+
{
613+
index: tc.index,
614+
id: tc.id,
615+
type: "function",
616+
function: {
617+
name: tc.function.name,
618+
arguments: tc.function.arguments,
619+
},
620+
},
621+
],
622+
},
623+
finish_reason: null,
624+
},
625+
],
626+
})
627+
}
628+
629+
if (parsed.isComplete && results.length === 0) {
630+
results.push({
631+
...baseResult,
632+
choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }],
633+
})
634+
}
635+
636+
return results
637+
}
638+
639+
if (baseResult.choices[0]?.delta?.tool_calls) {
640+
return [baseResult]
641+
}
642+
643+
if (baseResult.choices[0]?.finish_reason) {
644+
return [baseResult]
645+
}
646+
647+
return [baseResult]
648+
},
649+
650+
reset() {
651+
kimiParser.reset()
652+
pendingToolCallChunks = []
653+
},
654+
}
655+
}

0 commit comments

Comments
 (0)