Skip to content

Commit 16b4038

Browse files
committed
fix(ai): absorb content after the finish reason
1 parent d0a9028 commit 16b4038

4 files changed

Lines changed: 56 additions & 36 deletions

File tree

‎packages/ai/src/protocols/mistral-chat.ts‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -656,15 +656,11 @@ const step = Effect.fn("MistralChat.step")(function* (state: ParserState, event:
656656
}
657657
const events: LLMEvent[] = []
658658
const usage = mapUsage(event.usage) ?? state.usage
659-
if (state.finishReason) {
660-
if (hasLateContent(event))
661-
return yield* ProviderShared.eventError(
662-
ADAPTER,
663-
"Mistral Chat received content after the finish reason",
664-
ProviderShared.encodeJson(event),
665-
)
666-
return [{ ...state, usage }, events] as const
667-
}
659+
// Trailing deltas after a terminal `finish_reason` are real model output:
660+
// absorb them into the stream instead of failing the response. The single
661+
// terminal finish event is emitted once at stream end, so late content still
662+
// precedes it.
663+
if (state.finishReason && !hasLateContent(event)) return [{ ...state, usage }, events] as const
668664
const choice = event.choices?.[0]
669665
const withContent = choice?.delta?.content == null ? state : appendContent(state, events, choice.delta.content)
670666
const withTools = yield* appendTools(withContent, events, choice?.delta?.tool_calls ?? [])
@@ -685,7 +681,10 @@ const step = Effect.fn("MistralChat.step")(function* (state: ParserState, event:
685681
})
686682
}
687683
const incomplete = finishReason.normalized === "length" || finishReason.normalized === "content-filter"
688-
if (!incomplete && Object.keys(withTools.pendingTools).length > 0)
684+
// Only the first terminal arrival rejects unidentifiable tool deltas; late
685+
// frames after a finish absorb malformed tool identities silently instead of
686+
// failing the response.
687+
if (state.finishReason === undefined && !incomplete && Object.keys(withTools.pendingTools).length > 0)
689688
return yield* ProviderShared.eventError(
690689
ADAPTER,
691690
"Mistral Chat tool call delta is missing a name",
@@ -699,7 +698,7 @@ const step = Effect.fn("MistralChat.step")(function* (state: ParserState, event:
699698
{
700699
...withTools,
701700
tools: finished?.tools ?? withTools.tools,
702-
completedTools: finished?.events ?? withTools.completedTools,
701+
completedTools: finished ? [...withTools.completedTools, ...finished.events] : withTools.completedTools,
703702
usage,
704703
finishReason,
705704
},

‎packages/ai/src/protocols/openai-chat.ts‎

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -988,15 +988,12 @@ const step = (state: ParserState, event: OpenAIChatEvent) =>
988988
reasoning !== undefined ||
989989
(Array.isArray(delta?.reasoning_details) && delta.reasoning_details.length > 0) ||
990990
toolDeltas.some((tool) => Boolean(tool.id) || Boolean(tool.function?.name) || Boolean(tool.function?.arguments))
991-
if (state.finishReason !== undefined) {
992-
if (hasLateContent)
993-
return yield* ProviderShared.eventError(
994-
ADAPTER,
995-
"OpenAI Chat received content after the finish reason",
996-
ProviderShared.encodeJson(event),
997-
)
998-
return [{ ...state, usage }, events] as const
999-
}
991+
// Trailing deltas after a terminal `finish_reason` are real model output:
992+
// some OpenAI-compatible providers emit a last content or reasoning chunk
993+
// after the finish chunk. Absorb them into the stream instead of failing
994+
// the response; the single terminal finish event is emitted once at stream
995+
// end, so late content still precedes it.
996+
if (state.finishReason !== undefined && !hasLateContent) return [{ ...state, usage }, events] as const
1000997

1001998
const reasoningField = state.reasoningField ?? reasoning?.field
1002999
const detailDelta = Array.isArray(delta?.reasoning_details) ? delta.reasoning_details : undefined
@@ -1081,11 +1078,9 @@ const step = (state: ParserState, event: OpenAIChatEvent) =>
10811078
)
10821079

10831080
// Filtering or truncation terminates the response without confirming pending tool calls.
1081+
// Late frames after a complete finish finalize newly absorbed tool calls here too.
10841082
const finished =
1085-
finishReason !== undefined &&
1086-
!incompleteTools &&
1087-
state.finishReason === undefined &&
1088-
Object.keys(tools).length > 0
1083+
finishReason !== undefined && !incompleteTools && Object.keys(tools).length > 0
10891084
? yield* ToolStream.finishAll(ADAPTER, tools)
10901085
: undefined
10911086

@@ -1094,7 +1089,7 @@ const step = (state: ParserState, event: OpenAIChatEvent) =>
10941089
providerMetadataKey: state.providerMetadataKey,
10951090
tools: finished?.tools ?? tools,
10961091
pendingTools,
1097-
toolCallEvents: finished?.events ?? state.toolCallEvents,
1092+
toolCallEvents: finished ? [...state.toolCallEvents, ...finished.events] : state.toolCallEvents,
10981093
usage,
10991094
finishReason,
11001095
lifecycle,

‎packages/ai/test/provider/mistral-chat.test.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -673,9 +673,8 @@ describe("Mistral Chat", () => {
673673
Effect.provide(
674674
fixedResponse(sseEvents(chunk({}, "stop"), chunk({ content: [{ type: "text", text: "late" }] }))),
675675
),
676-
Effect.flip,
677676
)
678-
expect(lateContent.message).toContain("content after the finish reason")
677+
expect(lateContent.text).toBe("late")
679678
}),
680679
)
681680

‎packages/ai/test/provider/openai-compatible-chat.test.ts‎

Lines changed: 36 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -561,9 +561,23 @@ describe("OpenAI-compatible Chat route", () => {
561561
}),
562562
)
563563

564-
it.effect("rejects content after a terminal chunk", () =>
564+
it.effect("absorbs content after a terminal chunk", () =>
565565
Effect.gen(function* () {
566-
const error = yield* LLMClient.generate(request).pipe(
566+
const response = yield* LLMClient.generate(request).pipe(
567+
Effect.provide(
568+
fixedResponse(
569+
sseEvents(deltaChunk({ content: "Hello" }), deltaChunk({}, "stop"), deltaChunk({ content: " late" })),
570+
),
571+
),
572+
)
573+
574+
expect(response.text).toBe("Hello late")
575+
}),
576+
)
577+
578+
it.effect("finishes a tool call absorbed after a terminal chunk", () =>
579+
Effect.gen(function* () {
580+
const response = yield* LLMClient.generate(request).pipe(
567581
Effect.provide(
568582
fixedResponse(
569583
sseEvents(
@@ -573,15 +587,28 @@ describe("OpenAI-compatible Chat route", () => {
573587
),
574588
),
575589
),
576-
Effect.flip,
577590
)
578591

579-
expect(error.message).toContain("OpenAI Chat received content after the finish reason")
580-
expect(error.reason._tag).toBe("InvalidProviderOutput")
581-
if (error.reason._tag !== "InvalidProviderOutput") return
582-
expect(decodeJson(error.reason.body ?? "")).toMatchObject({
583-
choices: [{ delta: { tool_calls: [{ id: "call_1" }] } }],
584-
})
592+
expect(response.toolCalls).toMatchObject([{ id: "call_1", name: "lookup", input: {} }])
593+
}),
594+
)
595+
596+
it.effect("still drops unconfirmed tool calls after a content-filter finish", () =>
597+
Effect.gen(function* () {
598+
const response = yield* LLMClient.generate(request).pipe(
599+
Effect.provide(
600+
fixedResponse(
601+
sseEvents(
602+
deltaChunk({ tool_calls: [{ index: 0, id: "call_1", function: { name: "lookup", arguments: "{}" } }] }),
603+
deltaChunk({}, "content_filter"),
604+
deltaChunk({ tool_calls: [{ index: 1, id: "call_2", function: { name: "lookup", arguments: "{}" } }] }),
605+
),
606+
),
607+
),
608+
)
609+
610+
expect(response.toolCalls).toEqual([])
611+
expect(response.finishReason).toEqual({ normalized: "content-filter", raw: "content_filter" })
585612
}),
586613
)
587614
})

0 commit comments

Comments
 (0)