Skip to content

Commit 9204bca

Browse files
sebdanielssonclaude
andcommitted
feat(opencode): respect provider/model streaming:false to disable response streaming
Some OpenAI-compatible backends don't support streaming or return broken streamed output (e.g. self-hosted vLLM corrupting streamed tool-call args). The existing options.streaming config wasn't consumed, so there was no way to opt out. Honor options.streaming:false (per-model or per-provider) by adding the AI SDK's simulateStreamingMiddleware, which calls doGenerate (stream:false on the wire) and replays the result as a simulated stream, leaving the rest of the pipeline unchanged. Defaults to streaming on. Fixes anomalyco#785 Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
1 parent a4fed69 commit 9204bca

2 files changed

Lines changed: 77 additions & 1 deletion

File tree

‎packages/opencode/src/session/llm.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { SessionV1 } from "@opencode-ai/core/v1/session"
66
import { serviceUse } from "@opencode-ai/core/effect/service-use"
77
import { Context, Effect, Layer } from "effect"
88
import * as Stream from "effect/Stream"
9-
import { streamText, wrapLanguageModel, type ModelMessage, type Tool } from "ai"
9+
import { simulateStreamingMiddleware, streamText, wrapLanguageModel, type ModelMessage, type Tool } from "ai"
1010
import type { LLMEvent } from "@opencode-ai/llm"
1111
import { LLMClient } from "@opencode-ai/llm/route"
1212
import type { LLMClientService } from "@opencode-ai/llm/route"
@@ -273,6 +273,12 @@ const live: Layer.Layer<
273273
"llm.provider": input.model.providerID,
274274
"llm.model": input.model.id,
275275
})
276+
// Opt out of streaming per-model or per-provider with `options.streaming: false`.
277+
// Some backends corrupt or reject streamed responses (e.g. vLLM's gemma tool-call
278+
// parser duplicates characters in streamed tool args); simulateStreamingMiddleware
279+
// makes the model call doGenerate (stream:false on the wire) and re-emits a
280+
// simulated stream, so the rest of the pipeline is unchanged. Model-level wins.
281+
const disableStreaming = (input.model.options?.["streaming"] ?? item.options?.["streaming"]) === false
276282
// Default runtime path: AI SDK owns provider execution and tool dispatch;
277283
// LLMAISDK.toLLMEvents below normalizes fullStream parts for the processor.
278284
return {
@@ -339,6 +345,7 @@ const live: Layer.Layer<
339345
return args.params
340346
},
341347
},
348+
...(disableStreaming ? [simulateStreamingMiddleware()] : []),
342349
],
343350
}),
344351
experimental_telemetry: {

‎packages/opencode/test/session/llm.test.ts‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2000,4 +2000,73 @@ describe("session.llm.stream", () => {
20002000
}),
20012001
},
20022002
)
2003+
2004+
it.instance(
2005+
"disables streaming when options.streaming is false",
2006+
() =>
2007+
Effect.gen(function* () {
2008+
const fixture = loadFixture(vivgridFixture.providerID, vivgridFixture.modelID)
2009+
// Respond with a non-streaming chat completion. The provider can only parse
2010+
// this if it issued a doGenerate (stream:false) request — a streamed request
2011+
// would expect SSE and fail — so a clean drain proves streaming was disabled.
2012+
const request = waitRequest(
2013+
"/chat/completions",
2014+
new Response(
2015+
JSON.stringify({
2016+
id: "chatcmpl-nostream",
2017+
object: "chat.completion",
2018+
model: fixture.model.id,
2019+
choices: [{ index: 0, message: { role: "assistant", content: "Hello" }, finish_reason: "stop" }],
2020+
usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
2021+
}),
2022+
{ status: 200, headers: { "Content-Type": "application/json" } },
2023+
),
2024+
)
2025+
2026+
const resolved = yield* Provider.use.getModel(
2027+
ProviderV2.ID.make(vivgridFixture.providerID),
2028+
ModelV2.ID.make(fixture.model.id),
2029+
)
2030+
const sessionID = SessionID.make("session-test-no-stream")
2031+
const agent = {
2032+
name: "test",
2033+
mode: "primary",
2034+
options: {},
2035+
permission: [{ permission: "*", pattern: "*", action: "allow" }],
2036+
} satisfies Agent.Info
2037+
2038+
const user = {
2039+
id: MessageID.make("msg_user-no-stream"),
2040+
sessionID,
2041+
role: "user",
2042+
time: { created: Date.now() },
2043+
agent: agent.name,
2044+
model: { providerID: ProviderV2.ID.make(vivgridFixture.providerID), modelID: resolved.id },
2045+
} satisfies SessionV1.User
2046+
2047+
yield* drain({
2048+
user,
2049+
sessionID,
2050+
model: resolved,
2051+
agent,
2052+
system: ["You are a helpful assistant."],
2053+
messages: [{ role: "user", content: "Hello" }],
2054+
tools: {},
2055+
})
2056+
2057+
const capture = yield* Effect.promise(() => request)
2058+
expect(capture.url.pathname.endsWith("/chat/completions")).toBe(true)
2059+
expect(capture.body.stream).not.toBe(true)
2060+
}),
2061+
{
2062+
config: () => ({
2063+
enabled_providers: [vivgridFixture.providerID],
2064+
provider: {
2065+
[vivgridFixture.providerID]: {
2066+
options: { apiKey: "test-key", baseURL: `${state.server!.url.origin}/v1`, streaming: false },
2067+
},
2068+
},
2069+
}),
2070+
},
2071+
)
20032072
})

0 commit comments

Comments
 (0)