Skip to content

Commit 9a15bda

Browse files
committed
test(e2e): add LM Studio reasoning_content e2e guard
Adds an extension-host e2e suite (fixtures/lmstudio.json + suite/providers/lmstudio.test.ts) that replays an OpenAI-compatible stream carrying thinking only in delta.reasoning_content, covering the full path: aimock SSE -> openai SDK -> LmStudioHandler extractReasoningFromDelta -> say(reasoning) message. Regression guard for #1175 / #1075.
1 parent 871bb98 commit 9a15bda

2 files changed

Lines changed: 215 additions & 0 deletions

File tree

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"fixtures": [
3+
{
4+
"match": {
5+
"userMessage": "LMSTUDIO_E2E_THINK_BLOCK",
6+
"model": "qwen3.8-27b"
7+
},
8+
"response": {
9+
"reasoning": "LMSTUDIO_E2E_REASONING_PROBE I considered which country the question refers to and recalled its capital from geography knowledge before committing to an answer.",
10+
"toolCalls": [
11+
{
12+
"name": "attempt_completion",
13+
"arguments": "{\"result\":\"Paris\"}",
14+
"id": "call_lmstudio_e2e_think_001"
15+
}
16+
]
17+
}
18+
}
19+
]
20+
}
Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
import * as assert from "assert"
2+
3+
import { RooCodeEventName, type ClineMessage } from "@roo-code/types"
4+
5+
import { setDefaultSuiteTimeout } from "../test-utils"
6+
import { waitUntilCompleted } from "../utils"
7+
8+
// LM Studio streams Qwen3.x thinking as reasoning_content deltas on the
9+
// OpenAI-compatible endpoint (verified against a live local LM Studio server),
10+
// not as think tags inside content. Regression #1175 made LmStudioHandler
11+
// forward those deltas as reasoning chunks. This suite proves the full path
12+
// through the real extension host: real SSE wire format -> OpenAI SDK parsing
13+
// -> provider reasoning extraction -> Task say("reasoning") -> webview message,
14+
// which is exactly the boundary the provider unit tests (which mock the OpenAI
15+
// client) do not exercise.
16+
const LMSTUDIO_MODEL_ID = "qwen3.8-27b"
17+
const PROMPT_TAG = "LMSTUDIO_E2E_THINK_BLOCK"
18+
const REASONING_PROBE = "LMSTUDIO_E2E_REASONING_PROBE"
19+
const EXPECTED_ANSWER = "Paris"
20+
21+
type CapturedLmStudioRequest = {
22+
model?: string
23+
lastUserMessage: string
24+
}
25+
26+
function getRequestUrl(input: RequestInfo | URL): string {
27+
return typeof input === "string" ? input : input instanceof URL ? input.href : (input as Request).url
28+
}
29+
30+
function isUrlWithOrigin(rawUrl: string, expectedOrigin: string): boolean {
31+
try {
32+
return new URL(rawUrl).origin === expectedOrigin
33+
} catch {
34+
return false
35+
}
36+
}
37+
38+
function isChatCompletionsUrl(rawUrl: string): boolean {
39+
try {
40+
return new URL(rawUrl).pathname.endsWith("/chat/completions")
41+
} catch {
42+
return false
43+
}
44+
}
45+
46+
function installLmStudioRequestCapture(capture: CapturedLmStudioRequest[], baseUrl: string): () => void {
47+
const originalFetch = globalThis.fetch
48+
const targetOrigin = new URL(baseUrl).origin
49+
50+
globalThis.fetch = async function (input: RequestInfo | URL, init?: RequestInit): Promise<Response> {
51+
const url = getRequestUrl(input)
52+
53+
if (isUrlWithOrigin(url, targetOrigin) && isChatCompletionsUrl(url)) {
54+
const body = init?.body && typeof init.body === "string" ? JSON.parse(init.body) : {}
55+
const messages = Array.isArray(body.messages) ? body.messages : []
56+
const lastUser = [...messages].reverse().find((message: { role?: string }) => message.role === "user")
57+
const lastUserMessage =
58+
typeof lastUser?.content === "string" ? lastUser.content : JSON.stringify(lastUser?.content ?? "")
59+
60+
capture.push({ model: body.model, lastUserMessage })
61+
}
62+
63+
return originalFetch.call(globalThis, input, init as RequestInit)
64+
} as typeof globalThis.fetch
65+
66+
return () => {
67+
globalThis.fetch = originalFetch
68+
}
69+
}
70+
71+
suite("LM Studio provider", function () {
72+
setDefaultSuiteTimeout(this)
73+
74+
let restoreFetch: (() => void) | undefined
75+
const requests: CapturedLmStudioRequest[] = []
76+
77+
setup(function () {
78+
const aimockUrl = process.env.AIMOCK_URL
79+
// Only replay mode can serve the reasoning fixture: record mode has no
80+
// real LM Studio upstream to proxy to, and live runs have no aimock.
81+
const isReplay = !!aimockUrl && process.env.AIMOCK_RECORD !== "true"
82+
83+
if (!isReplay) {
84+
this.skip()
85+
}
86+
})
87+
88+
suiteSetup(() => {
89+
restoreFetch = installLmStudioRequestCapture(requests, process.env.AIMOCK_URL || "http://localhost:1234")
90+
})
91+
92+
suiteTeardown(async () => {
93+
restoreFetch?.()
94+
restoreFetch = undefined
95+
96+
// Restore the default OpenRouter config so subsequent suites are unaffected.
97+
await globalThis.api.setConfiguration({
98+
apiProvider: "openrouter" as const,
99+
openRouterApiKey:
100+
process.env.AIMOCK_URL && process.env.AIMOCK_RECORD !== "true"
101+
? "mock-key"
102+
: process.env.OPENROUTER_API_KEY!,
103+
openRouterModelId: "openai/gpt-4.1",
104+
...(process.env.AIMOCK_URL && { openRouterBaseUrl: `${process.env.AIMOCK_URL}/v1` }),
105+
})
106+
})
107+
108+
test("should surface the LM Studio thinking stream as a separate reasoning message", async () => {
109+
requests.length = 0
110+
111+
const api = globalThis.api
112+
const aimockUrl = process.env.AIMOCK_URL!
113+
114+
// LmStudioHandler appends /v1 itself, so the base URL is aimock's origin only.
115+
await api.setConfiguration({
116+
apiProvider: "lmstudio" as const,
117+
lmStudioBaseUrl: aimockUrl,
118+
lmStudioModelId: LMSTUDIO_MODEL_ID,
119+
})
120+
121+
const messages: ClineMessage[] = []
122+
const messageHandler = ({ message }: { message: ClineMessage }) => {
123+
if (message.type === "say" && message.partial === false) {
124+
messages.push(message)
125+
}
126+
}
127+
128+
api.on(RooCodeEventName.Message, messageHandler)
129+
130+
let taskId: string | undefined
131+
try {
132+
taskId = await api.startNewTask({
133+
configuration: { mode: "ask", autoApprovalEnabled: true, alwaysAllowModeSwitch: true },
134+
text: `${PROMPT_TAG}: What is the capital of France? Reply with only the city name.`,
135+
})
136+
137+
await waitUntilCompleted({ api, taskId, timeout: 120_000 })
138+
} finally {
139+
api.off(RooCodeEventName.Message, messageHandler)
140+
}
141+
142+
// The request must have gone to aimock carrying the LM Studio model id.
143+
const request = requests.find((entry) => entry.lastUserMessage.includes(PROMPT_TAG))
144+
assert.ok(
145+
request,
146+
`LM Studio provider should issue a chat completions request for the task prompt (saw ${requests.length} request(s))`,
147+
)
148+
assert.strictEqual(request.model, LMSTUDIO_MODEL_ID)
149+
150+
// The thinking stream must surface as a finalized reasoning message.
151+
const reasoningMessages = messages.filter((message) => message.say === "reasoning")
152+
assert.ok(
153+
reasoningMessages.length > 0,
154+
`Task should surface the LM Studio thinking stream as a reasoning message. Observed says: ${messages
155+
.map((message) => message.say)
156+
.join(", ")}`,
157+
)
158+
159+
const reasoningText = reasoningMessages.map((message) => message.text ?? "").join("")
160+
assert.ok(
161+
reasoningText.includes(REASONING_PROBE),
162+
`Reasoning message should contain the streamed thinking text. Got: ${reasoningText.slice(0, 300)}`,
163+
)
164+
165+
// The visible answer must be the plain completion result.
166+
const completionMessage = messages.find(
167+
({ say, text }) => (say === "completion_result" || say === "text") && text?.trim() === EXPECTED_ANSWER,
168+
)
169+
assert.ok(
170+
completionMessage,
171+
`Task should complete with the expected answer "${EXPECTED_ANSWER}". Observed: ${messages
172+
.map((message) => `${message.say}:${message.text?.slice(0, 80)}`)
173+
.join(" | ")}`,
174+
)
175+
176+
// Channel separation: thinking must not leak into the visible answer,
177+
// and the final answer must not appear inside the reasoning message.
178+
const visibleText = messages
179+
.filter(({ say }) => say === "completion_result" || say === "text")
180+
.map((message) => message.text ?? "")
181+
.join("")
182+
assert.ok(!visibleText.includes(REASONING_PROBE), "Thinking text must not leak into the visible answer")
183+
assert.ok(!reasoningText.includes(EXPECTED_ANSWER), "Final answer must not appear inside the reasoning message")
184+
185+
// Thinking streams before the answer.
186+
const reasoningIndex = messages.findIndex((message) => message.say === "reasoning")
187+
const completionIndex = messages.findIndex(
188+
({ say, text }) => (say === "completion_result" || say === "text") && text?.trim() === EXPECTED_ANSWER,
189+
)
190+
assert.ok(
191+
reasoningIndex !== -1 && completionIndex !== -1 && reasoningIndex < completionIndex,
192+
"Reasoning message should precede the completion message",
193+
)
194+
})
195+
})

0 commit comments

Comments
 (0)