|
| 1 | +import * as assert from "assert" |
| 2 | + |
| 3 | +import { RooCodeEventName, type ClineMessage } from "@roo-code/types" |
| 4 | + |
| 5 | +import { setDefaultSuiteTimeout } from "../test-utils" |
| 6 | +import { waitUntilCompleted } from "../utils" |
| 7 | + |
| 8 | +// LM Studio streams Qwen3.x thinking as reasoning_content deltas on the |
| 9 | +// OpenAI-compatible endpoint (verified against a live local LM Studio server), |
| 10 | +// not as think tags inside content. Regression #1175 made LmStudioHandler |
| 11 | +// forward those deltas as reasoning chunks. This suite proves the full path |
| 12 | +// through the real extension host: real SSE wire format -> OpenAI SDK parsing |
| 13 | +// -> provider reasoning extraction -> Task say("reasoning") -> webview message, |
| 14 | +// which is exactly the boundary the provider unit tests (which mock the OpenAI |
| 15 | +// client) do not exercise. |
| 16 | +const LMSTUDIO_MODEL_ID = "qwen3.8-27b" |
| 17 | +const PROMPT_TAG = "LMSTUDIO_E2E_THINK_BLOCK" |
| 18 | +const REASONING_PROBE = "LMSTUDIO_E2E_REASONING_PROBE" |
| 19 | +const EXPECTED_ANSWER = "Paris" |
| 20 | + |
| 21 | +type CapturedLmStudioRequest = { |
| 22 | + model?: string |
| 23 | + lastUserMessage: string |
| 24 | +} |
| 25 | + |
| 26 | +function getRequestUrl(input: RequestInfo | URL): string { |
| 27 | + return typeof input === "string" ? input : input instanceof URL ? input.href : (input as Request).url |
| 28 | +} |
| 29 | + |
| 30 | +function isUrlWithOrigin(rawUrl: string, expectedOrigin: string): boolean { |
| 31 | + try { |
| 32 | + return new URL(rawUrl).origin === expectedOrigin |
| 33 | + } catch { |
| 34 | + return false |
| 35 | + } |
| 36 | +} |
| 37 | + |
| 38 | +function isChatCompletionsUrl(rawUrl: string): boolean { |
| 39 | + try { |
| 40 | + return new URL(rawUrl).pathname.endsWith("/chat/completions") |
| 41 | + } catch { |
| 42 | + return false |
| 43 | + } |
| 44 | +} |
| 45 | + |
| 46 | +function installLmStudioRequestCapture(capture: CapturedLmStudioRequest[], baseUrl: string): () => void { |
| 47 | + const originalFetch = globalThis.fetch |
| 48 | + const targetOrigin = new URL(baseUrl).origin |
| 49 | + |
| 50 | + globalThis.fetch = async function (input: RequestInfo | URL, init?: RequestInit): Promise<Response> { |
| 51 | + const url = getRequestUrl(input) |
| 52 | + |
| 53 | + if (isUrlWithOrigin(url, targetOrigin) && isChatCompletionsUrl(url)) { |
| 54 | + const body = init?.body && typeof init.body === "string" ? JSON.parse(init.body) : {} |
| 55 | + const messages = Array.isArray(body.messages) ? body.messages : [] |
| 56 | + const lastUser = [...messages].reverse().find((message: { role?: string }) => message.role === "user") |
| 57 | + const lastUserMessage = |
| 58 | + typeof lastUser?.content === "string" ? lastUser.content : JSON.stringify(lastUser?.content ?? "") |
| 59 | + |
| 60 | + capture.push({ model: body.model, lastUserMessage }) |
| 61 | + } |
| 62 | + |
| 63 | + return originalFetch.call(globalThis, input, init as RequestInit) |
| 64 | + } as typeof globalThis.fetch |
| 65 | + |
| 66 | + return () => { |
| 67 | + globalThis.fetch = originalFetch |
| 68 | + } |
| 69 | +} |
| 70 | + |
| 71 | +suite("LM Studio provider", function () { |
| 72 | + setDefaultSuiteTimeout(this) |
| 73 | + |
| 74 | + let restoreFetch: (() => void) | undefined |
| 75 | + const requests: CapturedLmStudioRequest[] = [] |
| 76 | + |
| 77 | + setup(function () { |
| 78 | + const aimockUrl = process.env.AIMOCK_URL |
| 79 | + // Only replay mode can serve the reasoning fixture: record mode has no |
| 80 | + // real LM Studio upstream to proxy to, and live runs have no aimock. |
| 81 | + const isReplay = !!aimockUrl && process.env.AIMOCK_RECORD !== "true" |
| 82 | + |
| 83 | + if (!isReplay) { |
| 84 | + this.skip() |
| 85 | + } |
| 86 | + }) |
| 87 | + |
| 88 | + suiteSetup(() => { |
| 89 | + restoreFetch = installLmStudioRequestCapture(requests, process.env.AIMOCK_URL || "http://localhost:1234") |
| 90 | + }) |
| 91 | + |
| 92 | + suiteTeardown(async () => { |
| 93 | + restoreFetch?.() |
| 94 | + restoreFetch = undefined |
| 95 | + |
| 96 | + // Restore the default OpenRouter config so subsequent suites are unaffected. |
| 97 | + await globalThis.api.setConfiguration({ |
| 98 | + apiProvider: "openrouter" as const, |
| 99 | + openRouterApiKey: |
| 100 | + process.env.AIMOCK_URL && process.env.AIMOCK_RECORD !== "true" |
| 101 | + ? "mock-key" |
| 102 | + : process.env.OPENROUTER_API_KEY!, |
| 103 | + openRouterModelId: "openai/gpt-4.1", |
| 104 | + ...(process.env.AIMOCK_URL && { openRouterBaseUrl: `${process.env.AIMOCK_URL}/v1` }), |
| 105 | + }) |
| 106 | + }) |
| 107 | + |
| 108 | + test("should surface the LM Studio thinking stream as a separate reasoning message", async () => { |
| 109 | + requests.length = 0 |
| 110 | + |
| 111 | + const api = globalThis.api |
| 112 | + const aimockUrl = process.env.AIMOCK_URL! |
| 113 | + |
| 114 | + // LmStudioHandler appends /v1 itself, so the base URL is aimock's origin only. |
| 115 | + await api.setConfiguration({ |
| 116 | + apiProvider: "lmstudio" as const, |
| 117 | + lmStudioBaseUrl: aimockUrl, |
| 118 | + lmStudioModelId: LMSTUDIO_MODEL_ID, |
| 119 | + }) |
| 120 | + |
| 121 | + const messages: ClineMessage[] = [] |
| 122 | + const messageHandler = ({ message }: { message: ClineMessage }) => { |
| 123 | + if (message.type === "say" && message.partial === false) { |
| 124 | + messages.push(message) |
| 125 | + } |
| 126 | + } |
| 127 | + |
| 128 | + api.on(RooCodeEventName.Message, messageHandler) |
| 129 | + |
| 130 | + let taskId: string | undefined |
| 131 | + try { |
| 132 | + taskId = await api.startNewTask({ |
| 133 | + configuration: { mode: "ask", autoApprovalEnabled: true, alwaysAllowModeSwitch: true }, |
| 134 | + text: `${PROMPT_TAG}: What is the capital of France? Reply with only the city name.`, |
| 135 | + }) |
| 136 | + |
| 137 | + await waitUntilCompleted({ api, taskId, timeout: 120_000 }) |
| 138 | + } finally { |
| 139 | + api.off(RooCodeEventName.Message, messageHandler) |
| 140 | + } |
| 141 | + |
| 142 | + // The request must have gone to aimock carrying the LM Studio model id. |
| 143 | + const request = requests.find((entry) => entry.lastUserMessage.includes(PROMPT_TAG)) |
| 144 | + assert.ok( |
| 145 | + request, |
| 146 | + `LM Studio provider should issue a chat completions request for the task prompt (saw ${requests.length} request(s))`, |
| 147 | + ) |
| 148 | + assert.strictEqual(request.model, LMSTUDIO_MODEL_ID) |
| 149 | + |
| 150 | + // The thinking stream must surface as a finalized reasoning message. |
| 151 | + const reasoningMessages = messages.filter((message) => message.say === "reasoning") |
| 152 | + assert.ok( |
| 153 | + reasoningMessages.length > 0, |
| 154 | + `Task should surface the LM Studio thinking stream as a reasoning message. Observed says: ${messages |
| 155 | + .map((message) => message.say) |
| 156 | + .join(", ")}`, |
| 157 | + ) |
| 158 | + |
| 159 | + const reasoningText = reasoningMessages.map((message) => message.text ?? "").join("") |
| 160 | + assert.ok( |
| 161 | + reasoningText.includes(REASONING_PROBE), |
| 162 | + `Reasoning message should contain the streamed thinking text. Got: ${reasoningText.slice(0, 300)}`, |
| 163 | + ) |
| 164 | + |
| 165 | + // The visible answer must be the plain completion result. |
| 166 | + const completionMessage = messages.find( |
| 167 | + ({ say, text }) => (say === "completion_result" || say === "text") && text?.trim() === EXPECTED_ANSWER, |
| 168 | + ) |
| 169 | + assert.ok( |
| 170 | + completionMessage, |
| 171 | + `Task should complete with the expected answer "${EXPECTED_ANSWER}". Observed: ${messages |
| 172 | + .map((message) => `${message.say}:${message.text?.slice(0, 80)}`) |
| 173 | + .join(" | ")}`, |
| 174 | + ) |
| 175 | + |
| 176 | + // Channel separation: thinking must not leak into the visible answer, |
| 177 | + // and the final answer must not appear inside the reasoning message. |
| 178 | + const visibleText = messages |
| 179 | + .filter(({ say }) => say === "completion_result" || say === "text") |
| 180 | + .map((message) => message.text ?? "") |
| 181 | + .join("") |
| 182 | + assert.ok(!visibleText.includes(REASONING_PROBE), "Thinking text must not leak into the visible answer") |
| 183 | + assert.ok(!reasoningText.includes(EXPECTED_ANSWER), "Final answer must not appear inside the reasoning message") |
| 184 | + |
| 185 | + // Thinking streams before the answer. |
| 186 | + const reasoningIndex = messages.findIndex((message) => message.say === "reasoning") |
| 187 | + const completionIndex = messages.findIndex( |
| 188 | + ({ say, text }) => (say === "completion_result" || say === "text") && text?.trim() === EXPECTED_ANSWER, |
| 189 | + ) |
| 190 | + assert.ok( |
| 191 | + reasoningIndex !== -1 && completionIndex !== -1 && reasoningIndex < completionIndex, |
| 192 | + "Reasoning message should precede the completion message", |
| 193 | + ) |
| 194 | + }) |
| 195 | +}) |
0 commit comments