|
| 1 | +import * as assert from "assert" |
| 2 | + |
| 3 | +import { RooCodeEventName, providerIdentifiers, type ClineMessage } from "@roo-code/types" |
| 4 | + |
| 5 | +import { setDefaultSuiteTimeout } from "../test-utils" |
| 6 | +import { waitUntilCompleted } from "../utils" |
| 7 | + |
| 8 | +// LM Studio streams Qwen3.x thinking as reasoning_content deltas on the |
| 9 | +// OpenAI-compatible endpoint (verified against a live local LM Studio server), |
| 10 | +// not as think tags inside content. Regression #1175 made LmStudioHandler |
| 11 | +// forward those deltas as reasoning chunks. This suite proves the full path |
| 12 | +// through the real extension host: real SSE wire format -> OpenAI SDK parsing |
| 13 | +// -> provider reasoning extraction -> Task say("reasoning") -> webview message, |
| 14 | +// which is exactly the boundary the provider unit tests (which mock the OpenAI |
| 15 | +// client) do not exercise. |
| 16 | +const LMSTUDIO_MODEL_ID = "qwen3.8-27b" |
| 17 | +const PROMPT_TAG = "LMSTUDIO_E2E_THINK_BLOCK" |
| 18 | +const REASONING_PROBE = "LMSTUDIO_E2E_REASONING_PROBE" |
| 19 | +const EXPECTED_ANSWER = "Paris" |
| 20 | + |
| 21 | +type CapturedLmStudioRequest = { |
| 22 | + model?: string |
| 23 | + lastUserMessage: string |
| 24 | +} |
| 25 | + |
| 26 | +/** Returns the URL string for a fetch request input (string, URL, or Request). */ |
| 27 | +function getRequestUrl(input: RequestInfo | URL): string { |
| 28 | + return typeof input === "string" ? input : input instanceof URL ? input.href : input.url |
| 29 | +} |
| 30 | + |
| 31 | +/** True if the raw URL parses and its origin equals the expected origin. */ |
| 32 | +function isUrlWithOrigin(rawUrl: string, expectedOrigin: string): boolean { |
| 33 | + try { |
| 34 | + return new URL(rawUrl).origin === expectedOrigin |
| 35 | + } catch { |
| 36 | + return false |
| 37 | + } |
| 38 | +} |
| 39 | + |
| 40 | +/** True if the raw URL points at an OpenAI-compatible /chat/completions endpoint. */ |
| 41 | +function isChatCompletionsUrl(rawUrl: string): boolean { |
| 42 | + try { |
| 43 | + return new URL(rawUrl).pathname.endsWith("/chat/completions") |
| 44 | + } catch { |
| 45 | + return false |
| 46 | + } |
| 47 | +} |
| 48 | + |
| 49 | +/** |
| 50 | + * Installs a global fetch capture that records the model and last user message of every |
| 51 | + * chat completions request sent to the given base URL. Returns a function that restores |
| 52 | + * the original fetch. |
| 53 | + */ |
| 54 | +function installLmStudioRequestCapture(capture: CapturedLmStudioRequest[], baseUrl: string): () => void { |
| 55 | + const originalFetch = globalThis.fetch |
| 56 | + const targetOrigin = new URL(baseUrl).origin |
| 57 | + |
| 58 | + globalThis.fetch = async function (input: RequestInfo | URL, init?: RequestInit): Promise<Response> { |
| 59 | + const url = getRequestUrl(input) |
| 60 | + |
| 61 | + if (isUrlWithOrigin(url, targetOrigin) && isChatCompletionsUrl(url)) { |
| 62 | + const body = init?.body && typeof init.body === "string" ? JSON.parse(init.body) : {} |
| 63 | + const messages = Array.isArray(body.messages) ? body.messages : [] |
| 64 | + const lastUser = [...messages].reverse().find((message: { role?: string }) => message.role === "user") |
| 65 | + const lastUserMessage = |
| 66 | + typeof lastUser?.content === "string" ? lastUser.content : JSON.stringify(lastUser?.content ?? "") |
| 67 | + |
| 68 | + capture.push({ model: body.model, lastUserMessage }) |
| 69 | + } |
| 70 | + |
| 71 | + return originalFetch.call(globalThis, input, init) |
| 72 | + } |
| 73 | + |
| 74 | + return () => { |
| 75 | + globalThis.fetch = originalFetch |
| 76 | + } |
| 77 | +} |
| 78 | + |
| 79 | +suite("LM Studio provider", function () { |
| 80 | + setDefaultSuiteTimeout(this) |
| 81 | + |
| 82 | + // Only replay mode can serve the reasoning fixture: record mode has no |
| 83 | + // real LM Studio upstream to proxy to, and live runs have no aimock. |
| 84 | + // Computed once at suite scope so the suite hooks can be guarded as well: |
| 85 | + // a skipped suite must not touch shared fetch or provider configuration. |
| 86 | + const isReplay = !!process.env.AIMOCK_URL && process.env.AIMOCK_RECORD !== "true" |
| 87 | + |
| 88 | + let restoreFetch: (() => void) | undefined |
| 89 | + const requests: CapturedLmStudioRequest[] = [] |
| 90 | + |
| 91 | + /** Skips the suite when aimock replay mode is not active (no fixture source). */ |
| 92 | + setup(function () { |
| 93 | + if (!isReplay) { |
| 94 | + this.skip() |
| 95 | + } |
| 96 | + |
| 97 | + // Fresh per-test buffer: late requests from prior tasks must not satisfy |
| 98 | + // this test's assertions, which are scoped by PROMPT_TAG. |
| 99 | + requests.length = 0 |
| 100 | + }) |
| 101 | + |
| 102 | + /** Captures chat completions requests sent to the aimock origin for later assertions. */ |
| 103 | + suiteSetup(() => { |
| 104 | + if (!isReplay) { |
| 105 | + return |
| 106 | + } |
| 107 | + |
| 108 | + restoreFetch = installLmStudioRequestCapture(requests, process.env.AIMOCK_URL!) |
| 109 | + }) |
| 110 | + |
| 111 | + /** Restores the original fetch and the default OpenRouter provider configuration. */ |
| 112 | + suiteTeardown(async () => { |
| 113 | + if (!isReplay) { |
| 114 | + return |
| 115 | + } |
| 116 | + |
| 117 | + restoreFetch?.() |
| 118 | + restoreFetch = undefined |
| 119 | + |
| 120 | + // Restore the default OpenRouter config so subsequent suites are unaffected. |
| 121 | + // Explicitly clear the LM Studio fields: setConfiguration only updates |
| 122 | + // supplied keys, so omitting them would leave them set for later suites. |
| 123 | + await globalThis.api.setConfiguration({ |
| 124 | + apiProvider: providerIdentifiers.openrouter, |
| 125 | + lmStudioBaseUrl: undefined, |
| 126 | + lmStudioModelId: undefined, |
| 127 | + openRouterApiKey: "mock-key", |
| 128 | + openRouterModelId: "openai/gpt-4.1", |
| 129 | + openRouterBaseUrl: `${process.env.AIMOCK_URL}/v1`, |
| 130 | + }) |
| 131 | + }) |
| 132 | + |
| 133 | + /** |
| 134 | + * Runs an ask-mode task against the aimock LM Studio fixture and asserts the reasoning |
| 135 | + * stream surfaces as a separate finalized reasoning message ahead of the completion. |
| 136 | + */ |
| 137 | + test("should surface the LM Studio thinking stream as a separate reasoning message", async () => { |
| 138 | + const api = globalThis.api |
| 139 | + const aimockUrl = process.env.AIMOCK_URL! |
| 140 | + |
| 141 | + // LmStudioHandler appends /v1 itself, so the base URL is aimock's origin only. |
| 142 | + await api.setConfiguration({ |
| 143 | + apiProvider: providerIdentifiers.lmstudio, |
| 144 | + lmStudioBaseUrl: aimockUrl, |
| 145 | + lmStudioModelId: LMSTUDIO_MODEL_ID, |
| 146 | + }) |
| 147 | + |
| 148 | + const messages: ClineMessage[] = [] |
| 149 | + const messageHandler = ({ message }: { message: ClineMessage }) => { |
| 150 | + if (message.type === "say" && message.partial === false) { |
| 151 | + messages.push(message) |
| 152 | + } |
| 153 | + } |
| 154 | + |
| 155 | + api.on(RooCodeEventName.Message, messageHandler) |
| 156 | + |
| 157 | + try { |
| 158 | + await waitUntilCompleted({ |
| 159 | + api, |
| 160 | + timeout: 120_000, |
| 161 | + start: () => |
| 162 | + api.startNewTask({ |
| 163 | + configuration: { mode: "ask", autoApprovalEnabled: true, alwaysAllowModeSwitch: true }, |
| 164 | + text: `${PROMPT_TAG}: What is the capital of France? Reply with only the city name.`, |
| 165 | + }), |
| 166 | + }) |
| 167 | + } finally { |
| 168 | + api.off(RooCodeEventName.Message, messageHandler) |
| 169 | + } |
| 170 | + |
| 171 | + // The request must have gone to aimock carrying the LM Studio model id. |
| 172 | + const request = requests.find((entry) => entry.lastUserMessage.includes(PROMPT_TAG)) |
| 173 | + assert.ok( |
| 174 | + request, |
| 175 | + `LM Studio provider should issue a chat completions request for the task prompt (saw ${requests.length} request(s))`, |
| 176 | + ) |
| 177 | + assert.strictEqual(request.model, LMSTUDIO_MODEL_ID) |
| 178 | + |
| 179 | + // The thinking stream must surface as a finalized reasoning message. |
| 180 | + const reasoningMessages = messages.filter((message) => message.say === "reasoning") |
| 181 | + assert.ok( |
| 182 | + reasoningMessages.length > 0, |
| 183 | + `Task should surface the LM Studio thinking stream as a reasoning message. Observed says: ${messages |
| 184 | + .map((message) => message.say) |
| 185 | + .join(", ")}`, |
| 186 | + ) |
| 187 | + |
| 188 | + const reasoningText = reasoningMessages.map((message) => message.text ?? "").join("") |
| 189 | + assert.ok( |
| 190 | + reasoningText.includes(REASONING_PROBE), |
| 191 | + `Reasoning message should contain the streamed thinking text. Got: ${reasoningText.slice(0, 300)}`, |
| 192 | + ) |
| 193 | + |
| 194 | + // The visible answer must be the plain completion result. |
| 195 | + const completionMessage = messages.find( |
| 196 | + ({ say, text }) => (say === "completion_result" || say === "text") && text?.trim() === EXPECTED_ANSWER, |
| 197 | + ) |
| 198 | + assert.ok( |
| 199 | + completionMessage, |
| 200 | + `Task should complete with the expected answer "${EXPECTED_ANSWER}". Observed: ${messages |
| 201 | + .map((message) => `${message.say}:${message.text?.slice(0, 80)}`) |
| 202 | + .join(" | ")}`, |
| 203 | + ) |
| 204 | + |
| 205 | + // Channel separation: thinking must not leak into the visible answer, |
| 206 | + // and the final answer must not appear inside the reasoning message. |
| 207 | + const visibleText = messages |
| 208 | + .filter(({ say }) => say === "completion_result" || say === "text") |
| 209 | + .map((message) => message.text ?? "") |
| 210 | + .join("") |
| 211 | + assert.ok(!visibleText.includes(REASONING_PROBE), "Thinking text must not leak into the visible answer") |
| 212 | + assert.ok(!reasoningText.includes(EXPECTED_ANSWER), "Final answer must not appear inside the reasoning message") |
| 213 | + |
| 214 | + // Thinking streams before the answer. |
| 215 | + const reasoningIndex = messages.findIndex((message) => message.say === "reasoning") |
| 216 | + const completionIndex = messages.findIndex( |
| 217 | + ({ say, text }) => (say === "completion_result" || say === "text") && text?.trim() === EXPECTED_ANSWER, |
| 218 | + ) |
| 219 | + assert.ok( |
| 220 | + reasoningIndex !== -1 && completionIndex !== -1 && reasoningIndex < completionIndex, |
| 221 | + "Reasoning message should precede the completion message", |
| 222 | + ) |
| 223 | + }) |
| 224 | +}) |
0 commit comments