|
| 1 | +import * as assert from "assert" |
| 2 | +import { createServer, type IncomingMessage, type ServerResponse } from "http" |
| 3 | + |
| 4 | +import { RooCodeEventName, type ClineMessage } from "@roo-code/types" |
| 5 | + |
| 6 | +import { setDefaultSuiteTimeout } from "./test-utils" |
| 7 | +import { waitUntilCompleted } from "./utils" |
| 8 | + |
| 9 | +/** |
| 10 | + * DTE addendum: set_thinking_effort mid-task workflow. |
| 11 | + * |
| 12 | + * Exercises the real extension-host boundary end to end with aimock fixtures: |
| 13 | + * the model calls set_thinking_effort mid-task (no approval gate), the |
| 14 | + * SetThinkingEffortTool display say is emitted, and the FOLLOWING API request |
| 15 | + * carries the applied effort in the OpenRouter reasoning envelope. |
| 16 | + * |
| 17 | + * Determinism notes: |
| 18 | + * - Runs against aimock only (replay or record); skips when aimock is absent, |
| 19 | + * so no API keys are required. |
| 20 | + * - The capture proxy is the local 127.0.0.1 pattern from |
| 21 | + * anthropic-opus-4-7.test.ts: it intercepts the OpenRouter-compatible |
| 22 | + * chat/completions POST (so request shapes can be asserted) and forwards it |
| 23 | + * to aimock for the fixture-driven SSE responses. |
| 24 | + * - The OpenRouter model catalog is resolved by the shared model-cache layer |
| 25 | + * (fetchers/modelCache.ts) from the public OpenRouter endpoint, exactly like |
| 26 | + * the other provider suites. openai/gpt-5 advertises "reasoning" in |
| 27 | + * supported_parameters, so the fetcher resolves supportsReasoningEffort and |
| 28 | + * the dynamicThinkingEffort gate exposes the tool. |
| 29 | + * - The mid-task tool call is dispatched by name in presentAssistantMessage |
| 30 | + * (a hard-coded case, not the request-declared tool list), and the tool |
| 31 | + * executor re-checks the model capability after the first request has loaded |
| 32 | + * the catalog, so the flow is correct even if the first request's tool list |
| 33 | + * was built before the catalog fetch resolved. |
| 34 | + */ |
| 35 | + |
| 36 | +const DTE_MODEL_ID = "openai/gpt-5" |
| 37 | +const APPLY_MARKER = "DTE_E2E_EFFORT_APPLY" |
| 38 | +const SET_EFFORT_TOOL_CALL_ID = "call_dte_e2e_001" |
| 39 | +const COMPLETION_EXPECTED = "42" |
| 40 | + |
| 41 | +type DteReasoningEnvelope = { |
| 42 | + effort?: string |
| 43 | + max_tokens?: number |
| 44 | + exclude?: boolean |
| 45 | +} |
| 46 | + |
| 47 | +type CapturedDteRequest = { |
| 48 | + model?: string |
| 49 | + reasoning: DteReasoningEnvelope | undefined |
| 50 | + carriesSetEffortToolResult: boolean |
| 51 | + lastUserMessage: string |
| 52 | +} |
| 53 | + |
| 54 | +type OpenRouterChatCompletionBody = { |
| 55 | + model?: string |
| 56 | + reasoning?: DteReasoningEnvelope |
| 57 | + messages?: Array<{ role?: string; content?: unknown }> |
| 58 | +} |
| 59 | + |
| 60 | +const ALLOWED_PROXY_HOSTS = new Set(["127.0.0.1", "localhost"]) |
| 61 | +const CHAT_COMPLETIONS_PATH = "/v1/chat/completions" |
| 62 | +const HOP_BY_HOP = new Set([ |
| 63 | + "connection", |
| 64 | + "keep-alive", |
| 65 | + "transfer-encoding", |
| 66 | + "te", |
| 67 | + "trailer", |
| 68 | + "upgrade", |
| 69 | + "proxy-connection", |
| 70 | + "proxy-authenticate", |
| 71 | + "proxy-authorization", |
| 72 | + "host", |
| 73 | + "content-length", |
| 74 | +]) |
| 75 | + |
| 76 | +function isChatCompletionsUrl(rawUrl: string): boolean { |
| 77 | + try { |
| 78 | + return new URL(rawUrl).pathname.endsWith(CHAT_COMPLETIONS_PATH) |
| 79 | + } catch { |
| 80 | + return false |
| 81 | + } |
| 82 | +} |
| 83 | + |
| 84 | +function readRequestBody(req: IncomingMessage): Promise<string> { |
| 85 | + return new Promise((resolve, reject) => { |
| 86 | + const chunks: Buffer[] = [] |
| 87 | + req.on("data", (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))) |
| 88 | + req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))) |
| 89 | + req.on("error", reject) |
| 90 | + }) |
| 91 | +} |
| 92 | + |
| 93 | +function writeResponseHeaders(target: ServerResponse, source: Response) { |
| 94 | + const headers: Record<string, string> = {} |
| 95 | + source.headers.forEach((value, key) => { |
| 96 | + const lower = key.toLowerCase() |
| 97 | + // fetch() automatically decompresses the body, so strip content-encoding to |
| 98 | + // prevent the SDK from attempting a second decompression. Also strip |
| 99 | + // content-length since the decoded body length differs from the compressed one. |
| 100 | + if (lower !== "content-length" && lower !== "content-encoding") { |
| 101 | + headers[key] = value |
| 102 | + } |
| 103 | + }) |
| 104 | + target.writeHead(source.status, headers) |
| 105 | +} |
| 106 | + |
| 107 | +async function pipeFetchResponse(target: ServerResponse, source: Response) { |
| 108 | + writeResponseHeaders(target, source) |
| 109 | + |
| 110 | + if (!source.body) { |
| 111 | + target.end() |
| 112 | + return |
| 113 | + } |
| 114 | + |
| 115 | + const reader = source.body.getReader() |
| 116 | + while (true) { |
| 117 | + const { done, value } = await reader.read() |
| 118 | + if (done) { |
| 119 | + break |
| 120 | + } |
| 121 | + target.write(value) |
| 122 | + } |
| 123 | + |
| 124 | + target.end() |
| 125 | +} |
| 126 | + |
| 127 | +function resolveAllowedUpstreamUrl(baseUrl: string): URL { |
| 128 | + const upstreamBase = new URL(baseUrl) |
| 129 | + |
| 130 | + if (!ALLOWED_PROXY_HOSTS.has(upstreamBase.hostname) || upstreamBase.protocol !== "http:") { |
| 131 | + throw new Error("Unexpected OpenRouter proxy target: " + upstreamBase.origin) |
| 132 | + } |
| 133 | + |
| 134 | + return new URL(CHAT_COMPLETIONS_PATH, upstreamBase) |
| 135 | +} |
| 136 | + |
| 137 | +/** |
| 138 | + * Serves a loopback capture proxy for the OpenRouter-compatible |
| 139 | + * chat/completions endpoint: captures each request body for assertions and |
| 140 | + * forwards it unchanged to the upstream (aimock in replay/record mode). |
| 141 | + */ |
| 142 | +async function withOpenRouterCaptureProxy<T>( |
| 143 | + upstreamUrl: string, |
| 144 | + run: (args: { proxyUrl: string; requests: CapturedDteRequest[] }) => Promise<T>, |
| 145 | +): Promise<T> { |
| 146 | + const requests: CapturedDteRequest[] = [] |
| 147 | + const upstreamTarget = resolveAllowedUpstreamUrl(upstreamUrl) |
| 148 | + let proxyError: Error | undefined |
| 149 | + |
| 150 | + const server = createServer(async (req, res) => { |
| 151 | + try { |
| 152 | + const requestUrl = req.url ?? "/" |
| 153 | + |
| 154 | + if (!isChatCompletionsUrl("http://127.0.0.1" + requestUrl)) { |
| 155 | + res.writeHead(404) |
| 156 | + res.end("Not found") |
| 157 | + return |
| 158 | + } |
| 159 | + |
| 160 | + const bodyText = await readRequestBody(req) |
| 161 | + const body = JSON.parse(bodyText) as OpenRouterChatCompletionBody |
| 162 | + const lastUser = [...(body.messages ?? [])].reverse().find((message) => message.role === "user") |
| 163 | + const lastUserMessage = |
| 164 | + typeof lastUser?.content === "string" ? lastUser.content : JSON.stringify(lastUser?.content ?? "") |
| 165 | + |
| 166 | + requests.push({ |
| 167 | + model: body.model, |
| 168 | + reasoning: body.reasoning, |
| 169 | + carriesSetEffortToolResult: JSON.stringify(body.messages ?? []).includes(SET_EFFORT_TOOL_CALL_ID), |
| 170 | + lastUserMessage, |
| 171 | + }) |
| 172 | + |
| 173 | + const forwardHeaders: Record<string, string> = {} |
| 174 | + for (const [key, value] of Object.entries(req.headers)) { |
| 175 | + if (!HOP_BY_HOP.has(key.toLowerCase()) && value !== undefined) { |
| 176 | + forwardHeaders[key] = Array.isArray(value) ? value.join(", ") : value |
| 177 | + } |
| 178 | + } |
| 179 | + |
| 180 | + const upstream = await fetch(upstreamTarget, { |
| 181 | + method: req.method, |
| 182 | + headers: forwardHeaders, |
| 183 | + body: bodyText, |
| 184 | + }) |
| 185 | + |
| 186 | + await pipeFetchResponse(res, upstream) |
| 187 | + } catch (error) { |
| 188 | + proxyError = error instanceof Error ? error : new Error(String(error)) |
| 189 | + console.error("OpenRouter proxy request failed:", proxyError) |
| 190 | + if (!res.headersSent) { |
| 191 | + res.writeHead(502) |
| 192 | + res.end("Capture proxy error") |
| 193 | + } else if (!res.writableEnded) { |
| 194 | + res.destroy() |
| 195 | + } |
| 196 | + } |
| 197 | + }) |
| 198 | + |
| 199 | + await new Promise<void>((resolve) => { |
| 200 | + server.listen(0, "127.0.0.1", () => resolve()) |
| 201 | + }) |
| 202 | + |
| 203 | + const address = server.address() |
| 204 | + if (address === null || typeof address === "string") { |
| 205 | + server.close() |
| 206 | + throw new Error("Capture proxy failed to bind a loopback port") |
| 207 | + } |
| 208 | + |
| 209 | + const proxyUrl = "http://127.0.0.1:" + address.port |
| 210 | + |
| 211 | + try { |
| 212 | + const result = await run({ proxyUrl, requests }) |
| 213 | + if (proxyError) { |
| 214 | + throw proxyError |
| 215 | + } |
| 216 | + return result |
| 217 | + } finally { |
| 218 | + await new Promise<void>((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))) |
| 219 | + } |
| 220 | +} |
| 221 | + |
| 222 | +suite("set_thinking_effort mid-task workflow (DTE addendum)", function () { |
| 223 | + setDefaultSuiteTimeout(this) |
| 224 | + |
| 225 | + // Restore the default OpenRouter configuration (and switch the experiment off) |
| 226 | + // so subsequent suites are unaffected. |
| 227 | + suiteTeardown(async () => { |
| 228 | + const aimockUrl = process.env.AIMOCK_URL |
| 229 | + const isRecord = process.env.AIMOCK_RECORD === "true" |
| 230 | + await globalThis.api.setConfiguration({ |
| 231 | + apiProvider: "openrouter" as const, |
| 232 | + openRouterApiKey: aimockUrl && !isRecord ? "mock-key" : process.env.OPENROUTER_API_KEY!, |
| 233 | + openRouterModelId: "openai/gpt-4.1", |
| 234 | + ...(aimockUrl && { openRouterBaseUrl: `${aimockUrl}/v1` }), |
| 235 | + experiments: { dynamicThinkingEffort: false }, |
| 236 | + }) |
| 237 | + }) |
| 238 | + |
| 239 | + test("Should apply set_thinking_effort mid-task, emit the display say, and send the applied effort on the next request", async function () { |
| 240 | + const api = globalThis.api |
| 241 | + const aimockUrl = process.env.AIMOCK_URL |
| 242 | + |
| 243 | + // Deterministic, key-free: aimock replay/record only. A live run would need |
| 244 | + // a real model that deterministically emits the tool call. |
| 245 | + if (!aimockUrl) { |
| 246 | + this.skip() |
| 247 | + } |
| 248 | + |
| 249 | + await withOpenRouterCaptureProxy(aimockUrl, async ({ proxyUrl, requests }) => { |
| 250 | + // OpenRouter provider, a model that advertises per-request reasoning |
| 251 | + // effort, and the dynamicThinkingEffort experiment enabled. |
| 252 | + await api.setConfiguration({ |
| 253 | + apiProvider: "openrouter" as const, |
| 254 | + openRouterApiKey: "mock-key", |
| 255 | + openRouterModelId: DTE_MODEL_ID, |
| 256 | + openRouterBaseUrl: `${proxyUrl}/v1`, |
| 257 | + enableReasoningEffort: true, |
| 258 | + experiments: { dynamicThinkingEffort: true }, |
| 259 | + }) |
| 260 | + |
| 261 | + const messages: ClineMessage[] = [] |
| 262 | + const onMessage = ({ message }: { message: ClineMessage }) => { |
| 263 | + if (message.type === "say" && message.partial === false) { |
| 264 | + messages.push(message) |
| 265 | + } |
| 266 | + } |
| 267 | + api.on(RooCodeEventName.Message, onMessage) |
| 268 | + |
| 269 | + const taskId = await api.startNewTask({ |
| 270 | + configuration: { mode: "ask", alwaysAllowModeSwitch: true, autoApprovalEnabled: true }, |
| 271 | + text: APPLY_MARKER + ": answer the math question", |
| 272 | + }) |
| 273 | + |
| 274 | + await waitUntilCompleted({ api, taskId }) |
| 275 | + api.off(RooCodeEventName.Message, onMessage) |
| 276 | + |
| 277 | + // (a) Real boundary: the task completes with the math answer after the |
| 278 | + // mid-task tool round trip. |
| 279 | + const completion = messages.find( |
| 280 | + ({ say, text }) => |
| 281 | + (say === "completion_result" || say === "text") && text?.trim() === COMPLETION_EXPECTED, |
| 282 | + ) |
| 283 | + assert.ok(completion, "Task should complete with '" + COMPLETION_EXPECTED + "' after set_thinking_effort") |
| 284 | + |
| 285 | + // (b) Real boundary: the SetThinkingEffortTool display say carries the |
| 286 | + // applied effort (not a refusal). |
| 287 | + const effortSays = messages.filter( |
| 288 | + ({ say, text }) => say === "tool" && typeof text === "string" && text.includes("thinkingEffort"), |
| 289 | + ) |
| 290 | + const appliedSay = effortSays.find(({ text }) => text?.includes('"high"')) |
| 291 | + assert.ok(appliedSay, "SetThinkingEffortTool should emit a 'tool' say carrying the applied effort") |
| 292 | + const effortPayload = JSON.parse(appliedSay.text ?? "") as { |
| 293 | + tool?: string |
| 294 | + effort?: string |
| 295 | + reason?: string |
| 296 | + refusal?: string |
| 297 | + } |
| 298 | + assert.strictEqual( |
| 299 | + effortPayload.tool, |
| 300 | + "thinkingEffort", |
| 301 | + "display say should identify the thinkingEffort event", |
| 302 | + ) |
| 303 | + assert.strictEqual(effortPayload.effort, "high", "display say should carry the applied 'high' effort") |
| 304 | + assert.strictEqual(effortPayload.reason, "multi-step math", "display say should carry the model's reason") |
| 305 | + assert.strictEqual( |
| 306 | + effortPayload.refusal, |
| 307 | + undefined, |
| 308 | + "the effort change should have been applied, not refused", |
| 309 | + ) |
| 310 | + |
| 311 | + // (c) Real boundary: the request AFTER the tool round trip carries the |
| 312 | + // applied effort in the OpenRouter reasoning envelope. |
| 313 | + const preToolRequest = requests.find( |
| 314 | + (request) => !request.carriesSetEffortToolResult && request.lastUserMessage.includes(APPLY_MARKER), |
| 315 | + ) |
| 316 | + assert.ok(preToolRequest, "Should have captured the pre-tool request containing the task prompt") |
| 317 | + assert.strictEqual(preToolRequest.model, DTE_MODEL_ID) |
| 318 | + assert.notStrictEqual( |
| 319 | + preToolRequest.reasoning?.effort, |
| 320 | + "high", |
| 321 | + "the baseline request should not already carry the 'high' effort", |
| 322 | + ) |
| 323 | + |
| 324 | + const postToolRequest = requests.find((request) => request.carriesSetEffortToolResult) |
| 325 | + assert.ok(postToolRequest, "The follow-up request should carry the set_thinking_effort tool result") |
| 326 | + assert.strictEqual(postToolRequest.model, DTE_MODEL_ID) |
| 327 | + assert.ok(postToolRequest.reasoning, "Post-tool request should carry a reasoning envelope") |
| 328 | + assert.strictEqual( |
| 329 | + postToolRequest.reasoning.effort, |
| 330 | + "high", |
| 331 | + "Post-tool request should send the applied 'high' effort", |
| 332 | + ) |
| 333 | + }) |
| 334 | + }) |
| 335 | +}) |
0 commit comments