Skip to content

Commit e83af72

Browse files
committed
test(e2e): set_thinking_effort mid-task workflow (DTE addendum)
1 parent 19954d3 commit e83af72

2 files changed

Lines changed: 368 additions & 0 deletions

File tree

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
{
2+
"fixtures": [
3+
{
4+
"match": {
5+
"sequenceIndex": 0,
6+
"userMessage": "DTE_E2E_EFFORT_APPLY: answer the math question"
7+
},
8+
"response": {
9+
"toolCalls": [
10+
{
11+
"name": "set_thinking_effort",
12+
"arguments": "{\"effort\": \"high\", \"reason\": \"multi-step math\"}",
13+
"id": "call_dte_e2e_001"
14+
}
15+
]
16+
}
17+
},
18+
{
19+
"match": {
20+
"toolCallId": "call_dte_e2e_001"
21+
},
22+
"response": {
23+
"toolCalls": [
24+
{
25+
"name": "attempt_completion",
26+
"arguments": "{\"result\": \"42\"}",
27+
"id": "call_dte_e2e_002"
28+
}
29+
]
30+
}
31+
}
32+
]
33+
}
Lines changed: 335 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,335 @@
1+
import * as assert from "assert"
2+
import { createServer, type IncomingMessage, type ServerResponse } from "http"
3+
4+
import { RooCodeEventName, type ClineMessage } from "@roo-code/types"
5+
6+
import { setDefaultSuiteTimeout } from "./test-utils"
7+
import { waitUntilCompleted } from "./utils"
8+
9+
/**
10+
* DTE addendum: set_thinking_effort mid-task workflow.
11+
*
12+
* Exercises the real extension-host boundary end to end with aimock fixtures:
13+
* the model calls set_thinking_effort mid-task (no approval gate), the
14+
* SetThinkingEffortTool display say is emitted, and the FOLLOWING API request
15+
* carries the applied effort in the OpenRouter reasoning envelope.
16+
*
17+
* Determinism notes:
18+
* - Runs against aimock only (replay or record); skips when aimock is absent,
19+
* so no API keys are required.
20+
* - The capture proxy is the local 127.0.0.1 pattern from
21+
* anthropic-opus-4-7.test.ts: it intercepts the OpenRouter-compatible
22+
* chat/completions POST (so request shapes can be asserted) and forwards it
23+
* to aimock for the fixture-driven SSE responses.
24+
* - The OpenRouter model catalog is resolved by the shared model-cache layer
25+
* (fetchers/modelCache.ts) from the public OpenRouter endpoint, exactly like
26+
* the other provider suites. openai/gpt-5 advertises "reasoning" in
27+
* supported_parameters, so the fetcher resolves supportsReasoningEffort and
28+
* the dynamicThinkingEffort gate exposes the tool.
29+
* - The mid-task tool call is dispatched by name in presentAssistantMessage
30+
* (a hard-coded case, not the request-declared tool list), and the tool
31+
* executor re-checks the model capability after the first request has loaded
32+
* the catalog, so the flow is correct even if the first request's tool list
33+
* was built before the catalog fetch resolved.
34+
*/
35+
36+
const DTE_MODEL_ID = "openai/gpt-5"
37+
const APPLY_MARKER = "DTE_E2E_EFFORT_APPLY"
38+
const SET_EFFORT_TOOL_CALL_ID = "call_dte_e2e_001"
39+
const COMPLETION_EXPECTED = "42"
40+
41+
type DteReasoningEnvelope = {
42+
effort?: string
43+
max_tokens?: number
44+
exclude?: boolean
45+
}
46+
47+
type CapturedDteRequest = {
48+
model?: string
49+
reasoning: DteReasoningEnvelope | undefined
50+
carriesSetEffortToolResult: boolean
51+
lastUserMessage: string
52+
}
53+
54+
type OpenRouterChatCompletionBody = {
55+
model?: string
56+
reasoning?: DteReasoningEnvelope
57+
messages?: Array<{ role?: string; content?: unknown }>
58+
}
59+
60+
const ALLOWED_PROXY_HOSTS = new Set(["127.0.0.1", "localhost"])
61+
const CHAT_COMPLETIONS_PATH = "/v1/chat/completions"
62+
const HOP_BY_HOP = new Set([
63+
"connection",
64+
"keep-alive",
65+
"transfer-encoding",
66+
"te",
67+
"trailer",
68+
"upgrade",
69+
"proxy-connection",
70+
"proxy-authenticate",
71+
"proxy-authorization",
72+
"host",
73+
"content-length",
74+
])
75+
76+
function isChatCompletionsUrl(rawUrl: string): boolean {
77+
try {
78+
return new URL(rawUrl).pathname.endsWith(CHAT_COMPLETIONS_PATH)
79+
} catch {
80+
return false
81+
}
82+
}
83+
84+
function readRequestBody(req: IncomingMessage): Promise<string> {
85+
return new Promise((resolve, reject) => {
86+
const chunks: Buffer[] = []
87+
req.on("data", (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)))
88+
req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8")))
89+
req.on("error", reject)
90+
})
91+
}
92+
93+
function writeResponseHeaders(target: ServerResponse, source: Response) {
94+
const headers: Record<string, string> = {}
95+
source.headers.forEach((value, key) => {
96+
const lower = key.toLowerCase()
97+
// fetch() automatically decompresses the body, so strip content-encoding to
98+
// prevent the SDK from attempting a second decompression. Also strip
99+
// content-length since the decoded body length differs from the compressed one.
100+
if (lower !== "content-length" && lower !== "content-encoding") {
101+
headers[key] = value
102+
}
103+
})
104+
target.writeHead(source.status, headers)
105+
}
106+
107+
async function pipeFetchResponse(target: ServerResponse, source: Response) {
108+
writeResponseHeaders(target, source)
109+
110+
if (!source.body) {
111+
target.end()
112+
return
113+
}
114+
115+
const reader = source.body.getReader()
116+
while (true) {
117+
const { done, value } = await reader.read()
118+
if (done) {
119+
break
120+
}
121+
target.write(value)
122+
}
123+
124+
target.end()
125+
}
126+
127+
function resolveAllowedUpstreamUrl(baseUrl: string): URL {
128+
const upstreamBase = new URL(baseUrl)
129+
130+
if (!ALLOWED_PROXY_HOSTS.has(upstreamBase.hostname) || upstreamBase.protocol !== "http:") {
131+
throw new Error("Unexpected OpenRouter proxy target: " + upstreamBase.origin)
132+
}
133+
134+
return new URL(CHAT_COMPLETIONS_PATH, upstreamBase)
135+
}
136+
137+
/**
138+
* Serves a loopback capture proxy for the OpenRouter-compatible
139+
* chat/completions endpoint: captures each request body for assertions and
140+
* forwards it unchanged to the upstream (aimock in replay/record mode).
141+
*/
142+
async function withOpenRouterCaptureProxy<T>(
143+
upstreamUrl: string,
144+
run: (args: { proxyUrl: string; requests: CapturedDteRequest[] }) => Promise<T>,
145+
): Promise<T> {
146+
const requests: CapturedDteRequest[] = []
147+
const upstreamTarget = resolveAllowedUpstreamUrl(upstreamUrl)
148+
let proxyError: Error | undefined
149+
150+
const server = createServer(async (req, res) => {
151+
try {
152+
const requestUrl = req.url ?? "/"
153+
154+
if (!isChatCompletionsUrl("http://127.0.0.1" + requestUrl)) {
155+
res.writeHead(404)
156+
res.end("Not found")
157+
return
158+
}
159+
160+
const bodyText = await readRequestBody(req)
161+
const body = JSON.parse(bodyText) as OpenRouterChatCompletionBody
162+
const lastUser = [...(body.messages ?? [])].reverse().find((message) => message.role === "user")
163+
const lastUserMessage =
164+
typeof lastUser?.content === "string" ? lastUser.content : JSON.stringify(lastUser?.content ?? "")
165+
166+
requests.push({
167+
model: body.model,
168+
reasoning: body.reasoning,
169+
carriesSetEffortToolResult: JSON.stringify(body.messages ?? []).includes(SET_EFFORT_TOOL_CALL_ID),
170+
lastUserMessage,
171+
})
172+
173+
const forwardHeaders: Record<string, string> = {}
174+
for (const [key, value] of Object.entries(req.headers)) {
175+
if (!HOP_BY_HOP.has(key.toLowerCase()) && value !== undefined) {
176+
forwardHeaders[key] = Array.isArray(value) ? value.join(", ") : value
177+
}
178+
}
179+
180+
const upstream = await fetch(upstreamTarget, {
181+
method: req.method,
182+
headers: forwardHeaders,
183+
body: bodyText,
184+
})
185+
186+
await pipeFetchResponse(res, upstream)
187+
} catch (error) {
188+
proxyError = error instanceof Error ? error : new Error(String(error))
189+
console.error("OpenRouter proxy request failed:", proxyError)
190+
if (!res.headersSent) {
191+
res.writeHead(502)
192+
res.end("Capture proxy error")
193+
} else if (!res.writableEnded) {
194+
res.destroy()
195+
}
196+
}
197+
})
198+
199+
await new Promise<void>((resolve) => {
200+
server.listen(0, "127.0.0.1", () => resolve())
201+
})
202+
203+
const address = server.address()
204+
if (address === null || typeof address === "string") {
205+
server.close()
206+
throw new Error("Capture proxy failed to bind a loopback port")
207+
}
208+
209+
const proxyUrl = "http://127.0.0.1:" + address.port
210+
211+
try {
212+
const result = await run({ proxyUrl, requests })
213+
if (proxyError) {
214+
throw proxyError
215+
}
216+
return result
217+
} finally {
218+
await new Promise<void>((resolve, reject) => server.close((error) => (error ? reject(error) : resolve())))
219+
}
220+
}
221+
222+
suite("set_thinking_effort mid-task workflow (DTE addendum)", function () {
223+
setDefaultSuiteTimeout(this)
224+
225+
// Restore the default OpenRouter configuration (and switch the experiment off)
226+
// so subsequent suites are unaffected.
227+
suiteTeardown(async () => {
228+
const aimockUrl = process.env.AIMOCK_URL
229+
const isRecord = process.env.AIMOCK_RECORD === "true"
230+
await globalThis.api.setConfiguration({
231+
apiProvider: "openrouter" as const,
232+
openRouterApiKey: aimockUrl && !isRecord ? "mock-key" : process.env.OPENROUTER_API_KEY!,
233+
openRouterModelId: "openai/gpt-4.1",
234+
...(aimockUrl && { openRouterBaseUrl: `${aimockUrl}/v1` }),
235+
experiments: { dynamicThinkingEffort: false },
236+
})
237+
})
238+
239+
test("Should apply set_thinking_effort mid-task, emit the display say, and send the applied effort on the next request", async function () {
240+
const api = globalThis.api
241+
const aimockUrl = process.env.AIMOCK_URL
242+
243+
// Deterministic, key-free: aimock replay/record only. A live run would need
244+
// a real model that deterministically emits the tool call.
245+
if (!aimockUrl) {
246+
this.skip()
247+
}
248+
249+
await withOpenRouterCaptureProxy(aimockUrl, async ({ proxyUrl, requests }) => {
250+
// OpenRouter provider, a model that advertises per-request reasoning
251+
// effort, and the dynamicThinkingEffort experiment enabled.
252+
await api.setConfiguration({
253+
apiProvider: "openrouter" as const,
254+
openRouterApiKey: "mock-key",
255+
openRouterModelId: DTE_MODEL_ID,
256+
openRouterBaseUrl: `${proxyUrl}/v1`,
257+
enableReasoningEffort: true,
258+
experiments: { dynamicThinkingEffort: true },
259+
})
260+
261+
const messages: ClineMessage[] = []
262+
const onMessage = ({ message }: { message: ClineMessage }) => {
263+
if (message.type === "say" && message.partial === false) {
264+
messages.push(message)
265+
}
266+
}
267+
api.on(RooCodeEventName.Message, onMessage)
268+
269+
const taskId = await api.startNewTask({
270+
configuration: { mode: "ask", alwaysAllowModeSwitch: true, autoApprovalEnabled: true },
271+
text: APPLY_MARKER + ": answer the math question",
272+
})
273+
274+
await waitUntilCompleted({ api, taskId })
275+
api.off(RooCodeEventName.Message, onMessage)
276+
277+
// (a) Real boundary: the task completes with the math answer after the
278+
// mid-task tool round trip.
279+
const completion = messages.find(
280+
({ say, text }) =>
281+
(say === "completion_result" || say === "text") && text?.trim() === COMPLETION_EXPECTED,
282+
)
283+
assert.ok(completion, "Task should complete with '" + COMPLETION_EXPECTED + "' after set_thinking_effort")
284+
285+
// (b) Real boundary: the SetThinkingEffortTool display say carries the
286+
// applied effort (not a refusal).
287+
const effortSays = messages.filter(
288+
({ say, text }) => say === "tool" && typeof text === "string" && text.includes("thinkingEffort"),
289+
)
290+
const appliedSay = effortSays.find(({ text }) => text?.includes('"high"'))
291+
assert.ok(appliedSay, "SetThinkingEffortTool should emit a 'tool' say carrying the applied effort")
292+
const effortPayload = JSON.parse(appliedSay.text ?? "") as {
293+
tool?: string
294+
effort?: string
295+
reason?: string
296+
refusal?: string
297+
}
298+
assert.strictEqual(
299+
effortPayload.tool,
300+
"thinkingEffort",
301+
"display say should identify the thinkingEffort event",
302+
)
303+
assert.strictEqual(effortPayload.effort, "high", "display say should carry the applied 'high' effort")
304+
assert.strictEqual(effortPayload.reason, "multi-step math", "display say should carry the model's reason")
305+
assert.strictEqual(
306+
effortPayload.refusal,
307+
undefined,
308+
"the effort change should have been applied, not refused",
309+
)
310+
311+
// (c) Real boundary: the request AFTER the tool round trip carries the
312+
// applied effort in the OpenRouter reasoning envelope.
313+
const preToolRequest = requests.find(
314+
(request) => !request.carriesSetEffortToolResult && request.lastUserMessage.includes(APPLY_MARKER),
315+
)
316+
assert.ok(preToolRequest, "Should have captured the pre-tool request containing the task prompt")
317+
assert.strictEqual(preToolRequest.model, DTE_MODEL_ID)
318+
assert.notStrictEqual(
319+
preToolRequest.reasoning?.effort,
320+
"high",
321+
"the baseline request should not already carry the 'high' effort",
322+
)
323+
324+
const postToolRequest = requests.find((request) => request.carriesSetEffortToolResult)
325+
assert.ok(postToolRequest, "The follow-up request should carry the set_thinking_effort tool result")
326+
assert.strictEqual(postToolRequest.model, DTE_MODEL_ID)
327+
assert.ok(postToolRequest.reasoning, "Post-tool request should carry a reasoning envelope")
328+
assert.strictEqual(
329+
postToolRequest.reasoning.effort,
330+
"high",
331+
"Post-tool request should send the applied 'high' effort",
332+
)
333+
})
334+
})
335+
})

0 commit comments

Comments
 (0)