Skip to content

Commit 37c8040

Browse files
committed
test(e2e): set_thinking_effort switching workflow (DTE addendum)
Adds a second DTE e2e suite that drives one task through a scripted switching sequence (baseline -> applied -> no-op -> applied -> oscillation refusal) against openai/gpt-5.1, asserting the per-request OpenRouter reasoning envelope plus the display says and tool results. Extracts the shared OpenRouter capture proxy from thinking-effort-tool.test.ts into thinking-effort-proxy.ts and switches that suite's request lookups to raw-body tool-call-id matching. Fixtures are scoped by model + hasToolResult + unique turnIndex because aimock's toolCallId matcher only inspects the last message and post-tool requests end with a fresh user message.
1 parent 1270e7a commit 37c8040

4 files changed

Lines changed: 522 additions & 184 deletions

File tree

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
{
2+
"fixtures": [
3+
{
4+
"match": {
5+
"model": "openai/gpt-5.1",
6+
"hasToolResult": false,
7+
"turnIndex": 0
8+
},
9+
"response": {
10+
"toolCalls": [
11+
{
12+
"name": "set_thinking_effort",
13+
"arguments": "{\"effort\": \"medium\", \"reason\": \"start at medium\"}",
14+
"id": "call_dte_sw_001"
15+
}
16+
]
17+
}
18+
},
19+
{
20+
"match": {
21+
"model": "openai/gpt-5.1",
22+
"hasToolResult": true,
23+
"turnIndex": 1
24+
},
25+
"response": {
26+
"toolCalls": [
27+
{
28+
"name": "set_thinking_effort",
29+
"arguments": "{\"effort\": \"medium\", \"reason\": \"confirm current level\"}",
30+
"id": "call_dte_sw_002"
31+
}
32+
]
33+
}
34+
},
35+
{
36+
"match": {
37+
"model": "openai/gpt-5.1",
38+
"hasToolResult": true,
39+
"turnIndex": 2
40+
},
41+
"response": {
42+
"toolCalls": [
43+
{
44+
"name": "set_thinking_effort",
45+
"arguments": "{\"effort\": \"high\", \"reason\": \"raise to high\"}",
46+
"id": "call_dte_sw_003"
47+
}
48+
]
49+
}
50+
},
51+
{
52+
"match": {
53+
"model": "openai/gpt-5.1",
54+
"hasToolResult": true,
55+
"turnIndex": 3
56+
},
57+
"response": {
58+
"toolCalls": [
59+
{
60+
"name": "set_thinking_effort",
61+
"arguments": "{\"effort\": \"medium\", \"reason\": \"try returning to medium\"}",
62+
"id": "call_dte_sw_004"
63+
}
64+
]
65+
}
66+
},
67+
{
68+
"match": {
69+
"model": "openai/gpt-5.1",
70+
"hasToolResult": true,
71+
"turnIndex": 4
72+
},
73+
"response": {
74+
"toolCalls": [
75+
{
76+
"name": "attempt_completion",
77+
"arguments": "{\"result\": \"DTE_E2E_SWITCH_DONE\"}",
78+
"id": "call_dte_sw_005"
79+
}
80+
]
81+
}
82+
}
83+
]
84+
}
Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
import { createServer, type IncomingMessage, type ServerResponse } from "http"
2+
3+
/**
4+
* Shared loopback capture proxy for the DTE e2e suites
5+
* (thinking-effort-tool / thinking-effort-switching).
6+
*
7+
* Pattern from anthropic-opus-4-7.test.ts: it intercepts the
8+
* OpenRouter-compatible chat/completions POST so request shapes can be
9+
* asserted (model, reasoning envelope, message content), then forwards the
10+
* request unchanged to the upstream — aimock in replay/record mode — which
11+
* answers with the fixture-driven SSE.
12+
*/
13+
14+
export type DteReasoningEnvelope = {
15+
effort?: string
16+
max_tokens?: number
17+
exclude?: boolean
18+
}
19+
20+
export type CapturedDteRequest = {
21+
model?: string
22+
reasoning: DteReasoningEnvelope | undefined
23+
/** Raw JSON body, so assertions can inspect any part of the wire request (e.g. tool result text). */
24+
bodyText: string
25+
lastUserMessage: string
26+
}
27+
28+
type OpenRouterChatCompletionBody = {
29+
model?: string
30+
reasoning?: DteReasoningEnvelope
31+
messages?: Array<{ role?: string; content?: unknown }>
32+
}
33+
34+
const ALLOWED_PROXY_HOSTS = new Set(["127.0.0.1", "localhost"])
35+
const CHAT_COMPLETIONS_PATH = "/v1/chat/completions"
36+
const HOP_BY_HOP = new Set([
37+
"connection",
38+
"keep-alive",
39+
"transfer-encoding",
40+
"te",
41+
"trailer",
42+
"upgrade",
43+
"proxy-connection",
44+
"proxy-authenticate",
45+
"proxy-authorization",
46+
"host",
47+
"content-length",
48+
])
49+
50+
function isChatCompletionsUrl(rawUrl: string): boolean {
51+
try {
52+
return new URL(rawUrl).pathname.endsWith(CHAT_COMPLETIONS_PATH)
53+
} catch {
54+
return false
55+
}
56+
}
57+
58+
function readRequestBody(req: IncomingMessage): Promise<string> {
59+
return new Promise((resolve, reject) => {
60+
const chunks: Buffer[] = []
61+
req.on("data", (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)))
62+
req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8")))
63+
req.on("error", reject)
64+
})
65+
}
66+
67+
function writeResponseHeaders(target: ServerResponse, source: Response) {
68+
const headers: Record<string, string> = {}
69+
source.headers.forEach((value, key) => {
70+
const lower = key.toLowerCase()
71+
// fetch() automatically decompresses the body, so strip content-encoding to
72+
// prevent the SDK from attempting a second decompression. Also strip
73+
// content-length since the decoded body length differs from the compressed one.
74+
if (lower !== "content-length" && lower !== "content-encoding") {
75+
headers[key] = value
76+
}
77+
})
78+
target.writeHead(source.status, headers)
79+
}
80+
81+
async function pipeFetchResponse(target: ServerResponse, source: Response) {
82+
writeResponseHeaders(target, source)
83+
84+
if (!source.body) {
85+
target.end()
86+
return
87+
}
88+
89+
const reader = source.body.getReader()
90+
while (true) {
91+
const { done, value } = await reader.read()
92+
if (done) {
93+
break
94+
}
95+
target.write(value)
96+
}
97+
98+
target.end()
99+
}
100+
101+
function resolveAllowedUpstreamUrl(baseUrl: string): URL {
102+
const upstreamBase = new URL(baseUrl)
103+
104+
if (!ALLOWED_PROXY_HOSTS.has(upstreamBase.hostname) || upstreamBase.protocol !== "http:") {
105+
throw new Error("Unexpected OpenRouter proxy target: " + upstreamBase.origin)
106+
}
107+
108+
return new URL(CHAT_COMPLETIONS_PATH, upstreamBase)
109+
}
110+
111+
/**
112+
* Serves a loopback capture proxy for the OpenRouter-compatible
113+
* chat/completions endpoint: captures each request body for assertions and
114+
* forwards it unchanged to the upstream (aimock in replay/record mode).
115+
*/
116+
export async function withOpenRouterCaptureProxy<T>(
117+
upstreamUrl: string,
118+
run: (args: { proxyUrl: string; requests: CapturedDteRequest[] }) => Promise<T>,
119+
): Promise<T> {
120+
const requests: CapturedDteRequest[] = []
121+
const upstreamTarget = resolveAllowedUpstreamUrl(upstreamUrl)
122+
let proxyError: Error | undefined
123+
124+
const server = createServer(async (req, res) => {
125+
try {
126+
const requestUrl = req.url ?? "/"
127+
128+
if (!isChatCompletionsUrl("http://127.0.0.1" + requestUrl)) {
129+
res.writeHead(404)
130+
res.end("Not found")
131+
return
132+
}
133+
134+
const bodyText = await readRequestBody(req)
135+
const body = JSON.parse(bodyText) as OpenRouterChatCompletionBody
136+
const lastUser = [...(body.messages ?? [])].reverse().find((message) => message.role === "user")
137+
const lastUserMessage =
138+
typeof lastUser?.content === "string" ? lastUser.content : JSON.stringify(lastUser?.content ?? "")
139+
140+
requests.push({
141+
model: body.model,
142+
reasoning: body.reasoning,
143+
bodyText,
144+
lastUserMessage,
145+
})
146+
147+
const forwardHeaders: Record<string, string> = {}
148+
for (const [key, value] of Object.entries(req.headers)) {
149+
if (!HOP_BY_HOP.has(key.toLowerCase()) && value !== undefined) {
150+
forwardHeaders[key] = Array.isArray(value) ? value.join(", ") : value
151+
}
152+
}
153+
154+
const upstream = await fetch(upstreamTarget, {
155+
method: req.method,
156+
headers: forwardHeaders,
157+
body: bodyText,
158+
})
159+
160+
await pipeFetchResponse(res, upstream)
161+
} catch (error) {
162+
proxyError = error instanceof Error ? error : new Error(String(error))
163+
console.error("OpenRouter proxy request failed:", proxyError)
164+
if (!res.headersSent) {
165+
res.writeHead(502)
166+
res.end("Capture proxy error")
167+
} else if (!res.writableEnded) {
168+
res.destroy()
169+
}
170+
}
171+
})
172+
173+
await new Promise<void>((resolve) => {
174+
server.listen(0, "127.0.0.1", () => resolve())
175+
})
176+
177+
const address = server.address()
178+
if (address === null || typeof address === "string") {
179+
server.close()
180+
throw new Error("Capture proxy failed to bind a loopback port")
181+
}
182+
183+
const proxyUrl = "http://127.0.0.1:" + address.port
184+
185+
try {
186+
const result = await run({ proxyUrl, requests })
187+
if (proxyError) {
188+
throw proxyError
189+
}
190+
return result
191+
} finally {
192+
await new Promise<void>((resolve, reject) => server.close((error) => (error ? reject(error) : resolve())))
193+
}
194+
}

0 commit comments

Comments
 (0)