Skip to content

Commit 27a2e97

Browse files
easonliang28easonliang28
committed
merge dte-3: predicate-scoped DTE e2e fixtures (CodeRabbit 3862663852 closeout)
2 parents 0902687 + 0b335c6 commit 27a2e97

6 files changed

Lines changed: 178 additions & 144 deletions

File tree

‎apps/vscode-e2e/fixtures/thinking-effort-switching.json‎

Lines changed: 0 additions & 89 deletions
This file was deleted.

‎apps/vscode-e2e/fixtures/thinking-effort-tool.json‎

Lines changed: 0 additions & 36 deletions
This file was deleted.

‎apps/vscode-e2e/src/fixtures/thinking-effort.ts‎

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
import { LLMock } from "@copilotkit/aimock"
2+
import type { ChatCompletionRequest } from "@copilotkit/aimock"
3+
4+
// DTE e2e fixtures for the thinking-effort-tool and thinking-effort-switching
5+
// suites. Replaces the former JSON fixtures (fixtures/thinking-effort-*.json):
6+
// post-tool requests end with a role:user message (fresh <environment_details>
7+
// is appended after the tool result), so aimock's toolCallId matcher — which
8+
// requires the LAST message to be role:tool — can never bind those continuation
9+
// turns, and JSON fixtures cannot carry predicates. Each turn is instead scoped
10+
// by a predicate that searches the whole request for its own flow identifiers:
11+
// the unique prompt marker for the baseline turn and the previous turn's unique
12+
// tool call id for the continuations (same pattern as deepseek-v4.ts). No other
13+
// suite can serve these responses and these suites cannot match unrelated turns.
14+
15+
const SWITCH_MODEL = "openai/gpt-5.1"
16+
const APPLY_MODEL = "openai/gpt-5"
17+
const SWITCH_MARKER = "DTE_E2E_SWITCH"
18+
const APPLY_MARKER = "DTE_E2E_EFFORT_APPLY"
19+
const SWITCH_DONE = "DTE_E2E_SWITCH_DONE"
20+
21+
// Post-tool requests carry the tool result of the PREVIOUS turn and nothing
22+
// appends another tool result before the next API call, so the LAST role:tool
23+
// message is exactly the call whose result this request follows.
24+
const lastToolCallId = (req: ChatCompletionRequest): string | undefined => {
25+
const messages = Array.isArray(req?.messages) ? req.messages : []
26+
return messages.filter((message) => message?.role === "tool").at(-1)?.tool_call_id
27+
}
28+
29+
// aimock's userMessage matcher only inspects the LAST user message and joins
30+
// only the type:"text" content parts (getTextContent in aimock's router) — the
31+
// predicate replicates that semantics for the baseline turns.
32+
const lastUserMessageContains = (req: ChatCompletionRequest, text: string): boolean => {
33+
const userMessages = req.messages?.filter((message) => message.role === "user") ?? []
34+
const last = userMessages.at(-1)
35+
if (!last) return false
36+
const content =
37+
typeof last.content === "string"
38+
? last.content
39+
: (last.content ?? [])
40+
.filter((part): part is { type: "text"; text: string } => part?.type === "text")
41+
.map((part) => part.text)
42+
.join("")
43+
return content.includes(text)
44+
}
45+
46+
export function addThinkingEffortFixtures(mock: InstanceType<typeof LLMock>) {
47+
// --- thinking-effort-switching suite (openai/gpt-5.1) ---
48+
// Baseline turn: bound to this suite's unique prompt marker.
49+
mock.addFixture({
50+
match: {
51+
model: SWITCH_MODEL,
52+
predicate: (req: ChatCompletionRequest) => lastUserMessageContains(req, SWITCH_MARKER),
53+
},
54+
response: {
55+
toolCalls: [
56+
{
57+
name: "set_thinking_effort",
58+
arguments: JSON.stringify({ effort: "medium", reason: "start at medium" }),
59+
id: "call_dte_sw_001",
60+
},
61+
],
62+
},
63+
})
64+
65+
// Continuations: each turn binds to the previous turn's unique tool call id,
66+
// so a future flow on the same model cannot serve these responses.
67+
const switchingContinuations: Array<{
68+
afterCallId: string
69+
effort: string
70+
reason: string
71+
responseCallId: string
72+
}> = [
73+
{
74+
afterCallId: "call_dte_sw_001",
75+
effort: "medium",
76+
reason: "confirm current level",
77+
responseCallId: "call_dte_sw_002",
78+
},
79+
{ afterCallId: "call_dte_sw_002", effort: "high", reason: "raise to high", responseCallId: "call_dte_sw_003" },
80+
{
81+
afterCallId: "call_dte_sw_003",
82+
effort: "medium",
83+
reason: "try returning to medium",
84+
responseCallId: "call_dte_sw_004",
85+
},
86+
]
87+
for (const continuation of switchingContinuations) {
88+
mock.addFixture({
89+
match: {
90+
model: SWITCH_MODEL,
91+
predicate: (req: ChatCompletionRequest) => lastToolCallId(req) === continuation.afterCallId,
92+
},
93+
response: {
94+
toolCalls: [
95+
{
96+
name: "set_thinking_effort",
97+
arguments: JSON.stringify({
98+
effort: continuation.effort,
99+
reason: continuation.reason,
100+
}),
101+
id: continuation.responseCallId,
102+
},
103+
],
104+
},
105+
})
106+
}
107+
108+
// Final turn: after the refused oscillation call, the task completes.
109+
mock.addFixture({
110+
match: {
111+
model: SWITCH_MODEL,
112+
predicate: (req: ChatCompletionRequest) => lastToolCallId(req) === "call_dte_sw_004",
113+
},
114+
response: {
115+
toolCalls: [
116+
{
117+
name: "attempt_completion",
118+
arguments: JSON.stringify({ result: SWITCH_DONE }),
119+
id: "call_dte_sw_005",
120+
},
121+
],
122+
},
123+
})
124+
125+
// --- thinking-effort-tool suite (openai/gpt-5) ---
126+
// Baseline turn: bound to this suite's unique prompt marker.
127+
mock.addFixture({
128+
match: {
129+
model: APPLY_MODEL,
130+
predicate: (req: ChatCompletionRequest) => lastUserMessageContains(req, APPLY_MARKER),
131+
},
132+
response: {
133+
toolCalls: [
134+
{
135+
name: "set_thinking_effort",
136+
arguments: JSON.stringify({ effort: "high", reason: "multi-step math" }),
137+
id: "call_dte_e2e_001",
138+
},
139+
],
140+
},
141+
})
142+
143+
// Continuation: binds to the set_thinking_effort call's unique tool call id.
144+
mock.addFixture({
145+
match: {
146+
model: APPLY_MODEL,
147+
predicate: (req: ChatCompletionRequest) => lastToolCallId(req) === "call_dte_e2e_001",
148+
},
149+
response: {
150+
toolCalls: [
151+
{
152+
name: "attempt_completion",
153+
arguments: JSON.stringify({ result: "42" }),
154+
id: "call_dte_e2e_002",
155+
},
156+
],
157+
},
158+
})
159+
}

‎apps/vscode-e2e/src/runTest.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import { addListFilesResultFixtures } from "./fixtures/list-files"
1919
import { addReadFileResultFixtures } from "./fixtures/read-file"
2020
import { addSearchFilesResultFixtures } from "./fixtures/search-files"
2121
import { addDteNewTaskEffortFixtures, addSubtaskFixtures } from "./fixtures/subtasks"
22+
import { addThinkingEffortFixtures } from "./fixtures/thinking-effort"
2223
import { addUseMcpToolResultFixtures } from "./fixtures/use-mcp-tool"
2324
import { addWriteToFileResultFixtures } from "./fixtures/write-to-file"
2425
import { createScenarioWorkspace, removeScenarioWorkspace } from "./restart/scenarioWorkspace"
@@ -141,6 +142,7 @@ async function main() {
141142
addSearchFilesResultFixtures(mock)
142143
addSubtaskFixtures(mock)
143144
addDteNewTaskEffortFixtures(mock)
145+
addThinkingEffortFixtures(mock)
144146
addUseMcpToolResultFixtures(mock)
145147
addWriteToFileResultFixtures(mock)
146148
addDeepSeekV4Fixtures(mock)

‎apps/vscode-e2e/src/suite/thinking-effort-switching.test.ts‎

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -36,21 +36,17 @@ import { waitUntilCompleted, waitFor } from "./utils"
3636
* replaces the whole provider profile (ProviderSettingsManager.saveConfig),
3737
* so the baseline is deterministic and cannot inherit state from other
3838
* suites; "low" is distinct from every level the tool applies here.
39-
* - Fixture matching (apps/vscode-e2e/fixtures/thinking-effort-switching.json):
39+
* - Fixture matching (apps/vscode-e2e/src/fixtures/thinking-effort.ts):
4040
* post-tool requests end with a role:user message (fresh environment details
4141
* are appended after the tool result), so aimock's toolCallId matcher — which
42-
* inspects only the LAST message — can never match them; the fixtures are
43-
* scoped to this flow with the match keys a JSON fixture can express: the
44-
* first turn binds to this suite's unique prompt marker (userMessage
45-
* "DTE_E2E_SWITCH"), and every follow-up turn additionally binds to the
46-
* post-tool continuation shape (userMessage "<environment_details>": after
47-
* each tool result the request's last user message is the fresh
48-
* environment-details block), on top of model + hasToolResult + turnIndex.
49-
* aimock's selectByTurnIndex picks the highest turnIndex <= the request's
50-
* assistant count, and each turnIndex is unique per fixture, so every
51-
* request matches exactly one fixture. Note the fixture file is plain JSON:
52-
* aimock's fixture loader uses JSON.parse and SKIPS the whole file on parse
53-
* errors, so no // comments may be added to it.
42+
* requires the LAST message to be role:tool — can never bind the continuation
43+
* turns, and a JSON fixture cannot carry a predicate. The fixtures live in a
44+
* JS module added via addThinkingEffortFixtures (same pattern as
45+
* deepseek-v4.ts): the baseline turn binds to this suite's unique prompt
46+
* marker ("DTE_E2E_SWITCH"), and every continuation binds to the previous
47+
* turn's unique tool call id (call_dte_sw_001 → 002 → 003 → 004 →
48+
* completion), so no other suite can serve these responses and this suite
49+
* cannot match unrelated turns.
5450
*/
5551

5652
const SWITCH_MODEL_ID = "openai/gpt-5.1"

‎apps/vscode-e2e/src/suite/thinking-effort-tool.test.ts‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,14 +31,16 @@ import { waitUntilCompleted } from "./utils"
3131
* executor re-checks the model capability after the first request has loaded
3232
* the catalog, so the flow is correct even if the first request's tool list
3333
* was built before the catalog fetch resolved.
34-
* - Fixture matching (apps/vscode-e2e/fixtures/thinking-effort-tool.json): the
34+
* - Fixture matching (apps/vscode-e2e/src/fixtures/thinking-effort.ts): the
3535
* post-tool request ends with a role:user message (fresh environment details
3636
* are appended after the tool result), so aimock's toolCallId matcher — which
37-
* inspects only the LAST message — can never match it. The follow-up request is
38-
* scoped by the DTE-only model + hasToolResult instead, with turnIndex 1 as a
39-
* tie-break. Note the fixture file is plain JSON: aimock's fixture-loader uses
40-
* JSON.parse and SKIPS the whole file on parse errors, so no // comments may be
41-
* added to it.
37+
* requires the LAST message to be role:tool — can never bind it, and a JSON
38+
* fixture cannot carry a predicate. The fixtures live in a JS module added via
39+
* addThinkingEffortFixtures (same pattern as deepseek-v4.ts): the baseline turn
40+
* binds to this suite's unique prompt marker ("DTE_E2E_EFFORT_APPLY") and the
41+
* follow-up binds to the set_thinking_effort call's unique tool call id
42+
* ("call_dte_e2e_001"), so no other suite can serve these responses and this
43+
* suite cannot match unrelated turns.
4244
*/
4345

4446
const DTE_MODEL_ID = "openai/gpt-5"

0 commit comments

Comments
 (0)