Skip to content

Commit 87b57cb

Browse files
simurg79Bertan Ari
andauthored
Update VS Code LM model metadata and regression tests (#2255)
Co-authored-by: Bertan Ari <[email protected]>
1 parent 159c42e commit 87b57cb

4 files changed

Lines changed: 434 additions & 30 deletions

File tree

‎packages/types/src/__tests__/vscode-llm.spec.ts‎

Lines changed: 86 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,7 @@ import { vscodeLlmModels, vscodeLlmDefaultModelId } from "../providers/vscode-ll
33

44
describe("vscodeLlmModels", () => {
55
it("exposes the opus-4.8 row with its measured maxInputTokens and contextWindow", () => {
6-
// claude-opus-4.8 intentionally diverges: maxInputTokens (197897) is the enforced ceiling the
7-
// UI reads, contextWindow (679560) the advertised window. Assert the on-disk literals as a tripwire.
6+
// The two values differ for this row, so pin the exact stored values rather than forcing equality.
87
expect(vscodeLlmModels).toHaveProperty("claude-opus-4.8")
98
expect(vscodeLlmModels["claude-opus-4.8"].contextWindow).toBe(679560)
109
expect(vscodeLlmModels["claude-opus-4.8"].maxInputTokens).toBe(197897)
@@ -16,13 +15,16 @@ describe("vscodeLlmModels", () => {
1615
expect(vscodeLlmModels["gemini-2.5-pro"].maxInputTokens).toBe(108594)
1716
})
1817
it.each([
19-
["claude-opus-5", "Claude Opus 5", 935793, 197897],
18+
["claude-opus-5.5", "Claude Opus 5.5", 871793, 677108],
19+
["claude-opus-5", "Claude Opus 5", 935793, 680456],
2020
["claude-sonnet-5", "Claude Sonnet 5", 925449, 925449],
2121
["gpt-5.6-luna", "GPT-5.6 Luna", 199753, 199753],
2222
["gpt-5.6-sol", "GPT-5.6 Sol", 271785, 271785],
2323
["gpt-5.6-terra", "GPT-5.6 Terra", 271785, 271785],
24-
["gemini-3.6-flash", "Gemini 3.6 Flash", 935793, 197897],
25-
] as const)("preserves PR #13 row %s (%s) with PR #14 ceilings", (family, name, contextWindow, maxInputTokens) => {
24+
["gemini-3.8-flash", "Gemini 3.8 Flash", 982833, 955113],
25+
["gemini-3.7-flash", "Gemini 3.7 Flash", 935793, 935783],
26+
["gemini-3.6-flash", "Gemini 3.6 Flash", 935793, 935785],
27+
] as const)("preserves measured row %s (%s)", (family, name, contextWindow, maxInputTokens) => {
2628
expect(vscodeLlmModels[family]).toEqual({
2729
contextWindow,
2830
supportsImages: true,
@@ -38,7 +40,85 @@ describe("vscodeLlmModels", () => {
3840
})
3941

4042
it.each([
41-
["claude-opus-5", "claude-opus-4.8"],
43+
["grok-4.5", "Grok 4.5", 199783],
44+
["grok-4.6", "Grok 4.6", 199784],
45+
] as const)("keeps %s image input disabled pending evidence", (family, name, maxInputTokens) => {
46+
expect(vscodeLlmModels[family]).toEqual({
47+
contextWindow: 424794,
48+
supportsImages: false,
49+
supportsPromptCache: false,
50+
inputPrice: 0,
51+
outputPrice: 0,
52+
family,
53+
version: family,
54+
name,
55+
supportsToolCalling: true,
56+
maxInputTokens,
57+
})
58+
})
59+
60+
it("records Astra's measured budget minus a 5K safety margin without unmeasured prices", () => {
61+
expect(vscodeLlmModels["gpt-6-astra"]).toEqual({
62+
contextWindow: 921793,
63+
supportsImages: true,
64+
supportsPromptCache: false,
65+
family: "gpt-6-astra",
66+
version: "gpt-6-astra",
67+
name: "GPT-6 Astra",
68+
supportsToolCalling: true,
69+
maxInputTokens: 266789,
70+
})
71+
expect(vscodeLlmModels["gpt-6-astra"]).not.toHaveProperty("inputPrice")
72+
expect(vscodeLlmModels["gpt-6-astra"]).not.toHaveProperty("outputPrice")
73+
})
74+
75+
it.each([
76+
["claude-sonnet-5.5", "Claude Sonnet 5.5", 871793, 680552, true],
77+
["claude-haiku-5.5", "Claude Haiku 5.5", 871793, 99784, true],
78+
["grok-4.7", "Grok 4.7", 424794, 199751, false],
79+
["gemini-4-argon", "Gemini 4 Argon", 982833, 953064, true],
80+
["vscModelF", "MAI-Code-2-Flash (Preview)", 921790, 921736, true],
81+
["oswe-vscode-modelD", "MAI-Code-1.1-Flash", 127790, 127783, true],
82+
["gpt-6.1-sol", "GPT-6.1 Sol", 921793, 269903, true],
83+
["gpt-5.6-sol-fast", "GPT-5.6 Sol Fast (Internal only)", 921793, 269903, true],
84+
["gpt-6.1-terra", "GPT-6.1 Terra", 871793, 269123, true],
85+
["gpt-6-luna", "GPT-6 Luna", 871793, 269122, true],
86+
["gpt-6-sol", "GPT-6 Sol", 871793, 269122, true],
87+
["claude-fable-5.1", "Claude Fable 5.1", 935793, 660686, true],
88+
["claude-fable-5", "Claude Fable 5", 935793, 658785, true],
89+
] as const)(
90+
"records the October capture for %s without unmeasured prices",
91+
(family, name, contextWindow, maxInputTokens, supportsImages) => {
92+
expect(vscodeLlmModels[family]).toEqual({
93+
contextWindow,
94+
maxInputTokens,
95+
supportsImages,
96+
supportsPromptCache: false,
97+
family,
98+
version: family,
99+
name,
100+
supportsToolCalling: true,
101+
})
102+
expect(vscodeLlmModels[family]).not.toHaveProperty("inputPrice")
103+
expect(vscodeLlmModels[family]).not.toHaveProperty("outputPrice")
104+
},
105+
)
106+
107+
it("assigns Fable 5.5 the requested Fable 5.1 values rather than claiming a measurement", () => {
108+
expect(vscodeLlmModels["claude-fable-5.5"]).toEqual({
109+
...vscodeLlmModels["claude-fable-5.1"],
110+
family: "claude-fable-5.5",
111+
version: "claude-fable-5.5",
112+
name: "Claude Fable 5.5",
113+
})
114+
expect(vscodeLlmModels["claude-fable-5.5"].contextWindow).toBe(935793)
115+
expect(vscodeLlmModels["claude-fable-5.5"].maxInputTokens).toBe(660686)
116+
})
117+
118+
it.each([
119+
["claude-opus-5.5", "claude-opus-5", "claude-opus-4.8"],
120+
["gpt-6-astra", "gpt-5.6-luna"],
121+
["grok-4.6", "grok-4.5", "gemini-3.8-flash", "gemini-3.7-flash", "gemini-3.6-flash"],
42122
["claude-sonnet-5", "claude-sonnet-4.6"],
43123
["gpt-5.6-luna", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.5"],
44124
["gemini-3.6-flash", "gemini-3.1-pro-preview"],

‎packages/types/src/providers/vscode-llm.ts‎

Lines changed: 232 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -49,12 +49,24 @@ export type VscodeLlmModelId = keyof typeof vscodeLlmModels
4949

5050
export const vscodeLlmDefaultModelId: VscodeLlmModelId = "claude-sonnet-4.5"
5151

52-
// The UI and condense gate use maxInputTokens; contextWindow preserves the advertised window.
53-
// https://github.com/simurg79/Roo-Code/pull/13: sonnet-5 and gpt-5.6 were binary-searched.
54-
// https://github.com/simurg79/Roo-Code/pull/14: opus-5 and gemini-3.6-flash retain advertised
55-
// 935793 windows but conservatively reuse the 197897 ceiling measured on other models to avoid
56-
// request overflow; neither model's actual input ceiling has been measured.
52+
// The UI and condense gate use assigned maxInputTokens budgets; contextWindow keeps advertised windows.
53+
// Use measured acceptances, optionally lowered by documented safety margins; Fable 5.5 alone borrows 5.1's values.
54+
// Measurement details: https://github.com/simurg79/Roo-Code/pull/26.
5755
export const vscodeLlmModels = {
56+
// Refusals left the 2026-09-23 bracket open (677108 accepted, 695778 rejected); use the lower bound.
57+
// Image/tool inputs succeeded; caching false and zero prices remain unverified defaults.
58+
"claude-opus-5.5": {
59+
contextWindow: 871793,
60+
supportsImages: true,
61+
supportsPromptCache: false,
62+
inputPrice: 0,
63+
outputPrice: 0,
64+
family: "claude-opus-5.5",
65+
version: "claude-opus-5.5",
66+
name: "Claude Opus 5.5",
67+
supportsToolCalling: true,
68+
maxInputTokens: 677108,
69+
},
5870
"claude-opus-5": {
5971
contextWindow: 935793,
6072
supportsImages: true,
@@ -65,7 +77,7 @@ export const vscodeLlmModels = {
6577
version: "claude-opus-5",
6678
name: "Claude Opus 5",
6779
supportsToolCalling: true,
68-
maxInputTokens: 197897,
80+
maxInputTokens: 680456,
6981
},
7082
"claude-opus-4.8": {
7183
contextWindow: 679560,
@@ -115,6 +127,156 @@ export const vscodeLlmModels = {
115127
supportsToolCalling: true,
116128
maxInputTokens: 167790,
117129
},
130+
// Rows below through oswe-vscode-modelD: measured 2026-10-28, VS Code 1.142.0/copilot; maxInputTokens is
131+
// an explicitly accepted size (the smaller when repeated). Prices and caching are unmeasured.
132+
"claude-sonnet-5.5": {
133+
contextWindow: 871793,
134+
supportsImages: true,
135+
supportsPromptCache: false,
136+
family: "claude-sonnet-5.5",
137+
version: "claude-sonnet-5.5",
138+
name: "Claude Sonnet 5.5",
139+
supportsToolCalling: true,
140+
maxInputTokens: 680552,
141+
},
142+
"claude-haiku-5.5": {
143+
contextWindow: 871793,
144+
supportsImages: true,
145+
supportsPromptCache: false,
146+
family: "claude-haiku-5.5",
147+
version: "claude-haiku-5.5",
148+
name: "Claude Haiku 5.5",
149+
supportsToolCalling: true,
150+
maxInputTokens: 99784,
151+
},
152+
"grok-4.7": {
153+
contextWindow: 424794,
154+
// Image request failed with a nondiagnostic HTTP 400: disabled pending evidence.
155+
supportsImages: false,
156+
supportsPromptCache: false,
157+
family: "grok-4.7",
158+
version: "grok-4.7",
159+
name: "Grok 4.7",
160+
supportsToolCalling: true,
161+
maxInputTokens: 199751,
162+
},
163+
"gemini-4-argon": {
164+
contextWindow: 982833,
165+
supportsImages: true,
166+
supportsPromptCache: false,
167+
family: "gemini-4-argon",
168+
version: "gemini-4-argon",
169+
name: "Gemini 4 Argon",
170+
supportsToolCalling: true,
171+
// Accepted at the advertised target; no observed rejection, so only a lower bound.
172+
maxInputTokens: 953064,
173+
},
174+
// Provider lookups need the live family key, not the user-facing id mai-code-2-flash.
175+
vscModelF: {
176+
contextWindow: 921790,
177+
supportsImages: true,
178+
supportsPromptCache: false,
179+
family: "vscModelF",
180+
version: "vscModelF",
181+
name: "MAI-Code-2-Flash (Preview)",
182+
supportsToolCalling: true,
183+
maxInputTokens: 921736,
184+
},
185+
// Live family string; the user-facing id is mai-code-1.1-flash.
186+
"oswe-vscode-modelD": {
187+
contextWindow: 127790,
188+
supportsImages: true,
189+
supportsPromptCache: false,
190+
family: "oswe-vscode-modelD",
191+
version: "oswe-vscode-modelD",
192+
name: "MAI-Code-1.1-Flash",
193+
supportsToolCalling: true,
194+
maxInputTokens: 127783,
195+
},
196+
// Captured 2026-10-28 UTC (Fable 5: 2026-10-29 UTC), VS Code 1.142.0/copilot.
197+
// Image/tool requests were accepted, not verified for understanding or invocation.
198+
// Prices and caching are unmeasured; caching false is a schema default.
199+
"gpt-6.1-sol": {
200+
contextWindow: 921793,
201+
supportsImages: true,
202+
supportsPromptCache: false,
203+
family: "gpt-6.1-sol",
204+
version: "gpt-6.1-sol",
205+
name: "GPT-6.1 Sol",
206+
supportsToolCalling: true,
207+
maxInputTokens: 269903,
208+
},
209+
"gpt-5.6-sol-fast": {
210+
contextWindow: 921793,
211+
supportsImages: true,
212+
supportsPromptCache: false,
213+
family: "gpt-5.6-sol-fast",
214+
version: "gpt-5.6-sol-fast",
215+
name: "GPT-5.6 Sol Fast (Internal only)",
216+
supportsToolCalling: true,
217+
maxInputTokens: 269903,
218+
},
219+
"gpt-6.1-terra": {
220+
contextWindow: 871793,
221+
supportsImages: true,
222+
supportsPromptCache: false,
223+
family: "gpt-6.1-terra",
224+
version: "gpt-6.1-terra",
225+
name: "GPT-6.1 Terra",
226+
supportsToolCalling: true,
227+
maxInputTokens: 269123,
228+
},
229+
"gpt-6-luna": {
230+
contextWindow: 871793,
231+
supportsImages: true,
232+
supportsPromptCache: false,
233+
family: "gpt-6-luna",
234+
version: "gpt-6-luna",
235+
name: "GPT-6 Luna",
236+
supportsToolCalling: true,
237+
maxInputTokens: 269122,
238+
},
239+
"gpt-6-sol": {
240+
contextWindow: 871793,
241+
supportsImages: true,
242+
supportsPromptCache: false,
243+
family: "gpt-6-sol",
244+
version: "gpt-6-sol",
245+
name: "GPT-6 Sol",
246+
supportsToolCalling: true,
247+
maxInputTokens: 269122,
248+
},
249+
"claude-fable-5.1": {
250+
contextWindow: 935793,
251+
supportsImages: true,
252+
supportsPromptCache: false,
253+
family: "claude-fable-5.1",
254+
version: "claude-fable-5.1",
255+
name: "Claude Fable 5.1",
256+
supportsToolCalling: true,
257+
maxInputTokens: 660686,
258+
},
259+
"claude-fable-5": {
260+
contextWindow: 935793,
261+
supportsImages: true,
262+
supportsPromptCache: false,
263+
family: "claude-fable-5",
264+
version: "claude-fable-5",
265+
name: "Claude Fable 5",
266+
supportsToolCalling: true,
267+
maxInputTokens: 658785,
268+
},
269+
// Values copied from claude-fable-5.1 at the user's request; not measured for claude-fable-5.5.
270+
"claude-fable-5.5": {
271+
contextWindow: 935793,
272+
supportsImages: true,
273+
supportsPromptCache: false,
274+
family: "claude-fable-5.5",
275+
version: "claude-fable-5.5",
276+
name: "Claude Fable 5.5",
277+
supportsToolCalling: true,
278+
maxInputTokens: 660686,
279+
},
118280
"claude-sonnet-5": {
119281
contextWindow: 925449,
120282
supportsImages: true,
@@ -163,6 +325,18 @@ export const vscodeLlmModels = {
163325
supportsToolCalling: true,
164326
maxInputTokens: 135790,
165327
},
328+
// The 2026-09-23 trials accepted 271789 and rejected 271790; subtract a 5K safety margin.
329+
// Image/tool inputs succeeded; caching false is unverified, and optional prices are unmeasured.
330+
"gpt-6-astra": {
331+
contextWindow: 921793,
332+
supportsImages: true,
333+
supportsPromptCache: false,
334+
family: "gpt-6-astra",
335+
version: "gpt-6-astra",
336+
name: "GPT-6 Astra",
337+
supportsToolCalling: true,
338+
maxInputTokens: 266789,
339+
},
166340
"gpt-5.6-luna": {
167341
contextWindow: 199753,
168342
supportsImages: true,
@@ -271,6 +445,57 @@ export const vscodeLlmModels = {
271445
supportsToolCalling: true,
272446
maxInputTokens: 12078,
273447
},
448+
"grok-4.6": {
449+
contextWindow: 424794,
450+
// Image trials failed with nondiagnostic HTTP 400s; disable pending evidence.
451+
supportsImages: false,
452+
supportsPromptCache: false,
453+
inputPrice: 0,
454+
outputPrice: 0,
455+
family: "grok-4.6",
456+
version: "grok-4.6",
457+
name: "Grok 4.6",
458+
supportsToolCalling: true,
459+
maxInputTokens: 199784,
460+
},
461+
"grok-4.5": {
462+
contextWindow: 424794,
463+
// As with Grok 4.6, image capability remains unverified.
464+
supportsImages: false,
465+
supportsPromptCache: false,
466+
inputPrice: 0,
467+
outputPrice: 0,
468+
family: "grok-4.5",
469+
version: "grok-4.5",
470+
name: "Grok 4.5",
471+
supportsToolCalling: true,
472+
maxInputTokens: 199783,
473+
},
474+
"gemini-3.8-flash": {
475+
contextWindow: 982833,
476+
supportsImages: true,
477+
supportsPromptCache: false,
478+
inputPrice: 0,
479+
outputPrice: 0,
480+
family: "gemini-3.8-flash",
481+
version: "gemini-3.8-flash",
482+
name: "Gemini 3.8 Flash",
483+
supportsToolCalling: true,
484+
// No rejection was observed; this is an accepted lower bound, not an exact ceiling.
485+
maxInputTokens: 955113,
486+
},
487+
"gemini-3.7-flash": {
488+
contextWindow: 935793,
489+
supportsImages: true,
490+
supportsPromptCache: false,
491+
inputPrice: 0,
492+
outputPrice: 0,
493+
family: "gemini-3.7-flash",
494+
version: "gemini-3.7-flash",
495+
name: "Gemini 3.7 Flash",
496+
supportsToolCalling: true,
497+
maxInputTokens: 935783,
498+
},
274499
"gemini-3.6-flash": {
275500
contextWindow: 935793,
276501
supportsImages: true,
@@ -281,7 +506,7 @@ export const vscodeLlmModels = {
281506
version: "gemini-3.6-flash",
282507
name: "Gemini 3.6 Flash",
283508
supportsToolCalling: true,
284-
maxInputTokens: 197897,
509+
maxInputTokens: 935785,
285510
},
286511
"gemini-3.1-pro-preview": {
287512
contextWindow: 197897,

0 commit comments

Comments
 (0)