Skip to content

Commit 0852093

Browse files
JamesRobert20xavier-arosemena
authored andcommitted
fix(zoo-gateway): stop inventing UI cost from default model prices (Zoo-Code-Org#1339)
* fix(zoo-gateway): stop inventing UI cost from default model prices Stale in-memory model maps and defaultModelInfo $/token fallback made the panel overstate spend vs gateway billing after new catalog models (e.g. Alibaba promo). * fix(zoo-gateway): negative-cache missing models and cover unset usage.cost Avoid refetching the catalog on every request when an id is still absent after a fetch; retry after a short TTL. Add coverage for usage chunks that omit cost. * fix(zoo-gateway): negative-cache empty catalog misses too An empty getModels() result still records a missing-id attempt; the retry guard no longer requires a non-empty models map. * fix(zoo-gateway): refresh catalog before negative-caching a miss getModels can return a stale shared catalog for non-auth-scoped providers. Force refreshModels when the selected id is still missing so newly listed models are not blocked for the five-minute negative-cache window. * fix(zoo-gateway): include refreshModels in RouterProvider test mocks fetchModel now calls refreshModels on catalog misses; provider specs that mocked only getModels broke under CI. (cherry picked from commit 6ad8a6e)
1 parent 1850da0 commit 0852093

10 files changed

Lines changed: 101 additions & 15 deletions

‎src/api/providers/__tests__/kenari.spec.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ vitest.mock("../fetchers/modelCache", () => ({
3434
},
3535
}),
3636
),
37+
refreshModels: vitest.fn(async (options) => {
38+
const { getModels } = await import("../fetchers/modelCache")
39+
return getModels(options)
40+
}),
3741
getModelsFromCache: vitest.fn().mockReturnValue(undefined),
3842
}))
3943

‎src/api/providers/__tests__/kimi-code.spec.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,10 @@ vi.mock("../../../integrations/kimi-code/oauth", () => ({
1616
},
1717
}))
1818

19-
vi.mock("../fetchers/modelCache", () => ({ getModels: mockGetModels }))
19+
vi.mock("../fetchers/modelCache", () => ({
20+
getModels: mockGetModels,
21+
refreshModels: mockGetModels,
22+
}))
2023

2124
describe("KimiCodeHandler", () => {
2225
beforeEach(() => {

‎src/api/providers/__tests__/lite-llm.spec.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,10 @@ vi.mock("../fetchers/modelCache", () => ({
5858
"vertex_ai/gemini-3-pro": { ...litellmDefaultModelInfo, maxTokens: 8192 },
5959
})
6060
}),
61+
refreshModels: vi.fn(async (options) => {
62+
const { getModels } = await import("../fetchers/modelCache")
63+
return getModels(options)
64+
}),
6165
getModelsFromCache: vi.fn().mockReturnValue(undefined),
6266
}))
6367

‎src/api/providers/__tests__/nanogpt.spec.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,15 @@ vi.mock("../fetchers/modelCache", () => ({
2323
supportsReasoningEffort: ["low", "medium", "high"],
2424
},
2525
}),
26+
refreshModels: vi.fn().mockResolvedValue({
27+
"model:thinking": {
28+
maxTokens: 128000,
29+
contextWindow: 1050000,
30+
supportsImages: true,
31+
supportsPromptCache: false,
32+
supportsReasoningEffort: ["low", "medium", "high"],
33+
},
34+
}),
2635
getModelsFromCache: vi.fn(),
2736
}))
2837

‎src/api/providers/__tests__/opencode-go.spec.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,12 @@ vitest.mock("../fetchers/modelCache", () => ({
3636
"qwen3.7-max": { ...opencodeGoModels["qwen3.7-max"] },
3737
})
3838
}),
39+
refreshModels: vitest.fn().mockImplementation(function () {
40+
return Promise.resolve({
41+
"glm-5.1": { ...opencodeGoModels["glm-5.1"] },
42+
"qwen3.7-max": { ...opencodeGoModels["qwen3.7-max"] },
43+
})
44+
}),
3945
getModelsFromCache: vitest.fn().mockReturnValue(undefined),
4046
}))
4147

‎src/api/providers/__tests__/openrouter.spec.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,10 @@ vitest.mock("../fetchers/modelCache", () => ({
9090
},
9191
})
9292
}),
93+
refreshModels: vitest.fn(async (options) => {
94+
const { getModels } = await import("../fetchers/modelCache")
95+
return getModels(options)
96+
}),
9397
}))
9498

9599
describe("OpenRouterHandler", () => {

‎src/api/providers/__tests__/requesty.spec.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,10 @@ vitest.mock("../fetchers/modelCache", () => ({
9696
},
9797
})
9898
}),
99+
refreshModels: vitest.fn(async (options) => {
100+
const { getModels } = await import("../fetchers/modelCache")
101+
return getModels(options)
102+
}),
99103
}))
100104

101105
describe("RequestyHandler", () => {

‎src/api/providers/__tests__/unbound.spec.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,10 @@ vi.mock("../fetchers/modelCache", () => ({
3232
description: "GPT-4o",
3333
},
3434
}),
35+
refreshModels: vi.fn(async (options) => {
36+
const { getModels } = await import("../fetchers/modelCache")
37+
return getModels(options)
38+
}),
3539
}))
3640

3741
describe("UnboundHandler", () => {

‎src/api/providers/__tests__/vercel-ai-gateway.spec.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,10 @@ vitest.mock("../fetchers/modelCache", () => ({
9999
},
100100
})
101101
}),
102+
refreshModels: vitest.fn(async (options) => {
103+
const { getModels } = await import("../fetchers/modelCache")
104+
return getModels(options)
105+
}),
102106
getModelsFromCache: vitest.fn().mockReturnValue(undefined),
103107
}))
104108

‎src/api/providers/router-provider.ts‎

Lines changed: 58 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import { type ModelInfo, type ModelRecord } from "@roo-code/types"
55
import { ApiHandlerOptions, RouterName } from "../../shared/api"
66

77
import { BaseProvider } from "./base-provider"
8-
import { getModels, getModelsFromCache } from "./fetchers/modelCache"
8+
import { getModels, getModelsFromCache, refreshModels } from "./fetchers/modelCache"
99

1010
import { DEFAULT_HEADERS, NOT_PROVIDED } from "./constants"
1111

@@ -51,25 +51,58 @@ export abstract class RouterProvider extends BaseProvider {
5151
}
5252

5353
private modelFetchPromise?: Promise<{ id: string; info: ModelInfo }>
54+
/** Last catalog refresh attempt per missing model id (ms), for negative caching. */
55+
private missingModelRefreshAt = new Map<string, number>()
56+
private static readonly MISSING_MODEL_RETRY_MS = 5 * 60 * 1000
5457

5558
public async fetchModel() {
56-
if (Object.keys(this.models).length > 0) {
59+
// Refetch when the selected model is missing — a stale non-empty map
60+
// would otherwise keep serving defaultModelInfo prices for cost estimates.
61+
const id = this.modelId || this.defaultModelId
62+
if (this.models[id]) {
63+
return this.getModel()
64+
}
65+
66+
// After a catalog fetch that still lacks this id, don't hammer getModels
67+
// on every createMessage; retry only after the negative-cache window.
68+
const lastMissingAttempt = this.missingModelRefreshAt.get(id)
69+
if (
70+
lastMissingAttempt !== undefined &&
71+
Date.now() - lastMissingAttempt < RouterProvider.MISSING_MODEL_RETRY_MS
72+
) {
5773
return this.getModel()
5874
}
5975

6076
if (!this.modelFetchPromise) {
61-
this.modelFetchPromise = getModels({
77+
const fetchOptions = {
6278
provider: this.name,
6379
apiKey: this.apiKey,
6480
baseUrl: this.client.baseURL,
65-
})
66-
.then((models) => {
81+
}
82+
83+
this.modelFetchPromise = (async () => {
84+
let models = await getModels(fetchOptions)
85+
this.models = models
86+
87+
// getModels may return a shared cached catalog that predates this
88+
// model. Force a provider refresh before recording a miss so
89+
// newly listed models are not blocked for MISSING_MODEL_RETRY_MS.
90+
// Auth-scoped providers already bypass that cache in getModels;
91+
// refreshModels is then a no-op extra live fetch only on true misses.
92+
if (!models[id]) {
93+
models = await refreshModels(fetchOptions)
6794
this.models = models
68-
return this.getModel()
69-
})
70-
.finally(() => {
71-
this.modelFetchPromise = undefined
72-
})
95+
}
96+
97+
if (models[id]) {
98+
this.missingModelRefreshAt.delete(id)
99+
} else {
100+
this.missingModelRefreshAt.set(id, Date.now())
101+
}
102+
return this.getModel()
103+
})().finally(() => {
104+
this.modelFetchPromise = undefined
105+
})
73106
}
74107

75108
return this.modelFetchPromise
@@ -107,10 +140,21 @@ export abstract class RouterProvider extends BaseProvider {
107140
return { id, info: cachedModels[id] }
108141
}
109142

110-
// Last resort: preserve the configured model ID (falling back to the default
111-
// only when none is configured) so an as-yet-unfetched model isn't silently
112-
// swapped for the hardcoded default. info still comes from defaults since we
113-
// have no fetched or cached metadata for the configured model at this point.
143+
// Last resort: keep the configured id so we don't swap models, but zero
144+
// prices so we don't bill the UI with defaultModelInfo's $/token rates.
145+
if (id !== this.defaultModelId) {
146+
return {
147+
id,
148+
info: {
149+
...this.defaultModelInfo,
150+
inputPrice: 0,
151+
outputPrice: 0,
152+
cacheWritesPrice: 0,
153+
cacheReadsPrice: 0,
154+
},
155+
}
156+
}
157+
114158
return { id, info: this.defaultModelInfo }
115159
}
116160

0 commit comments

Comments
 (0)