Skip to content

Commit effc83e

Browse files
daniel-lxsdaniel-lxs
committed
fix: implement model cache refresh to prevent stale disk cache
- Add refreshModels() function to force fresh API fetch bypassing cache - Add initializeModelCacheRefresh() for background refresh on extension load - Update flushModels() with optional refresh parameter - Refresh public providers (OpenRouter, Glama, Vercel AI Gateway) on startup - Update manual refresh triggers to actually fetch fresh data - Use atomic writes to keep cache available during refresh Previously, the disk cache was written once and never refreshed, causing stale model info (pricing, context windows, new models) to persist indefinitely. This fix ensures the cache is refreshed on extension load and whenever users manually refresh models.
1 parent 038f830 commit effc83e

File tree

4 files changed

+131
-20
lines changed

4 files changed

+131
-20
lines changed

‎src/api/providers/fetchers/lmstudio.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ export const forceFullModelDetailsLoad = async (baseUrl: string, modelId: string
1818

1919
const client = new LMStudioClient({ baseUrl: lmsUrl })
2020
await client.llm.model(modelId)
21-
await flushModels("lmstudio")
22-
await getModels({ provider: "lmstudio" }) // Force cache update now.
21+
// Flush and refresh cache to get updated model details
22+
await flushModels("lmstudio", true)
2323

2424
// Mark this model as having full details loaded.
2525
modelsWithLoadedDetails.add(modelId)

‎src/api/providers/fetchers/modelCache.ts‎

Lines changed: 115 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,13 +145,127 @@ export const getModels = async (options: GetModelsOptions): Promise<ModelRecord>
145145
}
146146
}
147147

148+
/**
149+
* Force-refresh models from API, bypassing cache.
150+
* Uses atomic writes so cache remains available during refresh.
151+
*
152+
* @param options - Provider options for fetching models
153+
* @returns Fresh models from API
154+
*/
155+
export const refreshModels = async (options: GetModelsOptions): Promise<ModelRecord> => {
156+
const { provider } = options
157+
158+
let models: ModelRecord
159+
160+
try {
161+
// Force fresh API fetch - skip getModelsFromCache() check
162+
switch (provider) {
163+
case "openrouter":
164+
models = await getOpenRouterModels()
165+
break
166+
case "requesty":
167+
models = await getRequestyModels(options.baseUrl, options.apiKey)
168+
break
169+
case "glama":
170+
models = await getGlamaModels()
171+
break
172+
case "unbound":
173+
models = await getUnboundModels(options.apiKey)
174+
break
175+
case "litellm":
176+
models = await getLiteLLMModels(options.apiKey, options.baseUrl)
177+
break
178+
case "ollama":
179+
models = await getOllamaModels(options.baseUrl, options.apiKey)
180+
break
181+
case "lmstudio":
182+
models = await getLMStudioModels(options.baseUrl)
183+
break
184+
case "deepinfra":
185+
models = await getDeepInfraModels(options.apiKey, options.baseUrl)
186+
break
187+
case "io-intelligence":
188+
models = await getIOIntelligenceModels(options.apiKey)
189+
break
190+
case "vercel-ai-gateway":
191+
models = await getVercelAiGatewayModels()
192+
break
193+
case "huggingface":
194+
models = await getHuggingFaceModels()
195+
break
196+
case "roo": {
197+
const rooBaseUrl =
198+
options.baseUrl ?? process.env.ROO_CODE_PROVIDER_URL ?? "https://api.roocode.com/proxy"
199+
models = await getRooModels(rooBaseUrl, options.apiKey)
200+
break
201+
}
202+
case "chutes":
203+
models = await getChutesModels(options.apiKey)
204+
break
205+
default: {
206+
const exhaustiveCheck: never = provider
207+
throw new Error(`Unknown provider: ${exhaustiveCheck}`)
208+
}
209+
}
210+
211+
// Update memory cache first
212+
memoryCache.set(provider, models)
213+
214+
// Atomically write to disk (safeWriteJson handles atomic writes)
215+
await writeModels(provider, models).catch((err) =>
216+
console.error(`[refreshModels] Error writing ${provider} models to disk:`, err),
217+
)
218+
219+
return models
220+
} catch (error) {
221+
console.debug(`[refreshModels] Failed to refresh ${provider}:`, error)
222+
// On error, return existing cache if available (graceful degradation)
223+
return getModelsFromCache(provider) || {}
224+
}
225+
}
226+
227+
/**
228+
* Initialize background model cache refresh.
229+
* Refreshes public provider caches without blocking or requiring auth.
230+
* Should be called once during extension activation.
231+
*/
232+
export async function initializeModelCacheRefresh(): Promise<void> {
233+
// Wait for extension to fully activate before refreshing
234+
setTimeout(async () => {
235+
// Providers that work without API keys
236+
const publicProviders: Array<{ provider: RouterName; options: GetModelsOptions }> = [
237+
{ provider: "openrouter", options: { provider: "openrouter" } },
238+
{ provider: "glama", options: { provider: "glama" } },
239+
{ provider: "vercel-ai-gateway", options: { provider: "vercel-ai-gateway" } },
240+
]
241+
242+
// Refresh each provider in background (fire and forget)
243+
for (const { options } of publicProviders) {
244+
refreshModels(options).catch(() => {
245+
// Silent fail - old cache remains available
246+
})
247+
248+
// Small delay between refreshes to avoid API rate limits
249+
await new Promise((resolve) => setTimeout(resolve, 500))
250+
}
251+
}, 2000)
252+
}
253+
148254
/**
149255
* Flush models memory cache for a specific router.
150256
*
151257
* @param router - The router to flush models for.
258+
* @param refresh - If true, immediately fetch fresh data from API
152259
*/
153-
export const flushModels = async (router: RouterName) => {
260+
export const flushModels = async (router: RouterName, refresh: boolean = false): Promise<void> => {
154261
memoryCache.del(router)
262+
263+
if (refresh) {
264+
// Trigger background refresh - don't await to avoid blocking
265+
refreshModels({ provider: router } as GetModelsOptions).catch((error) => {
266+
console.error(`[flushModels] Refresh failed for ${router}:`, error)
267+
})
268+
}
155269
}
156270

157271
/**

‎src/core/webview/webviewMessageHandler.ts‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -790,7 +790,7 @@ export const webviewMessageHandler = async (
790790
break
791791
case "flushRouterModels":
792792
const routerNameFlush: RouterName = toRouterName(message.text)
793-
await flushModels(routerNameFlush)
793+
await flushModels(routerNameFlush, true)
794794
break
795795
case "requestRouterModels":
796796
const { apiConfiguration } = await provider.getState()
@@ -932,8 +932,8 @@ export const webviewMessageHandler = async (
932932
// Specific handler for Ollama models only.
933933
const { apiConfiguration: ollamaApiConfig } = await provider.getState()
934934
try {
935-
// Flush cache first to ensure fresh models.
936-
await flushModels("ollama")
935+
// Flush cache and refresh to ensure fresh models.
936+
await flushModels("ollama", true)
937937

938938
const ollamaModels = await getModels({
939939
provider: "ollama",
@@ -954,8 +954,8 @@ export const webviewMessageHandler = async (
954954
// Specific handler for LM Studio models only.
955955
const { apiConfiguration: lmStudioApiConfig } = await provider.getState()
956956
try {
957-
// Flush cache first to ensure fresh models.
958-
await flushModels("lmstudio")
957+
// Flush cache and refresh to ensure fresh models.
958+
await flushModels("lmstudio", true)
959959

960960
const lmStudioModels = await getModels({
961961
provider: "lmstudio",
@@ -977,8 +977,8 @@ export const webviewMessageHandler = async (
977977
case "requestRooModels": {
978978
// Specific handler for Roo models only - flushes cache to ensure fresh auth token is used
979979
try {
980-
// Flush cache first to ensure fresh models with current auth state
981-
await flushModels("roo")
980+
// Flush cache and refresh to ensure fresh models with current auth state
981+
await flushModels("roo", true)
982982

983983
const rooModels = await getModels({
984984
provider: "roo",

‎src/extension.ts‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ import {
4040
CodeActionProvider,
4141
} from "./activate"
4242
import { initializeI18n } from "./i18n"
43-
import { flushModels, getModels } from "./api/providers/fetchers/modelCache"
43+
import { flushModels, getModels, initializeModelCacheRefresh } from "./api/providers/fetchers/modelCache"
4444

4545
/**
4646
* Built using https://github.com/microsoft/vscode-webview-ui-toolkit
@@ -145,17 +145,11 @@ export async function activate(context: vscode.ExtensionContext) {
145145
// Handle Roo models cache based on auth state
146146
const handleRooModelsCache = async () => {
147147
try {
148-
await flushModels("roo")
148+
// Flush and refresh cache on auth state changes
149+
await flushModels("roo", true)
149150

150151
if (data.state === "active-session") {
151-
// Reload models with the new auth token
152-
const sessionToken = cloudService?.authService?.getSessionToken()
153-
await getModels({
154-
provider: "roo",
155-
baseUrl: process.env.ROO_CODE_PROVIDER_URL ?? "https://api.roocode.com/proxy",
156-
apiKey: sessionToken,
157-
})
158-
cloudLogger(`[authStateChangedHandler] Reloaded Roo models cache for active session`)
152+
cloudLogger(`[authStateChangedHandler] Refreshed Roo models cache for active session`)
159153
} else {
160154
cloudLogger(`[authStateChangedHandler] Flushed Roo models cache on logout`)
161155
}
@@ -353,6 +347,9 @@ export async function activate(context: vscode.ExtensionContext) {
353347
})
354348
}
355349

350+
// Initialize background model cache refresh
351+
initializeModelCacheRefresh()
352+
356353
return new API(outputChannel, provider, socketPath, enableLogging)
357354
}
358355

0 commit comments

Comments
 (0)