Skip to content

Commit 93ac20e

Browse files
committed
fix(friendli): load model catalog lazily before requests
1 parent 6d1192d commit 93ac20e

5 files changed

Lines changed: 81 additions & 13 deletions

File tree

‎src/api/providers/__tests__/friendli.spec.ts‎

Lines changed: 42 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
import { Anthropic } from "@anthropic-ai/sdk"
44
import OpenAI from "openai"
55

6-
import { friendliDefaultModelId, friendliModels } from "@roo-code/types"
6+
import { friendliDefaultModelId, friendliModels, openAiModelInfoSaneDefaults } from "@roo-code/types"
77

88
import { buildApiHandler } from "../../index"
99
import { getModelMaxOutputTokens } from "../../../shared/api"
@@ -87,13 +87,51 @@ describe("FriendliHandler", () => {
8787
expect(OpenAI).toHaveBeenCalledWith(expect.objectContaining({ apiKey: friendliApiKey }))
8888
})
8989

90-
it("should use fetched model info, keeping a model id missing from the static list", async () => {
90+
it("should construct without an API key and without fetching the catalog", () => {
91+
expect(() => new FriendliHandler({})).not.toThrow()
92+
expect(mockGetModels).not.toHaveBeenCalled()
93+
})
94+
95+
it("should load the catalog once before building requests for fetched-only models", async () => {
96+
const info = { maxTokens: 8192, contextWindow: 100_000, supportsPromptCache: false }
97+
mockGetModels.mockResolvedValueOnce({ "vendor/new-model": info })
98+
mockCreate.mockResolvedValue({ choices: [{ message: { content: "ok" } }] })
99+
const handlerWithModel = new FriendliHandler({ apiModelId: "vendor/new-model", friendliApiKey: "test-key" })
100+
101+
await handlerWithModel.completePrompt("a")
102+
await handlerWithModel.completePrompt("b")
103+
104+
expect(mockGetModels).toHaveBeenCalledTimes(1)
105+
expect(handlerWithModel.getModel()).toEqual(expect.objectContaining({ id: "vendor/new-model", info }))
106+
})
107+
108+
it("should wait for the catalog before streaming", async () => {
91109
const info = { maxTokens: 8192, contextWindow: 100_000, supportsPromptCache: false }
92110
mockGetModels.mockResolvedValueOnce({ "vendor/new-model": info })
93111
const handlerWithModel = new FriendliHandler({ apiModelId: "vendor/new-model", friendliApiKey: "test-key" })
94112

95-
await vi.waitFor(() => expect(handlerWithModel.getModel().info).toEqual(info))
96-
expect(handlerWithModel.getModel().id).toBe("vendor/new-model")
113+
await handlerWithModel.createMessage("system", []).next()
114+
115+
expect(mockCreate.mock.calls[0][0]).toEqual(expect.objectContaining({ max_tokens: 8192 }))
116+
})
117+
118+
it("should fall back to static models when the catalog is empty", async () => {
119+
mockCreate.mockResolvedValue({ choices: [{ message: { content: "ok" } }] })
120+
await handler.completePrompt("a")
121+
122+
expect(handler.getModel().info).toEqual(friendliModels[friendliDefaultModelId])
123+
})
124+
125+
it("should keep an unknown model id with default metadata when the catalog fails", async () => {
126+
mockGetModels.mockRejectedValueOnce(new Error("Network error"))
127+
mockCreate.mockResolvedValue({ choices: [{ message: { content: "ok" } }] })
128+
const handlerWithModel = new FriendliHandler({ apiModelId: "vendor/unknown", friendliApiKey: "test-key" })
129+
130+
await handlerWithModel.completePrompt("a")
131+
132+
expect(handlerWithModel.getModel()).toEqual(
133+
expect.objectContaining({ id: "vendor/unknown", info: openAiModelInfoSaneDefaults }),
134+
)
97135
})
98136

99137
it("fetchModels should load the public /v1/models catalog", async () => {

‎src/api/providers/fetchers/__tests__/friendli.spec.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,10 @@ describe("getFriendliModels", () => {
3434
},
3535
})
3636

37-
const models = await getFriendliModels()
37+
const signal = new AbortController().signal
38+
const models = await getFriendliModels({ signal })
3839

40+
expect(axios.get).toHaveBeenCalledWith("https://api.friendli.ai/serverless/v1/models", { signal })
3941
expect(Object.keys(models)).toEqual(["zai-org/GLM-5.3", "google/gemma-4-31B-it"])
4042
expect(models["zai-org/GLM-5.3"]).toEqual({
4143
maxTokens: 131_072,

‎src/api/providers/friendli.ts‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import { hasUsableTimeout, toSdkTimeoutMs } from "./utils/abort-signal"
2222
import { NOT_PROVIDED } from "./constants"
2323
import { getModels } from "./fetchers/modelCache"
2424
import { getFriendliModels } from "./fetchers/friendli"
25+
import type { ApiStream } from "../transform/stream"
2526
import type { ApiHandlerCreateMessageMetadata, CompletePromptOptions } from "../index"
2627

2728
/**
@@ -66,6 +67,7 @@ type FriendliChatCompletionNonStreamingParams = Omit<
6667
*/
6768
export class FriendliHandler extends BaseOpenAiCompatibleProvider<FriendliModelId> {
6869
private models: ModelRecord = {}
70+
private modelsLoad?: Promise<void>
6971

7072
constructor(options: ApiHandlerOptions) {
7173
super({
@@ -77,10 +79,22 @@ export class FriendliHandler extends BaseOpenAiCompatibleProvider<FriendliModelI
7779
providerModels: friendliModels,
7880
defaultTemperature: 0.6,
7981
})
82+
}
8083

81-
getModels({ provider: providerIdentifiers.friendli }, this)
82-
.then((models) => (this.models = models))
84+
private loadModels(): Promise<void> {
85+
this.modelsLoad ??= getModels({ provider: providerIdentifiers.friendli }, this)
86+
.then((models) => void (this.models = models))
8387
.catch(() => {}) // static friendliModels remain the fallback
88+
return this.modelsLoad
89+
}
90+
91+
override async *createMessage(
92+
systemPrompt: string,
93+
messages: Anthropic.Messages.MessageParam[],
94+
metadata?: ApiHandlerCreateMessageMetadata,
95+
): ApiStream {
96+
await this.loadModels()
97+
yield* super.createMessage(systemPrompt, messages, metadata)
8498
}
8599

86100
async fetchModels(_options: GetModelsOptions, signal?: AbortSignal): Promise<ModelRecord> {
@@ -197,6 +211,7 @@ export class FriendliHandler extends BaseOpenAiCompatibleProvider<FriendliModelI
197211
}
198212

199213
override async completePrompt(prompt: string, options?: CompletePromptOptions): Promise<string> {
214+
await this.loadModels()
200215
const { id: modelId } = this.getModel()
201216
const friendliExtra = this.buildFriendliReasoningParams()
202217

‎webview-ui/src/components/ui/hooks/__tests__/useSelectedModel.spec.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1893,6 +1893,16 @@ describe("useSelectedModel", () => {
18931893
expect(result.current.info).toEqual(friendliModels["zai-org/GLM-5.1"])
18941894
})
18951895

1896+
it("should use static model info before router models load", () => {
1897+
mockUseRouterModels.mockReturnValue(createRouterModelsResult(undefined, { isLoading: true }))
1898+
const apiConfiguration: ProviderSettings = { apiProvider: providerIdentifiers.friendli }
1899+
1900+
const { result } = renderHook(() => useSelectedModel(apiConfiguration), { wrapper: createWrapper() })
1901+
1902+
expect(result.current.id).toBe(friendliDefaultModelId)
1903+
expect(result.current.info).toEqual(friendliModels[friendliDefaultModelId])
1904+
})
1905+
18961906
it("should prefer fetched router model info", () => {
18971907
const info = { maxTokens: 8192, contextWindow: 100_000, supportsPromptCache: false }
18981908
mockUseRouterModels.mockReturnValue(createRouterModelsResult({ friendli: { "vendor/new-model": info } }))

‎webview-ui/src/components/ui/hooks/useSelectedModel.ts‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -133,12 +133,15 @@ export const useSelectedModel = (apiConfiguration: ProviderSettings) => {
133133
// before the selection is resolved.
134134
const hasValidRouterData =
135135
needRouterModels && dynamicProvider
136-
? dynamicProvider === providerIdentifiers.litellm || dynamicProvider === providerIdentifiers.ioIntelligence
137-
? !routerModels.isLoading
138-
: routerModels.data &&
139-
routerModels.data[dynamicProvider] !== undefined &&
140-
typeof routerModels.data[dynamicProvider] === "object" &&
141-
!routerModels.isLoading
136+
? dynamicProvider === providerIdentifiers.friendli
137+
? true // static friendliModels cover the window before /v1/models data arrives
138+
: dynamicProvider === providerIdentifiers.litellm ||
139+
dynamicProvider === providerIdentifiers.ioIntelligence
140+
? !routerModels.isLoading
141+
: routerModels.data &&
142+
routerModels.data[dynamicProvider] !== undefined &&
143+
typeof routerModels.data[dynamicProvider] === "object" &&
144+
!routerModels.isLoading
142145
: true
143146

144147
const isReady =

0 commit comments

Comments
 (0)