Skip to content

Commit 18861b2

Browse files
authored
feat(friendli): fetch models from /v1/models (#2331)
* feat(friendli): fetch models from /v1/models * test(friendli): cover model catalog paths * fix(friendli): load model catalog lazily before requests * fix(friendli): stop on cancellation during catalog loading * fix(friendli): keep toggle-only reasoning and prototype-safe model IDs * fix(friendli): use the fetched catalog in the chat model picker * fix(friendli): share catalog loading with task preparation and honor cancellation - expose ensureModelFetched so context management reads the fetched context window - completePrompt: catalog wait and request share one timeoutMs deadline - forward the abort signal to the stream and normalize SDK aborts - accept null reasoning_options in the /v1/models schema - tests: image support, null reasoning_options, settle/stream cancellation, ApiOptions picker wiring, same-mount ModelSelector catalog swap * fix(friendli): bridge the caller's abort signal per stream The OpenAI SDK never removes its abort listener from the signal it receives, so passing the task's long-lived signal leaked one listener per request. Hand the SDK a per-request signal and detach it when the stream ends. - tests: listener cleanup across reused, failed and early-exited streams - tests: an empty catalog is retried and the next nonempty one is kept * fix(friendli): bridge the caller's abort signal in completePrompt completePrompt passed the caller's signal straight to the SDK, which never removes its abort listener, so reusing one signal leaked a listener per call. Use a per-request signal and detach it on success or failure. - tests: repeated keyless completePrompt calls reusing one signal - tests: effort default falls back to the first supported value without high * fix(friendli): guard webview model lookups against inherited properties useSelectedModel indexed both Friendli catalogs directly, so an ID such as toString or constructor resolved to an inherited function and the task header showed a one-token context window. Use own-property reads and fall back to openAiModelInfoSaneDefaults, matching the backend getModel(). - tests: toString/constructor/__proto__/unknown IDs; fetched __proto__ ID * fix(friendli): default empty model IDs and keep __proto__ IDs across refetches - useSelectedModel: an empty apiModelId resolves to the default Friendli model, matching the backend getModel() - useRouterModels: skip structural sharing when a catalog has an own __proto__ model ID; replaceEqualDeep copies into {}, where that key swaps the prototype - tests: empty ID, empty-then-__proto__ refetch through useQuery, and keyless requests sending the placeholder key and surfacing a 401 * test(friendli): cover ordinary IDs beside __proto__ and fetched default metadata - useRouterModels: the refetched catalog keeps both __proto__ and an ordinary ID - useSelectedModel: an empty model ID uses fetched metadata for the default model * test(friendli): cover spent-deadline guard, provider filter, and static-only ID validation * test(friendli): assert default context window when context_length is missing
1 parent a715715 commit 18861b2

23 files changed

Lines changed: 1088 additions & 53 deletions

‎packages/types/src/__tests__/provider-identifiers.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ describe("provider identifiers", () => {
114114
providerIdentifiers.nanogpt,
115115
providerIdentifiers.ioIntelligence,
116116
providerIdentifiers.kimiCode,
117+
providerIdentifiers.friendli,
117118
])
118119
expect(localProviders).toEqual([providerIdentifiers.ollama, providerIdentifiers.lmstudio])
119120
expect(internalProviders).toEqual([providerIdentifiers.vscodeLm, providerIdentifiers.githubCopilot])

‎packages/types/src/provider-settings.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ export const dynamicProviders = [
102102
providerIdentifiers.nanogpt,
103103
providerIdentifiers.ioIntelligence,
104104
providerIdentifiers.kimiCode,
105+
providerIdentifiers.friendli,
105106
] as const
106107

107108
export type DynamicProvider = (typeof dynamicProviders)[number]

‎src/api/providers/__tests__/friendli.spec.ts‎

Lines changed: 379 additions & 7 deletions
Large diffs are not rendered by default.
Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
// npx vitest run api/providers/fetchers/__tests__/friendli.spec.ts
2+
3+
import axios from "axios"
4+
5+
import { openAiModelInfoSaneDefaults } from "@roo-code/types"
6+
7+
import { getFriendliModels } from "../friendli"
8+
9+
vi.mock("axios")
10+
11+
describe("getFriendliModels", () => {
12+
it("maps chat models from /v1/models", async () => {
13+
vi.mocked(axios.get).mockResolvedValueOnce({
14+
data: {
15+
data: [
16+
{
17+
id: "zai-org/GLM-5.3",
18+
mode: "chat",
19+
description: "GLM",
20+
context_length: 1_048_576,
21+
max_completion_tokens: 131_072,
22+
pricing: { input: "0.00000126", output: "0.00000396", input_cache_read: "0.000000234" },
23+
input_modalities: ["text", "image"],
24+
reasoning_options: [
25+
{ type: "effort", values: ["low", "high", "max", "ultracode"] },
26+
{ type: "budget_tokens", min: -1, max: 1_048_576 },
27+
],
28+
},
29+
{
30+
id: "google/gemma-4-31B-it",
31+
mode: "chat",
32+
pricing: { input: "0.00000014", output: "0.0000004" },
33+
},
34+
{ id: "zai-org/GLM-5.2", mode: "chat", input_modalities: ["text"] },
35+
{ id: "some/embedding", mode: "embedding" },
36+
],
37+
},
38+
})
39+
40+
const signal = new AbortController().signal
41+
const models = await getFriendliModels({ signal })
42+
43+
expect(axios.get).toHaveBeenCalledWith("https://api.friendli.ai/serverless/v1/models", { signal })
44+
expect(Object.keys(models)).toEqual(["zai-org/GLM-5.3", "google/gemma-4-31B-it", "zai-org/GLM-5.2"])
45+
expect(models["zai-org/GLM-5.3"]).toEqual({
46+
maxTokens: 131_072,
47+
contextWindow: 1_048_576,
48+
supportsImages: true,
49+
supportsPromptCache: true,
50+
supportsMaxTokens: true,
51+
inputPrice: expect.closeTo(1.26),
52+
outputPrice: expect.closeTo(3.96),
53+
cacheReadsPrice: expect.closeTo(0.234),
54+
supportsReasoningEffort: ["low", "high", "max"],
55+
reasoningEffort: "high",
56+
description: "GLM",
57+
})
58+
expect(models["google/gemma-4-31B-it"].supportsReasoningEffort).toBeUndefined()
59+
expect(models["google/gemma-4-31B-it"].supportsPromptCache).toBe(false)
60+
// A model without context_length gets the sane default window.
61+
expect(models["google/gemma-4-31B-it"].contextWindow).toBe(openAiModelInfoSaneDefaults.contextWindow)
62+
// Missing input_modalities and a text-only list both mean no image input.
63+
expect(models["google/gemma-4-31B-it"].supportsImages).toBe(false)
64+
expect(models["zai-org/GLM-5.2"].supportsImages).toBe(false)
65+
})
66+
67+
it("accepts null reasoning_options and keeps the catalog", async () => {
68+
vi.mocked(axios.get).mockResolvedValueOnce({
69+
data: {
70+
data: [
71+
{ id: "vendor/fetched-only", reasoning_options: null },
72+
{ id: "zai-org/GLM-5.1", reasoning_options: [{ type: "toggle" }] },
73+
],
74+
},
75+
})
76+
77+
const models = await getFriendliModels()
78+
79+
expect(Object.keys(models)).toEqual(["vendor/fetched-only", "zai-org/GLM-5.1"])
80+
expect(models["vendor/fetched-only"].supportsReasoningEffort).toBeUndefined()
81+
expect(models["vendor/fetched-only"].supportsReasoningBinary).toBeUndefined()
82+
})
83+
84+
it("maps toggle-only reasoning to supportsReasoningBinary without inventing an effort", async () => {
85+
vi.mocked(axios.get).mockResolvedValueOnce({
86+
data: {
87+
data: [
88+
{
89+
id: "zai-org/GLM-5.1",
90+
reasoning_options: [{ type: "toggle" }, { type: "budget_tokens", min: -1, max: 202_752 }],
91+
},
92+
{
93+
id: "zai-org/GLM-5.2",
94+
reasoning_options: [{ type: "toggle" }, { type: "effort", values: ["high", "max"] }],
95+
},
96+
],
97+
},
98+
})
99+
100+
const models = await getFriendliModels()
101+
102+
expect(models["zai-org/GLM-5.1"].supportsReasoningBinary).toBe(true)
103+
expect(models["zai-org/GLM-5.1"].supportsReasoningEffort).toBeUndefined()
104+
expect(models["zai-org/GLM-5.1"].reasoningEffort).toBeUndefined()
105+
// A toggle next to effort levels keeps the effort selector.
106+
expect(models["zai-org/GLM-5.2"].supportsReasoningBinary).toBeUndefined()
107+
expect(models["zai-org/GLM-5.2"].supportsReasoningEffort).toEqual(["high", "max"])
108+
})
109+
110+
it("defaults to the first supported effort when high is not offered", async () => {
111+
vi.mocked(axios.get).mockResolvedValueOnce({
112+
data: {
113+
data: [
114+
{
115+
id: "vendor/no-high",
116+
reasoning_options: [{ type: "effort", values: ["ultracode", "low", "medium", "max"] }],
117+
},
118+
],
119+
},
120+
})
121+
122+
const models = await getFriendliModels()
123+
124+
expect(models["vendor/no-high"].supportsReasoningEffort).toEqual(["low", "medium", "max"])
125+
expect(models["vendor/no-high"].reasoningEffort).toBe("low")
126+
})
127+
128+
it("keeps reserved upstream IDs as own keys", async () => {
129+
vi.mocked(axios.get).mockResolvedValueOnce({
130+
data: {
131+
data: [
132+
{ id: "__proto__", context_length: 1000 },
133+
{ id: "toString", context_length: 2000 },
134+
],
135+
},
136+
})
137+
138+
const models = await getFriendliModels()
139+
140+
expect(Object.getPrototypeOf(models)).toBeNull()
141+
expect(Object.keys(models)).toEqual(["__proto__", "toString"])
142+
expect(models["__proto__"].contextWindow).toBe(1000)
143+
expect(models["toString"].contextWindow).toBe(2000)
144+
expect(models["valueOf"]).toBeUndefined()
145+
})
146+
147+
it("rethrows cancellation instead of returning an empty record", async () => {
148+
const controller = new AbortController()
149+
controller.abort()
150+
vi.mocked(axios.get).mockRejectedValueOnce(new Error("canceled"))
151+
152+
await expect(getFriendliModels({ signal: controller.signal })).rejects.toMatchObject({ name: "AbortError" })
153+
})
154+
155+
it("returns an empty record on an invalid response", async () => {
156+
vi.spyOn(console, "error").mockImplementation(() => {})
157+
vi.mocked(axios.get).mockResolvedValueOnce({ data: { invalid: true } })
158+
159+
expect(await getFriendliModels()).toEqual({})
160+
})
161+
162+
it("returns an empty record on request failure", async () => {
163+
vi.spyOn(console, "error").mockImplementation(() => {})
164+
vi.mocked(axios.get).mockRejectedValueOnce(new Error("Network error"))
165+
166+
expect(await getFriendliModels()).toEqual({})
167+
})
168+
})
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
import axios from "axios"
2+
import { z } from "zod"
3+
4+
import {
5+
type ModelInfo,
6+
type ReasoningEffortExtended,
7+
openAiModelInfoSaneDefaults,
8+
reasoningEffortsExtended,
9+
} from "@roo-code/types"
10+
11+
import { parseApiPrice } from "../../../shared/cost"
12+
import { throwIfAborted } from "../utils/abort-signal"
13+
14+
const friendliModelSchema = z.object({
15+
id: z.string(),
16+
mode: z.string().optional(),
17+
description: z.string().optional(),
18+
context_length: z.number().optional(),
19+
max_completion_tokens: z.number().optional(),
20+
pricing: z
21+
.object({
22+
input: z.string().optional(),
23+
output: z.string().optional(),
24+
input_cache_read: z.string().optional(),
25+
})
26+
.optional(),
27+
input_modalities: z.array(z.string()).optional(),
28+
reasoning_options: z.array(z.object({ type: z.string(), values: z.array(z.string()).optional() })).nullish(),
29+
})
30+
31+
type FriendliModel = z.infer<typeof friendliModelSchema>
32+
33+
const friendliModelsResponseSchema = z.object({ data: z.array(friendliModelSchema) })
34+
35+
const parseFriendliModel = (model: FriendliModel): ModelInfo => {
36+
const efforts = model.reasoning_options
37+
?.find((option) => option.type === "effort")
38+
?.values?.filter((v): v is ReasoningEffortExtended =>
39+
(reasoningEffortsExtended as readonly string[]).includes(v),
40+
)
41+
// A toggle without effort levels (e.g. GLM-5.1, Gemma) is an on/off switch, not an effort scale.
42+
const toggleOnly = !efforts?.length && !!model.reasoning_options?.some((option) => option.type === "toggle")
43+
44+
return {
45+
maxTokens: model.max_completion_tokens,
46+
contextWindow: model.context_length ?? openAiModelInfoSaneDefaults.contextWindow,
47+
supportsImages: model.input_modalities?.includes("image") ?? false,
48+
supportsPromptCache: model.pricing?.input_cache_read !== undefined,
49+
supportsMaxTokens: true,
50+
inputPrice: parseApiPrice(model.pricing?.input),
51+
outputPrice: parseApiPrice(model.pricing?.output),
52+
cacheReadsPrice: parseApiPrice(model.pricing?.input_cache_read),
53+
...(efforts?.length && {
54+
supportsReasoningEffort: efforts,
55+
reasoningEffort: efforts.includes("high") ? "high" : efforts[0],
56+
}),
57+
...(toggleOnly && { supportsReasoningBinary: true }),
58+
description: model.description,
59+
}
60+
}
61+
62+
/** Fetches the public (no auth) Friendli model catalog. */
63+
export async function getFriendliModels(opts?: { signal?: AbortSignal }): Promise<Record<string, ModelInfo>> {
64+
// Null prototype: upstream IDs such as "__proto__" or "toString" must stay plain keys.
65+
const models: Record<string, ModelInfo> = Object.create(null)
66+
67+
try {
68+
const response = await axios.get("https://api.friendli.ai/serverless/v1/models", { signal: opts?.signal })
69+
const result = friendliModelsResponseSchema.safeParse(response.data)
70+
71+
if (!result.success) {
72+
console.error(`Friendli models response is invalid: ${JSON.stringify(result.error.format())}`)
73+
return models
74+
}
75+
76+
for (const model of result.data.data) {
77+
if (model.mode && model.mode !== "chat") continue
78+
models[model.id] = parseFriendliModel(model)
79+
}
80+
} catch (error) {
81+
throwIfAborted(opts?.signal)
82+
console.error(`Error fetching Friendli models: ${error instanceof Error ? error.message : String(error)}`)
83+
}
84+
85+
return models
86+
}

0 commit comments

Comments
 (0)