Skip to content

Commit 06b4d56

Browse files
Alaundoclaude
Alaundo
and
claude
committed
fix(plugins): read top-level context_length/context_window/context_size on self-hosted model discovery
discoverOpenAICompatibleLocalModels only read contextWindow from model.meta.n_ctx_train (llama.cpp-specific), falling back to the hardcoded 128k default for any provider that advertises context length as a top-level field instead. Cortecs.ai's /v1/models response for glm-5.2 reports context_size: 1048576 (the model's real 1M-token window), but that field was ignored, so the catalog under-reported it as 128k/131k. This caused unnecessarily aggressive/premature compaction on sessions that had far more real headroom than the catalog believed. Fixes #109367 Co-Authored-By: Claude Opus 4.7 <[email protected]>
1 parent 6e19c36 commit 06b4d56

2 files changed

Lines changed: 98 additions & 1 deletion

File tree

‎src/plugins/provider-self-hosted-setup.test.ts‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -439,6 +439,85 @@ describe("discoverOpenAICompatibleLocalModels", () => {
439439
expect(propsRelease).toHaveBeenCalledOnce();
440440
});
441441

442+
it("reads provider-advertised context_size when meta.n_ctx_train is absent", async () => {
443+
const modelsRelease = vi.fn(async () => undefined);
444+
const propsRelease = vi.fn(async () => undefined);
445+
fetchWithSsrFGuardMock.mockResolvedValueOnce({
446+
response: new Response(
447+
JSON.stringify({
448+
data: [
449+
{
450+
id: "glm-5.2",
451+
context_size: 1_048_576,
452+
},
453+
],
454+
}),
455+
{ status: 200 },
456+
),
457+
finalUrl: "https://api.cortecs.ai/v1/models",
458+
release: modelsRelease,
459+
});
460+
fetchWithSsrFGuardMock.mockResolvedValueOnce({
461+
response: new Response("not found", { status: 404 }),
462+
finalUrl: "https://api.cortecs.ai/props",
463+
release: propsRelease,
464+
});
465+
466+
const models = await discoverOpenAICompatibleLocalModels({
467+
baseUrl: "https://api.cortecs.ai/v1",
468+
apiKey: "self-hosted-test-key",
469+
label: "cortecs",
470+
env: {},
471+
});
472+
473+
expect(models).toEqual([
474+
{
475+
id: "glm-5.2",
476+
name: "glm-5.2",
477+
reasoning: false,
478+
input: ["text"],
479+
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
480+
contextWindow: 1_048_576,
481+
maxTokens: 8192,
482+
},
483+
]);
484+
});
485+
486+
it("prefers context_length over context_window and context_size when multiple are present", async () => {
487+
const modelsRelease = vi.fn(async () => undefined);
488+
const propsRelease = vi.fn(async () => undefined);
489+
fetchWithSsrFGuardMock.mockResolvedValueOnce({
490+
response: new Response(
491+
JSON.stringify({
492+
data: [
493+
{
494+
id: "some-model",
495+
context_length: 200_000,
496+
context_window: 100_000,
497+
context_size: 50_000,
498+
},
499+
],
500+
}),
501+
{ status: 200 },
502+
),
503+
finalUrl: "http://127.0.0.1:8000/v1/models",
504+
release: modelsRelease,
505+
});
506+
fetchWithSsrFGuardMock.mockResolvedValueOnce({
507+
response: new Response("not found", { status: 404 }),
508+
finalUrl: "http://127.0.0.1:8000/props",
509+
release: propsRelease,
510+
});
511+
512+
const models = await discoverOpenAICompatibleLocalModels({
513+
baseUrl: "http://127.0.0.1:8000/v1",
514+
label: "generic",
515+
env: {},
516+
});
517+
518+
expect(models[0]?.contextWindow).toBe(200_000);
519+
});
520+
442521
it("preserves explicit configured context windows ahead of llama.cpp /props", async () => {
443522
const release = vi.fn(async () => undefined);
444523
fetchWithSsrFGuardMock.mockResolvedValueOnce({

‎src/plugins/provider-self-hosted-setup.ts‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,13 @@ const SELF_HOSTED_DISCOVERY_JSON_MAX_BYTES = 16 * 1024 * 1024;
4949
type OpenAICompatModelsResponse = {
5050
data?: Array<{
5151
id?: string;
52+
// Some OpenAI-compatible catalogs (e.g. cortecs.ai) advertise the model's
53+
// context window directly on the row instead of nesting it under `meta`.
54+
// Field naming isn't standardized across providers, so accept the common
55+
// variants seen in the wild.
56+
context_length?: unknown;
57+
context_window?: unknown;
58+
context_size?: unknown;
5259
meta?: {
5360
n_ctx_train?: unknown;
5461
};
@@ -222,7 +229,15 @@ export async function discoverOpenAICompatibleLocalModels(params: {
222229
if (!modelId) {
223230
return [];
224231
}
225-
return [{ id: modelId, meta: model.meta }];
232+
return [
233+
{
234+
id: modelId,
235+
meta: model.meta,
236+
context_length: model.context_length,
237+
context_window: model.context_window,
238+
context_size: model.context_size,
239+
},
240+
];
226241
});
227242
const runtimeContextTokensByModelId = new Map<string, number>();
228243
if (params.contextWindow === undefined) {
@@ -257,6 +272,9 @@ export async function discoverOpenAICompatibleLocalModels(params: {
257272
contextWindow:
258273
params.contextWindow ??
259274
readPositiveInteger(model.meta?.n_ctx_train) ??
275+
readPositiveInteger(model.context_length) ??
276+
readPositiveInteger(model.context_window) ??
277+
readPositiveInteger(model.context_size) ??
260278
SELF_HOSTED_DEFAULT_CONTEXT_WINDOW,
261279
maxTokens: params.maxTokens ?? SELF_HOSTED_DEFAULT_MAX_TOKENS,
262280
};

0 commit comments

Comments
 (0)