Skip to content

Commit abc5805

Browse files
committed
fix(fireworks): omit non-positive max_tokens for custom models
Custom models use sane-default metadata with maxTokens: -1 (unlimited), which Fireworks rejects with '400 max_tokens must be non-negative'. Add an opt-out supportsUnlimitedMaxTokens flag to BaseOpenAiCompatibleProvider (default true, preserving behavior for other providers) and set it to false for Fireworks so max_tokens is omitted.
1 parent 45825e9 commit abc5805

4 files changed

Lines changed: 54 additions & 1 deletion

File tree

‎src/api/providers/__tests__/base-openai-compatible-provider.spec.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -343,6 +343,18 @@ describe("BaseOpenAiCompatibleProvider", () => {
343343
)
344344
})
345345

346+
it("passes max_tokens through unchanged by default (providers may accept -1)", async () => {
347+
mockCreate.mockImplementationOnce(() => asyncStreamFrom([]))
348+
349+
const customHandler = new TestOpenAiCompatibleProvider("test-api-key", {
350+
apiModelId: "some/custom-model-not-in-list",
351+
})
352+
353+
await collectStream(customHandler.createMessage("system prompt", []))
354+
355+
expect(mockCreate.mock.calls[0][0].max_tokens).toBe(-1)
356+
})
357+
346358
it("completePrompt sends the custom model id verbatim to the API", async () => {
347359
mockCreate.mockResolvedValueOnce({ choices: [{ message: { content: "ok" } }] })
348360

‎src/api/providers/__tests__/fireworks.spec.ts‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,31 @@ describe("FireworksHandler", () => {
9898
expect(model.info).toEqual(expect.objectContaining(fireworksModels[testModelId]))
9999
})
100100

101+
it("should omit max_tokens for a custom model because Fireworks rejects -1", async () => {
102+
// Regression: custom models get sane-default metadata with maxTokens: -1, which
103+
// Fireworks rejects with `400 max_tokens must be non-negative`.
104+
const handlerWithCustomModel = new FireworksHandler({
105+
apiModelId: "accounts/fireworks/models/deepseek-v4p1-flash",
106+
fireworksApiKey: "test-fireworks-api-key",
107+
})
108+
109+
await collectStream(handlerWithCustomModel.createMessage("system prompt", []))
110+
111+
expect(mockCreate.mock.calls[0][0].model).toBe("accounts/fireworks/models/deepseek-v4p1-flash")
112+
expect(mockCreate.mock.calls[0][0].max_tokens).toBeUndefined()
113+
})
114+
115+
it("should still send a positive max_tokens for a known model", async () => {
116+
const handlerWithModel = new FireworksHandler({
117+
apiModelId: "accounts/fireworks/models/kimi-k2-instruct",
118+
fireworksApiKey: "test-fireworks-api-key",
119+
})
120+
121+
await collectStream(handlerWithModel.createMessage("system prompt", []))
122+
123+
expect(mockCreate.mock.calls[0][0].max_tokens).toBe(16384)
124+
})
125+
101126
it.each([
102127
{
103128
modelId: "accounts/fireworks/models/glm-5p1" as const,

‎src/api/providers/base-openai-compatible-provider.ts‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,12 @@ type BaseOpenAiCompatibleProviderOptions<ModelName extends string> = ApiHandlerO
2121
defaultProviderModelId: ModelName
2222
providerModels: Record<ModelName, ModelInfo>
2323
defaultTemperature?: number
24+
/**
25+
* Whether the provider API accepts `max_tokens: -1` (meaning "unlimited"), which is what
26+
* custom models with sane-default metadata produce. Providers that reject non-positive
27+
* values set this to `false`, and `max_tokens` is then omitted. Defaults to `true`.
28+
*/
29+
supportsUnlimitedMaxTokens?: boolean
2430
}
2531

2632
export abstract class BaseOpenAiCompatibleProvider<ModelName extends string>
@@ -32,6 +38,7 @@ export abstract class BaseOpenAiCompatibleProvider<ModelName extends string>
3238
protected readonly defaultTemperature: number
3339
protected readonly defaultProviderModelId: ModelName
3440
protected readonly providerModels: Record<ModelName, ModelInfo>
41+
protected readonly supportsUnlimitedMaxTokens: boolean
3542

3643
protected readonly options: ApiHandlerOptions
3744

@@ -43,6 +50,7 @@ export abstract class BaseOpenAiCompatibleProvider<ModelName extends string>
4350
defaultProviderModelId,
4451
providerModels,
4552
defaultTemperature,
53+
supportsUnlimitedMaxTokens,
4654
...options
4755
}: BaseOpenAiCompatibleProviderOptions<ModelName>) {
4856
super()
@@ -52,6 +60,7 @@ export abstract class BaseOpenAiCompatibleProvider<ModelName extends string>
5260
this.defaultProviderModelId = defaultProviderModelId
5361
this.providerModels = providerModels
5462
this.defaultTemperature = defaultTemperature ?? 0
63+
this.supportsUnlimitedMaxTokens = supportsUnlimitedMaxTokens ?? true
5564

5665
this.options = options
5766

@@ -76,13 +85,17 @@ export abstract class BaseOpenAiCompatibleProvider<ModelName extends string>
7685
const { id: model, info } = this.getModel()
7786

7887
// Centralized cap: clamp to 20% of the context window (unless provider-specific exceptions apply)
79-
const max_tokens =
88+
const computedMaxTokens =
8089
getModelMaxOutputTokens({
8190
modelId: model,
8291
model: info,
8392
settings: this.options,
8493
format: "openai",
8594
}) ?? undefined
95+
const max_tokens =
96+
!this.supportsUnlimitedMaxTokens && computedMaxTokens !== undefined && computedMaxTokens <= 0
97+
? undefined
98+
: computedMaxTokens
8699

87100
const temperature = this.options.modelTemperature ?? info.defaultTemperature ?? this.defaultTemperature
88101

‎src/api/providers/fireworks.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ export class FireworksHandler extends BaseOpenAiCompatibleProvider<FireworksMode
1414
defaultProviderModelId: fireworksDefaultModelId,
1515
providerModels: fireworksModels,
1616
defaultTemperature: 0.5,
17+
// Fireworks rejects `max_tokens: -1` ("unlimited") with
18+
// `400 max_tokens must be non-negative`, so omit it instead.
19+
supportsUnlimitedMaxTokens: false,
1720
})
1821
}
1922
}

0 commit comments

Comments
 (0)