Skip to content

Commit 5de9656

Browse files
committed
fix(core): propagate resolved output limit into model request generation
The prepared model request only carried generation values set by hooks, so a model's configured limit.output never reached the provider. On Bedrock the request was sent without inferenceConfig.maxTokens and the service applied a much lower implicit budget; with adaptive thinking the whole budget could be consumed by reasoning, truncating responses at exactly 4,096 output tokens despite limit.output=128000. Project maxTokens with explicit precedence: hook/context override, model defaults, route defaults, then resolved.limit.output. Each protocol already lowers generation.maxTokens into its native field, so no protocol changes are needed. Closes anomalyco#46595 Related: anomalyco#47398
1 parent 437cdc0 commit 5de9656

3 files changed

Lines changed: 39 additions & 3 deletions

File tree

‎packages/core/src/session/model-request.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -249,7 +249,17 @@ export const layer = Layer.effect(
249249
messages: boundImages(unsupportedParts(shaped.messages, model.capabilities)),
250250
tools: Array.from(hooked, ([name, t]) => ({ ...t, name })),
251251
toolChoice: input.toolChoice,
252-
generation: Object.keys(generation).length === 0 ? undefined : generation,
252+
// Always project an output limit: hook override, then model defaults, then route defaults,
253+
// then the catalog limit. Without it providers apply their own implicit budget (Bedrock
254+
// truncates long adaptive-thinking turns at 4096 output tokens).
255+
generation: {
256+
...generation,
257+
maxTokens:
258+
generation.maxTokens ??
259+
model.model.defaults?.generation?.maxTokens ??
260+
model.model.route.defaults.generation?.maxTokens ??
261+
model.limit.output,
262+
},
253263
providerOptions: Object.keys(providerOptions).length === 0 ? undefined : providerOptions,
254264
})
255265

‎packages/core/test/session-compaction.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -407,7 +407,7 @@ it.effect("manual compaction summarizes short context instead of no-op", () =>
407407
"x-opencode-session": sessionID,
408408
"x-opencode-client": "opencode",
409409
})
410-
expect(requests[0]?.generation).toBeUndefined()
410+
expect(requests[0]?.generation).toEqual({ maxTokens: 32_000 })
411411
expect(JSON.stringify(requests[0]?.messages)).toContain("Manual compaction should include this short conversation.")
412412
expect(JSON.stringify(requests[0]?.messages)).toContain("Use Effect services and generators.")
413413
expect(JSON.stringify(requests[0]?.messages)).toContain("User shell pwd completed: /project")

‎packages/core/test/session-runner.test.ts‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2959,6 +2959,32 @@ describe("SessionRunnerLLM", () => {
29592959
expect(yield* s.context).not.toContainEqual(expect.objectContaining({ type: "compaction" }))
29602960
})
29612961

2962+
scenario("projects the resolved output limit into the request while preserving hook overrides", function* (s) {
2963+
const hooks = yield* PluginHooks.Service
2964+
s.currentModel = recoveryModel
2965+
yield* s.llm.push(TestLLM.text("Answer", "text-output-limit-default"))
2966+
yield* s.runPrompt("Hello")
2967+
2968+
// Without hooks the catalog limit reaches the provider instead of an implicit budget.
2969+
expect(s.requests).toHaveLength(1)
2970+
expect(s.requests[0]?.generation).toEqual({ maxTokens: 1_000 })
2971+
2972+
const hook = yield* hooks.register("session", "context", (event) =>
2973+
Effect.sync(() => {
2974+
event.options.maxTokens = 250
2975+
event.options.temperature = 0.2
2976+
}),
2977+
)
2978+
s.requests.length = 0
2979+
yield* s.llm.push(TestLLM.text("Answer", "text-output-limit-hook"))
2980+
yield* s.runPrompt("Hello again")
2981+
2982+
// A hook value wins over the catalog limit; other generation fields are preserved.
2983+
expect(s.requests).toHaveLength(1)
2984+
expect(s.requests[0]?.generation).toEqual({ maxTokens: 250, temperature: 0.2 })
2985+
yield* hook.dispose
2986+
})
2987+
29622988
scenario("stops after required automatic compaction fails", function* (s) {
29632989
yield* s.llm.push(TestLLM.textWithUsage("Earlier answer", "text-before-failed-compaction", 3_950))
29642990
yield* s.runPrompt("Earlier question ".repeat(180))
@@ -2973,7 +2999,7 @@ describe("SessionRunnerLLM", () => {
29732999
expect(yield* Effect.exit(s.resume)).toMatchObject({ _tag: "Failure" })
29743000

29753001
expect(s.requests).toHaveLength(1)
2976-
expect(s.requests[0]?.generation).toBeUndefined()
3002+
expect(s.requests[0]?.generation).toEqual({ maxTokens: 50 })
29773003
expect(yield* s.context).toContainEqual(
29783004
expect.objectContaining({
29793005
type: "compaction",

0 commit comments

Comments
 (0)