Skip to content

Commit 6709f56

Browse files
authored
fix(core): give compaction summaries the same output limit as normal turns (#57564)
1 parent e98ab91 commit 6709f56

2 files changed

Lines changed: 13 additions & 13 deletions

File tree

‎packages/core/src/session/compaction.ts‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,8 @@ const RESERVE_MIN = 16_000
101101
const RESERVE_SHARE = 0.15
102102
/** A common window size, assumed for the compaction request when the model's window is unknown. */
103103
const UNKNOWN_WINDOW = 200_000
104+
/** A summary gets at least this much output, or the room left if less, even when the estimate margin leaves less. */
105+
const SUMMARY_OUTPUT_FLOOR = 32_000
104106
const IMAGE_TOKEN_ESTIMATE = 1_500
105107
const PDF_TOKEN_ESTIMATE = 2_000
106108

@@ -875,15 +877,20 @@ const hasMeasuredPrompt = (message: SessionMessage.Info, model: SessionContext.L
875877
* cannot shrink the conversation further. Lower the limit to the room the request being sent leaves in the window,
876878
* so the input and the requested output never exceed the window by our own estimate.
877879
*
878-
* Like `budget` in `prepare`, the estimate counts as measured, without the margin primary requests add for estimates:
879-
* `deliver` aims at about 90% of the window, so that margin would cut every summary near its target to the minimum.
880+
* The limit leaves the same margin for a low estimate that primary requests do. `deliver` aims at about 90% of the
881+
* window, so that margin alone would cut a summary near its target to the minimum; there it still gets up to
882+
* `SUMMARY_OUTPUT_FLOOR` of the room left without the margin.
880883
*/
881884
const fitOutput = (request: LLMRequest, limit: SessionContext.Loaded["model"]["limit"]) => {
882885
const current = request.generation?.maxTokens
883886
if (current === undefined) return request
887+
const estimated = estimateRequest(request)
884888
const maxTokens = Math.min(
885889
current,
886-
SessionModelRequest.outputLimit(limit, "compaction", { measured: estimateRequest(request), estimated: 0 }),
890+
Math.max(
891+
SessionModelRequest.outputLimit(limit, { measured: 0, estimated }),
892+
Math.min(SUMMARY_OUTPUT_FLOOR, SessionModelRequest.outputLimit(limit, { measured: estimated, estimated: 0 })),
893+
),
887894
)
888895
if (maxTokens === current) return request
889896
return LLMRequest.update(request, { generation: { ...request.generation, maxTokens } })

‎packages/core/src/session/model-request.ts‎

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,6 @@ const GENERATION_KEYS = new Set(Object.keys(GenerationOptions.fields))
4848
const OUTPUT_TOKEN_FALLBACK = 32_000
4949
// No reply needs more, however much the model allows.
5050
const OUTPUT_TOKEN_MAX = 256_000
51-
// A summary never needs more, and a request asking for more cannot be shrunk to fit a window the catalog overstates.
52-
const SUMMARY_OUTPUT_MAX = 32_000
5351
// Prompt text is estimated at about 4 characters per token, which can run low on dense text such as code.
5452
const ESTIMATE_ERROR = 0.15
5553
// Never ask for less; only reachable with automatic compaction off, since it keeps the window from filling this far.
@@ -84,13 +82,8 @@ export interface Input {
8482
}
8583

8684
/** The default output limit: the catalog limit, fitted to the room the prompt leaves in the context window. */
87-
export const outputLimit = (
88-
limit: Model.Info["limit"],
89-
kind: "primary" | "compaction",
90-
inputTokens?: Input["inputTokens"],
91-
) => {
92-
const model = Math.min(limit.output > 0 ? limit.output : OUTPUT_TOKEN_FALLBACK, OUTPUT_TOKEN_MAX)
93-
const requested = kind === "compaction" ? Math.min(model, SUMMARY_OUTPUT_MAX) : model
85+
export const outputLimit = (limit: Model.Info["limit"], inputTokens?: Input["inputTokens"]) => {
86+
const requested = Math.min(limit.output > 0 ? limit.output : OUTPUT_TOKEN_FALLBACK, OUTPUT_TOKEN_MAX)
9487
if (inputTokens === undefined || limit.context <= 0) return requested
9588
const room = limit.context - inputTokens.measured - Math.ceil(inputTokens.estimated * (1 + ESTIMATE_ERROR))
9689
return Math.min(requested, Math.max(OUTPUT_TOKEN_MIN, room))
@@ -258,7 +251,7 @@ export const layer = Layer.effect(
258251
messages: input.messages,
259252
options:
260253
kind === "primary" || kind === "compaction"
261-
? { maxTokens: outputLimit(model.limit, kind, input.inputTokens) }
254+
? { maxTokens: outputLimit(model.limit, input.inputTokens) }
262255
: {},
263256
},
264257
Object.fromEntries(Array.from(given, ([d, t]) => [t.name, d])),

0 commit comments

Comments
 (0)