Skip to content

Commit c35bc6f

Browse files
committed
fix(core): keep a user buffer authoritative and offset the reservation by the input limit
Rework of the ceiling reserve after review on #58492: - A user-set compaction buffer replaces the default reserve outright again; the output reservation no longer raises it, so buffer stays an override rather than advisory. - The output reservation only reserves room it needs inside the effective window: when the input limit sits below the context window, the part of the reservation that fits between the two limits no longer adds to the reserve. - Test boundaries updated to the new ceilings, plus cases for the 1M window from #58463 (compacts at 792,576) and for the buffer staying authoritative.
1 parent 7b0c3b4 commit c35bc6f

2 files changed

Lines changed: 37 additions & 16 deletions

File tree

‎packages/core/src/session/compaction.ts‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ import { SessionUsage } from "./usage.js"
4040

4141
export type Settings = {
4242
auto: boolean
43-
/** Tokens kept free below the model's limits before compacting. Unset keeps 15%, and at least 16k, free. */
43+
/** Tokens kept free below the model's limits before compacting. Unset keeps 15%, and at least 16k and the model's output reservation, free. */
4444
buffer?: number
4545
/** Tokens of recent conversation kept verbatim beside the summary. */
4646
keep: number
@@ -1098,19 +1098,23 @@ export const estimatePrompt = (context: SessionContext.Loaded) => {
10981098

10991099
/**
11001100
* The largest request the model takes while leaving room for its reply: `RESERVE_SHARE` of the window, or
1101-
* `RESERVE_MIN` when that is more, and never less than the output tokens a request reserves — otherwise a
1102-
* request that still fits overflows before automatic compaction runs. A summary request that must be shrunk
1103-
* is aimed at the same size. A window too small to give up `RESERVE_MIN` keeps `RESERVE_SHARE`.
1101+
* `RESERVE_MIN` when that is more, and never less than the output tokens a request reserves past what already
1102+
* fits below a lower input limit — otherwise a request that still fits overflows before automatic compaction
1103+
* runs. A summary request that must be shrunk is aimed at the same size. A window too small to give up
1104+
* `RESERVE_MIN` keeps `RESERVE_SHARE`. A user-set buffer replaces the default reserve outright.
11041105
*/
11051106
const calculateCeiling = (limit: SessionContext.Loaded["model"]["limit"], buffer: number | undefined) => {
11061107
// Unknown limits are reported as 0. An unknown input limit falls back to the context window; with no window at
11071108
// all, only a provider rejection can limit the request.
11081109
const window = limit.input || limit.context
11091110
if (window <= 0) return Number.POSITIVE_INFINITY
1111+
// The reservation leaves room in the context window, so the part of it that fits below a lower input limit
1112+
// needs no reserve of its own. An unknown context window cannot offset any of it.
1113+
const offset = Math.max(0, limit.context - window)
11101114
// The reservation raise is ignored when it would leave the window no room, such as an output limit copied from a larger model.
1111-
const reserved = Math.min(SessionModelRequest.outputReservation(limit), window - 1)
1115+
const reserved = Math.min(Math.max(0, SessionModelRequest.outputReservation(limit) - offset), window - 1)
11121116
// A buffer that leaves no room, such as one set for a larger model, is ignored.
1113-
if (buffer !== undefined && buffer < window) return window - Math.max(buffer, reserved)
1117+
if (buffer !== undefined && buffer < window) return window - buffer
11141118
return window - Math.max(Math.floor(window * RESERVE_SHARE), window >= 2 * RESERVE_MIN ? RESERVE_MIN : 0, reserved)
11151119
}
11161120

‎packages/core/test/session-compaction.test.ts‎

Lines changed: 27 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -318,7 +318,8 @@ it.effect("auto compaction estimates current content against the buffered prompt
318318
const due = (context: SessionContext.Loaded) =>
319319
compaction.compact({ reason: "auto", context }).pipe(Effect.map((outcome) => outcome.status !== "skipped"))
320320

321-
// 85% of the input limit, which takes precedence over the context window.
321+
// 85% of the input limit, which takes precedence over the context window. The output reservation fits
322+
// between the two limits, so it does not add to the reserve.
322323
const inputLimited = { context: 400_000, input: 272_000, output: 128_000 }
323324
expect(yield* due(input(231_199, inputLimited))).toBe(false)
324325
expect(yield* due(input(231_200, inputLimited))).toBe(true)
@@ -335,10 +336,16 @@ it.effect("auto compaction estimates current content against the buffered prompt
335336
expect(yield* due(input(83_999, contextLimited))).toBe(false)
336337
expect(yield* due(input(84_000, contextLimited))).toBe(true)
337338

338-
// The reply limit does not lower the ceiling.
339+
// The reply limit lowers the ceiling only past the default reserve.
339340
const outputLimited = { context: 100_000, output: 30_000 }
340-
expect(yield* due(input(83_999, outputLimited))).toBe(false)
341-
expect(yield* due(input(84_000, outputLimited))).toBe(true)
341+
expect(yield* due(input(69_999, outputLimited))).toBe(false)
342+
expect(yield* due(input(70_000, outputLimited))).toBe(true)
343+
344+
// A 15% buffer of a large window can fall below the 256k output reservation, so the reservation keeps the
345+
// ceiling at the request ceiling (#58463: a 1M window with 393,216 output compacts at 792,576, not 85%).
346+
const largeWindow = { context: 1_048_576, output: 393_216 }
347+
expect(yield* due(input(792_575, largeWindow))).toBe(false)
348+
expect(yield* due(input(792_576, largeWindow))).toBe(true)
342349

343350
// A buffer that leaves the model no room, such as one set for a larger model, falls back to the default reserve.
344351
yield* Effect.scoped(
@@ -349,6 +356,15 @@ it.effect("auto compaction estimates current content against the buffered prompt
349356
}),
350357
)
351358

359+
// A user-set buffer stays authoritative; the output reservation does not raise it.
360+
yield* Effect.scoped(
361+
Effect.gen(function* () {
362+
yield* compaction.transform((editor) => editor.configure({ buffer: 8_000 }))
363+
expect(yield* due(input(91_999, outputLimited))).toBe(false)
364+
expect(yield* due(input(92_000, outputLimited))).toBe(true)
365+
}),
366+
)
367+
352368
const assistant = input(89_000, contextLimited).messages[0]
353369
const tool = SessionMessage.AssistantTool.make({
354370
type: "tool",
@@ -599,7 +615,8 @@ it.effect("manual compaction summarizes short context instead of no-op", () =>
599615
"x-opencode-session": parentID,
600616
"x-opencode-client": "opencode",
601617
})
602-
expect(requests[0]?.generation).toEqual(GenerationOptions.make({ maxTokens: 30_000 }))
618+
// The reservation raises the default reserve to 32k of the 200k window, so the summary asks for all the room left.
619+
expect(requests[0]?.generation).toEqual(GenerationOptions.make({ maxTokens: 32_000 }))
603620
expect(JSON.stringify(requests[0]?.messages)).toContain("Manual compaction should include this short conversation.")
604621
expect(JSON.stringify(requests[0]?.messages)).toContain("Use Effect services and generators.")
605622
expect(JSON.stringify(requests[0]?.messages)).toContain("User shell pwd completed: /project")
@@ -681,7 +698,7 @@ it.effect("summary compaction past the ceiling keeps the conversation unchanged
681698
Effect.gen(function* () {
682699
requests = []
683700
const session = yield* insertSession(Session.ID.make("ses_compaction_near_window"))
684-
// About 182k estimated tokens: past the 170k ceiling, within the 184k left after reserving room for the summary.
701+
// About 182k estimated tokens: past the 168k ceiling, within the 184k left after reserving room for the summary.
685702
const messages = ["first", "second"].map((label) =>
686703
SessionMessage.User.make({
687704
id: SessionMessage.ID.create(),
@@ -706,7 +723,7 @@ it.effect(
706723
requests = []
707724
const session = yield* insertSession(Session.ID.make("ses_compaction_older_fallback"))
708725
// About 190k estimated tokens in total, past the 184k left for the summary. The 160k before the retained
709-
// 30k latest exchange fits the 170k ceiling.
726+
// 30k latest exchange fits the 168k ceiling.
710727
const messages = [
711728
["older", 640_000],
712729
["latest", 120_000],
@@ -754,7 +771,7 @@ it.effect("compaction keeps the output limit for a request that fits its target"
754771
Effect.gen(function* () {
755772
requests = []
756773
const session = yield* insertSession(Session.ID.make("ses_fitting_compaction"))
757-
// About 160k estimated tokens, under the 170k target, leaving 40k of the 200k window.
774+
// About 160k estimated tokens, under the 168k target, leaving 40k of the 200k window.
758775
const messages = [
759776
SessionMessage.User.make({
760777
id: SessionMessage.ID.create(),
@@ -766,7 +783,7 @@ it.effect("compaction keeps the output limit for a request that fits its target"
766783

767784
expect(yield* compactManually(session, messages)).toEqual({ status: "completed" })
768785
expect(requests).toHaveLength(1)
769-
expect(requests[0]?.generation).toEqual(GenerationOptions.make({ maxTokens: 30_000 }))
786+
expect(requests[0]?.generation).toEqual(GenerationOptions.make({ maxTokens: 32_000 }))
770787
}),
771788
)
772789

@@ -889,7 +906,7 @@ const measured = (text: string, input: number) =>
889906
})
890907

891908
/**
892-
* A 200k window compacts at 170k and starts a background summary at 85% of that, 144.5k. `near` ends past that
909+
* A 200k window compacts at 168k and starts a background summary at 85% of that, 142.8k. `near` ends past that
893910
* point; `due` continues it past the threshold.
894911
*/
895912
const precomputeHistory = () => {

0 commit comments

Comments
 (0)