Skip to content

Commit fd65892

Browse files
authored
fix(core): summarize the whole conversation during compaction (#57091)
1 parent 367c8fb commit fd65892

4 files changed

Lines changed: 189 additions & 59 deletions

File tree

‎packages/core/src/session/compaction.ts‎

Lines changed: 71 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ import { SessionUsage } from "./usage.js"
4040

4141
export type Settings = {
4242
auto: boolean
43-
/** Tokens kept free below the model's limits before compacting. Unset keeps 10% free. */
43+
/** Tokens kept free below the model's limits before compacting. Unset keeps 15%, and at least 16k, free. */
4444
buffer?: number
4545
/** Tokens of recent conversation kept verbatim beside the summary. */
4646
keep: number
@@ -95,6 +95,12 @@ const NOTHING_TO_COMPACT: Failure = { error: { type: "compaction.unavailable", m
9595
const SHRINK_STEPS = [0.7, 0.5, 0.35]
9696
// The least of the window kept free for the last reply before compaction and for the summary itself.
9797
const RESERVE_MIN = 16_000
98+
/**
99+
* Share of the window kept free before automatic compaction. Above `RESERVE_MIN`, the margin between it and
100+
* `summaryRoom` absorbs a large step past the trigger, so the whole conversation still goes out unchanged and reuses
101+
* the cache. Single steps grew by up to ~22k tokens at p99 in real sessions.
102+
*/
103+
const RESERVE_SHARE = 0.15
98104
/** A common window size, assumed for the compaction request when the model's window is unknown. */
99105
const UNKNOWN_WINDOW = 200_000
100106
const TOOL_OUTPUT_MAX_CHARS = 1_250
@@ -252,11 +258,12 @@ export const layer = Layer.effect(
252258

253259
/**
254260
* first: [E1 … E8][E9 E10] → [S1 + "E9 E10"]
255-
* again: [S1 + "E9 E10"][E11 … E18][E19] → [S2 + "E19"] S2 covers S1, E9 E10, and E11–E18
261+
* again: [S1 + "E9 E10"][E11 … E18][E19] → [S2 + "E19"] S2 covers the whole current conversation
256262
*
257-
* The model is sent the older part, then the summary prompt:
263+
* The model is sent the whole conversation, including the tail kept beside the summary, then the summary prompt.
264+
* Keeping the normal request's prefix intact lets the summary reuse its prompt cache:
258265
*
259-
* [system][S1 + "E9 E10"][E11 … E18][prompt] plus one nudge if the reply skips the template
266+
* [system][S1 + "E9 E10"][E11 … E18][E19][prompt] plus one nudge if the reply skips the template
260267
*
261268
* Later requests open with the checkpoint as one user message:
262269
*
@@ -275,14 +282,14 @@ export const layer = Layer.effect(
275282
keep: number,
276283
): Effect.fn.Return<Result, Failure> {
277284
const context = trigger.context
278-
const split = splitConversation(context.messages, keep)
279-
if (!split) return yield* Effect.fail(NOTHING_TO_COMPACT)
285+
const recent = recentConversation(context.messages, keep)
286+
if (recent === undefined) return yield* Effect.fail(NOTHING_TO_COMPACT)
280287

281288
const previous = previousCompaction(context.messages)
282289
const prompt = buildPrompt(previous !== undefined, previous?.summary.includes(LEGACY_HEADING) ?? false)
283290
const headings = SUMMARY_TEMPLATE.split("\n").filter((line) => line.startsWith("##"))
284291
const filled = (text: string) => text.split("\n").some((line) => headings.includes(line.trim()))
285-
const prepared = yield* prepare(context, split.older, budget)
292+
const prepared = yield* prepare(context, context.messages, budget)
286293

287294
// Hooks saw the request without the summary prompt, so it is appended here. A reply that ignores the
288295
// template gets one reminder before it counts as a failure.
@@ -293,14 +300,14 @@ export const layer = Layer.effect(
293300
context.model.limit,
294301
)
295302
const reply = yield* stream(context, prompted, prepared.options)
296-
if (filled(reply.text)) return { ...reply, recent: split.recent }
303+
if (filled(reply.text)) return { ...reply, recent: recent.text }
297304

298305
const nudged = fitOutput(
299306
LLMRequest.update(prompted, { messages: [...prompted.messages, Message.user(NUDGE)] }),
300307
context.model.limit,
301308
)
302309
const retry = yield* stream(context, nudged, prepared.options)
303-
if (filled(retry.text)) return { ...retry, recent: split.recent }
310+
if (filled(retry.text)) return { ...retry, recent: recent.text }
304311
return yield* Effect.fail<Failure>({
305312
error: {
306313
type: "compaction.failed",
@@ -312,7 +319,16 @@ export const layer = Layer.effect(
312319
})
313320

314321
const overhead = Token.estimate(prompt) + Token.estimate(NUDGE)
315-
return yield* deliver(trigger, prepared, split.recent, budget - overhead, send)
322+
// Automatic compaction starts past the ceiling, so the whole conversation is sent unchanged while the window can
323+
// still fit the summary. Below that, the history before the tail is sent instead, as before: unchanged when it fits,
324+
// which still reuses the cache for that shorter prefix, and as text otherwise. Dropping exchanges to make room
325+
// for the tail would leave them in neither the summary nor the tail. Where 15% of the window is under
326+
// `RESERVE_MIN`, the ceiling equals `summaryRoom`, so automatic compaction always takes this fallback.
327+
const fit = trigger.reason === "overflow" ? budget : Math.max(budget, summaryRoom(context.model.limit))
328+
return yield* deliver(trigger, prepared, recent.text, budget - overhead, send, {
329+
fit: fit - overhead,
330+
history: withoutTail(prepared.request, recent.from),
331+
})
316332
})
317333

318334
/**
@@ -378,7 +394,8 @@ export const layer = Layer.effect(
378394
})
379395

380396
/**
381-
* Sends the request as-is if it is estimated to fit `target`, and as text otherwise (see `flattenAndDropOldest`).
397+
* Sends the request as-is if it is estimated to fit `shrink.fit`, else `shrink.history` as-is if it fits `target`,
398+
* and otherwise `shrink.history` as text aimed at `target` (see `flattenAndDropOldest`).
382399
* Limits and estimates can be wrong, so until the provider rejects something as too long, a request that cannot
383400
* be made to fit is sent whole: unchanged, or as text once its media has to go. A "too long" rejection means the
384401
* estimate ran low, so the next attempts aim at 70%, 50%, then 35% of the first rejected request's estimate, and a
@@ -393,6 +410,7 @@ export const layer = Layer.effect(
393410
recent: string,
394411
target: number,
395412
send: (request: LLMRequest) => Effect.Effect<Result, AIError | Failure>,
413+
shrink: { readonly fit: number; readonly history: LLMRequest } = { fit: target, history: prepared.request },
396414
): Effect.fn.Return<Result, Failure> {
397415
const context = trigger.context
398416
// The runner opened the manual compaction's message when it delivered the `/compact` item.
@@ -405,17 +423,24 @@ export const layer = Layer.effect(
405423
let rejections = 0
406424
let rejected: number | undefined
407425
let asText = false
426+
let fit = shrink.fit
408427

409428
while (true) {
410-
const fits = !asText && estimateRequest(prepared.request) <= target
411-
const request = fits
412-
? prepared.request
413-
: (flattenAndDropOldest(prepared.request, context, target) ??
414-
(rejections > 0
415-
? undefined
416-
: asText
417-
? flattenAndDropOldest(prepared.request, context, Number.POSITIVE_INFINITY)
418-
: prepared.request))
429+
const unchanged = asText
430+
? undefined
431+
: estimateRequest(prepared.request) <= fit
432+
? prepared.request
433+
: estimateRequest(shrink.history) <= target
434+
? shrink.history
435+
: undefined
436+
const request =
437+
unchanged ??
438+
flattenAndDropOldest(shrink.history, context, target) ??
439+
(rejections > 0
440+
? undefined
441+
: asText
442+
? flattenAndDropOldest(shrink.history, context, Number.POSITIVE_INFINITY)
443+
: shrink.history)
419444
if (!request) {
420445
return yield* Effect.fail<Failure>({
421446
error: {
@@ -433,7 +458,7 @@ export const layer = Layer.effect(
433458
const error = toSessionError(cause)
434459
const tooLarge = cause.reason._tag === "InvalidRequest" && cause.reason.classification === "payload-too-large"
435460

436-
if (tooLarge && request === prepared.request) {
461+
if (tooLarge && (request === prepared.request || request === shrink.history)) {
437462
asText = true
438463
continue
439464
}
@@ -443,6 +468,7 @@ export const layer = Layer.effect(
443468
if (step === undefined) return yield* Effect.fail<Failure>({ error })
444469
rejected ??= estimateRequest(request)
445470
target = Math.floor(rejected * step)
471+
fit = target
446472
rejections++
447473
continue
448474
}
@@ -702,21 +728,25 @@ const transcript = (context: SessionContext.Loaded, messages: ReadonlyArray<Sess
702728
})
703729

704730
/**
705-
* `older` gets summarized; `recent`, the newest messages within `keep` tokens, is kept verbatim as text beside
706-
* the summary. Undefined when there is nothing to compact.
731+
* `text`, the newest messages within `keep` tokens, is kept verbatim beside the summary; `from` is the first of
732+
* them. Undefined when there is nothing to compact.
707733
*/
708-
const splitConversation = (messages: ReadonlyArray<SessionMessage.Info>, keep: number) => {
709-
const entries = messages.flatMap((message, index) => {
734+
const recentConversation = (messages: ReadonlyArray<SessionMessage.Info>, keep: number) => {
735+
const entries = messages.flatMap((message) => {
710736
const text = messageToText(message)
711-
return text ? [{ message, text, index }] : []
737+
return text ? [{ message, text }] : []
712738
})
713739
if (entries.length === 0) return undefined
714740

715741
const recent = entries.slice(recentStart(entries, keep, previousCompaction(messages)))
716-
return {
717-
older: messages.slice(0, recent[0]?.index ?? messages.length),
718-
recent: recent.map((entry) => entry.text).join("\n\n"),
719-
}
742+
return { text: recent.map((entry) => entry.text).join("\n\n"), from: recent[0]?.message.id }
743+
}
744+
745+
/** The request without the retained tail, which starts at the message with ID `from`. */
746+
const withoutTail = (request: LLMRequest, from: SessionMessage.ID | undefined) => {
747+
const index = from === undefined ? -1 : request.messages.findIndex((message) => message.id === from)
748+
if (index < 0) return request
749+
return LLMRequest.update(request, { messages: request.messages.slice(0, index) })
720750
}
721751

722752
const recentStart = (
@@ -734,12 +764,12 @@ const recentStart = (
734764
const userBoundary = entries.findLastIndex((entry, index) => index <= dropped && entry.message.type === "user")
735765
if (userBoundary > 0) return userBoundary
736766

737-
// Everything fits. Keep only the latest exchange so there is an older part left to summarize.
767+
// Everything fits. Keep only the latest exchange rather than duplicating the whole conversation beside the summary.
738768
const latestUser = entries.findLastIndex((entry) => entry.message.type === "user")
739769
if (latestUser > 0) return latestUser
740770

741-
// One exchange, nothing older. Summarize it all and keep nothing, unless a previous summary already
742-
// kept recent text, in which case keep everything and summarize only the summary before it.
771+
// One exchange, nothing older. Keep nothing, unless a previous checkpoint kept recent text: the new summary replaces
772+
// that text, so this exchange stays verbatim in its place.
743773
return previous?.recent ? 0 : entries.length
744774
}
745775

@@ -904,19 +934,23 @@ export const estimatePrompt = (context: SessionContext.Loaded) => {
904934
}
905935

906936
/**
907-
* The largest request the model takes while leaving room for its reply: 10% of the window, or `RESERVE_MIN` when that
908-
* is more. The summary request is capped at the same size, so its output limit is whatever the reserve leaves. A window
909-
* too small to give up `RESERVE_MIN` keeps 10%.
937+
* The largest request the model takes while leaving room for its reply: `RESERVE_SHARE` of the window, or
938+
* `RESERVE_MIN` when that is more. A summary request that must be shrunk is aimed at the same size. A window too small
939+
* to give up `RESERVE_MIN` keeps `RESERVE_SHARE`.
910940
*/
911941
const calculateCeiling = (limit: SessionContext.Loaded["model"]["limit"], buffer: number | undefined) => {
912942
// Unknown limits are reported as 0. An unknown input limit falls back to the context window; with no window at
913943
// all, only a provider rejection can limit the request.
914944
const window = limit.input || limit.context
915945
if (window <= 0) return Number.POSITIVE_INFINITY
916946
if (buffer !== undefined) return window - buffer
917-
return window - Math.max(Math.floor(window * 0.1), window >= 2 * RESERVE_MIN ? RESERVE_MIN : 0)
947+
return window - Math.max(Math.floor(window * RESERVE_SHARE), window >= 2 * RESERVE_MIN ? RESERVE_MIN : 0)
918948
}
919949

950+
/** The largest summary request that leaves `RESERVE_MIN` of the window for the summary. Zero for an unknown window. */
951+
const summaryRoom = (limit: SessionContext.Loaded["model"]["limit"]) =>
952+
Math.max(0, (limit.input || limit.context) - RESERVE_MIN)
953+
920954
/**
921955
* Another provider's count may describe a native window this model cannot replay. History then holds the original
922956
* messages that window stood for, which the count leaves out, so only the current provider's counts are trusted.

0 commit comments

Comments
 (0)