@@ -40,7 +40,7 @@ import { SessionUsage } from "./usage.js"
4040
4141export type Settings = {
4242 auto : boolean
43- /** Tokens kept free below the model's limits before compacting. Unset keeps 10% free. */
43+ /** Tokens kept free below the model's limits before compacting. Unset keeps 15%, and at least 16k, free. */
4444 buffer ?: number
4545 /** Tokens of recent conversation kept verbatim beside the summary. */
4646 keep : number
@@ -95,6 +95,12 @@ const NOTHING_TO_COMPACT: Failure = { error: { type: "compaction.unavailable", m
9595const SHRINK_STEPS = [ 0.7 , 0.5 , 0.35 ]
9696// The least of the window kept free for the last reply before compaction and for the summary itself.
9797const RESERVE_MIN = 16_000
98+ /**
99+ * Share of the window kept free before automatic compaction. Above `RESERVE_MIN`, the margin between it and
100+ * `summaryRoom` absorbs a large step past the trigger, so the whole conversation still goes out unchanged and reuses
101+ * the cache. Single steps grew by up to ~22k tokens at p99 in real sessions.
102+ */
103+ const RESERVE_SHARE = 0.15
98104/** A common window size, assumed for the compaction request when the model's window is unknown. */
99105const UNKNOWN_WINDOW = 200_000
100106const TOOL_OUTPUT_MAX_CHARS = 1_250
@@ -252,11 +258,12 @@ export const layer = Layer.effect(
252258
253259 /**
254260 * first: [E1 … E8][E9 E10] → [S1 + "E9 E10"]
255- * again: [S1 + "E9 E10"][E11 … E18][E19] → [S2 + "E19"] S2 covers S1, E9 E10, and E11–E18
261+ * again: [S1 + "E9 E10"][E11 … E18][E19] → [S2 + "E19"] S2 covers the whole current conversation
256262 *
257- * The model is sent the older part, then the summary prompt:
263+ * The model is sent the whole conversation, including the tail kept beside the summary, then the summary prompt.
264+ * Keeping the normal request's prefix intact lets the summary reuse its prompt cache:
258265 *
259- * [system][S1 + "E9 E10"][E11 … E18][prompt] plus one nudge if the reply skips the template
266+ * [system][S1 + "E9 E10"][E11 … E18][E19][ prompt] plus one nudge if the reply skips the template
260267 *
261268 * Later requests open with the checkpoint as one user message:
262269 *
@@ -275,14 +282,14 @@ export const layer = Layer.effect(
275282 keep : number ,
276283 ) : Effect . fn . Return < Result , Failure > {
277284 const context = trigger . context
278- const split = splitConversation ( context . messages , keep )
279- if ( ! split ) return yield * Effect . fail ( NOTHING_TO_COMPACT )
285+ const recent = recentConversation ( context . messages , keep )
286+ if ( recent === undefined ) return yield * Effect . fail ( NOTHING_TO_COMPACT )
280287
281288 const previous = previousCompaction ( context . messages )
282289 const prompt = buildPrompt ( previous !== undefined , previous ?. summary . includes ( LEGACY_HEADING ) ?? false )
283290 const headings = SUMMARY_TEMPLATE . split ( "\n" ) . filter ( ( line ) => line . startsWith ( "##" ) )
284291 const filled = ( text : string ) => text . split ( "\n" ) . some ( ( line ) => headings . includes ( line . trim ( ) ) )
285- const prepared = yield * prepare ( context , split . older , budget )
292+ const prepared = yield * prepare ( context , context . messages , budget )
286293
287294 // Hooks saw the request without the summary prompt, so it is appended here. A reply that ignores the
288295 // template gets one reminder before it counts as a failure.
@@ -293,14 +300,14 @@ export const layer = Layer.effect(
293300 context . model . limit ,
294301 )
295302 const reply = yield * stream ( context , prompted , prepared . options )
296- if ( filled ( reply . text ) ) return { ...reply , recent : split . recent }
303+ if ( filled ( reply . text ) ) return { ...reply , recent : recent . text }
297304
298305 const nudged = fitOutput (
299306 LLMRequest . update ( prompted , { messages : [ ...prompted . messages , Message . user ( NUDGE ) ] } ) ,
300307 context . model . limit ,
301308 )
302309 const retry = yield * stream ( context , nudged , prepared . options )
303- if ( filled ( retry . text ) ) return { ...retry , recent : split . recent }
310+ if ( filled ( retry . text ) ) return { ...retry , recent : recent . text }
304311 return yield * Effect . fail < Failure > ( {
305312 error : {
306313 type : "compaction.failed" ,
@@ -312,7 +319,16 @@ export const layer = Layer.effect(
312319 } )
313320
314321 const overhead = Token . estimate ( prompt ) + Token . estimate ( NUDGE )
315- return yield * deliver ( trigger , prepared , split . recent , budget - overhead , send )
322+ // Automatic compaction starts past the ceiling, so the whole conversation is sent unchanged while the window can
323+ // still fit the summary. Below that, the history before the tail is sent instead, as before: unchanged when it fits,
324+ // which still reuses the cache for that shorter prefix, and as text otherwise. Dropping exchanges to make room
325+ // for the tail would leave them in neither the summary nor the tail. Where 15% of the window is under
326+ // `RESERVE_MIN`, the ceiling equals `summaryRoom`, so automatic compaction always takes this fallback.
327+ const fit = trigger . reason === "overflow" ? budget : Math . max ( budget , summaryRoom ( context . model . limit ) )
328+ return yield * deliver ( trigger , prepared , recent . text , budget - overhead , send , {
329+ fit : fit - overhead ,
330+ history : withoutTail ( prepared . request , recent . from ) ,
331+ } )
316332 } )
317333
318334 /**
@@ -378,7 +394,8 @@ export const layer = Layer.effect(
378394 } )
379395
380396 /**
381- * Sends the request as-is if it is estimated to fit `target`, and as text otherwise (see `flattenAndDropOldest`).
397+ * Sends the request as-is if it is estimated to fit `shrink.fit`, else `shrink.history` as-is if it fits `target`,
398+ * and otherwise `shrink.history` as text aimed at `target` (see `flattenAndDropOldest`).
382399 * Limits and estimates can be wrong, so until the provider rejects something as too long, a request that cannot
383400 * be made to fit is sent whole: unchanged, or as text once its media has to go. A "too long" rejection means the
384401 * estimate ran low, so the next attempts aim at 70%, 50%, then 35% of the first rejected request's estimate, and a
@@ -393,6 +410,7 @@ export const layer = Layer.effect(
393410 recent : string ,
394411 target : number ,
395412 send : ( request : LLMRequest ) => Effect . Effect < Result , AIError | Failure > ,
413+ shrink : { readonly fit : number ; readonly history : LLMRequest } = { fit : target , history : prepared . request } ,
396414 ) : Effect . fn . Return < Result , Failure > {
397415 const context = trigger . context
398416 // The runner opened the manual compaction's message when it delivered the `/compact` item.
@@ -405,17 +423,24 @@ export const layer = Layer.effect(
405423 let rejections = 0
406424 let rejected : number | undefined
407425 let asText = false
426+ let fit = shrink . fit
408427
409428 while ( true ) {
410- const fits = ! asText && estimateRequest ( prepared . request ) <= target
411- const request = fits
412- ? prepared . request
413- : ( flattenAndDropOldest ( prepared . request , context , target ) ??
414- ( rejections > 0
415- ? undefined
416- : asText
417- ? flattenAndDropOldest ( prepared . request , context , Number . POSITIVE_INFINITY )
418- : prepared . request ) )
429+ const unchanged = asText
430+ ? undefined
431+ : estimateRequest ( prepared . request ) <= fit
432+ ? prepared . request
433+ : estimateRequest ( shrink . history ) <= target
434+ ? shrink . history
435+ : undefined
436+ const request =
437+ unchanged ??
438+ flattenAndDropOldest ( shrink . history , context , target ) ??
439+ ( rejections > 0
440+ ? undefined
441+ : asText
442+ ? flattenAndDropOldest ( shrink . history , context , Number . POSITIVE_INFINITY )
443+ : shrink . history )
419444 if ( ! request ) {
420445 return yield * Effect . fail < Failure > ( {
421446 error : {
@@ -433,7 +458,7 @@ export const layer = Layer.effect(
433458 const error = toSessionError ( cause )
434459 const tooLarge = cause . reason . _tag === "InvalidRequest" && cause . reason . classification === "payload-too-large"
435460
436- if ( tooLarge && request === prepared . request ) {
461+ if ( tooLarge && ( request === prepared . request || request === shrink . history ) ) {
437462 asText = true
438463 continue
439464 }
@@ -443,6 +468,7 @@ export const layer = Layer.effect(
443468 if ( step === undefined ) return yield * Effect . fail < Failure > ( { error } )
444469 rejected ??= estimateRequest ( request )
445470 target = Math . floor ( rejected * step )
471+ fit = target
446472 rejections ++
447473 continue
448474 }
@@ -702,21 +728,25 @@ const transcript = (context: SessionContext.Loaded, messages: ReadonlyArray<Sess
702728 } )
703729
704730/**
705- * `older` gets summarized; `recent` , the newest messages within `keep` tokens, is kept verbatim as text beside
706- * the summary . Undefined when there is nothing to compact.
731+ * `text` , the newest messages within `keep` tokens, is kept verbatim beside the summary; `from` is the first of
732+ * them . Undefined when there is nothing to compact.
707733 */
708- const splitConversation = ( messages : ReadonlyArray < SessionMessage . Info > , keep : number ) => {
709- const entries = messages . flatMap ( ( message , index ) => {
734+ const recentConversation = ( messages : ReadonlyArray < SessionMessage . Info > , keep : number ) => {
735+ const entries = messages . flatMap ( ( message ) => {
710736 const text = messageToText ( message )
711- return text ? [ { message, text, index } ] : [ ]
737+ return text ? [ { message, text } ] : [ ]
712738 } )
713739 if ( entries . length === 0 ) return undefined
714740
715741 const recent = entries . slice ( recentStart ( entries , keep , previousCompaction ( messages ) ) )
716- return {
717- older : messages . slice ( 0 , recent [ 0 ] ?. index ?? messages . length ) ,
718- recent : recent . map ( ( entry ) => entry . text ) . join ( "\n\n" ) ,
719- }
742+ return { text : recent . map ( ( entry ) => entry . text ) . join ( "\n\n" ) , from : recent [ 0 ] ?. message . id }
743+ }
744+
745+ /** The request without the retained tail, which starts at the message with ID `from`. */
746+ const withoutTail = ( request : LLMRequest , from : SessionMessage . ID | undefined ) => {
747+ const index = from === undefined ? - 1 : request . messages . findIndex ( ( message ) => message . id === from )
748+ if ( index < 0 ) return request
749+ return LLMRequest . update ( request , { messages : request . messages . slice ( 0 , index ) } )
720750}
721751
722752const recentStart = (
@@ -734,12 +764,12 @@ const recentStart = (
734764 const userBoundary = entries . findLastIndex ( ( entry , index ) => index <= dropped && entry . message . type === "user" )
735765 if ( userBoundary > 0 ) return userBoundary
736766
737- // Everything fits. Keep only the latest exchange so there is an older part left to summarize .
767+ // Everything fits. Keep only the latest exchange rather than duplicating the whole conversation beside the summary .
738768 const latestUser = entries . findLastIndex ( ( entry ) => entry . message . type === "user" )
739769 if ( latestUser > 0 ) return latestUser
740770
741- // One exchange, nothing older. Summarize it all and keep nothing, unless a previous summary already
742- // kept recent text, in which case keep everything and summarize only the summary before it .
771+ // One exchange, nothing older. Keep nothing, unless a previous checkpoint kept recent text: the new summary replaces
772+ // that text, so this exchange stays verbatim in its place .
743773 return previous ?. recent ? 0 : entries . length
744774}
745775
@@ -904,19 +934,23 @@ export const estimatePrompt = (context: SessionContext.Loaded) => {
904934}
905935
906936/**
907- * The largest request the model takes while leaving room for its reply: 10% of the window, or `RESERVE_MIN` when that
908- * is more. The summary request is capped at the same size, so its output limit is whatever the reserve leaves . A window
909- * too small to give up `RESERVE_MIN` keeps 10% .
937+ * The largest request the model takes while leaving room for its reply: `RESERVE_SHARE` of the window, or
938+ * `RESERVE_MIN` when that is more. A summary request that must be shrunk is aimed at the same size . A window too small
939+ * to give up `RESERVE_MIN` keeps `RESERVE_SHARE` .
910940 */
911941const calculateCeiling = ( limit : SessionContext . Loaded [ "model" ] [ "limit" ] , buffer : number | undefined ) => {
912942 // Unknown limits are reported as 0. An unknown input limit falls back to the context window; with no window at
913943 // all, only a provider rejection can limit the request.
914944 const window = limit . input || limit . context
915945 if ( window <= 0 ) return Number . POSITIVE_INFINITY
916946 if ( buffer !== undefined ) return window - buffer
917- return window - Math . max ( Math . floor ( window * 0.1 ) , window >= 2 * RESERVE_MIN ? RESERVE_MIN : 0 )
947+ return window - Math . max ( Math . floor ( window * RESERVE_SHARE ) , window >= 2 * RESERVE_MIN ? RESERVE_MIN : 0 )
918948}
919949
950+ /** The largest summary request that leaves `RESERVE_MIN` of the window for the summary. Zero for an unknown window. */
951+ const summaryRoom = ( limit : SessionContext . Loaded [ "model" ] [ "limit" ] ) =>
952+ Math . max ( 0 , ( limit . input || limit . context ) - RESERVE_MIN )
953+
920954/**
921955 * Another provider's count may describe a native window this model cannot replay. History then holds the original
922956 * messages that window stood for, which the count leaves out, so only the current provider's counts are trusted.
0 commit comments