@@ -288,11 +288,17 @@ export const layer = Layer.effect(
288288 // template gets one reminder before it counts as a failure.
289289 const send = ( request : LLMRequest ) =>
290290 Effect . gen ( function * ( ) {
291- const prompted = LLMRequest . update ( request , { messages : [ ...request . messages , Message . user ( prompt ) ] } )
291+ const prompted = fitOutput (
292+ LLMRequest . update ( request , { messages : [ ...request . messages , Message . user ( prompt ) ] } ) ,
293+ context . model . limit ,
294+ )
292295 const reply = yield * stream ( context , prompted , prepared . options )
293296 if ( filled ( reply . text ) ) return { ...reply , recent : split . recent }
294297
295- const nudged = LLMRequest . update ( prompted , { messages : [ ...prompted . messages , Message . user ( NUDGE ) ] } )
298+ const nudged = fitOutput (
299+ LLMRequest . update ( prompted , { messages : [ ...prompted . messages , Message . user ( NUDGE ) ] } ) ,
300+ context . model . limit ,
301+ )
296302 const retry = yield * stream ( context , nudged , prepared . options )
297303 if ( filled ( retry . text ) ) return { ...retry , recent : split . recent }
298304 return yield * Effect . fail < Failure > ( {
@@ -922,6 +928,25 @@ const hasMeasuredPrompt = (message: SessionMessage.Info, model: SessionContext.L
922928 message . tokens !== undefined &&
923929 message . tokens . input + message . tokens . cache . read + message . tokens . cache . write > 0
924930
931+ /**
932+ * `prepare` sizes the output limit for a request of `budget` tokens, but `deliver` can send a larger one when it
933+ * cannot shrink the conversation further. Lower the limit to the room the request being sent leaves in the window,
934+ * so the input and the requested output never exceed the window by our own estimate.
935+ *
936+ * Like `budget` in `prepare`, the estimate counts as measured, without the margin primary requests add for estimates:
937+ * `deliver` aims at about 90% of the window, so that margin would cut every summary near its target to the minimum.
938+ */
939+ const fitOutput = ( request : LLMRequest , limit : SessionContext . Loaded [ "model" ] [ "limit" ] ) => {
940+ const current = request . generation ?. maxTokens
941+ if ( current === undefined ) return request
942+ const maxTokens = Math . min (
943+ current ,
944+ SessionModelRequest . outputLimit ( limit , "compaction" , { measured : estimateRequest ( request ) , estimated : 0 } ) ,
945+ )
946+ if ( maxTokens === current ) return request
947+ return LLMRequest . update ( request , { generation : { ...request . generation , maxTokens } } )
948+ }
949+
925950const estimateRequest = ( request : Pick < LLMRequest , "system" | "tools" | "messages" > ) =>
926951 request . system . reduce ( ( sum , part ) => sum + Token . estimate ( part . text ) , 0 ) +
927952 request . tools . reduce ( ( sum , tool ) => sum + estimateTool ( tool ) , 0 ) +
0 commit comments