Skip to content

Commit 5f53236

Browse files
author
Eva
committed
fix(buzz-agent): reserve the provider's actual summary completion cap in the handoff input budget
Review follow-up (PR #5248, atishpatel): the previous commit doubled OpenRouter's top-level max_tokens to 16384 (text budget + reasoning budget) but build_handoff_prompt still reserved only 8192 output tokens when sizing the input. At the maximum constructed prompt, input plus the actual completion allowance could exceed the configured context window by 8192 — making the summarizer itself context-reject exactly when a handoff is needed. Introduce summary_completion_cap(provider, budget): 2x for OpenRouter (where reasoning gets its own equal budget), unchanged for Anthropic, OpenAI, Databricks, and DatabricksV2 (whose summary bodies request the caller's budget as-is). openrouter_summary_body and build_handoff_prompt both derive from it, so the wire cap and the input reservation cannot drift apart again. A regression test pins the join: budget + fixed prompt + actual completion cap <= window at the 1-byte/token upper bound, and asserts the old single reservation observably violates it. Also per review suggestion: HANDOFF_SYSTEM_PROMPT now says 'Keep the visible plain-text summary under N tokens' with N derived from HANDOFF_MAX_OUTPUT_TOKENS via format!, removing the duplicated literal and the ambiguity about whether hidden reasoning counts against the figure. Verification: - cargo test -p buzz-agent (425 unit + 110 integration, all pass) - cargo fmt / clippy clean Co-authored-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Signed-off-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz>
1 parent bb2fedd commit 5f53236

2 files changed

Lines changed: 108 additions & 12 deletions

File tree

‎crates/buzz-agent/src/handoff.rs‎

Lines changed: 85 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ use crate::config::{
33
HANDOFF_MAX_OUTPUT_TOKENS, HANDOFF_MAX_TOOL_NAMES, HANDOFF_MIN_PROMPT_BUDGET_BYTES,
44
HANDOFF_ORIGINAL_TASK_MAX_BYTES, MAX_CONTEXT_RECOVERIES_PER_RUN,
55
};
6+
use crate::llm::summary_completion_cap;
67
use crate::types::HistoryItem;
78

89
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -35,10 +36,21 @@ pub(crate) enum ContextRecovery {
3536
Exhausted,
3637
}
3738

38-
const HANDOFF_SYSTEM_PROMPT: &str = "You are generating a context handoff summary for the next \
39-
turn of an autonomous agent. Be concise but thorough. Cover: what the original task was, what \
40-
you accomplished, key decisions made, what remains, and one concrete next step. Output plain \
41-
text only — no tool calls, no JSON. Stay under 8192 tokens.";
39+
/// System prompt for the handoff summarizer. `LazyLock` + `format!` so the
40+
/// token figure is derived from [`HANDOFF_MAX_OUTPUT_TOKENS`] instead of a
41+
/// duplicated literal, and "visible plain-text summary" makes explicit that
42+
/// the limit is on summary text, not on any hidden reasoning the model does
43+
/// first (which is budgeted separately on the wire — see
44+
/// `openrouter_summary_body`).
45+
static HANDOFF_SYSTEM_PROMPT: std::sync::LazyLock<String> = std::sync::LazyLock::new(|| {
46+
format!(
47+
"You are generating a context handoff summary for the next turn of an autonomous agent. \
48+
Be concise but thorough. Cover: what the original task was, what you accomplished, key \
49+
decisions made, what remains, and one concrete next step. Output plain text only — no \
50+
tool calls, no JSON. Keep the visible plain-text summary under \
51+
{HANDOFF_MAX_OUTPUT_TOKENS} tokens."
52+
)
53+
});
4254

4355
impl RunCtx<'_> {
4456
pub(crate) async fn maybe_handoff(&mut self, handoff_attempts: &mut usize) -> HandoffOutcome {
@@ -170,7 +182,7 @@ impl RunCtx<'_> {
170182
_ = self.cancel.changed() => return HandoffOutcome::Cancelled,
171183
r = self.llm.summarize(
172184
self.cfg,
173-
HANDOFF_SYSTEM_PROMPT,
185+
&HANDOFF_SYSTEM_PROMPT,
174186
&prompt,
175187
HANDOFF_MAX_OUTPUT_TOKENS,
176188
self.effective_model,
@@ -331,7 +343,7 @@ impl RunCtx<'_> {
331343
Some(explicit) => explicit.saturating_sub(fixed_bytes),
332344
None => handoff_prompt_budget_bytes(
333345
self.cfg.max_context_tokens,
334-
HANDOFF_MAX_OUTPUT_TOKENS,
346+
summary_completion_cap(self.cfg.provider, HANDOFF_MAX_OUTPUT_TOKENS),
335347
fixed_bytes,
336348
),
337349
};
@@ -513,8 +525,9 @@ fn byte_fallback_threshold(
513525
mod tests {
514526
use super::{
515527
byte_fallback_threshold, estimate_tokens_from_bytes, handoff_prompt_budget_bytes,
516-
token_threshold,
528+
summary_completion_cap, token_threshold, HANDOFF_SYSTEM_PROMPT,
517529
};
530+
use crate::config::{Provider, HANDOFF_MAX_OUTPUT_TOKENS};
518531

519532
#[test]
520533
fn handoff_prompt_budget_reserves_summary_output_and_fixed_prompt() {
@@ -526,6 +539,71 @@ mod tests {
526539
assert_eq!(handoff_prompt_budget_bytes(1_000, 2_000, 10_000), 0);
527540
}
528541

542+
/// OpenRouter's summary request grants reasoning an equal budget on top of
543+
/// the visible-text budget, so its completion cap is 2× the handoff text
544+
/// budget; the input budget must reserve that doubled cap. At the
545+
/// 1-byte/token upper bound, prompt bytes bound prompt tokens, so the join
546+
/// to pin is: (budget + fixed prompt) + actual completion cap ≤ window.
547+
/// Reserving only `HANDOFF_MAX_OUTPUT_TOKENS` would break this by exactly
548+
/// one extra reasoning budget at the maximum constructed prompt.
549+
#[test]
550+
fn openrouter_prompt_budget_reserves_doubled_completion_cap() {
551+
let cap = summary_completion_cap(Provider::OpenRouter, HANDOFF_MAX_OUTPUT_TOKENS);
552+
assert_eq!(
553+
cap,
554+
2 * HANDOFF_MAX_OUTPUT_TOKENS,
555+
"OpenRouter doubles: text + reasoning"
556+
);
557+
let window = 200_000u64;
558+
let fixed = 1_000usize;
559+
let budget = handoff_prompt_budget_bytes(window, cap, fixed);
560+
assert_eq!(budget, 182_616); // 200_000 - 16_384 - 1_000
561+
let max_prompt_tokens = estimate_tokens_from_bytes(budget + fixed);
562+
assert!(
563+
max_prompt_tokens + u64::from(cap) <= window,
564+
"input + completion allowance must fit the configured window"
565+
);
566+
// The old single reservation violates the same join — the regression
567+
// this guards against.
568+
let stale_budget = handoff_prompt_budget_bytes(window, HANDOFF_MAX_OUTPUT_TOKENS, fixed);
569+
assert!(
570+
estimate_tokens_from_bytes(stale_budget + fixed) + u64::from(cap) > window,
571+
"reserving only the text budget must be observable as an overflow here"
572+
);
573+
}
574+
575+
/// Anthropic/OpenAI/Databricks summary bodies request exactly the caller's
576+
/// budget, so their input reservation is unchanged.
577+
#[test]
578+
fn non_openrouter_completion_cap_is_the_callers_budget() {
579+
for provider in [
580+
Provider::Anthropic,
581+
Provider::OpenAi,
582+
Provider::Databricks,
583+
Provider::DatabricksV2,
584+
] {
585+
assert_eq!(
586+
summary_completion_cap(provider, HANDOFF_MAX_OUTPUT_TOKENS),
587+
HANDOFF_MAX_OUTPUT_TOKENS
588+
);
589+
}
590+
}
591+
592+
/// The prompt's token figure is derived from `HANDOFF_MAX_OUTPUT_TOKENS`
593+
/// and names the *visible plain-text summary* as its target, so hidden
594+
/// reasoning (budgeted separately on the wire) is not the referent.
595+
#[test]
596+
fn handoff_system_prompt_derives_limit_and_targets_visible_text() {
597+
let expected = format!(
598+
"Keep the visible plain-text summary under {HANDOFF_MAX_OUTPUT_TOKENS} tokens."
599+
);
600+
assert!(
601+
HANDOFF_SYSTEM_PROMPT.contains(&expected),
602+
"prompt must derive its token figure from HANDOFF_MAX_OUTPUT_TOKENS: {}",
603+
*HANDOFF_SYSTEM_PROMPT
604+
);
605+
}
606+
529607
#[test]
530608
fn token_threshold_uses_fraction_when_output_is_small() {
531609
// 200k window, 1k output. fractional = 0.9*200000 = 180000;

‎crates/buzz-agent/src/llm.rs‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2237,12 +2237,30 @@ pub(crate) fn build_token_source(cfg: &Config) -> Result<Arc<dyn TokenSource>, A
22372237
}
22382238
}
22392239

2240+
/// Completion-token cap that [`Llm::summarize`] actually requests from
2241+
/// `provider`, given the caller's visible-text budget. OpenRouter grants
2242+
/// reasoning a separate, equal budget on top of the text budget (see
2243+
/// [`openrouter_summary_body`]), so its top-level cap is double the caller's
2244+
/// budget; every other provider requests the caller's budget unchanged.
2245+
/// Callers that reserve output headroom in an input budget
2246+
/// (`handoff_prompt_budget_bytes`) must reserve THIS value, not the text
2247+
/// budget — otherwise input plus the actual completion allowance can exceed
2248+
/// the configured context window.
2249+
pub(crate) fn summary_completion_cap(provider: Provider, max_output_tokens: u32) -> u32 {
2250+
match provider {
2251+
Provider::OpenRouter => max_output_tokens.saturating_mul(2),
2252+
Provider::Anthropic | Provider::OpenAi | Provider::Databricks | Provider::DatabricksV2 => {
2253+
max_output_tokens
2254+
}
2255+
}
2256+
}
2257+
22402258
/// Build the request body for `Llm::summarize` on `Provider::OpenRouter`.
22412259
/// Extracted so tests can assert on the actual wire shape instead of a
2242-
/// hand-rolled literal — summaries never carry `reasoning` (see
2243-
/// `apply_openrouter_mutations`, which the summary path never calls).
2244-
/// It spells the token limit `max_tokens` directly for the same reason: the
2245-
/// mutation that renames it is never applied here.
2260+
/// hand-rolled literal — summaries never carry config-driven reasoning
2261+
/// *effort* (see `apply_openrouter_mutations`, which the summary path never
2262+
/// calls). It spells the token limit `max_tokens` directly for the same
2263+
/// reason: the mutation that renames it is never applied here.
22462264
fn openrouter_summary_body(
22472265
effective_model: &str,
22482266
system_prompt: &str,
@@ -2264,7 +2282,7 @@ fn openrouter_summary_body(
22642282
json!({
22652283
"model": effective_model,
22662284
"stream": false,
2267-
"max_tokens": max_output_tokens.saturating_mul(2),
2285+
"max_tokens": summary_completion_cap(Provider::OpenRouter, max_output_tokens),
22682286
"reasoning": {
22692287
"max_tokens": max_output_tokens,
22702288
"exclude": true,

0 commit comments

Comments
 (0)