Skip to content

Commit df7fad6

Browse files
loganjLarry
andcommitted
test(agent): await asynchronously collected recovery diagnostics
Wait on the harness stderr collector rather than treating the ACP stdout response as a drain barrier. Preserve budget and floor discrimination and corroborate exactly four provider requests. Exercise the same budget scenario with collection gated until after stdout, and prove absent diagnostics time out. Co-authored-by: Larry <627498bd4bd1f281a16431e3c6cce3b5c25b6692798c78672298aefbf2f8f8b5@buzz.block.builderlab.xyz> Signed-off-by: Logan Johnson <[email protected]>
1 parent b78d093 commit df7fad6

2 files changed

Lines changed: 107 additions & 5 deletions

File tree

‎crates/buzz-agent/tests/common/mod.rs‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ use std::time::Duration;
1919
use serde_json::{json, Value};
2020
use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
2121
use tokio::net::TcpListener;
22-
use tokio::sync::Mutex;
22+
use tokio::sync::{oneshot, Mutex, Notify};
2323

2424
pub struct CapturingLlm {
2525
pub url: String,
@@ -107,11 +107,22 @@ pub struct Harness {
107107
stdin: tokio::process::ChildStdin,
108108
stdout: BufReader<tokio::process::ChildStdout>,
109109
stderr: Arc<StdMutex<String>>,
110+
stderr_changed: Arc<Notify>,
110111
next_id: i64,
111112
}
112113

113114
impl Harness {
114115
pub async fn spawn_with_env(base_url: &str, extra: &[(&str, &str)]) -> Self {
116+
Self::spawn_with_stderr_gate(base_url, extra, None).await
117+
}
118+
119+
/// Delay stderr collection until released, to exercise stdout/stderr ordering
120+
/// without changing the child or relying on scheduler timing.
121+
pub async fn spawn_with_stderr_gate(
122+
base_url: &str,
123+
extra: &[(&str, &str)],
124+
stderr_gate: Option<oneshot::Receiver<()>>,
125+
) -> Self {
115126
let bin = env!("CARGO_BIN_EXE_buzz-agent");
116127
let mut cmd = tokio::process::Command::new(bin);
117128
cmd.env("BUZZ_AGENT_PROVIDER", "openai")
@@ -135,7 +146,14 @@ impl Harness {
135146
let stderr = child.stderr.take().unwrap();
136147
let stderr_buf = Arc::new(StdMutex::new(String::new()));
137148
let stderr_out = Arc::clone(&stderr_buf);
149+
let stderr_changed = Arc::new(Notify::new());
150+
let changed = Arc::clone(&stderr_changed);
138151
tokio::spawn(async move {
152+
if let Some(gate) = stderr_gate {
153+
// Dropping the sender (e.g. on assertion failure) also unblocks
154+
// collection, rather than leaving a detached reader waiting.
155+
let _ = gate.await;
156+
}
139157
let mut reader = BufReader::new(stderr);
140158
let mut line = String::new();
141159
loop {
@@ -150,13 +168,15 @@ impl Harness {
150168
if let Ok(mut out) = stderr_out.lock() {
151169
out.push_str(&line);
152170
}
171+
changed.notify_waiters();
153172
}
154173
});
155174
Self {
156175
child,
157176
stdin,
158177
stdout,
159178
stderr: stderr_buf,
179+
stderr_changed,
160180
next_id: 1,
161181
}
162182
}
@@ -228,9 +248,36 @@ impl Harness {
228248
let _ = self.child.start_kill();
229249
}
230250

251+
/// Snapshot only: receiving a response on stdout does not drain stderr.
231252
pub fn stderr_text(&self) -> String {
232253
self.stderr.lock().map(|s| s.clone()).unwrap_or_default()
233254
}
255+
256+
/// Wait for a diagnostic in the independently collected stderr stream.
257+
/// Returns the matching snapshot so subsequent assertions see its prefix.
258+
pub async fn wait_for_stderr(&self, needle: &str, timeout: Duration) -> String {
259+
tokio::time::timeout(timeout, async {
260+
loop {
261+
let changed = self.stderr_changed.notified();
262+
tokio::pin!(changed);
263+
// Register before inspecting the buffer: a line collected between
264+
// the snapshot and await must not become a lost wakeup.
265+
changed.as_mut().enable();
266+
let stderr = self.stderr_text();
267+
if stderr.contains(needle) {
268+
return stderr;
269+
}
270+
changed.await;
271+
}
272+
})
273+
.await
274+
.unwrap_or_else(|_| {
275+
panic!(
276+
"timed out waiting for stderr diagnostic {needle:?}; stderr={}",
277+
self.stderr_text()
278+
)
279+
})
280+
}
234281
}
235282

236283
pub fn openai_text(content: &str) -> Value {

‎crates/buzz-agent/tests/regressions.rs‎

Lines changed: 59 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2706,14 +2706,38 @@ async fn ordinary_400_stays_terminal_and_triggers_no_recovery() {
27062706
/// part of the assertion.
27072707
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
27082708
async fn context_recovery_budget_exhaustion_surfaces_the_error() {
2709+
assert_context_recovery_budget_exhaustion(false).await;
2710+
}
2711+
2712+
/// The same real provider/ACP scenario with stderr collection held until after
2713+
/// the stdout response. The old immediate snapshot cannot observe the budget.
2714+
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2715+
async fn context_recovery_budget_exhaustion_waits_for_delayed_stderr() {
2716+
assert_context_recovery_budget_exhaustion(true).await;
2717+
}
2718+
2719+
#[tokio::test]
2720+
#[should_panic(expected = "timed out waiting for stderr diagnostic")]
2721+
async fn stderr_diagnostic_wait_is_bounded_when_absent() {
2722+
let llm = spawn_capturing_llm(vec![]).await;
2723+
let h = Harness::spawn(&llm.url).await;
2724+
h.wait_for_stderr(
2725+
"diagnostic that is never emitted",
2726+
Duration::from_millis(20),
2727+
)
2728+
.await;
2729+
}
2730+
2731+
async fn assert_context_recovery_budget_exhaustion(delay_stderr: bool) {
2732+
let (release_stderr, stderr_gate) = tokio::sync::oneshot::channel();
27092733
// Enough canned 400s that the queue is never the thing that stops the loop;
27102734
// the fallback response is also a 400-shaped body under this helper only if
27112735
// queued, so keep the queue generously long.
27122736
let responses: Vec<(u16, Value)> = (0..40)
27132737
.map(|_| (400, openai_context_length_error()))
27142738
.collect();
27152739
let llm = spawn_capturing_llm_with_status(responses).await;
2716-
let mut h = Harness::spawn_with_env(
2740+
let mut h = Harness::spawn_with_stderr_gate(
27172741
&llm.url,
27182742
&[
27192743
("BUZZ_AGENT_MAX_CONTEXT_TOKENS", "200000"),
@@ -2723,6 +2747,7 @@ async fn context_recovery_budget_exhaustion_surfaces_the_error() {
27232747
),
27242748
("BUZZ_AGENT_MAX_HANDOFFS", "0"),
27252749
],
2750+
delay_stderr.then_some(stderr_gate),
27262751
)
27272752
.await;
27282753
let sid = init_session(&mut h, json!([])).await;
@@ -2751,8 +2776,27 @@ async fn context_recovery_budget_exhaustion_surfaces_the_error() {
27512776
// floor produce a surfaced error, so the assertion above passes either way
27522777
// — and the floor can fire on the first rung without the budget ever being
27532778
// consumed, which would make this test silently exercise a different
2754-
// mechanism than its name claims. Pin the budget explicitly.
2755-
let stderr = h.stderr_text();
2779+
// mechanism than its name claims. Pin the budget explicitly. Stdout is not
2780+
// a barrier for the independent stderr collector.
2781+
let stderr = {
2782+
let wait = h.wait_for_stderr("context recovery budget spent", Duration::from_secs(5));
2783+
tokio::pin!(wait);
2784+
if delay_stderr {
2785+
assert!(
2786+
!h.stderr_text().contains("context recovery budget spent"),
2787+
"the old immediate snapshot must miss the held diagnostic"
2788+
);
2789+
// Prove the actual wait stays pending before releasing the collector,
2790+
// without a sleep or depending on how quickly either task runs.
2791+
std::future::poll_fn(|cx| {
2792+
assert!(std::future::Future::poll(wait.as_mut(), cx).is_pending());
2793+
std::task::Poll::Ready(())
2794+
})
2795+
.await;
2796+
release_stderr.send(()).expect("release stderr collection");
2797+
}
2798+
wait.await
2799+
};
27562800
assert!(
27572801
stderr.contains("context recovery budget spent"),
27582802
"the per-run recovery BUDGET must be what stops the loop here, not the prompt floor; \
@@ -2767,6 +2811,15 @@ async fn context_recovery_budget_exhaustion_surfaces_the_error() {
27672811
"expected all 3 recovery rungs to be attempted before giving up, saw {rungs} — \
27682812
stderr={stderr}"
27692813
);
2814+
assert!(
2815+
!stderr.contains("context recovery would shrink"),
2816+
"the prompt floor must not stop this fixture: {stderr}"
2817+
);
2818+
assert_eq!(
2819+
llm.captured.lock().await.len(),
2820+
4,
2821+
"expected the rejected completion plus exactly three failed summaries"
2822+
);
27702823
h.shutdown().await;
27712824
}
27722825

@@ -2816,7 +2869,9 @@ async fn small_history_context_400_refuses_rescue_at_the_prompt_floor() {
28162869
r0.get("error").is_some(),
28172870
"a context 400 with no shrinkable history must surface the error, got: {r0}"
28182871
);
2819-
let stderr = h.stderr_text();
2872+
let stderr = h
2873+
.wait_for_stderr("context recovery would shrink", Duration::from_secs(5))
2874+
.await;
28202875
assert!(
28212876
stderr.contains("below the") && stderr.contains("floor"),
28222877
"the prompt-budget FLOOR must be what stops this, not the recovery budget; got: {stderr}"

0 commit comments

Comments
 (0)