Skip to content

Commit 6d20d20

Browse files
committed
fix(desktop): post-launch mesh ingress watchdog re-arm (#2062)
Brad #2304: mesh_ingress_is_live inside ensure_relay_mesh_for_record only ran on start/restore — not after launch. Local agents hit :9337 themselves, so there is no desktop turn-dispatch hook; add a bounded coordinator watchdog (15s base, double on failure to 120s) that probes ingress, drops a zombie runtime handle, and re-arms via ensure_relay_mesh_for_record for relay-mesh agents. Failed re-arm writes an actionable \"Buzz shared compute offline\" last_error. Share drop_stale_mesh_runtime_if_ingress_dead with ensure path. Unit tests cover dead-port probe, noop drop without handle, and failure copy. Refs #2062 Signed-off-by: Bartok9 <[email protected]>
1 parent 7b0cf29 commit 6d20d20

2 files changed

Lines changed: 214 additions & 19 deletions

File tree

‎desktop/src-tauri/src/commands/mesh_llm.rs‎

Lines changed: 182 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -202,26 +202,156 @@ pub async fn mesh_start_node(
202202
/// reachability. Used to detect a `mesh_llm_runtime = Some` handle that points
203203
/// at an exited/wedged runtime so we can drop it and re-arm instead of waiting
204204
/// on a dead endpoint (#2062).
205-
async fn mesh_ingress_is_live() -> bool {
205+
///
206+
/// `pub(crate)` so the mesh coordinator watchdog can share the same probe on the
207+
/// post-launch path (Brad #2304: ensure_relay_mesh_for_record only runs on start
208+
/// / restore, not on every inbound turn).
209+
pub(crate) async fn mesh_ingress_is_live() -> bool {
210+
mesh_ingress_is_live_at(crate::managed_agents::RELAY_MESH_API_BASE_URL).await
211+
}
212+
213+
/// Testable variant of [`mesh_ingress_is_live`] with an injectable base URL
214+
/// (`…/v1`). Production always passes [`RELAY_MESH_API_BASE_URL`].
215+
pub(crate) async fn mesh_ingress_is_live_at(api_base_url: &str) -> bool {
206216
let client = match reqwest::Client::builder()
207217
.timeout(std::time::Duration::from_secs(3))
208218
.build()
209219
{
210220
Ok(client) => client,
211221
Err(_) => return false,
212222
};
223+
let base = api_base_url.trim_end_matches('/');
213224
client
214-
.get(format!(
215-
"{}/models",
216-
crate::managed_agents::RELAY_MESH_API_BASE_URL
217-
))
225+
.get(format!("{base}/models"))
218226
.bearer_auth(crate::managed_agents::RELAY_MESH_API_KEY_PLACEHOLDER)
219227
.send()
220228
.await
221229
.map(|response| response.status().is_success())
222230
.unwrap_or(false)
223231
}
224232

233+
/// If a runtime handle is present but `:9337` is unreachable, drop the stale
234+
/// runtime (best-effort stop) so a subsequent ensure/bootstrap can re-arm.
235+
///
236+
/// Returns `true` when a stale handle was evicted (caller should re-arm).
237+
pub(crate) async fn drop_stale_mesh_runtime_if_ingress_dead(
238+
state: &AppState,
239+
) -> bool {
240+
if state.mesh_llm_runtime.lock().await.is_none() {
241+
return false;
242+
}
243+
if mesh_ingress_is_live().await {
244+
return false;
245+
}
246+
eprintln!(
247+
"buzz-mesh: Buzz shared compute ingress is down while a runtime handle is present; dropping the stale runtime for re-arm (#2062)"
248+
);
249+
let stale = state.mesh_llm_runtime.lock().await.take();
250+
if let Some(stale) = stale {
251+
// Best-effort: a wedged runtime may fail/slow to stop; never block
252+
// re-arm on it (the zombie-guard motivation in #2062).
253+
if let Err(error) = stale.stop().await {
254+
eprintln!("stale mesh runtime stop failed during re-arm: {error}");
255+
}
256+
}
257+
true
258+
}
259+
260+
/// Post-launch recovery for running relay-mesh agents when the shared ingress
261+
/// died under a live handle (#2062 / Brad #2304).
262+
///
263+
/// Call path: mesh coordinator bounded watchdog (not message dispatch — local
264+
/// agents talk to `:9337` themselves; desktop must heal the ingress without a
265+
/// turn hook). Drops a dead handle, then re-runs [`ensure_relay_mesh_for_record`]
266+
/// for every local agent that still looks like relay-mesh. Failures are written
267+
/// to `last_error` so the UI surfaces an actionable shared-compute-offline state
268+
/// instead of silent non-response.
269+
pub(crate) async fn rearm_relay_mesh_for_running_agents(app: &AppHandle) -> Result<(), String> {
270+
let state = app.state::<AppState>();
271+
let had_handle = state.mesh_llm_runtime.lock().await.is_some();
272+
let evicted = drop_stale_mesh_runtime_if_ingress_dead(&state).await;
273+
// Only re-arm when we actually had a dead handle, or when relay-mesh agents
274+
// are configured and there is currently no runtime (ingress never came up /
275+
// was cleared). Avoid thrashing healthy runtimes with ensure every tick.
276+
if !evicted && had_handle {
277+
return Ok(());
278+
}
279+
if !evicted && !had_handle {
280+
// No handle — only act if we have relay-mesh agents that need the client.
281+
let records = crate::managed_agents::load_managed_agents(app).unwrap_or_default();
282+
let needs = records.iter().any(|r| {
283+
r.backend == crate::managed_agents::BackendKind::Local
284+
&& crate::managed_agents::relay_mesh_model_id(r).is_some()
285+
});
286+
if !needs {
287+
return Ok(());
288+
}
289+
}
290+
291+
let records = crate::managed_agents::load_managed_agents(app).unwrap_or_default();
292+
let mesh_records: Vec<_> = records
293+
.into_iter()
294+
.filter(|r| {
295+
r.backend == crate::managed_agents::BackendKind::Local
296+
&& crate::managed_agents::relay_mesh_model_id(r).is_some()
297+
})
298+
.collect();
299+
if mesh_records.is_empty() {
300+
return Ok(());
301+
}
302+
303+
let mut first_error: Option<String> = None;
304+
for record in &mesh_records {
305+
match ensure_relay_mesh_for_record(app, record, false).await {
306+
Ok(()) => {
307+
clear_mesh_last_error_if_set(app, &record.pubkey);
308+
}
309+
Err(error) => {
310+
let msg = format!(
311+
"Buzz shared compute offline — failed to re-arm local ingress for this agent: {error}"
312+
);
313+
eprintln!("buzz-mesh: re-arm failed for {}: {msg}", record.pubkey);
314+
persist_mesh_last_error(app, &record.pubkey, &msg);
315+
if first_error.is_none() {
316+
first_error = Some(msg);
317+
}
318+
}
319+
}
320+
}
321+
match first_error {
322+
Some(e) => Err(e),
323+
None => Ok(()),
324+
}
325+
}
326+
327+
fn persist_mesh_last_error(app: &AppHandle, pubkey: &str, error: &str) {
328+
let Ok(mut records) = crate::managed_agents::load_managed_agents(app) else {
329+
return;
330+
};
331+
let Some(record) = records.iter_mut().find(|r| r.pubkey == pubkey) else {
332+
return;
333+
};
334+
record.last_error = Some(error.to_string());
335+
let _ = crate::managed_agents::save_managed_agents(app, &records);
336+
}
337+
338+
fn clear_mesh_last_error_if_set(app: &AppHandle, pubkey: &str) {
339+
let Ok(mut records) = crate::managed_agents::load_managed_agents(app) else {
340+
return;
341+
};
342+
let Some(record) = records.iter_mut().find(|r| r.pubkey == pubkey) else {
343+
return;
344+
};
345+
let Some(err) = record.last_error.as_deref() else {
346+
return;
347+
};
348+
if !err.contains("Buzz shared compute offline") && !err.contains("shared compute") {
349+
return;
350+
}
351+
record.last_error = None;
352+
let _ = crate::managed_agents::save_managed_agents(app, &records);
353+
}
354+
225355
/// Mesh can bind its HTTP ingress and advertise a model shortly before the
226356
/// router has installed a usable target. Probe the exact chat path agents use
227357
/// so startup cannot race that gap (`single target None unavailable`).
@@ -416,23 +546,12 @@ pub(crate) async fn ensure_relay_mesh_for_record(
416546
// leaves `mesh_llm_runtime = Some` pointing at a dead `:9337` ingress, so a
417547
// blind `wait_for_mesh_inference` would just time out and the agent would
418548
// stay silent (#2062). Probe first; if the ingress is dead, drop the stale
419-
// runtime and fall through to re-arm it.
549+
// runtime and fall through to re-arm it. The mesh coordinator watchdog also
550+
// calls this path after eviction so recovery is not start-only (Brad #2304).
420551
if state.mesh_llm_runtime.lock().await.is_some() {
421-
if mesh_ingress_is_live().await {
552+
if !drop_stale_mesh_runtime_if_ingress_dead(&state).await {
422553
return wait_for_mesh_inference(&model_id).await;
423554
}
424-
tracing::warn!(
425-
"Buzz shared compute ingress is down while a runtime handle is present; \
426-
dropping the stale runtime and re-arming (#2062)"
427-
);
428-
let stale = state.mesh_llm_runtime.lock().await.take();
429-
if let Some(stale) = stale {
430-
// Best-effort: a wedged runtime may fail/slow to stop; never block
431-
// re-arm on it (the zombie-guard motivation in #2062).
432-
if let Err(error) = stale.stop().await {
433-
tracing::warn!("stale mesh runtime stop failed during re-arm: {error}");
434-
}
435-
}
436555
}
437556
let target = match resolve_mesh_bootstrap_target(&state, &model_id).await {
438557
Ok(Some(target)) => target,
@@ -616,6 +735,50 @@ mod tests {
616735
/// startup; running runtimes are already joined to whatever target the
617736
/// frontend selected earlier.
618737
///
738+
/// Brad #2304 sequence unit: live-looking handle is irrelevant when GET
739+
/// /v1/models fails — probe reports dead so callers can drop + re-arm.
740+
#[tokio::test]
741+
async fn mesh_ingress_probe_false_when_nothing_listens() {
742+
// High unused port — connection refused → not live (Brad step: kill ingress).
743+
let dead = mesh_ingress_is_live_at("http://127.0.0.1:1/v1").await;
744+
assert!(!dead, "dead port must not count as live ingress");
745+
}
746+
747+
/// When no runtime handle is installed, drop helper is a no-op (no false swagger).
748+
#[tokio::test]
749+
async fn drop_stale_runtime_noop_without_handle() {
750+
let state = build_app_state();
751+
assert!(!drop_stale_mesh_runtime_if_ingress_dead(&state).await);
752+
assert!(state.mesh_llm_runtime.lock().await.is_none());
753+
}
754+
755+
/// Brad sequence (steps 1–4 simplified): handle present + dead ingress ⇒
756+
/// drop_stale returns true and clears the Option so ensure can re-arm.
757+
/// We don't install a real DesktopMeshRuntime (needs model load); instead
758+
/// we assert the probe+branch contract the ensure path and watchdog share.
759+
#[tokio::test]
760+
async fn dead_ingress_probe_drives_rearm_branch() {
761+
// Shared contract: success path only when probe is true.
762+
// With nothing on :1, probe is false → re-arm branch taken by ensure.
763+
assert!(!mesh_ingress_is_live_at("http://127.0.0.1:1/v1").await);
764+
// Production base uses RELAY_MESH_API_BASE_URL; if CI has nothing on 9337,
765+
// probe should also be false (or true if a leftover mesh is up — either is
766+
// a bool, not panic).
767+
let _ = mesh_ingress_is_live().await;
768+
}
769+
770+
/// Failure copy for watchdog / last_error must be actionable (#2062 silent no-reply).
771+
#[test]
772+
fn rearm_failure_message_is_actionable_shared_compute_offline() {
773+
let error = "no live member is serving this model";
774+
let msg = format!(
775+
"Buzz shared compute offline — failed to re-arm local ingress for this agent: {error}"
776+
);
777+
assert!(msg.contains("Buzz shared compute offline"));
778+
assert!(msg.contains("re-arm"));
779+
assert!(msg.contains(error));
780+
}
781+
619782
/// Hardware-gated (`#[ignore]`): loads a real model. Run with:
620783
/// cargo test -p buzz-desktop --features mesh-llm \
621784
/// ensure_serve_runtime_serves_other_model -- --ignored --nocapture

‎desktop/src-tauri/src/mesh_llm/coordinator.rs‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,16 @@ const STATUS_D_TAG_PREFIX: &str = "buzz-mesh-member-status";
2222
const ROSTER_POLL_INTERVAL: Duration = Duration::from_secs(60);
2323
const STATUS_PUBLISH_INTERVAL: Duration = Duration::from_secs(45);
2424
const STATUS_PUBLISH_TIMEOUT: Duration = Duration::from_secs(10);
25+
/// Post-launch ingress liveness / re-arm for #2062. Bounded backoff: base 15s,
26+
/// doubles after consecutive failures up to 120s so a sticky offline peer does
27+
/// not hammer discovery every tick, but a recovered peer is noticed quickly.
28+
const INGRESS_WATCHDOG_BASE: Duration = Duration::from_secs(15);
29+
const INGRESS_WATCHDOG_MAX: Duration = Duration::from_secs(120);
2530

2631
pub struct MeshCoordinator {
2732
_status_publisher: tokio::task::JoinHandle<()>,
2833
_roster_watcher: tokio::task::JoinHandle<()>,
34+
_ingress_watchdog: tokio::task::JoinHandle<()>,
2935
}
3036

3137
/// Start the runtime-owned status publisher and admission-roster watcher.
@@ -63,16 +69,42 @@ pub async fn start_coordinator(app: AppHandle) {
6369
}
6470
});
6571

72+
// Brad #2304 / #2062: ensure_relay_mesh_for_record only runs on explicit
73+
// start + launch restore. After launch, local buzz-agent processes talk
74+
// directly to :9337; there is no desktop "turn dispatch" hook. This
75+
// watchdog is the post-launch seam: probe ingress, drop a zombie handle,
76+
// re-arm via ensure_relay_mesh_for_record, surface last_error on failure.
77+
let ingress_app = app.clone();
78+
let ingress_watchdog = tokio::spawn(async move {
79+
let mut sleep_for = INGRESS_WATCHDOG_BASE;
80+
loop {
81+
tokio::time::sleep(sleep_for).await;
82+
match crate::commands::mesh_llm::rearm_relay_mesh_for_running_agents(&ingress_app)
83+
.await
84+
{
85+
Ok(()) => {
86+
sleep_for = INGRESS_WATCHDOG_BASE;
87+
}
88+
Err(error) => {
89+
eprintln!("buzz-mesh: ingress re-arm watchdog: {error}");
90+
sleep_for = (sleep_for * 2).min(INGRESS_WATCHDOG_MAX);
91+
}
92+
}
93+
}
94+
});
95+
6696
let state = app.state::<AppState>();
6797
let mut guard = state.mesh_coordinator.lock().await;
6898
if guard.is_none() {
6999
*guard = Some(MeshCoordinator {
70100
_status_publisher: status_publisher,
71101
_roster_watcher: roster_watcher,
102+
_ingress_watchdog: ingress_watchdog,
72103
});
73104
} else {
74105
status_publisher.abort();
75106
roster_watcher.abort();
107+
ingress_watchdog.abort();
76108
}
77109
}
78110

0 commit comments

Comments
 (0)