Skip to content

Commit 8d0099f

Browse files
committed
fix(proxy): retry with backoff on transient accept errors instead of exiting
The proxy accept loop unconditionally broke on any accept() error, permanently killing the proxy while the sandbox continued to report Ready. Replace the break with a sleep-and-continue pattern: EMFILE/ENFILE errors get exponential backoff (100ms to 3.2s) to let file descriptors drain, and all other accept errors get a fixed 100ms backoff matching the existing metadata_server and edge_tunnel patterns. Closes #2337 Signed-off-by: Sean Burdine <[email protected]> Signed-off-by: politerealism <[email protected]>
1 parent 9a4f8a8 commit 8d0099f

1 file changed

Lines changed: 25 additions & 3 deletions

File tree

  • crates/openshell-supervisor-network/src

‎crates/openshell-supervisor-network/src/proxy.rs‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -265,9 +265,11 @@ impl ProxyHandle {
265265
}
266266
}
267267

268+
let mut consecutive_fd_errors: u32 = 0;
268269
loop {
269270
match listener.accept().await {
270271
Ok((stream, _addr)) => {
272+
consecutive_fd_errors = 0;
271273
let opa = opa_engine.clone();
272274
let cache = identity_cache.clone();
273275
let spid = entrypoint_pid.clone();
@@ -314,14 +316,34 @@ impl ProxyHandle {
314316
});
315317
}
316318
Err(err) => {
319+
// EMFILE (24) / ENFILE (23) indicate FD exhaustion —
320+
// back off exponentially to let connections drain.
321+
let is_fd_exhaustion = matches!(
322+
err.raw_os_error(),
323+
Some(24) | Some(23)
324+
);
325+
let (severity, backoff) = if is_fd_exhaustion {
326+
consecutive_fd_errors = consecutive_fd_errors.saturating_add(1);
327+
let backoff_ms = 100u64
328+
.saturating_mul(
329+
1u64 << consecutive_fd_errors.min(6).saturating_sub(1),
330+
)
331+
.min(5_000);
332+
(SeverityId::Medium, std::time::Duration::from_millis(backoff_ms))
333+
} else {
334+
(SeverityId::Low, std::time::Duration::from_millis(100))
335+
};
317336
let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx())
318337
.activity(ActivityId::Fail)
319-
.severity(SeverityId::Low)
338+
.severity(severity)
320339
.status(StatusId::Failure)
321-
.message(format!("Proxy accept error: {err}"))
340+
.message(format!(
341+
"Proxy accept error (retrying in {}ms): {err}",
342+
backoff.as_millis(),
343+
))
322344
.build();
323345
ocsf_emit!(event);
324-
break;
346+
tokio::time::sleep(backoff).await;
325347
}
326348
}
327349
}

0 commit comments

Comments
 (0)