Skip to content

Commit ddae69b

Browse files
committed
fix(proxy): expand transient error allowlist and fix clippy/test issues
Widen the accept-error classification to cover all Linux accept(2) transient errnos (ENETDOWN, EPROTO, ENOPROTOOPT, EHOSTDOWN, EHOSTUNREACH, EOPNOTSUPP, ENETUNREACH, ENONET behind cfg gate). Rename fd-exhaustion helpers to resource-pressure to reflect ENOBUFS and ENOMEM coverage. Fix clippy lints in the integration test (borrow_as_ptr, collection_is_never_read, while_let_loop) and make post-EMFILE recovery verification cross-platform by handling Linux's stale backlog behavior. Signed-off-by: Quinn Burdine <[email protected]> Signed-off-by: politerealism <[email protected]>
1 parent 39705af commit ddae69b

2 files changed

Lines changed: 200 additions & 47 deletions

File tree

‎crates/openshell-supervisor-network/src/proxy.rs‎

Lines changed: 178 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -265,12 +265,12 @@ impl ProxyHandle {
265265
}
266266
}
267267

268-
let mut consecutive_fd_errors: u32 = 0;
268+
let mut consecutive_resource_errors: u32 = 0;
269269
let mut consecutive_unknown_errors: u32 = 0;
270270
loop {
271271
match listener.accept().await {
272272
Ok((stream, _addr)) => {
273-
consecutive_fd_errors = 0;
273+
consecutive_resource_errors = 0;
274274
consecutive_unknown_errors = 0;
275275
let opa = opa_engine.clone();
276276
let cache = identity_cache.clone();
@@ -320,7 +320,7 @@ impl ProxyHandle {
320320
Err(err) => {
321321
let outcome = handle_accept_error(
322322
&err,
323-
&mut consecutive_fd_errors,
323+
&mut consecutive_resource_errors,
324324
&mut consecutive_unknown_errors,
325325
);
326326

@@ -375,9 +375,24 @@ enum AcceptErrorClass {
375375
#[cfg(unix)]
376376
fn classify_accept_error(err: &std::io::Error) -> AcceptErrorClass {
377377
match err.raw_os_error() {
378-
Some(libc::EMFILE | libc::ENFILE | libc::ECONNABORTED | libc::ECONNRESET | libc::EINTR) => {
379-
AcceptErrorClass::Transient
380-
}
378+
Some(
379+
libc::EMFILE
380+
| libc::ENFILE
381+
| libc::ENOBUFS
382+
| libc::ENOMEM
383+
| libc::ECONNABORTED
384+
| libc::ECONNRESET
385+
| libc::EINTR
386+
| libc::ENETDOWN
387+
| libc::EPROTO
388+
| libc::ENOPROTOOPT
389+
| libc::EHOSTDOWN
390+
| libc::EHOSTUNREACH
391+
| libc::EOPNOTSUPP
392+
| libc::ENETUNREACH,
393+
) => AcceptErrorClass::Transient,
394+
#[cfg(target_os = "linux")]
395+
Some(libc::ENONET) => AcceptErrorClass::Transient,
381396
Some(libc::EBADF | libc::EINVAL | libc::ENOTSOCK) => AcceptErrorClass::Terminal,
382397
_ => AcceptErrorClass::Unknown,
383398
}
@@ -389,12 +404,15 @@ fn classify_accept_error(_err: &std::io::Error) -> AcceptErrorClass {
389404
}
390405

391406
#[cfg(unix)]
392-
fn is_fd_exhaustion_error(err: &std::io::Error) -> bool {
393-
matches!(err.raw_os_error(), Some(libc::EMFILE | libc::ENFILE))
407+
fn is_resource_pressure_error(err: &std::io::Error) -> bool {
408+
matches!(
409+
err.raw_os_error(),
410+
Some(libc::EMFILE | libc::ENFILE | libc::ENOBUFS | libc::ENOMEM)
411+
)
394412
}
395413

396414
#[cfg(not(unix))]
397-
fn is_fd_exhaustion_error(_err: &std::io::Error) -> bool {
415+
fn is_resource_pressure_error(_err: &std::io::Error) -> bool {
398416
false
399417
}
400418

@@ -418,7 +436,7 @@ struct AcceptErrorOutcome {
418436

419437
fn handle_accept_error(
420438
err: &std::io::Error,
421-
consecutive_fd_errors: &mut u32,
439+
consecutive_resource_errors: &mut u32,
422440
consecutive_unknown_errors: &mut u32,
423441
) -> AcceptErrorOutcome {
424442
let class = classify_accept_error(err);
@@ -454,9 +472,9 @@ fn handle_accept_error(
454472
}
455473
AcceptErrorClass::Transient => {
456474
*consecutive_unknown_errors = 0;
457-
if is_fd_exhaustion_error(err) {
458-
*consecutive_fd_errors = consecutive_fd_errors.saturating_add(1);
459-
let backoff = accept_backoff(*consecutive_fd_errors);
475+
if is_resource_pressure_error(err) {
476+
*consecutive_resource_errors = consecutive_resource_errors.saturating_add(1);
477+
let backoff = accept_backoff(*consecutive_resource_errors);
460478
AcceptErrorOutcome {
461479
severity: SeverityId::Medium,
462480
message: format!(
@@ -9979,23 +9997,34 @@ network_policies:
99799997

99809998
#[cfg(unix)]
99819999
#[test]
9982-
fn is_fd_exhaustion_detects_emfile() {
10000+
fn is_resource_pressure_detects_emfile() {
998310001
let err = std::io::Error::from_raw_os_error(libc::EMFILE);
9984-
assert!(is_fd_exhaustion_error(&err));
10002+
assert!(is_resource_pressure_error(&err));
998510003
}
998610004

998710005
#[cfg(unix)]
998810006
#[test]
9989-
fn is_fd_exhaustion_detects_enfile() {
10007+
fn is_resource_pressure_detects_enfile() {
999010008
let err = std::io::Error::from_raw_os_error(libc::ENFILE);
9991-
assert!(is_fd_exhaustion_error(&err));
10009+
assert!(is_resource_pressure_error(&err));
10010+
}
10011+
10012+
#[cfg(unix)]
10013+
#[test]
10014+
fn is_resource_pressure_detects_memory_pressure() {
10015+
assert!(is_resource_pressure_error(
10016+
&std::io::Error::from_raw_os_error(libc::ENOBUFS)
10017+
));
10018+
assert!(is_resource_pressure_error(
10019+
&std::io::Error::from_raw_os_error(libc::ENOMEM)
10020+
));
999210021
}
999310022

999410023
#[cfg(unix)]
999510024
#[test]
9996-
fn is_fd_exhaustion_rejects_other_errors() {
10025+
fn is_resource_pressure_rejects_other_errors() {
999710026
let err = std::io::Error::from_raw_os_error(libc::ECONNABORTED);
9998-
assert!(!is_fd_exhaustion_error(&err));
10027+
assert!(!is_resource_pressure_error(&err));
999910028
}
1000010029

1000110030
#[cfg(unix)]
@@ -10057,43 +10086,43 @@ network_policies:
1005710086
#[cfg(unix)]
1005810087
#[test]
1005910088
fn handle_accept_error_terminal_exits_immediately() {
10060-
let mut fd = 0;
10089+
let mut res = 0;
1006110090
let mut unk = 0;
1006210091
let err = std::io::Error::from_raw_os_error(libc::EBADF);
10063-
let outcome = handle_accept_error(&err, &mut fd, &mut unk);
10092+
let outcome = handle_accept_error(&err, &mut res, &mut unk);
1006410093
assert!(outcome.backoff.is_none());
1006510094
assert_eq!(outcome.severity, SeverityId::High);
1006610095
}
1006710096

1006810097
#[cfg(unix)]
1006910098
#[test]
1007010099
fn handle_accept_error_transient_retries_indefinitely() {
10071-
let mut fd = 0;
10100+
let mut res = 0;
1007210101
let mut unk = 0;
1007310102
let err = std::io::Error::from_raw_os_error(libc::EMFILE);
1007410103
for i in 1..=20 {
10075-
let outcome = handle_accept_error(&err, &mut fd, &mut unk);
10104+
let outcome = handle_accept_error(&err, &mut res, &mut unk);
1007610105
assert!(outcome.backoff.is_some(), "should retry on attempt {i}");
1007710106
assert_eq!(outcome.severity, SeverityId::Medium);
1007810107
}
10079-
assert_eq!(fd, 20);
10108+
assert_eq!(res, 20);
1008010109
}
1008110110

1008210111
#[cfg(unix)]
1008310112
#[test]
1008410113
fn handle_accept_error_unknown_exits_after_limit() {
10085-
let mut fd = 0;
10114+
let mut res = 0;
1008610115
let mut unk = 0;
1008710116
let err = std::io::Error::from_raw_os_error(libc::EPERM);
1008810117
for i in 1..=MAX_CONSECUTIVE_UNKNOWN_ERRORS {
10089-
let outcome = handle_accept_error(&err, &mut fd, &mut unk);
10118+
let outcome = handle_accept_error(&err, &mut res, &mut unk);
1009010119
assert!(
1009110120
outcome.backoff.is_some(),
1009210121
"should retry on attempt {i}/{MAX_CONSECUTIVE_UNKNOWN_ERRORS}",
1009310122
);
1009410123
assert_eq!(outcome.severity, SeverityId::Medium);
1009510124
}
10096-
let outcome = handle_accept_error(&err, &mut fd, &mut unk);
10125+
let outcome = handle_accept_error(&err, &mut res, &mut unk);
1009710126
assert!(
1009810127
outcome.backoff.is_none(),
1009910128
"should exit after limit exceeded"
@@ -10104,47 +10133,161 @@ network_policies:
1010410133
#[cfg(unix)]
1010510134
#[test]
1010610135
fn handle_accept_error_transient_resets_unknown_counter() {
10107-
let mut fd = 0;
10136+
let mut res = 0;
1010810137
let mut unk = 0;
1010910138
let unknown_err = std::io::Error::from_raw_os_error(libc::EPERM);
1011010139
let transient_err = std::io::Error::from_raw_os_error(libc::ECONNABORTED);
1011110140

1011210141
// Accumulate unknowns up to the limit.
1011310142
for _ in 1..=MAX_CONSECUTIVE_UNKNOWN_ERRORS {
10114-
handle_accept_error(&unknown_err, &mut fd, &mut unk);
10143+
handle_accept_error(&unknown_err, &mut res, &mut unk);
1011510144
}
1011610145
assert_eq!(unk, MAX_CONSECUTIVE_UNKNOWN_ERRORS);
1011710146

1011810147
// A transient error resets the unknown counter.
10119-
let outcome = handle_accept_error(&transient_err, &mut fd, &mut unk);
10148+
let outcome = handle_accept_error(&transient_err, &mut res, &mut unk);
1012010149
assert!(outcome.backoff.is_some());
1012110150
assert_eq!(unk, 0);
1012210151

1012310152
// Unknown errors can retry again from zero.
10124-
let outcome = handle_accept_error(&unknown_err, &mut fd, &mut unk);
10153+
let outcome = handle_accept_error(&unknown_err, &mut res, &mut unk);
1012510154
assert!(outcome.backoff.is_some());
1012610155
assert_eq!(unk, 1);
1012710156
}
1012810157

1012910158
#[cfg(unix)]
1013010159
#[test]
1013110160
fn handle_accept_error_fd_exhaustion_uses_exponential_backoff() {
10132-
let mut fd = 0;
10161+
let mut res = 0;
1013310162
let mut unk = 0;
1013410163
let err = std::io::Error::from_raw_os_error(libc::EMFILE);
1013510164

10136-
let b1 = handle_accept_error(&err, &mut fd, &mut unk)
10165+
let b1 = handle_accept_error(&err, &mut res, &mut unk)
1013710166
.backoff
1013810167
.unwrap();
10139-
let b2 = handle_accept_error(&err, &mut fd, &mut unk)
10168+
let b2 = handle_accept_error(&err, &mut res, &mut unk)
1014010169
.backoff
1014110170
.unwrap();
10142-
let b3 = handle_accept_error(&err, &mut fd, &mut unk)
10171+
let b3 = handle_accept_error(&err, &mut res, &mut unk)
1014310172
.backoff
1014410173
.unwrap();
1014510174

1014610175
assert_eq!(b1.as_millis(), 100);
1014710176
assert_eq!(b2.as_millis(), 200);
1014810177
assert_eq!(b3.as_millis(), 400);
1014910178
}
10179+
10180+
#[cfg(unix)]
10181+
#[test]
10182+
fn classify_accept_error_network_errors_are_transient() {
10183+
for errno in [
10184+
libc::ENETDOWN,
10185+
libc::EPROTO,
10186+
libc::ENOPROTOOPT,
10187+
libc::EHOSTDOWN,
10188+
libc::EHOSTUNREACH,
10189+
libc::EOPNOTSUPP,
10190+
libc::ENETUNREACH,
10191+
] {
10192+
assert_eq!(
10193+
classify_accept_error(&std::io::Error::from_raw_os_error(errno)),
10194+
AcceptErrorClass::Transient,
10195+
"errno {errno} should be transient",
10196+
);
10197+
}
10198+
}
10199+
10200+
#[cfg(target_os = "linux")]
10201+
#[test]
10202+
fn classify_accept_error_enonet_is_transient() {
10203+
assert_eq!(
10204+
classify_accept_error(&std::io::Error::from_raw_os_error(libc::ENONET)),
10205+
AcceptErrorClass::Transient,
10206+
);
10207+
}
10208+
10209+
#[cfg(unix)]
10210+
#[test]
10211+
fn classify_accept_error_resource_pressure_is_transient() {
10212+
assert_eq!(
10213+
classify_accept_error(&std::io::Error::from_raw_os_error(libc::ENOBUFS)),
10214+
AcceptErrorClass::Transient,
10215+
);
10216+
assert_eq!(
10217+
classify_accept_error(&std::io::Error::from_raw_os_error(libc::ENOMEM)),
10218+
AcceptErrorClass::Transient,
10219+
);
10220+
}
10221+
10222+
#[cfg(unix)]
10223+
#[test]
10224+
fn handle_accept_error_non_resource_transient_uses_fixed_backoff() {
10225+
let mut res = 0;
10226+
let mut unk = 0;
10227+
let err = std::io::Error::from_raw_os_error(libc::ECONNABORTED);
10228+
10229+
let o1 = handle_accept_error(&err, &mut res, &mut unk);
10230+
let o2 = handle_accept_error(&err, &mut res, &mut unk);
10231+
10232+
assert_eq!(o1.severity, SeverityId::Low);
10233+
assert_eq!(o1.backoff.unwrap().as_millis(), 100);
10234+
assert_eq!(o2.backoff.unwrap().as_millis(), 100);
10235+
assert_eq!(res, 0);
10236+
}
10237+
10238+
#[cfg(unix)]
10239+
#[test]
10240+
fn handle_accept_error_unknown_uses_exponential_backoff() {
10241+
let mut res = 0;
10242+
let mut unk = 0;
10243+
let err = std::io::Error::from_raw_os_error(libc::EPERM);
10244+
10245+
let b1 = handle_accept_error(&err, &mut res, &mut unk)
10246+
.backoff
10247+
.unwrap();
10248+
let b2 = handle_accept_error(&err, &mut res, &mut unk)
10249+
.backoff
10250+
.unwrap();
10251+
let b3 = handle_accept_error(&err, &mut res, &mut unk)
10252+
.backoff
10253+
.unwrap();
10254+
10255+
assert_eq!(b1.as_millis(), 100);
10256+
assert_eq!(b2.as_millis(), 200);
10257+
assert_eq!(b3.as_millis(), 400);
10258+
}
10259+
10260+
#[cfg(unix)]
10261+
#[test]
10262+
fn handle_accept_error_resource_counter_persists_across_mixed_transient() {
10263+
let mut res = 0;
10264+
let mut unk = 0;
10265+
let resource_err = std::io::Error::from_raw_os_error(libc::EMFILE);
10266+
let transient_err = std::io::Error::from_raw_os_error(libc::ECONNABORTED);
10267+
10268+
let o1 = handle_accept_error(&resource_err, &mut res, &mut unk);
10269+
assert_eq!(res, 1);
10270+
assert_eq!(o1.backoff.unwrap().as_millis(), 100);
10271+
10272+
let o2 = handle_accept_error(&transient_err, &mut res, &mut unk);
10273+
assert_eq!(res, 1);
10274+
assert_eq!(o2.backoff.unwrap().as_millis(), 100);
10275+
10276+
let o3 = handle_accept_error(&resource_err, &mut res, &mut unk);
10277+
assert_eq!(res, 2);
10278+
assert_eq!(o3.backoff.unwrap().as_millis(), 200);
10279+
}
10280+
10281+
#[cfg(unix)]
10282+
#[test]
10283+
fn handle_accept_error_terminal_leaves_counters_unchanged() {
10284+
let mut res = 3;
10285+
let mut unk = 2;
10286+
let err = std::io::Error::from_raw_os_error(libc::EBADF);
10287+
10288+
let outcome = handle_accept_error(&err, &mut res, &mut unk);
10289+
assert!(outcome.backoff.is_none());
10290+
assert_eq!(res, 3);
10291+
assert_eq!(unk, 2);
10292+
}
1015010293
}

0 commit comments

Comments
 (0)