@@ -265,12 +265,12 @@ impl ProxyHandle {
265265 }
266266 }
267267
268- let mut consecutive_fd_errors : u32 = 0 ;
268+ let mut consecutive_resource_errors : u32 = 0 ;
269269 let mut consecutive_unknown_errors: u32 = 0 ;
270270 loop {
271271 match listener. accept ( ) . await {
272272 Ok ( ( stream, _addr) ) => {
273- consecutive_fd_errors = 0 ;
273+ consecutive_resource_errors = 0 ;
274274 consecutive_unknown_errors = 0 ;
275275 let opa = opa_engine. clone ( ) ;
276276 let cache = identity_cache. clone ( ) ;
@@ -320,7 +320,7 @@ impl ProxyHandle {
320320 Err ( err) => {
321321 let outcome = handle_accept_error (
322322 & err,
323- & mut consecutive_fd_errors ,
323+ & mut consecutive_resource_errors ,
324324 & mut consecutive_unknown_errors,
325325 ) ;
326326
@@ -375,9 +375,24 @@ enum AcceptErrorClass {
375375#[ cfg( unix) ]
376376fn classify_accept_error ( err : & std:: io:: Error ) -> AcceptErrorClass {
377377 match err. raw_os_error ( ) {
378- Some ( libc:: EMFILE | libc:: ENFILE | libc:: ECONNABORTED | libc:: ECONNRESET | libc:: EINTR ) => {
379- AcceptErrorClass :: Transient
380- }
378+ Some (
379+ libc:: EMFILE
380+ | libc:: ENFILE
381+ | libc:: ENOBUFS
382+ | libc:: ENOMEM
383+ | libc:: ECONNABORTED
384+ | libc:: ECONNRESET
385+ | libc:: EINTR
386+ | libc:: ENETDOWN
387+ | libc:: EPROTO
388+ | libc:: ENOPROTOOPT
389+ | libc:: EHOSTDOWN
390+ | libc:: EHOSTUNREACH
391+ | libc:: EOPNOTSUPP
392+ | libc:: ENETUNREACH ,
393+ ) => AcceptErrorClass :: Transient ,
394+ #[ cfg( target_os = "linux" ) ]
395+ Some ( libc:: ENONET ) => AcceptErrorClass :: Transient ,
381396 Some ( libc:: EBADF | libc:: EINVAL | libc:: ENOTSOCK ) => AcceptErrorClass :: Terminal ,
382397 _ => AcceptErrorClass :: Unknown ,
383398 }
@@ -389,12 +404,15 @@ fn classify_accept_error(_err: &std::io::Error) -> AcceptErrorClass {
389404}
390405
391406#[ cfg( unix) ]
392- fn is_fd_exhaustion_error ( err : & std:: io:: Error ) -> bool {
393- matches ! ( err. raw_os_error( ) , Some ( libc:: EMFILE | libc:: ENFILE ) )
407+ fn is_resource_pressure_error ( err : & std:: io:: Error ) -> bool {
408+ matches ! (
409+ err. raw_os_error( ) ,
410+ Some ( libc:: EMFILE | libc:: ENFILE | libc:: ENOBUFS | libc:: ENOMEM )
411+ )
394412}
395413
396414#[ cfg( not( unix) ) ]
397- fn is_fd_exhaustion_error ( _err : & std:: io:: Error ) -> bool {
415+ fn is_resource_pressure_error ( _err : & std:: io:: Error ) -> bool {
398416 false
399417}
400418
@@ -418,7 +436,7 @@ struct AcceptErrorOutcome {
418436
419437fn handle_accept_error (
420438 err : & std:: io:: Error ,
421- consecutive_fd_errors : & mut u32 ,
439+ consecutive_resource_errors : & mut u32 ,
422440 consecutive_unknown_errors : & mut u32 ,
423441) -> AcceptErrorOutcome {
424442 let class = classify_accept_error ( err) ;
@@ -454,9 +472,9 @@ fn handle_accept_error(
454472 }
455473 AcceptErrorClass :: Transient => {
456474 * consecutive_unknown_errors = 0 ;
457- if is_fd_exhaustion_error ( err) {
458- * consecutive_fd_errors = consecutive_fd_errors . saturating_add ( 1 ) ;
459- let backoff = accept_backoff ( * consecutive_fd_errors ) ;
475+ if is_resource_pressure_error ( err) {
476+ * consecutive_resource_errors = consecutive_resource_errors . saturating_add ( 1 ) ;
477+ let backoff = accept_backoff ( * consecutive_resource_errors ) ;
460478 AcceptErrorOutcome {
461479 severity : SeverityId :: Medium ,
462480 message : format ! (
@@ -9979,23 +9997,34 @@ network_policies:
99799997
99809998 #[ cfg( unix) ]
99819999 #[ test]
9982- fn is_fd_exhaustion_detects_emfile ( ) {
10000+ fn is_resource_pressure_detects_emfile ( ) {
998310001 let err = std:: io:: Error :: from_raw_os_error ( libc:: EMFILE ) ;
9984- assert ! ( is_fd_exhaustion_error ( & err) ) ;
10002+ assert ! ( is_resource_pressure_error ( & err) ) ;
998510003 }
998610004
998710005 #[ cfg( unix) ]
998810006 #[ test]
9989- fn is_fd_exhaustion_detects_enfile ( ) {
10007+ fn is_resource_pressure_detects_enfile ( ) {
999010008 let err = std:: io:: Error :: from_raw_os_error ( libc:: ENFILE ) ;
9991- assert ! ( is_fd_exhaustion_error( & err) ) ;
10009+ assert ! ( is_resource_pressure_error( & err) ) ;
10010+ }
10011+
10012+ #[ cfg( unix) ]
10013+ #[ test]
10014+ fn is_resource_pressure_detects_memory_pressure ( ) {
10015+ assert ! ( is_resource_pressure_error(
10016+ & std:: io:: Error :: from_raw_os_error( libc:: ENOBUFS )
10017+ ) ) ;
10018+ assert ! ( is_resource_pressure_error(
10019+ & std:: io:: Error :: from_raw_os_error( libc:: ENOMEM )
10020+ ) ) ;
999210021 }
999310022
999410023 #[ cfg( unix) ]
999510024 #[ test]
9996- fn is_fd_exhaustion_rejects_other_errors ( ) {
10025+ fn is_resource_pressure_rejects_other_errors ( ) {
999710026 let err = std:: io:: Error :: from_raw_os_error ( libc:: ECONNABORTED ) ;
9998- assert ! ( !is_fd_exhaustion_error ( & err) ) ;
10027+ assert ! ( !is_resource_pressure_error ( & err) ) ;
999910028 }
1000010029
1000110030 #[ cfg( unix) ]
@@ -10057,43 +10086,43 @@ network_policies:
1005710086 #[ cfg( unix) ]
1005810087 #[ test]
1005910088 fn handle_accept_error_terminal_exits_immediately ( ) {
10060- let mut fd = 0 ;
10089+ let mut res = 0 ;
1006110090 let mut unk = 0 ;
1006210091 let err = std:: io:: Error :: from_raw_os_error ( libc:: EBADF ) ;
10063- let outcome = handle_accept_error ( & err, & mut fd , & mut unk) ;
10092+ let outcome = handle_accept_error ( & err, & mut res , & mut unk) ;
1006410093 assert ! ( outcome. backoff. is_none( ) ) ;
1006510094 assert_eq ! ( outcome. severity, SeverityId :: High ) ;
1006610095 }
1006710096
1006810097 #[ cfg( unix) ]
1006910098 #[ test]
1007010099 fn handle_accept_error_transient_retries_indefinitely ( ) {
10071- let mut fd = 0 ;
10100+ let mut res = 0 ;
1007210101 let mut unk = 0 ;
1007310102 let err = std:: io:: Error :: from_raw_os_error ( libc:: EMFILE ) ;
1007410103 for i in 1 ..=20 {
10075- let outcome = handle_accept_error ( & err, & mut fd , & mut unk) ;
10104+ let outcome = handle_accept_error ( & err, & mut res , & mut unk) ;
1007610105 assert ! ( outcome. backoff. is_some( ) , "should retry on attempt {i}" ) ;
1007710106 assert_eq ! ( outcome. severity, SeverityId :: Medium ) ;
1007810107 }
10079- assert_eq ! ( fd , 20 ) ;
10108+ assert_eq ! ( res , 20 ) ;
1008010109 }
1008110110
1008210111 #[ cfg( unix) ]
1008310112 #[ test]
1008410113 fn handle_accept_error_unknown_exits_after_limit ( ) {
10085- let mut fd = 0 ;
10114+ let mut res = 0 ;
1008610115 let mut unk = 0 ;
1008710116 let err = std:: io:: Error :: from_raw_os_error ( libc:: EPERM ) ;
1008810117 for i in 1 ..=MAX_CONSECUTIVE_UNKNOWN_ERRORS {
10089- let outcome = handle_accept_error ( & err, & mut fd , & mut unk) ;
10118+ let outcome = handle_accept_error ( & err, & mut res , & mut unk) ;
1009010119 assert ! (
1009110120 outcome. backoff. is_some( ) ,
1009210121 "should retry on attempt {i}/{MAX_CONSECUTIVE_UNKNOWN_ERRORS}" ,
1009310122 ) ;
1009410123 assert_eq ! ( outcome. severity, SeverityId :: Medium ) ;
1009510124 }
10096- let outcome = handle_accept_error ( & err, & mut fd , & mut unk) ;
10125+ let outcome = handle_accept_error ( & err, & mut res , & mut unk) ;
1009710126 assert ! (
1009810127 outcome. backoff. is_none( ) ,
1009910128 "should exit after limit exceeded"
@@ -10104,47 +10133,161 @@ network_policies:
1010410133 #[ cfg( unix) ]
1010510134 #[ test]
1010610135 fn handle_accept_error_transient_resets_unknown_counter ( ) {
10107- let mut fd = 0 ;
10136+ let mut res = 0 ;
1010810137 let mut unk = 0 ;
1010910138 let unknown_err = std:: io:: Error :: from_raw_os_error ( libc:: EPERM ) ;
1011010139 let transient_err = std:: io:: Error :: from_raw_os_error ( libc:: ECONNABORTED ) ;
1011110140
1011210141 // Accumulate unknowns up to the limit.
1011310142 for _ in 1 ..=MAX_CONSECUTIVE_UNKNOWN_ERRORS {
10114- handle_accept_error ( & unknown_err, & mut fd , & mut unk) ;
10143+ handle_accept_error ( & unknown_err, & mut res , & mut unk) ;
1011510144 }
1011610145 assert_eq ! ( unk, MAX_CONSECUTIVE_UNKNOWN_ERRORS ) ;
1011710146
1011810147 // A transient error resets the unknown counter.
10119- let outcome = handle_accept_error ( & transient_err, & mut fd , & mut unk) ;
10148+ let outcome = handle_accept_error ( & transient_err, & mut res , & mut unk) ;
1012010149 assert ! ( outcome. backoff. is_some( ) ) ;
1012110150 assert_eq ! ( unk, 0 ) ;
1012210151
1012310152 // Unknown errors can retry again from zero.
10124- let outcome = handle_accept_error ( & unknown_err, & mut fd , & mut unk) ;
10153+ let outcome = handle_accept_error ( & unknown_err, & mut res , & mut unk) ;
1012510154 assert ! ( outcome. backoff. is_some( ) ) ;
1012610155 assert_eq ! ( unk, 1 ) ;
1012710156 }
1012810157
1012910158 #[ cfg( unix) ]
1013010159 #[ test]
1013110160 fn handle_accept_error_fd_exhaustion_uses_exponential_backoff ( ) {
10132- let mut fd = 0 ;
10161+ let mut res = 0 ;
1013310162 let mut unk = 0 ;
1013410163 let err = std:: io:: Error :: from_raw_os_error ( libc:: EMFILE ) ;
1013510164
10136- let b1 = handle_accept_error ( & err, & mut fd , & mut unk)
10165+ let b1 = handle_accept_error ( & err, & mut res , & mut unk)
1013710166 . backoff
1013810167 . unwrap ( ) ;
10139- let b2 = handle_accept_error ( & err, & mut fd , & mut unk)
10168+ let b2 = handle_accept_error ( & err, & mut res , & mut unk)
1014010169 . backoff
1014110170 . unwrap ( ) ;
10142- let b3 = handle_accept_error ( & err, & mut fd , & mut unk)
10171+ let b3 = handle_accept_error ( & err, & mut res , & mut unk)
1014310172 . backoff
1014410173 . unwrap ( ) ;
1014510174
1014610175 assert_eq ! ( b1. as_millis( ) , 100 ) ;
1014710176 assert_eq ! ( b2. as_millis( ) , 200 ) ;
1014810177 assert_eq ! ( b3. as_millis( ) , 400 ) ;
1014910178 }
10179+
10180+ #[ cfg( unix) ]
10181+ #[ test]
10182+ fn classify_accept_error_network_errors_are_transient ( ) {
10183+ for errno in [
10184+ libc:: ENETDOWN ,
10185+ libc:: EPROTO ,
10186+ libc:: ENOPROTOOPT ,
10187+ libc:: EHOSTDOWN ,
10188+ libc:: EHOSTUNREACH ,
10189+ libc:: EOPNOTSUPP ,
10190+ libc:: ENETUNREACH ,
10191+ ] {
10192+ assert_eq ! (
10193+ classify_accept_error( & std:: io:: Error :: from_raw_os_error( errno) ) ,
10194+ AcceptErrorClass :: Transient ,
10195+ "errno {errno} should be transient" ,
10196+ ) ;
10197+ }
10198+ }
10199+
10200+ #[ cfg( target_os = "linux" ) ]
10201+ #[ test]
10202+ fn classify_accept_error_enonet_is_transient ( ) {
10203+ assert_eq ! (
10204+ classify_accept_error( & std:: io:: Error :: from_raw_os_error( libc:: ENONET ) ) ,
10205+ AcceptErrorClass :: Transient ,
10206+ ) ;
10207+ }
10208+
10209+ #[ cfg( unix) ]
10210+ #[ test]
10211+ fn classify_accept_error_resource_pressure_is_transient ( ) {
10212+ assert_eq ! (
10213+ classify_accept_error( & std:: io:: Error :: from_raw_os_error( libc:: ENOBUFS ) ) ,
10214+ AcceptErrorClass :: Transient ,
10215+ ) ;
10216+ assert_eq ! (
10217+ classify_accept_error( & std:: io:: Error :: from_raw_os_error( libc:: ENOMEM ) ) ,
10218+ AcceptErrorClass :: Transient ,
10219+ ) ;
10220+ }
10221+
10222+ #[ cfg( unix) ]
10223+ #[ test]
10224+ fn handle_accept_error_non_resource_transient_uses_fixed_backoff ( ) {
10225+ let mut res = 0 ;
10226+ let mut unk = 0 ;
10227+ let err = std:: io:: Error :: from_raw_os_error ( libc:: ECONNABORTED ) ;
10228+
10229+ let o1 = handle_accept_error ( & err, & mut res, & mut unk) ;
10230+ let o2 = handle_accept_error ( & err, & mut res, & mut unk) ;
10231+
10232+ assert_eq ! ( o1. severity, SeverityId :: Low ) ;
10233+ assert_eq ! ( o1. backoff. unwrap( ) . as_millis( ) , 100 ) ;
10234+ assert_eq ! ( o2. backoff. unwrap( ) . as_millis( ) , 100 ) ;
10235+ assert_eq ! ( res, 0 ) ;
10236+ }
10237+
10238+ #[ cfg( unix) ]
10239+ #[ test]
10240+ fn handle_accept_error_unknown_uses_exponential_backoff ( ) {
10241+ let mut res = 0 ;
10242+ let mut unk = 0 ;
10243+ let err = std:: io:: Error :: from_raw_os_error ( libc:: EPERM ) ;
10244+
10245+ let b1 = handle_accept_error ( & err, & mut res, & mut unk)
10246+ . backoff
10247+ . unwrap ( ) ;
10248+ let b2 = handle_accept_error ( & err, & mut res, & mut unk)
10249+ . backoff
10250+ . unwrap ( ) ;
10251+ let b3 = handle_accept_error ( & err, & mut res, & mut unk)
10252+ . backoff
10253+ . unwrap ( ) ;
10254+
10255+ assert_eq ! ( b1. as_millis( ) , 100 ) ;
10256+ assert_eq ! ( b2. as_millis( ) , 200 ) ;
10257+ assert_eq ! ( b3. as_millis( ) , 400 ) ;
10258+ }
10259+
10260+ #[ cfg( unix) ]
10261+ #[ test]
10262+ fn handle_accept_error_resource_counter_persists_across_mixed_transient ( ) {
10263+ let mut res = 0 ;
10264+ let mut unk = 0 ;
10265+ let resource_err = std:: io:: Error :: from_raw_os_error ( libc:: EMFILE ) ;
10266+ let transient_err = std:: io:: Error :: from_raw_os_error ( libc:: ECONNABORTED ) ;
10267+
10268+ let o1 = handle_accept_error ( & resource_err, & mut res, & mut unk) ;
10269+ assert_eq ! ( res, 1 ) ;
10270+ assert_eq ! ( o1. backoff. unwrap( ) . as_millis( ) , 100 ) ;
10271+
10272+ let o2 = handle_accept_error ( & transient_err, & mut res, & mut unk) ;
10273+ assert_eq ! ( res, 1 ) ;
10274+ assert_eq ! ( o2. backoff. unwrap( ) . as_millis( ) , 100 ) ;
10275+
10276+ let o3 = handle_accept_error ( & resource_err, & mut res, & mut unk) ;
10277+ assert_eq ! ( res, 2 ) ;
10278+ assert_eq ! ( o3. backoff. unwrap( ) . as_millis( ) , 200 ) ;
10279+ }
10280+
10281+ #[ cfg( unix) ]
10282+ #[ test]
10283+ fn handle_accept_error_terminal_leaves_counters_unchanged ( ) {
10284+ let mut res = 3 ;
10285+ let mut unk = 2 ;
10286+ let err = std:: io:: Error :: from_raw_os_error ( libc:: EBADF ) ;
10287+
10288+ let outcome = handle_accept_error ( & err, & mut res, & mut unk) ;
10289+ assert ! ( outcome. backoff. is_none( ) ) ;
10290+ assert_eq ! ( res, 3 ) ;
10291+ assert_eq ! ( unk, 2 ) ;
10292+ }
1015010293}
0 commit comments