Skip to content

Commit effd300

Browse files
committed
fix(extensions): fail fast on credential handshake rejection
Signed-off-by: Seth Jennings <[email protected]>
1 parent 44cab9d commit effd300

4 files changed

Lines changed: 114 additions & 21 deletions

File tree

‎crates/openshell-server/src/credentials.rs‎

Lines changed: 89 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1701,10 +1701,10 @@ where
17011701

17021702
match tokio::time::timeout(remaining, connect()).await {
17031703
Ok(Ok(connected)) => return Ok(connected),
1704-
Ok(Err(CredentialDriverReadinessError::Retryable(err))) => {
1705-
last_error = Some(err.to_string());
1704+
Ok(Err(error)) if error.is_retryable() => {
1705+
last_error = Some(error.into_error().to_string());
17061706
}
1707-
Ok(Err(CredentialDriverReadinessError::Terminal(err))) => return Err(err),
1707+
Ok(Err(error)) => return Err(error.into_error()),
17081708
Err(_) => {
17091709
return Err(Error::execution(format!(
17101710
"timed out waiting for credential driver '{driver_name}' to respond to GetCapabilities"
@@ -1728,14 +1728,44 @@ where
17281728
#[derive(Debug)]
17291729
enum CredentialDriverReadinessError {
17301730
Retryable(Error),
1731+
RpcStatus { driver_name: String, status: Status },
17311732
Terminal(Error),
17321733
}
17331734

17341735
#[cfg(unix)]
17351736
impl CredentialDriverReadinessError {
1737+
fn rpc_status(driver_name: &str, status: Status) -> Self {
1738+
Self::RpcStatus {
1739+
driver_name: driver_name.to_string(),
1740+
status,
1741+
}
1742+
}
1743+
1744+
fn is_retryable(&self) -> bool {
1745+
match self {
1746+
Self::Retryable(_) => true,
1747+
Self::RpcStatus { status, .. } => matches!(
1748+
status.code(),
1749+
tonic::Code::Unavailable
1750+
| tonic::Code::DeadlineExceeded
1751+
| tonic::Code::ResourceExhausted
1752+
| tonic::Code::Aborted
1753+
| tonic::Code::Internal
1754+
| tonic::Code::Unknown
1755+
),
1756+
Self::Terminal(_) => false,
1757+
}
1758+
}
1759+
17361760
fn into_error(self) -> Error {
17371761
match self {
17381762
Self::Retryable(error) | Self::Terminal(error) => error,
1763+
Self::RpcStatus {
1764+
driver_name,
1765+
status,
1766+
} => Error::config(format!(
1767+
"credential driver '{driver_name}' GetCapabilities failed: {status}"
1768+
)),
17391769
}
17401770
}
17411771
}
@@ -1760,8 +1790,7 @@ async fn connect_ready_credential_driver(
17601790
timeout,
17611791
client.get_capabilities(request),
17621792
)
1763-
.await
1764-
.map_err(CredentialDriverReadinessError::Retryable)?;
1793+
.await?;
17651794
let negotiated_extension = negotiate(
17661795
ExtensionFamily::Credentials,
17671796
driver_name,
@@ -1779,19 +1808,15 @@ async fn await_credential_driver_capabilities(
17791808
response: impl Future<
17801809
Output = Result<tonic::Response<GetCredentialDriverCapabilitiesResponse>, Status>,
17811810
>,
1782-
) -> CoreResult<GetCredentialDriverCapabilitiesResponse> {
1811+
) -> Result<GetCredentialDriverCapabilitiesResponse, CredentialDriverReadinessError> {
17831812
tokio::time::timeout(timeout, response)
17841813
.await
17851814
.map_err(|_| {
1786-
Error::config(format!(
1815+
CredentialDriverReadinessError::Retryable(Error::config(format!(
17871816
"credential driver '{driver_name}' GetCapabilities timed out"
1788-
))
1817+
)))
17891818
})?
1790-
.map_err(|status| {
1791-
Error::config(format!(
1792-
"credential driver '{driver_name}' GetCapabilities failed: {status}"
1793-
))
1794-
})
1819+
.map_err(|status| CredentialDriverReadinessError::rpc_status(driver_name, status))
17951820
.map(tonic::Response::into_inner)
17961821
}
17971822

@@ -2559,14 +2584,15 @@ socket_path = {socket_path_toml}
25592584
response,
25602585
)
25612586
.await
2562-
.unwrap_err();
2587+
.unwrap_err()
2588+
.into_error();
25632589

25642590
assert!(err.to_string().contains("GetCapabilities timed out"));
25652591
}
25662592

25672593
#[cfg(unix)]
25682594
#[tokio::test]
2569-
async fn launched_driver_protocol_incompatibility_is_terminal() {
2595+
async fn launched_driver_failed_precondition_is_terminal() {
25702596
let mut child = Command::new("sleep")
25712597
.arg("30")
25722598
.kill_on_drop(true)
@@ -2579,12 +2605,16 @@ socket_path = {socket_path_toml}
25792605
Path::new("/unused-test-socket"),
25802606
&mut child,
25812607
Duration::from_secs(30),
2582-
|| {
2583-
std::future::ready(Err(CredentialDriverReadinessError::Terminal(
2584-
Error::config(
2608+
|| async {
2609+
await_credential_driver_capabilities(
2610+
"enterprise-secrets",
2611+
Duration::from_secs(30),
2612+
std::future::ready(Err(Status::failed_precondition(
25852613
"credentials extension 'enterprise-secrets' uses unsupported protocol 2.0; gateway supports 1.0",
2586-
),
2587-
)))
2614+
))),
2615+
)
2616+
.await?;
2617+
unreachable!("failed-precondition response cannot produce capabilities")
25882618
},
25892619
)
25902620
.await
@@ -2594,6 +2624,45 @@ socket_path = {socket_path_toml}
25942624
assert!(started.elapsed() < Duration::from_secs(1));
25952625
}
25962626

2627+
#[cfg(unix)]
2628+
#[test]
2629+
fn credential_driver_rpc_status_retries_only_transient_failures() {
2630+
for code in [
2631+
Code::Unavailable,
2632+
Code::DeadlineExceeded,
2633+
Code::ResourceExhausted,
2634+
Code::Aborted,
2635+
Code::Internal,
2636+
Code::Unknown,
2637+
] {
2638+
assert!(
2639+
CredentialDriverReadinessError::rpc_status(
2640+
"enterprise-secrets",
2641+
Status::new(code, "not ready"),
2642+
)
2643+
.is_retryable(),
2644+
"{code:?} should be retried"
2645+
);
2646+
}
2647+
2648+
for code in [
2649+
Code::InvalidArgument,
2650+
Code::FailedPrecondition,
2651+
Code::PermissionDenied,
2652+
Code::Unauthenticated,
2653+
Code::Unimplemented,
2654+
] {
2655+
assert!(
2656+
!CredentialDriverReadinessError::rpc_status(
2657+
"enterprise-secrets",
2658+
Status::new(code, "incompatible"),
2659+
)
2660+
.is_retryable(),
2661+
"{code:?} should fail immediately"
2662+
);
2663+
}
2664+
}
2665+
25972666
#[test]
25982667
fn parse_driver_table_preserves_backend_config_without_transport_fields() {
25992668
let parsed = parse_driver_table(

‎crates/openshell-supervisor-middleware/src/response.rs‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1247,7 +1247,7 @@ mod tests {
12471247

12481248
async fn describe(
12491249
&self,
1250-
_request: tonic::Request<()>,
1250+
_request: tonic::Request<openshell_core::proto::MiddlewareDescribeRequest>,
12511251
) -> Result<tonic::Response<MiddlewareManifest>, tonic::Status> {
12521252
Ok(tonic::Response::new(response_manifest(
12531253
"test/remote-response",
@@ -1377,6 +1377,12 @@ mod tests {
13771377
}),
13781378
}],
13791379
expected_audience: String::new(),
1380+
extension: Some(openshell_core::extension_protocol::extension_metadata(
1381+
openshell_core::extension_protocol::ExtensionFamily::SupervisorMiddleware,
1382+
"openshell/test-response-middleware",
1383+
"test",
1384+
[],
1385+
)),
13801386
}
13811387
}
13821388

@@ -1717,6 +1723,12 @@ mod tests {
17171723
request_timeout: None,
17181724
}],
17191725
expected_audience: String::new(),
1726+
extension: Some(openshell_core::extension_protocol::extension_metadata(
1727+
openshell_core::extension_protocol::ExtensionFamily::SupervisorMiddleware,
1728+
name,
1729+
"test",
1730+
[],
1731+
)),
17201732
}
17211733
}
17221734

‎crates/openshell-supervisor-network/src/l7/rest.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3609,6 +3609,12 @@ mod tests {
36093609
request_timeout: None,
36103610
}],
36113611
expected_audience: String::new(),
3612+
extension: Some(openshell_core::extension_protocol::extension_metadata(
3613+
openshell_core::extension_protocol::ExtensionFamily::SupervisorMiddleware,
3614+
"openshell/test-response-relay",
3615+
"test",
3616+
[],
3617+
)),
36123618
}
36133619
}
36143620

‎crates/openshell-supervisor-network/src/proxy.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6856,6 +6856,12 @@ process:
68566856
request_timeout: None,
68576857
}],
68586858
expected_audience: String::new(),
6859+
extension: Some(openshell_core::extension_protocol::extension_metadata(
6860+
openshell_core::extension_protocol::ExtensionFamily::SupervisorMiddleware,
6861+
"openshell/test-forward-response",
6862+
"test",
6863+
[],
6864+
)),
68596865
}
68606866
}
68616867

0 commit comments

Comments
 (0)