Skip to content

Commit 34e7be6

Browse files
authored
fix(middleware): preserve HTTP request failure causes
Retain typed local deadline and remote failure causes through HTTP hook diagnostics and request handling. Track local upstream write progress through the existing live writer, including partial failures. Preserve response selection, diagnostic text and request retries. Refs #4144 Build on PR #4359. Caller and event exposure remains with the companion tracing work. Signed-off-by: Shiju <[email protected]>
1 parent f37687a commit 34e7be6

11 files changed

Lines changed: 727 additions & 75 deletions

File tree

‎crates/openshell-supervisor-middleware/src/http_hooks/pipeline.rs‎

Lines changed: 88 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,10 @@ use openshell_core::proto::{
4141

4242
use crate::headers::{self, HeaderAuthority};
4343
use crate::{
44-
ChainRunner, DescribedChainEntry, HttpHookVersion, MAX_MIDDLEWARE_CHAIN_TIMEOUT,
45-
MAX_MIDDLEWARE_PAYLOAD_BYTES, MiddlewareDenial, MiddlewareDiagnosticPolicy,
46-
MiddlewareInvocation, MiddlewareSessionPermit, NamespacedFinding, OnError,
47-
TransformedBodyPolicy, middleware_denial_reason, safe_reason,
44+
ChainRunner, DescribedChainEntry, HttpHookVersion, HttpRequestFailureKind,
45+
MAX_MIDDLEWARE_CHAIN_TIMEOUT, MAX_MIDDLEWARE_PAYLOAD_BYTES, MiddlewareDenial,
46+
MiddlewareDiagnosticPolicy, MiddlewareInvocation, MiddlewareSessionPermit, NamespacedFinding,
47+
OnError, TransformedBodyPolicy, middleware_denial_reason, safe_reason,
4848
};
4949

5050
/// Largest normalized body chunk a stage sends or receives.
@@ -197,6 +197,8 @@ pub enum HttpStageOutcome {
197197
/// Audit record for one stage and phase. Carries no service-provided text.
198198
#[derive(Debug, Clone, PartialEq, Eq)]
199199
pub struct HttpStageInvocation {
200+
/// Failure cause retained without service text or status metadata.
201+
pub failure_kind: Option<HttpRequestFailureKind>,
200202
pub config_name: String,
201203
pub implementation: String,
202204
pub outcome: HttpStageOutcome,
@@ -213,6 +215,8 @@ pub struct HttpStageInvocation {
213215
/// Findings, metadata, and invocations for one exchange.
214216
#[derive(Debug, Default)]
215217
pub struct HttpStageDiagnostics {
218+
/// Terminal cause, including failures with no failed stage invocation.
219+
pub failure_kind: Option<HttpRequestFailureKind>,
216220
pub findings: Vec<NamespacedFinding>,
217221
pub metadata: BTreeMap<String, BTreeMap<String, String>>,
218222
pub invocations: Vec<HttpStageInvocation>,
@@ -221,6 +225,7 @@ pub struct HttpStageDiagnostics {
221225
impl HttpStageDiagnostics {
222226
/// Append `other`, keeping chain order.
223227
pub fn extend(&mut self, other: Self) {
228+
self.failure_kind = other.failure_kind.or(self.failure_kind);
224229
self.findings.extend(other.findings);
225230
self.metadata.extend(other.metadata);
226231
self.invocations.extend(other.invocations);
@@ -248,6 +253,7 @@ impl HttpStageDiagnostics {
248253
.iter_mut()
249254
.find(|(existing, _)| existing.name == invocation.config_name)
250255
{
256+
existing.failure_kind = invocation.failure_kind.or(existing.failure_kind);
251257
existing.failed |= invocation.failed;
252258
existing.transformed |= invocation.transformed;
253259
if denied {
@@ -258,6 +264,7 @@ impl HttpStageDiagnostics {
258264
}
259265
applied.push((
260266
MiddlewareInvocation {
267+
failure_kind: invocation.failure_kind,
261268
name: invocation.config_name.clone(),
262269
implementation: invocation.implementation.clone(),
263270
decision: if denied {
@@ -286,6 +293,8 @@ impl HttpStageDiagnostics {
286293
/// The exchange failed closed or was denied.
287294
#[derive(Debug)]
288295
pub struct HttpMiddlewareFailure {
296+
/// Cause preserved independently of sandbox failure codes and diagnostics.
297+
pub kind: Option<HttpRequestFailureKind>,
289298
/// Platform-owned reason: `middleware_denied:<config>[:<code>]` for a
290299
/// denial, `middleware_failed: <reason>` for a failure, and
291300
/// `middleware_cancelled: <reason>` when the input producer went away.
@@ -1558,13 +1567,17 @@ async fn run_checkpoint(
15581567
.ok()
15591568
.flatten()
15601569
.is_none_or(|previous| previous != body);
1561-
if replaced && let Some(reason) = transformed_body_denial(body_policy, &body) {
1570+
if replaced && let Some((reason, kind)) = transformed_body_denial(body_policy, &body) {
15621571
return Err(TaskError::Failed(HttpMiddlewareFailure {
1572+
kind,
15631573
reason,
15641574
denial: None,
15651575
failure: None,
15661576
end_reason: MiddlewareSessionEndReason::PolicyDenial,
1567-
diagnostics: Box::default(),
1577+
diagnostics: Box::new(HttpStageDiagnostics {
1578+
failure_kind: kind,
1579+
..Default::default()
1580+
}),
15681581
}));
15691582
}
15701583
let (header_mutations, output_body_bytes) = start.expect("a started stage output");
@@ -2009,15 +2022,21 @@ async fn drive_buffered(
20092022
}
20102023

20112024
/// The body policy's verdict on a replaced body, or `None` when it passes.
2012-
fn transformed_body_denial(policy: TransformedBodyPolicy<'_>, body: &[u8]) -> Option<String> {
2025+
fn transformed_body_denial(
2026+
policy: TransformedBodyPolicy<'_>,
2027+
body: &[u8],
2028+
) -> Option<(String, Option<HttpRequestFailureKind>)> {
20132029
let TransformedBodyPolicy::Reevaluate(validate) = policy else {
20142030
return None;
20152031
};
20162032
match validate(body) {
2017-
Ok(reason) => reason,
2018-
Err(error) => Some(format!(
2019-
"transformed_body_policy_evaluation_failed: {}",
2020-
safe_reason(&error.to_string())
2033+
Ok(reason) => reason.map(|reason| (reason, None)),
2034+
Err(error) => Some((
2035+
format!(
2036+
"transformed_body_policy_evaluation_failed: {}",
2037+
safe_reason(&error.to_string())
2038+
),
2039+
Some(HttpRequestFailureKind::PolicyEvaluation),
20212040
)),
20222041
}
20232042
}
@@ -2179,7 +2198,16 @@ async fn drive_stream(
21792198
held.send_replace(false);
21802199
}
21812200
StageResult::OutputChunk(chunk) if started => {
2182-
if chunk.data.is_empty() || chunk.data.len() > chunk_limit {
2201+
if chunk.data.len() > chunk_limit {
2202+
return Err(entry_failure_with_code(
2203+
entry,
2204+
"stream_output_chunk_invalid",
2205+
HttpFailureCode::LifecycleViolation,
2206+
HttpRequestFailureKind::Capacity,
2207+
)
2208+
.into());
2209+
}
2210+
if chunk.data.is_empty() {
21832211
return Err(entry_failure(entry, "stream_output_chunk_invalid").into());
21842212
}
21852213
output_bytes = output_bytes.saturating_add(chunk.data.len());
@@ -2266,19 +2294,27 @@ pub(super) fn classify_result(
22662294
/// Classify an RPC status. Every status fails the exchange closed; the reason
22672295
/// is platform-owned for external services.
22682296
fn status_failure(entry: &DescribedChainEntry, status: &tonic::Status) -> HttpMiddlewareFailure {
2269-
match status.code() {
2270-
tonic::Code::DeadlineExceeded => entry_failure(entry, "middleware_timeout"),
2271-
// The service does not implement the HTTP hook RPC its binding
2272-
// selected.
2273-
tonic::Code::Unimplemented => entry_failure(entry, "middleware_unimplemented"),
2274-
// The reason may carry an in-process service's status text, so the
2275-
// sandbox sees only that the middleware failed.
2276-
_ => entry_failure_with_code(
2277-
entry,
2278-
&diagnostic_policy(entry).error_reason(status),
2297+
let (reason, code) = match status.code() {
2298+
tonic::Code::DeadlineExceeded => {
2299+
("middleware_timeout".to_string(), HttpFailureCode::Timeout)
2300+
}
2301+
// The service does not implement the HTTP hook RPC its binding selected.
2302+
tonic::Code::Unimplemented => (
2303+
"middleware_unimplemented".to_string(),
2304+
HttpFailureCode::LifecycleViolation,
2305+
),
2306+
// In-process status text stays in diagnostics, never the sandbox code.
2307+
_ => (
2308+
diagnostic_policy(entry).error_reason(status),
22792309
HttpFailureCode::Unavailable,
22802310
),
2281-
}
2311+
};
2312+
entry_failure_with_code(
2313+
entry,
2314+
&reason,
2315+
code,
2316+
HttpRequestFailureKind::RemoteStatus(status.code()),
2317+
)
22822318
}
22832319

22842320
fn diagnostic_policy(entry: &DescribedChainEntry) -> MiddlewareDiagnosticPolicy {
@@ -2298,6 +2334,7 @@ fn mutation_failure(
22982334
entry,
22992335
&diagnostic_policy(entry).header_mutation_error_reason(error),
23002336
HttpFailureCode::HeaderMutation(error.code()),
2337+
HttpRequestFailureKind::HeaderMutation,
23012338
)
23022339
}
23032340

@@ -2413,6 +2450,7 @@ pub(super) fn denial(
24132450
.invocations
24142451
.push(invocation(entry, HttpStageOutcome::Deny, reason_code));
24152452
HttpMiddlewareFailure {
2453+
kind: None,
24162454
reason: middleware_denial_reason(&denial.config_name, denial.reason_code.as_deref()),
24172455
denial: Some(denial),
24182456
failure: None,
@@ -2423,22 +2461,33 @@ pub(super) fn denial(
24232461

24242462
/// The stage failed and the exchange fails closed.
24252463
pub fn entry_failure(entry: &DescribedChainEntry, reason: &str) -> HttpMiddlewareFailure {
2426-
entry_failure_with_code(entry, reason, HttpFailureCode::of_reason(reason))
2464+
entry_failure_with_code(
2465+
entry,
2466+
reason,
2467+
HttpFailureCode::of_reason(reason),
2468+
HttpRequestFailureKind::of_platform_reason(reason),
2469+
)
24272470
}
24282471

24292472
/// The stage failed with `reason`, which the sandbox sees as `code`.
24302473
fn entry_failure_with_code(
24312474
entry: &DescribedChainEntry,
24322475
reason: &str,
24332476
code: HttpFailureCode,
2477+
kind: HttpRequestFailureKind,
24342478
) -> HttpMiddlewareFailure {
2435-
let mut diagnostics = HttpStageDiagnostics::default();
2479+
let mut diagnostics = HttpStageDiagnostics {
2480+
failure_kind: Some(kind),
2481+
..Default::default()
2482+
};
24362483
diagnostics.invocations.push(HttpStageInvocation {
2484+
failure_kind: Some(kind),
24372485
failed: true,
24382486
failure_reason: Some(reason.to_string()),
24392487
..invocation(entry, HttpStageOutcome::FailClosed, None)
24402488
});
24412489
HttpMiddlewareFailure {
2490+
kind: Some(kind),
24422491
reason: format!("middleware_failed: {reason}"),
24432492
denial: None,
24442493
failure: Some(Box::new(MiddlewareFailureDetail {
@@ -2452,22 +2501,32 @@ fn entry_failure_with_code(
24522501

24532502
/// The exchange failed for a reason no single stage owns.
24542503
pub fn platform_failure(reason: &str) -> HttpMiddlewareFailure {
2504+
let kind = HttpRequestFailureKind::InvalidResult;
24552505
HttpMiddlewareFailure {
2506+
kind: Some(kind),
24562507
reason: format!("middleware_failed: {reason}"),
24572508
denial: None,
24582509
failure: None,
24592510
end_reason: MiddlewareSessionEndReason::MiddlewareFailure,
2460-
diagnostics: Box::default(),
2511+
diagnostics: Box::new(HttpStageDiagnostics {
2512+
failure_kind: Some(kind),
2513+
..Default::default()
2514+
}),
24612515
}
24622516
}
24632517

24642518
fn cancelled(reason: &str) -> HttpMiddlewareFailure {
2519+
let kind = HttpRequestFailureKind::Cancelled;
24652520
HttpMiddlewareFailure {
2521+
kind: Some(kind),
24662522
reason: format!("middleware_cancelled: {reason}"),
24672523
denial: None,
24682524
failure: None,
24692525
end_reason: MiddlewareSessionEndReason::Cancellation,
2470-
diagnostics: Box::default(),
2526+
diagnostics: Box::new(HttpStageDiagnostics {
2527+
failure_kind: Some(kind),
2528+
..Default::default()
2529+
}),
24712530
}
24722531
}
24732532

@@ -2477,6 +2536,7 @@ pub fn invocation(
24772536
reason_code: Option<String>,
24782537
) -> HttpStageInvocation {
24792538
HttpStageInvocation {
2539+
failure_kind: None,
24802540
config_name: entry.entry.name.clone(),
24812541
implementation: entry.entry.implementation.clone(),
24822542
outcome,
@@ -2491,6 +2551,7 @@ pub fn invocation(
24912551

24922552
pub fn fail_open_invocation(entry: &DescribedChainEntry, reason: &str) -> HttpStageInvocation {
24932553
HttpStageInvocation {
2554+
failure_kind: Some(HttpRequestFailureKind::of_platform_reason(reason)),
24942555
failed: true,
24952556
failure_reason: Some(reason.to_string()),
24962557
..invocation(entry, HttpStageOutcome::FailOpen, None)

‎crates/openshell-supervisor-middleware/src/http_hooks/request.rs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -267,7 +267,13 @@ fn capacity_exhausted(headers: Vec<HttpHeader>) -> HttpRequestPreflightOutcome {
267267
allowed: false,
268268
reason: "middleware_failed: session_capacity_exhausted".into(),
269269
session_capacity_exhausted: true,
270-
..outcome(headers, HttpStageDiagnostics::default())
270+
..outcome(
271+
headers,
272+
HttpStageDiagnostics {
273+
failure_kind: Some(crate::HttpRequestFailureKind::Capacity),
274+
..Default::default()
275+
},
276+
)
271277
}
272278
}
273279

0 commit comments

Comments
 (0)