Skip to content

Commit 4f827ab

Browse files
committed
fix(supervisor-network): classify refusals and bound deny spans
Mark remaining inspection, capacity, admission and response refusals as Denied while preserving fail-open and I/O error classification. Add a process-wide deny-span budget alongside destination limits, with audit-preservation tests and matching documentation. Signed-off-by: Harry Turner <[email protected]>
1 parent 2bb7e01 commit 4f827ab

7 files changed

Lines changed: 305 additions & 25 deletions

File tree

‎crates/openshell-supervisor-network/src/l7/middleware.rs‎

Lines changed: 90 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ pub fn uninspectable_traffic_gate(
147147
/// Emit the detection finding for traffic a matching middleware chain cannot
148148
/// inspect: denied under a fail-closed chain, bypassed under fail-open.
149149
pub fn emit_middleware_uninspectable(ctx: &L7EvalContext, detail: &str, denied: bool) {
150-
let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx())
150+
let mut event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx())
151151
.severity(if denied {
152152
SeverityId::High
153153
} else {
@@ -167,9 +167,11 @@ pub fn emit_middleware_uninspectable(ctx: &L7EvalContext, detail: &str, denied:
167167
"Uninspectable traffic to host with required middleware; denied"
168168
} else {
169169
"Uninspectable traffic bypassed middleware (fail_open)"
170-
})
171-
.build();
172-
egress_emit!(event);
170+
});
171+
if denied {
172+
event = event.action(ActionId::Denied);
173+
}
174+
egress_emit!(event.build());
173175
}
174176

175177
pub fn emit_websocket_preflight_events(
@@ -185,7 +187,7 @@ pub fn emit_websocket_preflight_events(
185187
emit_websocket_saturation(ctx);
186188
}
187189
if outcome.session_capacity_exhausted {
188-
emit_middleware_session_capacity_exhausted(ctx);
190+
emit_middleware_session_capacity_exhausted(ctx, !outcome.allowed);
189191
}
190192
}
191193

@@ -454,8 +456,8 @@ fn emit_websocket_saturation(ctx: &L7EvalContext) {
454456
egress_emit!(event);
455457
}
456458

457-
fn emit_middleware_session_capacity_exhausted(ctx: &L7EvalContext) {
458-
let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx())
459+
fn emit_middleware_session_capacity_exhausted(ctx: &L7EvalContext, denied: bool) {
460+
let mut event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx())
459461
.severity(SeverityId::Medium)
460462
.finding_info(FindingInfo::new(
461463
"openshell.middleware.session_capacity_exhausted",
@@ -466,13 +468,16 @@ fn emit_middleware_session_capacity_exhausted(ctx: &L7EvalContext) {
466468
("host", ctx.host.as_str()),
467469
("operation", "websocket_message"),
468470
])
469-
.message("Persistent middleware session admission was refused at process capacity")
470-
.build();
471-
egress_emit!(event);
471+
.message("Persistent middleware session admission was refused at process capacity");
472+
if denied {
473+
event = event.action(ActionId::Denied);
474+
}
475+
egress_emit!(event.build());
472476
}
473477

474478
fn middleware_admission_exhausted_event(ctx: &L7EvalContext) -> openshell_ocsf::OcsfEvent {
475479
DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx())
480+
.action(ActionId::Denied)
476481
.severity(SeverityId::Medium)
477482
.finding_info(FindingInfo::new(
478483
"openshell.middleware.admission_exhausted",
@@ -796,7 +801,7 @@ pub(super) fn resolve_unbuffered_body(
796801
}
797802

798803
fn emit_middleware_body_unavailable(ctx: &L7EvalContext, denied: bool) {
799-
let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx())
804+
let mut event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx())
800805
.severity(if denied {
801806
SeverityId::High
802807
} else {
@@ -815,9 +820,11 @@ fn emit_middleware_body_unavailable(ctx: &L7EvalContext, denied: bool) {
815820
"Request body exceeded middleware inspection cap; denied"
816821
} else {
817822
"Request body exceeded middleware inspection cap; passed through (fail_open)"
818-
})
819-
.build();
820-
egress_emit!(event);
823+
});
824+
if denied {
825+
event = event.action(ActionId::Denied);
826+
}
827+
egress_emit!(event.build());
821828
}
822829

823830
/// Parse the raw header block into middleware-visible headers, preserving
@@ -1109,6 +1116,71 @@ mod trace_tests {
11091116
}
11101117
}
11111118

1119+
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1120+
async fn inspection_refusals_record_denial_but_fail_open_findings_do_not() {
1121+
for denied in [false, true] {
1122+
let (events, received) = exported("warn", || {
1123+
let ctx = L7EvalContext::default();
1124+
emit_middleware_uninspectable(&ctx, "unknown protocol", denied);
1125+
emit_middleware_body_unavailable(&ctx, denied);
1126+
})
1127+
.await;
1128+
assert_eq!(events.len(), 2);
1129+
for (event, uid) in events.iter().zip([
1130+
"openshell.middleware.traffic_uninspectable",
1131+
"openshell.middleware.body_unavailable",
1132+
]) {
1133+
assert_eq!(event["finding_info"]["uid"], uid);
1134+
assert_eq!(event["class_uid"], 2004);
1135+
assert_eq!(event["action_id"].as_u64(), denied.then_some(2));
1136+
assert_eq!(event["action"].as_str(), denied.then_some("Denied"));
1137+
if !denied {
1138+
assert!(event.get("action_id").is_none());
1139+
assert!(event.get("action").is_none());
1140+
}
1141+
}
1142+
// A finding-only refusal still creates no HTTP deny span.
1143+
assert_no_deny(&events, &received);
1144+
}
1145+
}
1146+
1147+
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1148+
async fn session_capacity_findings_follow_the_preflight_outcome() {
1149+
for allowed in [true, false] {
1150+
let (events, received) = exported("warn", || {
1151+
emit_websocket_preflight_events(
1152+
&L7EvalContext::default(),
1153+
&openshell_supervisor_middleware::WebSocketPreflightResult {
1154+
allowed,
1155+
terminal_reason: None,
1156+
reason: "middleware_session_capacity_exhausted".into(),
1157+
denial: None,
1158+
session: None,
1159+
findings: vec![],
1160+
metadata: std::collections::BTreeMap::new(),
1161+
invocations: vec![],
1162+
coverage: vec![],
1163+
saturated: false,
1164+
session_capacity_exhausted: true,
1165+
},
1166+
);
1167+
})
1168+
.await;
1169+
assert_eq!(events.len(), 1);
1170+
assert_eq!(
1171+
events[0]["finding_info"]["uid"],
1172+
"openshell.middleware.session_capacity_exhausted"
1173+
);
1174+
assert_eq!(events[0]["action_id"].as_u64(), (!allowed).then_some(2));
1175+
assert_eq!(events[0]["action"].as_str(), (!allowed).then_some("Denied"));
1176+
if allowed {
1177+
assert!(events[0].get("action_id").is_none());
1178+
assert!(events[0].get("action").is_none());
1179+
}
1180+
assert_no_deny(&events, &received);
1181+
}
1182+
}
1183+
11121184
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11131185
async fn http_chain_outcomes_group_findings_and_preserve_allowed_controls() {
11141186
for allowed in [true, false] {
@@ -1169,6 +1241,10 @@ mod tests {
11691241

11701242
let serialized = serde_json::to_string(&middleware_admission_exhausted_event(&ctx))
11711243
.expect("serialize admission event");
1244+
let event: serde_json::Value = serde_json::from_str(&serialized).unwrap();
1245+
assert_eq!(event["class_uid"], 2004);
1246+
assert_eq!(event["action_id"], 2);
1247+
assert_eq!(event["action"], "Denied");
11721248
assert!(serialized.contains("openshell.middleware.admission_exhausted"));
11731249
assert!(serialized.contains("api-policy"));
11741250
assert!(serialized.contains("api.example.test"));

‎crates/openshell-supervisor-network/src/l7/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -462,6 +462,7 @@ pub fn parse_l7_config(val: &regorus::Value) -> Option<L7EndpointConfig> {
462462

463463
pub(crate) fn emit_uninspected_credential_finding(host: &str, policy_name: &str, surface: &str) {
464464
let event = openshell_ocsf::DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx())
465+
.action(openshell_ocsf::ActionId::Denied)
465466
.severity(openshell_ocsf::SeverityId::High)
466467
.finding_info(openshell_ocsf::FindingInfo::new(
467468
"openshell.credentials.traffic_uninspectable",

‎crates/openshell-supervisor-network/src/l7/rest.rs‎

Lines changed: 54 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1536,6 +1536,7 @@ fn emit_uninspected_body_credential_denial(
15361536
.build();
15371537
crate::telemetry::egress_emit!(event);
15381538
let finding = openshell_ocsf::DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx())
1539+
.action(openshell_ocsf::ActionId::Denied)
15391540
.severity(openshell_ocsf::SeverityId::High)
15401541
.finding_info(openshell_ocsf::FindingInfo::new(
15411542
"openshell.credentials.traffic_uninspectable",
@@ -6977,6 +6978,43 @@ mod tests {
69776978
}
69786979
}
69796980

6981+
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6982+
async fn credential_refusal_findings_record_denial_without_http_spans() {
6983+
let request = L7Request {
6984+
action: "POST".into(),
6985+
target: "/api".into(),
6986+
query_params: HashMap::new(),
6987+
raw_header: vec![],
6988+
body_length: BodyLength::None,
6989+
};
6990+
let (events, received) = crate::telemetry::tests::exported("warn", || {
6991+
crate::l7::emit_uninspected_credential_finding("example.test", "p", "test");
6992+
emit_uninspected_body_credential_denial(
6993+
&request,
6994+
&RelayRequestOptions {
6995+
host: "example.test",
6996+
port: 443,
6997+
..Default::default()
6998+
},
6999+
BodyCredentialError::KnownUnavailable,
7000+
);
7001+
})
7002+
.await;
7003+
assert_eq!(events.len(), 3);
7004+
assert_eq!(
7005+
events
7006+
.iter()
7007+
.filter(|event| event["class_uid"] == 2004)
7008+
.count(),
7009+
2
7010+
);
7011+
for event in &events {
7012+
assert_eq!(event["action_id"], 2);
7013+
assert_eq!(event["action"], "Denied");
7014+
}
7015+
crate::telemetry::tests::assert_no_deny(&events, &received);
7016+
}
7017+
69807018
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
69817019
async fn response_trace_upstream_io_failure_is_not_an_enforced_denial() {
69827020
struct FailedReader;
@@ -7024,8 +7062,14 @@ mod tests {
70247062
});
70257063
})
70267064
.await;
7027-
assert!(events.iter().any(|event| event["finding_info"]["uid"]
7028-
== "openshell.middleware.http_response_failure"));
7065+
let finding = events
7066+
.iter()
7067+
.find(|event| {
7068+
event["finding_info"]["uid"] == "openshell.middleware.http_response_failure"
7069+
})
7070+
.expect("upstream I/O failure retains its diagnostic finding");
7071+
assert!(finding.get("action_id").is_none());
7072+
assert!(finding.get("action").is_none());
70297073
crate::telemetry::tests::assert_no_deny(&events, &received);
70307074
}
70317075
}
@@ -7085,10 +7129,14 @@ mod tests {
70857129
});
70867130
})
70877131
.await;
7088-
assert!(
7089-
events.iter().any(|event| event["finding_info"]["uid"]
7090-
== "openshell.middleware.http_response_failure")
7091-
);
7132+
let finding = events
7133+
.iter()
7134+
.find(|event| {
7135+
event["finding_info"]["uid"] == "openshell.middleware.http_response_failure"
7136+
})
7137+
.expect("downstream I/O failure retains its diagnostic finding");
7138+
assert!(finding.get("action_id").is_none());
7139+
assert!(finding.get("action").is_none());
70927140
crate::telemetry::tests::assert_no_deny(&events, &received);
70937141
}
70947142

‎crates/openshell-supervisor-network/src/l7/rest/http_response.rs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1034,6 +1034,7 @@ fn http_response_middleware_block_finding_event(
10341034
}
10351035
Some(
10361036
openshell_ocsf::DetectionFindingBuilder::new(ocsf_ctx())
1037+
.action(openshell_ocsf::ActionId::Denied)
10371038
.severity(openshell_ocsf::SeverityId::Medium)
10381039
.finding_info(openshell_ocsf::FindingInfo::new(
10391040
"openshell.middleware.http_response_blocked",
@@ -2239,6 +2240,14 @@ mod tests {
22392240
// A blocking invocation adds its own block finding.
22402241
assert_eq!(events.len(), if denied { 3 } else { 2 });
22412242
if denied {
2243+
let blocked = events
2244+
.iter()
2245+
.find(|event| {
2246+
event["finding_info"]["uid"] == "openshell.middleware.http_response_blocked"
2247+
})
2248+
.expect("blocked delivery emits its refusal finding");
2249+
assert_eq!(blocked["action_id"], 2);
2250+
assert_eq!(blocked["action"], "Denied");
22422251
assert_eq!(received.spans.len(), 1);
22432252
assert_deny_resolves(&events, &received, true);
22442253
} else {

‎crates/openshell-supervisor-network/src/telemetry.rs‎

Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,9 @@ fn deny_span(event: &HttpActivityEvent, parent: Option<&tracing::Span>) -> Optio
103103
const DENY_SPAN_BURST: u32 = 10;
104104
const DENY_SPAN_REFILL: Duration = Duration::from_secs(1);
105105
const DENY_SPAN_DESTINATIONS: usize = 1024;
106+
/// Bound aggregate span creation even when a workload rotates destinations.
107+
const DENY_SPAN_GLOBAL_BURST: u32 = 100;
108+
const DENY_SPAN_GLOBAL_REFILL: Duration = Duration::from_millis(10);
106109

107110
static LIMITER: LazyLock<Mutex<DenySpanLimiter>> = LazyLock::new(|| Mutex::new(default_limiter()));
108111

@@ -117,13 +120,16 @@ fn reset_deny_span_limiter() {
117120
.unwrap_or_else(std::sync::PoisonError::into_inner) = default_limiter();
118121
}
119122

120-
/// Per-destination token bucket bounding how many deny spans one workload can
121-
/// export. OCSF records are unaffected; a limited decision just has no span.
123+
/// Per-destination and process-wide token buckets bounding deny-span work.
124+
/// OCSF records are unaffected; a limited decision creates no additional span.
122125
struct DenySpanLimiter {
123126
burst: u32,
124127
refill: Duration,
125128
capacity: usize,
126129
buckets: HashMap<String, Bucket>,
130+
global_burst: u32,
131+
global_refill: Duration,
132+
global: Option<Bucket>,
127133
}
128134

129135
struct Bucket {
@@ -133,15 +139,42 @@ struct Bucket {
133139

134140
impl DenySpanLimiter {
135141
fn new(burst: u32, refill: Duration, capacity: usize) -> Self {
142+
Self::with_global_limit(
143+
burst,
144+
refill,
145+
capacity,
146+
DENY_SPAN_GLOBAL_BURST,
147+
DENY_SPAN_GLOBAL_REFILL,
148+
)
149+
}
150+
151+
fn with_global_limit(
152+
burst: u32,
153+
refill: Duration,
154+
capacity: usize,
155+
global_burst: u32,
156+
global_refill: Duration,
157+
) -> Self {
136158
Self {
137159
burst,
138160
refill,
139161
capacity,
140162
buckets: HashMap::new(),
163+
global_burst,
164+
global_refill,
165+
global: None,
141166
}
142167
}
143168

144169
fn allow(&mut self, destination: &str, now: Instant) -> bool {
170+
let global = self.global.get_or_insert(Bucket {
171+
tokens: self.global_burst,
172+
refilled: now,
173+
});
174+
global.refill(self.global_burst, self.global_refill, now);
175+
if global.tokens == 0 {
176+
return false;
177+
}
145178
if !self.buckets.contains_key(destination) && self.buckets.len() >= self.capacity {
146179
for bucket in self.buckets.values_mut() {
147180
bucket.refill(self.burst, self.refill, now);
@@ -161,7 +194,10 @@ impl DenySpanLimiter {
161194
});
162195
bucket.refill(self.burst, self.refill, now);
163196
let allowed = bucket.tokens > 0;
164-
bucket.tokens = bucket.tokens.saturating_sub(1);
197+
if allowed {
198+
bucket.tokens -= 1;
199+
global.tokens -= 1;
200+
}
165201
allowed
166202
}
167203
}

0 commit comments

Comments
 (0)