Skip to content

Commit 978f71b

Browse files
committed
feat(supervisor): export OCSF events as OTLP log records
Supervisor OCSF events reached only the sandbox's local JSONL file and the shorthand log lines sent to the gateway, so collectors had no structured copy of most of the security record. When the gateway has `[openshell.gateway.otlp.ocsf]` configured, compute drivers now pass its endpoint and schema version to supervisors, which export their own OCSF events directly as OTLP log records. Supervisors already have egress, so the gateway and its supervisor connection carry no extra load. Supervisors use the same `openshell-otel` exporter as the gateway, with a supervisor resource identity that includes `openshell.sandbox.id` and `openshell.gateway.name`, so telemetry from sandboxes of different gateways sharing a collector can be told apart. This applies to supervisor spans too. Drivers pass the settings to supervisors as `OPENSHELL_OTLP_OCSF_ENDPOINT`, `OPENSHELL_OTLP_OCSF_SCHEMA_VERSION`, and `OPENSHELL_GATEWAY_NAME`; the gateway passes them to the VM driver as arguments. Collector headers that the gateway uses for OCSF export (`OTEL_EXPORTER_OTLP_LOGS_HEADERS`, else `OTEL_EXPORTER_OTLP_HEADERS`) travel in each sandbox's supervisor authentication bundle, which only the supervisor reads, so authenticated collectors accept supervisor records without exposing credentials to the workload. The VM driver now sends SIGTERM and waits for the supervisor to exit before closing its liveness pipe. Closing the pipe first made the supervisor exit at once and drop queued records. Closes #4544 Refs #2892 Signed-off-by: Kris Hicks <[email protected]>
1 parent 582bcd8 commit 978f71b

29 files changed

Lines changed: 1007 additions & 68 deletions

File tree

‎crates/openshell-core/src/jwt.rs‎

Lines changed: 61 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,48 @@ mod session {
269269
}
270270
}
271271

272+
/// OTLP collector request headers, such as an authorization token, in
273+
/// the `OTEL_EXPORTER_OTLP_HEADERS` format. Redacted from diagnostics and
274+
/// cleared on drop.
275+
#[derive(Clone)]
276+
pub struct SecretOtlpHeaders(Zeroizing<String>);
277+
278+
impl SecretOtlpHeaders {
279+
#[must_use]
280+
pub fn new(value: String) -> Self {
281+
Self(Zeroizing::new(value))
282+
}
283+
284+
#[must_use]
285+
pub fn expose_secret(&self) -> &str {
286+
&self.0
287+
}
288+
}
289+
290+
impl fmt::Debug for SecretOtlpHeaders {
291+
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
292+
formatter.write_str("SecretOtlpHeaders([REDACTED])")
293+
}
294+
}
295+
296+
impl Serialize for SecretOtlpHeaders {
297+
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
298+
where
299+
S: serde::Serializer,
300+
{
301+
serializer.serialize_str(self.expose_secret())
302+
}
303+
}
304+
305+
impl<'de> Deserialize<'de> for SecretOtlpHeaders {
306+
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
307+
where
308+
D: serde::Deserializer<'de>,
309+
{
310+
String::deserialize(deserializer).map(Self::new)
311+
}
312+
}
313+
272314
/// Trusted launch input delivered only to `openshell-supervisor`.
273315
#[derive(Clone, Serialize, Deserialize)]
274316
#[serde(deny_unknown_fields)]
@@ -287,6 +329,10 @@ mod session {
287329
/// Never delivered to the workload. Missing only in older bundles.
288330
#[serde(default, skip_serializing_if = "Option::is_none")]
289331
pub ssh_host_private_key: Option<SecretSshHostKey>,
332+
/// Headers for the OCSF OTLP collector, present only when the gateway
333+
/// has them. Never delivered to the workload.
334+
#[serde(default, skip_serializing_if = "Option::is_none")]
335+
pub ocsf_otlp_headers: Option<SecretOtlpHeaders>,
290336
}
291337

292338
/// Gateway-created authentication input trusted by a compute driver.
@@ -373,6 +419,7 @@ mod session {
373419
.field("sandbox_token", &"[REDACTED]")
374420
.field("sandbox_expires_at", &self.sandbox_expires_at)
375421
.field("ssh_host_private_key", &self.ssh_host_private_key)
422+
.field("ocsf_otlp_headers", &self.ocsf_otlp_headers)
376423
.finish()
377424
}
378425
}
@@ -1080,6 +1127,7 @@ mod tests {
10801127
sandbox_token: pair.sandbox.token,
10811128
sandbox_expires_at: pair.sandbox.expires_at,
10821129
ssh_host_private_key: None,
1130+
ocsf_otlp_headers: None,
10831131
};
10841132
bundle.validate().expect("non-expiring auth bundle");
10851133
}
@@ -1109,6 +1157,9 @@ mod tests {
11091157
ssh_host_private_key: Some(SecretSshHostKey::new(
11101158
"private-ssh-host-key".to_string(),
11111159
)),
1160+
ocsf_otlp_headers: Some(SecretOtlpHeaders::new(
1161+
"authorization=collector-token".to_string(),
1162+
)),
11121163
};
11131164

11141165
let encoded = serde_json::to_vec(&bundle).expect("serialize auth bundle");
@@ -1128,18 +1179,23 @@ mod tests {
11281179
assert!(!debug.contains(bundle.gateway_token.expose_secret()));
11291180
assert!(!debug.contains(bundle.sandbox_token.expose_secret()));
11301181
assert!(!debug.contains("private-ssh-host-key"));
1131-
assert_eq!(debug.matches("[REDACTED]").count(), 3);
1182+
assert!(!debug.contains("collector-token"));
1183+
assert_eq!(debug.matches("[REDACTED]").count(), 4);
11321184
assert_eq!(
11331185
decoded.ssh_host_private_key.unwrap().expose_secret(),
11341186
"private-ssh-host-key"
11351187
);
1188+
assert_eq!(
1189+
decoded.ocsf_otlp_headers.unwrap().expose_secret(),
1190+
"authorization=collector-token"
1191+
);
11361192
let mut legacy = serde_json::to_value(&bundle).unwrap();
1137-
legacy
1138-
.as_object_mut()
1139-
.unwrap()
1140-
.remove("ssh_host_private_key");
1193+
let fields = legacy.as_object_mut().unwrap();
1194+
fields.remove("ssh_host_private_key");
1195+
fields.remove("ocsf_otlp_headers");
11411196
let legacy: SupervisorAuthBundle = serde_json::from_value(legacy).unwrap();
11421197
assert!(legacy.ssh_host_private_key.is_none());
1198+
assert!(legacy.ocsf_otlp_headers.is_none());
11431199
}
11441200

11451201
#[derive(Serialize)]

‎crates/openshell-core/src/sandbox_env.rs‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,36 @@ pub const LOG_LEVEL: &str = "OPENSHELL_LOG_LEVEL";
2929
/// OTLP/gRPC collector endpoint for supervisor trace export.
3030
pub const OTLP_ENDPOINT: &str = "OPENSHELL_OTLP_ENDPOINT";
3131

32+
/// OTLP/gRPC collector endpoint for supervisor OCSF event export.
33+
pub const OTLP_OCSF_ENDPOINT: &str = "OPENSHELL_OTLP_OCSF_ENDPOINT";
34+
35+
/// OCSF schema version that supervisors downgrade exported events to.
36+
pub const OTLP_OCSF_SCHEMA_VERSION: &str = "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION";
37+
38+
/// Name of the gateway that launched the sandbox, reported as
39+
/// `openshell.gateway.name` on supervisor telemetry.
40+
pub const GATEWAY_NAME: &str = "OPENSHELL_GATEWAY_NAME";
41+
42+
/// OCSF event export that a compute driver passes to its supervisors.
43+
#[derive(Clone, Debug, Eq, PartialEq)]
44+
pub struct OcsfOtlpExport {
45+
pub endpoint: String,
46+
pub schema_version: Option<String>,
47+
}
48+
49+
impl OcsfOtlpExport {
50+
/// The supervisor environment variables for this export.
51+
pub fn environment(&self) -> Vec<(&'static str, String)> {
52+
std::iter::once((OTLP_OCSF_ENDPOINT, self.endpoint.clone()))
53+
.chain(
54+
self.schema_version
55+
.clone()
56+
.map(|version| (OTLP_OCSF_SCHEMA_VERSION, version)),
57+
)
58+
.collect()
59+
}
60+
}
61+
3262
/// Versioned specification for the exact canonical main process.
3363
///
3464
/// Most drivers use JSON directly. Transports that cannot preserve spaces in
@@ -278,6 +308,26 @@ pub const OCI_IMAGE_USER: &str = "OPENSHELL_OCI_IMAGE_USER";
278308
mod tests {
279309
use super::*;
280310

311+
#[test]
312+
fn ocsf_export_environment_carries_the_schema_version_only_when_set() {
313+
let mut export = OcsfOtlpExport {
314+
endpoint: "http://siem:4317".into(),
315+
schema_version: None,
316+
};
317+
assert_eq!(
318+
export.environment(),
319+
[(OTLP_OCSF_ENDPOINT, "http://siem:4317".to_string())]
320+
);
321+
export.schema_version = Some("1.3".into());
322+
assert_eq!(
323+
export.environment(),
324+
[
325+
(OTLP_OCSF_ENDPOINT, "http://siem:4317".to_string()),
326+
(OTLP_OCSF_SCHEMA_VERSION, "1.3".to_string()),
327+
]
328+
);
329+
}
330+
281331
#[test]
282332
fn main_process_transport_preserves_argument_boundaries() {
283333
let spec = crate::proto::compute::v1::DriverSandboxSpec {

‎crates/openshell-driver-docker/src/lib.rs‎

Lines changed: 35 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,16 @@ pub struct DockerComputeConfig {
185185
#[serde(skip)]
186186
pub supervisor_otlp_endpoint: Option<String>,
187187

188+
/// OCSF event export passed to supervisors. The gateway supplies its own
189+
/// export; driver TOML cannot set it.
190+
#[serde(skip)]
191+
pub supervisor_ocsf_export: Option<openshell_core::sandbox_env::OcsfOtlpExport>,
192+
193+
/// Gateway name passed to supervisors for their telemetry. The gateway
194+
/// supplies its own name; driver TOML cannot set it.
195+
#[serde(skip)]
196+
pub supervisor_gateway_name: Option<String>,
197+
188198
/// Image containing the trusted `openshell-sandbox` binary.
189199
pub sandbox_runtime_image: Option<String>,
190200

@@ -284,6 +294,8 @@ impl Default for DockerComputeConfig {
284294
sandbox_label: "default".to_string(),
285295
grpc_endpoint: String::new(),
286296
supervisor_otlp_endpoint: None,
297+
supervisor_ocsf_export: None,
298+
supervisor_gateway_name: None,
287299
sandbox_runtime_image: None,
288300
supervisor_bin: None,
289301
supervisor_image: None,
@@ -319,6 +331,8 @@ struct DockerDriverRuntimeConfig {
319331
supervisor_image_id: String,
320332
supervisor_grpc_endpoint: String,
321333
supervisor_otlp_endpoint: Option<String>,
334+
supervisor_ocsf_export: Option<openshell_core::sandbox_env::OcsfOtlpExport>,
335+
supervisor_gateway_name: Option<String>,
322336
ssh_socket_path: String,
323337
guest_tls: Option<DockerGuestTlsPaths>,
324338
gpu: DockerGpuRuntimeCapabilities,
@@ -949,6 +963,8 @@ impl DockerComputeDriver {
949963
supervisor_image_id,
950964
supervisor_grpc_endpoint,
951965
supervisor_otlp_endpoint: docker_config.supervisor_otlp_endpoint.clone(),
966+
supervisor_ocsf_export: docker_config.supervisor_ocsf_export.clone(),
967+
supervisor_gateway_name: docker_config.supervisor_gateway_name.clone(),
952968
ssh_socket_path: docker_config.ssh_socket_path.clone(),
953969
guest_tls,
954970
gpu,
@@ -5103,7 +5119,7 @@ async fn spawn_docker_control_process(
51035119
openshell_core::telemetry::enabled_env_value()
51045120
),
51055121
];
5106-
environment.extend(supervisor_tracing_environment(config));
5122+
environment.extend(supervisor_otlp_environment(config));
51075123
if config.guest_tls.is_some() {
51085124
environment.push(format!(
51095125
"{}={SUPERVISOR_STATE_MOUNT_PATH}/tls/ca.pem",
@@ -5634,13 +5650,24 @@ fn docker_child_environment(sandbox: &DriverSandbox) -> HashMap<String, String>
56345650
environment
56355651
}
56365652

5637-
/// Environment that lets the supervisor export spans and join the current trace.
5638-
fn supervisor_tracing_environment(config: &DockerDriverRuntimeConfig) -> Vec<String> {
5639-
let Some(endpoint) = &config.supervisor_otlp_endpoint else {
5640-
return Vec::new();
5641-
};
5642-
std::iter::once((openshell_core::sandbox_env::OTLP_ENDPOINT, endpoint.clone()))
5643-
.chain(openshell_otel::current_trace_context_environment())
5653+
/// Environment that lets the supervisor export spans, joining the current
5654+
/// trace, and OCSF events, identifying the gateway that launched it.
5655+
fn supervisor_otlp_environment(config: &DockerDriverRuntimeConfig) -> Vec<String> {
5656+
let traces = config.supervisor_otlp_endpoint.iter().flat_map(|endpoint| {
5657+
std::iter::once((openshell_core::sandbox_env::OTLP_ENDPOINT, endpoint.clone()))
5658+
.chain(openshell_otel::current_trace_context_environment())
5659+
});
5660+
let ocsf = config
5661+
.supervisor_ocsf_export
5662+
.iter()
5663+
.flat_map(openshell_core::sandbox_env::OcsfOtlpExport::environment);
5664+
let gateway_name = config
5665+
.supervisor_gateway_name
5666+
.iter()
5667+
.map(|name| (openshell_core::sandbox_env::GATEWAY_NAME, name.clone()));
5668+
traces
5669+
.chain(ocsf)
5670+
.chain(gateway_name)
56445671
.map(|(name, value)| format!("{name}={value}"))
56455672
.collect()
56465673
}

‎crates/openshell-driver-docker/src/main.rs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,18 @@ struct Args {
4747
#[arg(long, env = "OPENSHELL_OTLP_ENDPOINT")]
4848
otlp_endpoint: Option<String>,
4949

50+
/// OTLP/gRPC collector endpoint for supervisor OCSF event export.
51+
#[arg(long, env = "OPENSHELL_OTLP_OCSF_ENDPOINT")]
52+
otlp_ocsf_endpoint: Option<String>,
53+
54+
/// OCSF schema version that supervisors downgrade exported events to.
55+
#[arg(
56+
long,
57+
env = "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION",
58+
requires = "otlp_ocsf_endpoint"
59+
)]
60+
otlp_ocsf_schema_version: Option<String>,
61+
5062
#[arg(long, env = "OPENSHELL_GATEWAY_NAME")]
5163
gateway_name: Option<String>,
5264
}
@@ -78,6 +90,13 @@ async fn main() -> Result<()> {
7890
docker_config.supervisor_image = Some(image);
7991
}
8092
docker_config.supervisor_otlp_endpoint = args.otlp_endpoint.clone();
93+
docker_config.supervisor_gateway_name = args.gateway_name.clone();
94+
docker_config.supervisor_ocsf_export = args.otlp_ocsf_endpoint.clone().map(|endpoint| {
95+
openshell_core::sandbox_env::OcsfOtlpExport {
96+
endpoint,
97+
schema_version: args.otlp_ocsf_schema_version.clone(),
98+
}
99+
});
81100
let driver = DockerComputeDriver::new(args.gateway_bind, &args.log_level, &docker_config)
82101
.await
83102
.into_diagnostic()?;

‎crates/openshell-driver-docker/src/tests.rs‎

Lines changed: 30 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ fn test_launch_authentication() -> Vec<u8> {
6464
sandbox_token: SecretJwt::parse("sandbox.token.value").unwrap(),
6565
sandbox_expires_at: i64::MAX,
6666
ssh_host_private_key: None,
67+
ocsf_otlp_headers: None,
6768
},
6869
gateway_id: "gateway-test".to_string(),
6970
verification_keys: vec![SessionVerificationKey {
@@ -212,6 +213,8 @@ fn runtime_config() -> DockerDriverRuntimeConfig {
212213
supervisor_image_id: "sha256:supervisor-test".to_string(),
213214
supervisor_grpc_endpoint: "https://host.openshell.internal:8443".to_string(),
214215
supervisor_otlp_endpoint: None,
216+
supervisor_ocsf_export: None,
217+
supervisor_gateway_name: None,
215218
ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(),
216219
guest_tls: Some(DockerGuestTlsPaths {
217220
ca: PathBuf::from("/tmp/ca.crt"),
@@ -1392,17 +1395,37 @@ fn docker_child_environment_strips_supervisor_control_keys() {
13921395
}
13931396

13941397
#[test]
1395-
fn supervisor_tracing_environment_requires_an_endpoint() {
1398+
fn supervisor_otlp_environment_carries_only_configured_signals() {
1399+
use openshell_core::sandbox_env::{
1400+
GATEWAY_NAME, OTLP_ENDPOINT, OTLP_OCSF_ENDPOINT, OTLP_OCSF_SCHEMA_VERSION, OcsfOtlpExport,
1401+
};
1402+
13961403
let mut config = runtime_config();
1397-
assert!(supervisor_tracing_environment(&config).is_empty());
1404+
assert!(supervisor_otlp_environment(&config).is_empty());
13981405

13991406
config.supervisor_otlp_endpoint = Some("http://127.0.0.1:4317".to_string());
14001407
assert_eq!(
1401-
supervisor_tracing_environment(&config),
1402-
[format!(
1403-
"{}=http://127.0.0.1:4317",
1404-
openshell_core::sandbox_env::OTLP_ENDPOINT
1405-
)]
1408+
supervisor_otlp_environment(&config),
1409+
[format!("{OTLP_ENDPOINT}=http://127.0.0.1:4317")]
1410+
);
1411+
1412+
config.supervisor_otlp_endpoint = None;
1413+
config.supervisor_ocsf_export = Some(OcsfOtlpExport {
1414+
endpoint: "http://siem:4317".to_string(),
1415+
schema_version: Some("1.3".to_string()),
1416+
});
1417+
assert_eq!(
1418+
supervisor_otlp_environment(&config),
1419+
[
1420+
format!("{OTLP_OCSF_ENDPOINT}=http://siem:4317"),
1421+
format!("{OTLP_OCSF_SCHEMA_VERSION}=1.3"),
1422+
]
1423+
);
1424+
1425+
config.supervisor_gateway_name = Some("prod-west".to_string());
1426+
assert_eq!(
1427+
supervisor_otlp_environment(&config).last(),
1428+
Some(&format!("{GATEWAY_NAME}=prod-west"))
14061429
);
14071430
}
14081431

‎crates/openshell-driver-kubernetes/src/config.rs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,15 @@ pub struct KubernetesComputeConfig {
248248
/// supplies its own export endpoint; driver TOML cannot set it.
249249
#[serde(skip)]
250250
pub supervisor_otlp_endpoint: Option<String>,
251+
/// OCSF event export passed to supervisors. The gateway supplies its own
252+
/// export; driver TOML cannot set it.
253+
#[serde(skip)]
254+
pub supervisor_ocsf_export: Option<openshell_core::sandbox_env::OcsfOtlpExport>,
255+
256+
/// Gateway name passed to supervisors for their telemetry. The gateway
257+
/// supplies its own name; driver TOML cannot set it.
258+
#[serde(skip)]
259+
pub supervisor_gateway_name: Option<String>,
251260
pub ssh_socket_path: String,
252261
pub client_tls_secret_name: String,
253262
pub host_gateway_ip: String,
@@ -370,6 +379,8 @@ impl Default for KubernetesComputeConfig {
370379
proxy_ca_bundle: None,
371380
grpc_endpoint: String::new(),
372381
supervisor_otlp_endpoint: None,
382+
supervisor_ocsf_export: None,
383+
supervisor_gateway_name: None,
373384
ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(),
374385
client_tls_secret_name: String::new(),
375386
host_gateway_ip: String::new(),

0 commit comments

Comments
 (0)